def _iter_with_header(file, parse, expected_file_types):
maybe_hdr = cbor.load(file)
if isinstance(maybe_hdr, list) and maybe_hdr[0] == 'CAR':
# we have a header
file_type = maybe_hdr[1][0]
if not file_type in expected_file_types:
# print( 'File type tag is expected to be ', (" ".join(expected_file_types)), 'but given file is of type ', file_type)
# print('Did not expect file of type', file_type)
raise WrongCarFileException(file_type, expected_file_types)
# read beginning of variable-length list
if (not file.read(1) == b'\x9f'):
raise BrokenCborFileException()
else:
yield parse(maybe_hdr)
while True:
try:
# Check for break symbol
if (peek_for_break(file)):
break
yield parse(cbor.load(file))
except EOFError:
break
def count_records(infile):
if infile.endswith('.gz'):
fp = gzip.open(infile, 'r')
else:
fp = open(infile, 'r')
count = 0
try:
while (1):
cbor.load(fp)
count += 1
except EOFError:
return count
def count_records(infile):
if infile.endswith('.gz'):
fp = gzip.open(infile, 'r')
else:
fp = open(infile, 'r')
count = 0
try:
while (1):
cbor.load(fp)
count += 1
except EOFError:
return count
def cbor_iter(fh):
while True:
try:
chunk = cbor.load(fh)
except EOFError:
break
yield FeatureCollection.from_dict(chunk)
def count_records_hacking(i,queue = queue):
# infile = '/Users/robinjoganah/Documents/TREC15/dataset/bhw.' + str(i) + '.cbor.gz'
infile = '/Users/robinjoganah/Documents/TREC15/dataset/hackforums-out-fix.' + str(i) + '.cbor.gz'
startTime = time.time()
print infile
if infile.endswith('.gz'):fp = gzip.open(infile, 'r')
else:fp = open(infile, 'r')
count = 0
try:
while (1):
r = cbor.load(fp)
key = r.get(u'key')
features = r.get(u'features')
if features:
items = features.get('items')
posts = []
for item in items:
posts.append(item.get('content'))
thread_name = item.get('thread_name') if(item.get('thread_name')) else ' '
posts = ' '.join(posts)
posts = posts.replace('\n', ' ')
queue.put((key,thread_name.encode('UTF-8'),posts.encode('UTF-8')))
count += 1
end = time.time()
if(count == 100):print str(end - startTime)
except EOFError:
end = time.time()
print str(end- startTime)
print count
def cbor_iter(fh):
while True:
try:
chunk = cbor.load(fh)
except EOFError:
break
yield FeatureCollection.from_dict(chunk)
def count_records_ebola(i,queue = queue):
infile = '/Users/robinjoganah/Documents/TREC15/dataset/ebola-web-01-2015-' + str(i) + '.cbor.gz'
startTime = time.time()
print infile
if infile.endswith('.gz'):
fp = gzip.open(infile, 'r')
else:
fp = open(infile, 'r')
count = 0
try:
while (1):
r = cbor.load(fp)
key = r.get(u'key')
title,content = strip_html_nltk(r.get('response').get('body'))
record = (str(key),str(title),str(content))
queue.put(record)
count += 1
end = time.time()
if(count == 100):print str(end - startTime)
except EOFError:
queue.put(STOP_TOKEN)
end = time.time()
print str(end- startTime)
print count
def _rpc(self, method_name, params):
'''Call a method on the server.
Calls ``method_name(*params)`` remotely, and returns the results
of that function call. Expected return types are primitives, lists,
and dictionaries.
:raise Exception: if the server response was a failure
'''
## it's really time and file-space consuming to log all the
## rpc data, but there are times when a developer needs to.
mlog = None
#mlog = logging.getLogger('cborrpc')
tryn = 0
delay = self._base_retry_seconds
self._message_count += 1
message = {
'id': self._message_count,
'method': method_name,
'params': params
}
if mlog is not None:
mlog.debug('request %r', message)
buf = cbor.dumps(message)
errormessage = None
while True:
try:
conn = self._conn()
conn.send(buf)
response = cbor.load(self.rfile)
if mlog is not None:
mlog.debug('response %r', response)
assert response['id'] == message['id']
if 'result' in response:
return response['result']
# From here on out we got a response, the server
# didn't have some weird intermittent error or
# non-connectivity, it gave us an error message. We
# don't retry that, we raise it to the user.
errormessage = response.get('error')
if errormessage and hasattr(errormessage, 'get'):
errormessage = errormessage.get('message')
if not errormessage:
errormessage = repr(response)
break
except Exception as ex:
if tryn < self._retries:
tryn += 1
logger.debug('ex in %r (%s), retrying %s in %s sec...',
method_name, ex, tryn, delay, exc_info=True)
self.close()
time.sleep(delay)
delay *= 2
continue
logger.error('failed in rpc %r %r', method_name, params,
exc_info=True)
raise
raise Exception(errormessage)
def to_json(cbor_file, output_file):
print('Converting to json')
with open(cbor_file, 'rb') as fi:
with open(output_file, 'w') as fo:
dic = cbor.load(fi)
dic_mapped = replace_dict_key_recursively(dic, CBOR_TO_JSON_MAPPER)
json.dump(dic_mapped, fo)
def read_cbor_message(self):
while True:
# 'self' is sufficiently 'file-like' to act as a load source.
# Throws EOFError on end of stream/timeout/lost-connection etc.
message = cbor.load(self)
# A message response (to a prior request)
if 'id' in message:
logger.info("Received msg: {}".format(message))
return message
# A log message - handle as normal
if 'log' in message:
response = message['log'].decode("utf-8")
log_methods = {
'E': device_logger.error,
'W': device_logger.warn,
'I': device_logger.info,
'D': device_logger.debug,
'V': device_logger.debug,
}
log_method = device_logger.error
if len(response) > 1 and response[1] == ' ':
lvl = response[0]
log_method = log_methods.get(lvl, device_logger.error)
log_method('>> {}'.format(response))
else:
# Unknown/unhandled/unexpected message
logger.error("Unhandled message received")
device_logger.error(message)
def run( args ):
time_data = dict(
start_ts = int( time.time() ),
end_ts = 0
)
sync_types = {}
syncdir_data = dict(
dir_data = collections.OrderedDict(),
dups = collections.OrderedDict(),
working = {}
)
starttime = int( time.time() )
total_records = 0
with open( args.infile, 'rb' ) as f:
try:
while (1):
total_records += 1
rec = cbor.load( f )
#logr.debug( 'Processing record: {0}'.format( rec ) )
process_start_end_times( rec, time_data )
count_sync_types( rec, sync_types )
try:
process_syncdir_stats( rec, syncdir_data )
except ( KeyError ) as e:
logr.warning( 'LogRecord={0}, Error={1}'.format(
total_records, e ) )
if total_records % 1000000 == 0:
elapsed_secs = int( time.time() ) - starttime
logr.info( 'Processed {0} records in {1} secs'.format(
total_records, elapsed_secs ) )
except ( EOFError ) as e:
pass
print_syncdir_summary( args, syncdir_data )
print_psync_summary( args, time_data, sync_types, total_records )
def get(self, page):
""" Lookup a page by name. Returns a Page or None """
offset = self.toc.get(page)
if offset is not None:
self.cbor.seek(offset)
return read_val(cbor.load(self.cbor))
return None
def __init__(self, filename):
with open(filename, 'rb') as f:
self.data = cbor.load(f)
self._bins = Bins(self.data['bins'])
if 'high_resolution' in self.data and self.data['high_resolution'] is not None:
self._high_resolution = Bins(self.data['high_resolution'])
else:
self._high_resolution = None
def load_variables(filename='demo_vars'):
infile = open(filename, 'rb')
variables = cbor.load(infile)
for var in variables:
setattr(mode_library, var, variables[var])
infile.close()
def main():
parser = argparse.ArgumentParser("Plot characteristics of a Terra world")
parser.add_argument("world", help="The world to plot, as a CBOR file")
args = parser.parse_args()
with open(args.world, "rb") as f:
world = cbor.load(f)
plot_elevations(world)
def __init__(self, fname):
"""
Read annotations from a file.
Arguments:
fname The name of the CBOR file. A table-of-contents file is
also expected to be present.
"""
self.cbor = open(fname, 'rb')
self.toc = cbor.load(open(fname + '.toc', 'rb'))
def _extract_encrypted0(cls, message, is_request):
if message.opt.object_security is None:
raise NotAProtectedMessage("No Object-Security option present")
# FIXME it's an error to have this in the wrong place
serialized = message.opt.object_security or message.payload
if USE_COMPRESSION:
if is_request:
# FIXME this will need a little reshaping when dealing with
# observe responses, which use the same compression but a
# 2-long array
if serialized[0] & 0b11000000 != 0:
raise ProtectionInvalid(
"Message does not look like a compressed request")
# the -1 on the first fragment keeps the cbor serializer from
# trying to decode ciphertext field with "excluded [...] type
# and length"
serialized = bytes((
0b10000000 | ((serialized[0] & 0b00111000) >> 3) - 1,
0b01000000 | (serialized[0] & 0b00000111),
)) + serialized[1:]
# this seems to be the easiest way to get the tail of the CBOR object
serialized = BytesIO(serialized)
try:
shortarray = cbor.load(serialized)
except ValueError:
raise ProtectionInvalid(
"Error parsing the compressed CBOR payload")
if not isinstance(shortarray, list) or len(shortarray) != 2 or \
not all(isinstance(x, bytes) for x in shortarray):
raise ProtectionInvalid(
"Compressed CBOR payload has wrong shape")
unprotected = {4: shortarray[1], 6: shortarray[0]}
ciphertext_and_tag = serialized.read()
return b'', {}, unprotected, ciphertext_and_tag
else:
return b'', {}, {}, serialized
else:
try:
encrypted0 = cbor.loads(serialized)
except ValueError:
raise ProtectionInvalid("Error parsing the CBOR payload")
if not isinstance(encrypted0, list) or len(encrypted0) != 3:
raise ProtectionInvalid(
"CBOR payload is not structured like Encrypt0")
try:
protected = cbor.loads(encrypted0[0])
except ValueError:
raise ProtectionInvalid(
"Error parsing the CBOR protected data")
return encrypted0[0], protected, encrypted0[1], encrypted0[2]
def cbor_iter(data):
'''Yields objects from a cbor stream.'''
while True:
try:
obj = cbor.load(data)
yield obj
except EOFError:
break
def cbor_iter(data):
'''Yields objects from a cbor stream.'''
while True:
try:
obj = cbor.load(data)
yield obj
except EOFError:
break
def process_file( infile ):
errors = collections.OrderedDict()
with open( infile, 'rb' ) as f:
try:
while True:
rec = cbor.load( f )
process_error_record( errors, rec )
except ( EOFError ):
pass
return errors
def process_file( infile ):
warnings = collections.OrderedDict()
with open( infile, 'rb' ) as f:
try:
while True:
rec = cbor.load( f )
process_warning( rec, warnings )
except ( EOFError ):
pass
return warnings
def _rpc(self, method_name, params):
'''Call a method on the server.
Calls ``method_name(*params)`` remotely, and returns the results
of that function call. Expected return types are primitives, lists,
and dictionaries.
:raise Exception: if the server response was a failure
'''
mlog = logging.getLogger('cborrpc')
tryn = 0
delay = self._base_retry_seconds
self._message_count += 1
message = {
'id': self._message_count,
'method': method_name,
'params': params
}
mlog.debug('request %r', message)
buf = cbor.dumps(message)
errormessage = None
while True:
try:
conn = self._conn()
conn.send(buf)
response = cbor.load(self.rfile)
mlog.debug('response %r', response)
assert response['id'] == message['id']
if 'result' in response:
return response['result']
# From here on out we got a response, the server
# didn't have some weird intermittent error or
# non-connectivity, it gave us an error message. We
# don't retry that, we raise it to the user.
errormessage = response.get('error')
if errormessage and hasattr(errormessage,'get'):
errormessage = errormessage.get('message')
if not errormessage:
errormessage = repr(response)
break
except Exception as ex:
if tryn < self._retries:
tryn += 1
logger.debug('ex in %r (%s), retrying %s in %s sec...',
method_name, ex, tryn, delay, exc_info=True)
self.close()
time.sleep(delay)
delay *= 2
continue
logger.error('failed in rpc %r %r', method_name, params,
exc_info=True)
raise
raise Exception(errormessage)
def read_msg_impl(self):
assert self._i_chunk_fh is not None
if not self.is_ok_raw_input:
self._i_chunk_fh = BufferedReader(self._i_chunk_fh)
while True:
try:
ob = cbor.load(self._i_chunk_fh)
msg = self.message(ob)
yield msg
except EOFError:
# okay. done.
return
def read_msg_impl(self):
assert self._i_chunk_fh is not None
if not self.is_ok_raw_input:
self._i_chunk_fh = BufferedReader(self._i_chunk_fh)
while True:
try:
ob = cbor.load(self._i_chunk_fh)
msg = self.message(ob)
yield msg
except EOFError:
# okay. done.
return
def _iter_with_header(file, parse, expected_file_types):
maybe_hdr = cbor.load(file)
if isinstance(maybe_hdr, list) and maybe_hdr[0] == 'CAR':
# we have a header
file_type = maybe_hdr[1][0]
assert file_type in expected_file_types
# read beginning of variable-length list
assert file.read(1) == b'\x9f'
else:
yield parse(maybe_hdr)
while True:
try:
# Check for break symbol
if (peek_for_break(file)):
break
yield parse(cbor.load(file))
except EOFError:
break
def read_data(path):
with open(path, 'rb') as stream:
try:
data_loaded = cbor.load(stream)
except IOError:
print('An error occurred trying to read the file.')
read_energy(data_loaded, path)
read_my_t(data_loaded, path)
read_entropy(data_loaded, path)
read_count(data_loaded, path)
read_pexc_tot(data_loaded, path)
read_density(data_loaded, path)
def _rpc(self, method_name, params):
mlog = logging.getLogger('cborrpc')
tryn = 0
delay = self._base_retry_seconds
self._message_count += 1
message = {
'id': self._message_count,
'method': method_name,
'params': params
}
mlog.debug('request %r', message)
buf = cbor.dumps(message)
errormessage = None
while True:
try:
conn = self._conn()
conn.send(buf)
response = cbor.load(self.rfile)
mlog.debug('response %r', response)
if 'result' in response:
return response['result']
errormessage = response.get('error')
if errormessage and hasattr(errormessage, 'get'):
errormessage = errormessage.get('message')
if not errormessage:
errormessage = repr(response)
break
except Exception as ex:
if tryn < self._retries:
tryn += 1
logger.debug('ex in %r (%s), retrying %s in %s sec...',
method_name,
ex,
tryn,
delay,
exc_info=True)
self.close()
time.sleep(delay)
delay *= 2
continue
logger.error('failed in rpc %r %r',
method_name,
params,
exc_info=True)
raise
raise Exception(errormessage)
return
def open_save_file(save_path):
try:
with open(save_path, 'r') as save_file:
try:
return json.load(save_file)
except ValueError as e:
logger.error("cannot decode JSON: %s", save_path)
try:
return cbor.load(save_file)
except ValueError as e:
logger.error("cannot decode save CBOR: %s", save_path)
return {}
except IOError as e:
logger.info(e)
return None
def usernames():
for path in args.input:
with gzip.open(path) as fh:
while True:
try:
rec = cbor.load(fh)
except EOFError:
break
if 'username' in rec:
for username in rec['username']:
if not isinstance(username, unicode):
username = username.decode('utf8')
yield username
if 'email' in rec:
for email in rec['email']:
uname = email.split('@')[0]
yield uname
def handle_wallet_import(args):
'interactive'
import urllib.request
import ssl
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
try:
rsp = urllib.request.urlopen('https://127.0.0.1:8090/api/v1/wallets',
context=ctx)
except urllib.error.URLError:
print('Please open Daedalus wallet.')
return
data = json.loads(rsp.read())
# load secrets
import cbor
import appdirs
root = appdirs.user_data_dir('Daedalus')
path = os.path.join(root, 'Secrets-1.0', 'secret.key')
secrets = {}
for item in cbor.load(open(path, 'rb'))[2]:
xpriv = item[0]
wid = AddressContent.pubkey(xpriv_to_xpub(xpriv)) \
.address() \
.encode_base58() \
.decode()
secrets[wid] = xpriv
candidates = []
for item in data['data']:
if item['id'] in secrets:
candidates.append((item['id'], item['name']))
for i, (wid, name) in enumerate(candidates):
print('[%d] %s' % (i + 1, name))
index = int(input('Select wallet [1]:') or 1) - 1
if create_wallet_with_xpriv(args, candidates[index][1],
secrets[candidates[index][0]]):
print('imported', candidates[index][1])
def get_content_type(directory):
files = get_all_files_in_directory(directory)
hasp_map = {}
errors = []
for item in files:
with open(item) as file2:
try:
data = cbor.load(file2)
json_data = json.loads(data)
content_type_array = json_data['response']['headers'][1]
hasp_map[json_data['key']] = {content_type_array[0]: content_type_array[1]}
except Exception as e:
errors.append(e.message)
result = json.dumps(hasp_map, indent=4)
with open(directory.split('/')[-1] + '.dump', 'w') as dumpfile:
for item in errors:
dumpfile.write(item)
dumpfile.write('\n')
return result
def count_records(infile):
if infile.endswith('.gz'):
fp = gzip.open(infile, 'r')
else:
fp = open(infile, 'r')
posts = 0
threads = 0
try:
while (1):
thread = cbor.load(fp)
threads += 1
features = thread.get(u'features')
if features:
items = features.get('items')
if items:
posts += len(items)
except EOFError:
return threads, posts
def get_content_type(directory):
files = get_all_files_in_directory(directory)
hasp_map = {}
errors = []
for item in files:
with open(item) as file2:
try:
data = cbor.load(file2)
json_data = json.loads(data)
content_type_array = json_data['response']['headers'][1]
hasp_map[json_data['key']] = {content_type_array[0]: content_type_array[1]}
except Exception as e:
errors.append(e.message)
result = json.dumps(hasp_map, indent=4)
with open(directory.split('/')[-1] + '.dump', 'w') as dumpfile:
for item in errors:
dumpfile.write(item)
dumpfile.write('\n')
return result
def getupdates(self, app, changelist=None, callback=None):
""" The client is pulling info from the server """
try:
self.__pause()
dfc_type, content_type = FORMATS[self.app_wire_format[app]]
final_updates = dfc_type()
if self.objectless_server:
# change this before callback
final_updates = dfc_type(
self.master_dataframe.get_record(changelist, app))
if app.startswith("CrawlerFrame_"):
crawler_name = app[len("CrawlerFrame_"):]
group_tpname = ("datamodel.search.{0}_datamodel.{0}Link".
format(crawler_name))
if group_tpname in final_updates["gc"]:
for link_key, link_changes in (
final_updates['gc'][group_tpname].iteritems()):
if "dims" in link_changes:
link_as_file = self.make_link_into_file(
link_key)
if os.path.exists(link_as_file):
# this means that the data
# already exist on disk
# so grab the data rather
# than downloading it
data = cbor.load(open(link_as_file, "rb"))
link_changes["dims"].update(data)
final_updates["stats"] = self.app_to_stats[app]
else:
if app in self.app_to_df:
final_updates = dfc_type(self.app_to_df[app].get_record())
self.app_to_df[app].clear_record()
if callback:
callback(app, final_updates.SerializeToString(), content_type)
else:
return final_updates.SerializeToString(), content_type
except Exception, e:
print "GU ERROR!!!", e, e.__class__.__name__
ex_type, ex, tb = sys.exc_info()
traceback.print_tb(tb)
raise
def __init__(self, filename):
with open(filename, 'rb') as stream:
if 'yaml' in filename:
try:
self.data = yaml.full_load(stream)
except IOError:
print('An error occurred trying to read the file.')
elif 'cbor' in filename:
try:
self.data = cbor.load(stream)
except IOError:
print('An error occurred trying to read the file.')
else:
print('What kind of file is this?!')
self._bins = Bins(self.data['bins'])
if 'high_resolution' in self.data and self.data[
'high_resolution'] is not None:
self._high_resolution = Bins(self.data['high_resolution'])
else:
self._high_resolution = None
def load_conf():
# try cache yaml result with cbor.
conf_path = os.path.join(CONF_DIR, 'configuration.yaml')
cache_path = os.path.join(CONF_DIR, 'cached_configuration.cbor')
if os.path.exists(cache_path) and \
os.path.getmtime(cache_path) > os.path.getmtime(conf_path):
# cache is good.
return cbor.load(open(cache_path, 'rb'))
with open(conf_path) as fp:
conf = yaml.load(fp)
fp = open(cache_path, 'wb')
try:
fp.write(cbor.dumps(conf))
except: # noqa[W291]
fp.close()
os.remove(cache_path)
raise
else:
fp.close()
return conf
def parseCommonCrawlFile(path): res = cbor.load(open(path, "rb")) return json.loads(res)
listfile = os.listdir(filePath)
#infile = "ebola-web-01-2016-1.cbor.gz"
n = 0
for i in listfile:
infile = filePath + i
if i.endswith('.gz'):
fp = gzip.open(infile, 'r')
else:
fp = open(sys.argv[1], 'r')
f = file(str(n) + "trec_polar_html.txt", "w+")
n += 1
try:
while 1:
r = cbor.load(fp)
#pp.pprint(r)
f.write("<DOC>\n<DOCNO>\n")
f.write(r["key"])
f.write("\n</DOCNO>\n")
f.write("<DOCCONTENT>\n")
content = r["response"]["body"].encode('utf-8')
#s = filters.solveSentence(content).encode('utf-8')
#f.write(s.encode('utf-8'))
f.write(content)
f.write("\n</DOCCONTENT>")
f.write("\n</DOC>\n")
f.write("\n")
except EOFError:
continue
count=0
if re.match(r'^\.', fileName):
continue
if re.match(r'.*.json$',fileName):
continue
if re.match(r'.*.py$',fileName):
continue
if re.match(r'.*.(sh|txt)$',fileName):
continue
# if countfiles==50:
# break
try:
with open(os.path.join(root,fileName), 'r') as temp:
rec=cbor.load(temp)
d = json.loads(rec)
content_Type=d["response"]["headers"][1][1]
if ";" in content_Type:
content_Type=content_Type.split(";")[0]
if d["response"]["status"]=="" or d["response"]["status"]=="200":
status="200"
if "=" in d["url"]:
keyword=d["url"].split("=")[1]
else:
continue
if keyword.lower() in d["response"]["body"].lower():
count=d["response"]["body"].count(keyword)
else:
continue
def read(self) -> Union[Dict, TreeNode]:
"""
I return the best representation the source format supports
pickle: TreeNode
else : Dict[inode -> properties]
"""
fn = self._path()
if self.filetype == FileType.PICKLE:
with open(fn, "rb") as f:
self.treenode = pickle.load(f)
return self.treenode
elif self.filetype == FileType.CSV:
self.id_dict = {}
with open(fn, "r") as f:
r = csv.DictReader(f)
for line in r:
# type conversion
for field in [k for k,v in Node._field_types.items() if v != str]:
line[field] = int(line[field])
self.id_dict[int(line['id'])] = Node(**line)
return self.id_dict
elif self.filetype == FileType.MSGPACK:
# TODO: This will fail with larger files - have to adjust max_xxx_len
with open(fn, "rb") as f:
self.id_dict = {}
for item in msgpack.unpack(f, raw=False):
self.id_dict[item['id']] = Node(**item)
return self.id_dict
elif self.filetype == FileType.JSON:
return self._json_read(fn, json.load)
elif self.filetype == FileType.UJSON:
return self._json_read(fn, ujson.load)
elif self.filetype == FileType.SIMPLEJSON:
# NOTE: simplejson includes key names when serializing NamedTuples
with open(fn, "r") as f:
self.id_dict = {}
if self.json_dict_list:
for item in simplejson.load(f):
self.id_dict[item['id']] = Node(**item)
else:
for v in simplejson.load(f).values():
self.id_dict[v['id']] = Node(**v)
return self.id_dict
elif self.filetype == FileType.CBOR2:
with open(fn, "rb") as f:
self.id_dict = {}
for item in cbor2.load(f):
self.id_dict[item['id']] = Node(**item)
return self.id_dict
elif self.filetype == FileType.CBOR:
with open(fn, "rb") as f:
self.id_dict = {}
for item in cbor.load(f):
self.id_dict[item['id']] = Node(**item)
return self.id_dict
elif self.filetype == FileType.RAPIDJSON:
self.id_dict = {}
with open(fn, "r") as f:
d = rapidjson.Decoder(number_mode=rapidjson.NM_NATIVE)(f)
if self.json_dict_list:
for item in d:
# safer cause key names are included, but slower
self.id_dict[item['id']] = Node(**item)
else:
# list(self.id_dict.values()) - produces a list of lists
for item in d:
self.id_dict[item[0]] = Node._make(item)
return self.id_dict
elif self.filetype == FileType.BSON:
self.id_dict = {}
with open(fn, "rb") as f:
for doc in decode_file_iter(f):
self.id_dict[doc['id']] = Node(**doc)
return self.id_dict
S_excess = np.zeros((col, row))
energy_data = np.zeros((col, row))
averaged_T = np.zeros((col, row))
T_inv = np.zeros((col, row))
T = np.zeros((col, row))
chem_potential = np.zeros((col, row))
number_data = np.zeros((col, row))
N_sites = 1
S_ideal = number_data*(1 + np.log(N_sites/number_data))
for frame in frames:
print('frame is', frame)
with open(frame,'rb') as stream:
data_loaded = cbor.load(stream)
hist = np.array(data_loaded['bins']['histogram'])
print('system', data_loaded['system'].keys())
print('system E', data_loaded['system']['E'])
print('bins', data_loaded['bins'].keys())
print(data_loaded.keys())
Sexcess = np.array(data_loaded['bins']['lnw'])
NE = data_loaded['bins']['num_E']
Nmax = data_loaded['bins']['max_N']
moves = data_loaded['moves']
number_data = np.array(data_loaded['bins']['histogram'])
#!/usr/bin/env python2.7
import cbor
import sys
import gzip
import pprint
pp = pprint.PrettyPrinter(indent=4)
infile = sys.argv[1]
if infile.endswith('.gz'):
fp = gzip.open(infile, 'r')
else:
fp = open(sys.argv[1], 'r')
try:
while (1):
r = cbor.load(fp)
pp.pprint(r)
except EOFError:
pass
except:
print >>sys.stderr, "Error reading", infile, sys.exc_info()[0]
#!/usr/bin/env python
import cbor
f = cbor.load(open("dump.cbor", "rb"))
print f
# f = cbor.loads(cbor.dumps(["123", {"a" : "bb"}, 45]))
# print f
def parse_message(self, input):
req_id = cbor.load(input)
data = cbor.load(input)
return _RedisResponse(req_id, data)
def parse_message(self, input):
dic = cbor.load(input)
return HttpRequest(dic["response_socket"], dic)
def _cbor_load(self, filei):
# no mutex yet, we're making a new object
# with self.mutex:
state = cbor.load(filei)
self.__setstate__(state)
def read_cbor(input_file):
cbor_dict = dict()
with open(input_file, 'rb') as fp:
data = cbor.load(fp)
traverse(data, cbor_dict)
return cbor_dict
description='Reconstruct parameter file from metadata')
parser.add_argument('--out',
help='Defaults to "parameter_reconstructed.toml"',
default='parameter_reconstructed.toml')
parser.add_argument('--format',
help='file format of simulation output',
default='MsgPack')
parser.add_argument('--index', help='Path to index file')
parser.add_argument('data', help='Path to simulation data', type=str)
args = parser.parse_args()
sim_setting = {}
index = np.fromfile(args.index, dtype=np.uint64)
with open(args.data, 'rb') as f:
if args.format == 'MsgPack':
buf = f.read(index[0])
sim_settings = msgpack.unpackb(buf, encoding='utf-8')
elif args.format == 'CBOR':
sim_settings = cbor.load(f).value
else:
print('ERROR: Unsupported format: {}'.format(args.format))
sys.exit(1)
print(sim_settings)
with open(args.out, 'w') as f:
toml.dump(sim_settings, f)
