def __init__(self, ctx: discord.ext.commands.Context = None, name=None): self.ctx = ctx self.name = name self.data = None if isinstance(ctx, str): self.filename = f"data/Bot/{self.ctx}.msgpack" try: ensure_file_existence(self.filename) with open(self.filename, "rb") as fp: if fp.read(): fp.seek(0) self.data = msgpack.unpack(fp, encoding="utf-8") else: self.data = {} except FileNotFoundError: self.data = {} else: self.filename = f"data/{self.ctx.guild.id}/{self.name}.msgpack" try: ensure_file_existence(self.filename) with open(self.filename, "rb") as fp: if fp.read(): fp.seek(0) self.data = msgpack.unpack(fp, encoding="utf-8") else: self.data = {} except FileNotFoundError: self.data = {}
def __init__(self, tsv_file_path, messagepack_tsv_path, transposed_tsv_file_path, transposed_messagepack_tsv_path): messagepack_tsv = open(messagepack_tsv_path + "/sample_data.msgpack", "rb") self.tsv_map = msgpack.unpack(messagepack_tsv, max_map_len=10000000, max_array_len=10000000) sample_file = open(messagepack_tsv_path + "/samples.msgpack", "rb") self.samples = msgpack.unpack(sample_file, max_map_len=10000000, max_array_len=10000000) transposed_map_file = open( transposed_messagepack_tsv_path + "/sample_data.msgpack", "rb") self.transposed_map = msgpack.unpack(transposed_map_file, max_map_len=10000000, max_array_len=10000000) transposed_samples_file = open( transposed_messagepack_tsv_path + "/samples.msgpack", "rb") self.transposed_samples = msgpack.unpack(transposed_samples_file, max_map_len=10000000, max_array_len=10000000) self.tsv_file = open(tsv_file_path, "rb") self.transposed_tsv_file = open(transposed_tsv_file_path, "rb") features_file = open(messagepack_tsv_path + "/features.msgpack", "rb") self.features = msgpack.unpack(features_file, max_map_len=10000000, max_array_len=10000000)
def merge(Origin, Destination): folders = [f[2:] for f in [i[0] for i in os.walk(Origin)][1:]] newWordDict = [] newWordCount = [] for f in range(len(folders)): print(''.join(("Merging folder " + str(f) + " out of ", str(len(folders)) + ": ", folders[f]))) with open(Path(Origin) / folders[f] / "WordDict.msgpack", "rb") as worddict, open( Path(Origin) / folders[f] / "WordCount.msgpack", "rb") as wordcount: wd = msgpack.unpack(worddict) wc = msgpack.unpack(wordcount) for i in range(len(wd)): if wd[i] in newWordDict: index = newWordDict.index(wd[i]) newWordCount[index] += wc[i] else: newWordDict.append(wd[i]) newWordCount.append(wc[i]) del wd del wc with open(Path(Destination) / "wordCount.msgpack", 'wb+') as outfile: msgpack.pack(newWordCount, outfile) with open(Path(Destination) / "wordDict.msgpack", 'wb+') as outfile: msgpack.pack(newWordDict, outfile)
def multiHot(Origin, Destination): print("Generating multihot arrays") folders = [f[2:] for f in [i[0] for i in os.walk(Origin)][1:]] for f in range(len(folders)): print(''.join(("Creating MultiHot of folder " + str(f) + " out of ", str(len(folders)) + ": ", folders[f]))) forumKeys.append([ f, folders[f], [1 if (f == k) else 0 for k in range(len(folders))] ]) with open(Path(folders[f]) / "Posts.msgpack", "rb") as posts, open( Path(Origin) / "WordDict.msgpack", "rb") as w: p, wD = msgpack.unpack(posts), msgpack.unpack(w) c, l = 0, len(p) r = [] for i in range(l): print(''.join((str(c), '/', str(l)))) r.append([[1 if (j in p[i][0]) else 0 for j in wD], [1 if (f == k) else 0 for k in range(len(folders))]]) c += 1 with open(Path(Destination) / folders[f] / "OneHot.msgpack", 'wb+') as outfile: msgpack.pack(r, outfile) del p del wD del r del c del l with open(Path(Destination) / "forumKeys.msgpack", 'wb+') as outfile: msgpack.pack(forumKeys, outfile)
def load_network(network_to_use): with open('./cache/' + network_to_use + '/vectors.msgpack', 'rb') as f: vectors = msgpack.unpack(f) with open('./cache/' + network_to_use + '/matrices.msgpack', 'rb') as f: matrices = msgpack.unpack(f) with open('./cache/' + network_to_use + '/dictionary.msgpack', 'rb') as f: cat_dict = msgpack.unpack(f) return vectors, matrices, cat_dict
def __init__(self, map_path, cost_weights, waypoints, directory, is_ego): self.limp_s = 0. self.is_limping = False self.is_ego = is_ego self.prev_traj = None self.prev_param = None self.prev_steer = 0. self.cost_weights = cost_weights self.waypoints = waypoints self.wheelbase = 0.3302 self.max_reacquire = 10 self.safe_speed = 2.5 self.CORNER_ON = False self.track_lad = 1.0 self.STEER_LP = 0.99 self.CURVATURE_THRESH = 20. self.WINDOW_SIZE = 3. self.TOP_POP_NUM = 3 lut_all = np.load(directory + 'mpc/lut_inuse.npz') self.lut_x = lut_all['x'] self.lut_y = lut_all['y'] self.lut_theta = lut_all['theta'] self.lut_kappa = lut_all['kappa'] self.lut = lut_all['lut'] step_sizes = [] step_sizes.append(self.lut_x[1] - self.lut_x[0]) step_sizes.append(self.lut_y[1] - self.lut_y[0]) step_sizes.append(self.lut_theta[1] - self.lut_theta[0]) step_sizes.append(self.lut_kappa[1] - self.lut_kappa[0]) self.lut_stepsizes = np.array(step_sizes) with open(directory + 'config.yaml', 'r') as yaml_stream: try: config = yaml.safe_load(yaml_stream) speed_lut_name = config['speed_lut_name'] range_lut_name = config['range_lut_name'] except yaml.YAMLError as ex: print(ex) speed_lut_temp = msgpack.unpack(open(directory + speed_lut_name, 'rb'), use_list=False) self.speed_lut_numba = Dict() for key, val in speed_lut_temp.items(): if key == b'resolution': continue self.speed_lut_numba[key] = val range_lut_temp = msgpack.unpack(open(directory + range_lut_name, 'rb'), use_list=False) self.range_lut_numba = Dict() for key, val in range_lut_temp.items(): if key == b'resolution': continue self.range_lut_numba[key] = val self.lut_resolution = float(speed_lut_temp[b'resolution'][0])
def save_metadata(pr: PullRequest, transposed_data_file, transposed_map_dir, out_file): printToLog("Saving metadata", pr) # In the transposed file, samples are actually features with open(os.path.join(transposed_map_dir, 'samples.msgpack'), 'rb') as samples_file: features = msgpack.unpack(samples_file) # Open the transposed data file so we can read feature values with open(os.path.join(transposed_map_dir, 'sample_data.msgpack'), 'rb') as map_file: data_map = msgpack.unpack(map_file) meta_dict = {} with open(transposed_data_file) as transposed_file: for i in range(len(features)): if i > 0 and i % 1000 == 0: printToLog("{}".format(i), pr) feature = features[i] feature_coordinates = data_map[feature] transposed_file.seek(feature_coordinates[0]) feature_values = [ x for x in transposed_file.read(feature_coordinates[1]).split( "\t") if x != "NA" ] feature_values = sorted(list(set(feature_values))) # Check whether we only had missing (NA) values if len(feature_values) == 0: meta_dict[feature] = {'options': ["NA"], 'numOptions': 1} else: float_values = convert_to_floats(feature_values) if not float_values: meta_dict[feature] = { 'options': feature_values, 'numOptions': len(feature_values) } else: meta_dict[feature] = { 'options': 'continuous', 'min': min(float_values), 'max': max(float_values) } metadata = {'meta': meta_dict} with open(out_file, 'wb') as fp: pickle.dump(metadata, fp)
def readinto(self, buffer): global _stream _clear_rx_buffer() msgpack.pack(("ri", self._ext_type, len(buffer)), _stream) _ready_to_read() # get actual number of bytes read OR handle error (if any) sz = msgpack.unpack(_stream, ext_hook=_ext_handler, use_list=False) # read data; not all versions of readinto support length argument mv = memoryview(buffer) # print("urpc client.readinto [sz]") _stream.readinto(mv[:sz]) # server sends an extra None assert msgpack.unpack(_stream) == None, "readinto expects terminating 'None'" return sz
def _call_mmtf(f): '''Call function for mmtf files''' if ".mmtf.gz" in f: name = f.split('/')[-1].split('.')[0].upper() data = gzip.open(f, 'rb') unpack = msgpack.unpack(data, raw=False) decoder = MmtfStructure(unpack) return (name, decoder) elif ".mmtf" in f: name = f.split('/')[-1].split('.')[0].upper() unpack = msgpack.unpack(open(f, "rb"), raw=False) decoder = MmtfStructure(unpack) return (name, decoder)
def unpack(package): """ Unpacks system data from exported package and saves to DB """ contents = msgpack.unpack(package) # load deps and check for missing deps, missing = dependency_check(contents['deps']) if len(missing) > 0: raise ImportDependencyMissing(missing) # start creating jobs for job in contents['jobs']: # transform directly stored object back job['hashes'] = list( map(lambda h: recreate_hash(h, job['hash_type']), job['hashes'])) if job.get('masks'): job['masks'] = list(map(recreate_mask, job['masks'])) newjob = FcJob() for field in JOB_EXPORTABLE_COLUMNS: setattr(newjob, field, job.get(field)) # manual labor now # # !!! If you added new DEPS to export, make sure to unpack them like these !!! # dep_rule = job.get('rulesFile') if dep_rule: newjob.rulesFile = deps[dep_rule[0]] dep_markov = job.get('markov') if dep_markov: newjob.markov = deps[dep_markov[0]] dep_left_dictionaries = job.get('left_dictionaries') if dep_left_dictionaries: for index in dep_left_dictionaries: jd = FcJobDictionary(is_left=1, dictionary=deps[index]) newjob.left_dictionaries.append(jd) dep_right_dictionaries = job.get('right_dictionaries') if dep_right_dictionaries: for index in dep_right_dictionaries: jd = FcJobDictionary(is_left=0, dictionary=deps[index]) newjob.right_dictionaries.append(jd) dep_pcfg = job.get('pcfg') if dep_pcfg: newjob.pcfg = deps[dep_pcfg[0]] # adding values for useless non-null fields newjob.indexes_verified = 0 newjob.current_index_2 = 0 # add owner newjob.permission_records.append( FcUserPermission(user_id=current_user.id, view=1, modify=1, operate=1, owner=1)) # save db.session.add(newjob) # end for loop over jobs try: db.session.commit() except exc.IntegrityError as e: db.session().rollback() raise e
def _decode(self, response): ct = response.getheader('Content-Type') ce = response.getheader('Content-Encoding') cl = response.getheader('Content-Length') if cl and not int(cl): LOG.debug("Empty response body") return "" if ce and ce == "gzip": # this reads the whole response in memory, but json.load() would do the same anyway... data = response.read() response = gzip.GzipFile(fileobj=StringIO(data)) if ct and 'application/json' in ct: LOG.debug("decoding %s", ct) return json.load(response) elif ct and 'application/x-msgpack' in ct: if msgpack: LOG.debug("decoding %s", ct) return msgpack.unpack(response) else: LOG.debug("not decoding %s, decoder is unavailable", ct) return response.read() return response.read()
def __read_gdf(self): if not os.path.exists(self.gdfpath): return {} with open(self.gdfpath, "rb") as stream: return msgpack.unpack(stream, encoding="utf-8")
def __init__(self, data_path, use_msgpack=False): self.net_as = SubnetTree.SubnetTree() self.d_asn = {} self.net_as_map = {} net_as_raw = None net_as_file = open(data_path, 'rb') if use_msgpack: import msgpack net_as_raw = msgpack.unpack(net_as_file) else: net_as_raw = json.load(net_as_file) net_as_file.close() for asn, v in net_as_raw.items(): ''' "11542": { "name": "EYEMG - EYEMG - interactive media group", "cc": "US", "timestamp": "20070524", "rir": "arin", "nets": {"208.79.156.0/22": {"cc": "US", "timestamp": "20070621", "rir": "arin"} }, "asn": "11542"} ''' self.d_asn[asn] = ASN(asn) self.d_asn[asn].fromdict(v) for net in self.d_asn[asn].nets.keys(): '''A single net may be announced in various ASNs''' self.net_as.insert(str(net), net) if not net in self.net_as_map: self.net_as_map[net] = [] if not asn in self.net_as_map[net]: self.net_as_map[net].append(asn)
def prepare_txn(self, transaction_id, do_cleanup=True): self._active_txn = True try: self.lock.upgrade() except (LockError, LockErrorT): # if upgrading the lock to exclusive fails, we do not have an # active transaction. this is important for "serve" mode, where # the repository instance lives on - even if exceptions happened. self._active_txn = False raise if not self.index or transaction_id is None: self.index = self.open_index(transaction_id) if transaction_id is None: self.segments = { } # XXX bad name: usage_count_of_segment_x = self.segments[x] self.compact = set( ) # XXX bad name: segments_needing_compaction = self.compact else: if do_cleanup: self.io.cleanup(transaction_id) with open(os.path.join(self.path, 'hints.%d' % transaction_id), 'rb') as fd: hints = msgpack.unpack(fd) if hints[b'version'] != 1: raise ValueError('Unknown hints file version: %d' % hints['version']) self.segments = hints[b'segments'] self.compact = set(hints[b'compact'])
def _load_token_statistics(file_name): with open_binary('akimous.resources', file_name) as f1: with lzma.open(f1, 'rb') as f2: return msgpack.unpack(f2, use_list=False, raw=False, strict_map_key=False)
def unpack(filename): """ Open and unpack a named msgpack file. """ with open(filename, 'rb') as infile: unpacked = msgpack.unpack(infile) return msgpack_transform(unpacked[1], unpacked[0])
def __init__(self, data=None): ''' @param data: file contents or open file handle, optionally gzipped @type data: bytes, stream or dict ''' if data is None: self._data = {} return if isinstance(data, dict): self._data = {} for key, value in data.items(): # discard non-required lists of length zero if key not in requiredfields and hasattr( value, '__len__') and len(value) == 0: continue self.set(key, value) return if isinstance(data, bytes): if data[:2] != b'\x1f\x8b': # gzip magic number self._set_data(msgpack.unpackb(data, **_KWARGS_UNPACK)) return import io, gzip data = gzip.GzipFile(fileobj=io.BytesIO(data)) self._set_data(msgpack.unpack(data, **_KWARGS_UNPACK))
def put(self): if request.content_type != "application/msgpack": raise InvalidUsage("Data should be sent as application/msgpack", 400) try: t = Timer() packed = BytesIO(request.get_data()) t.debug_reset("To BytesIO") data = msgpack.unpack(packed, encoding='utf-8') t.debug_reset("Unpack") payload = PayloadFromImport(data) t.debug_reset("Read payload") length, blob = payload.to_np_save() tags = payload.tags t.debug_reset("Repack payload") new_uuid = str(uuid4()) self.metadata.put(uuid=new_uuid, network=tags['network'], station=tags['station'], channel=tags['channel'], start=payload.start, end=payload.end, sampling_rate=tags['sampling_rate']) t.debug_reset("Put to Metadata") self.datastore.put(new_uuid, blob, length) t.debug_reset("Put to Datastore") except (DatastoreError, MetadataError, PayloadError) as e: raise InvalidUsage("Could not accept payload: {}".format(e))
def __init__(self, meta_file='./squad2_preprocessed/meta.msgpack'): with open(meta_file, 'rb') as f: meta = msgpack.unpack(f, encoding='utf8') self.vocab = meta['vocab'] self.vocab_tag = meta['vocab_tag'] self.vocab_ent = meta['vocab_ent'] self.embeddings = meta['embedding'] self.wv_cased = meta['wv_cased'] self.w2id = {w: i for i, w in enumerate(self.vocab)} self.tag2id = {w: i for i, w in enumerate(self.vocab_tag)} self.ent2id = {w: i for i, w in enumerate(self.vocab_ent)} self.nlp = spacy.load('en', parser=False) self.annotate = partial(annotate, wv_cased=self.wv_cased, init_nlp=self.nlp) self.annotate_single = partial(annotate_single, init_nlp=self.nlp) self.annotate_crossed = partial(annotate_crossed, wv_cased=self.wv_cased) self.to_id = partial(to_id, w2id=self.w2id, tag2id=self.tag2id, ent2id=self.ent2id)
def fetch_inst_id(self): """ Fetches the institute id of the RU """ for d in msgpack.unpack(urllib2.urlopen( "%s/list/institutes?format=msgpack" % self.url)): if d['name'] == 'Radboud Universiteit Nijmegen': return d['id'] assert False
def test_1chain_simplerestore(self): m = andrey.Markov(1, 1) m.teach('alpha beta conky delta') dump = msgpack.dumps(m.todict()) dic = msgpack.unpack(io.BytesIO(dump), encoding='utf-8') m2 = andrey.Markov.fromdict(dic) self.assertEqual('beta conky', m2.choose('alpha', continued=1))
def _load_metadata(self): meta_path = self.base_path / "metadata.msg" if not meta_path.exists(): _logger.info("Creating metadata of Waymo dataset (%s)...", self.phase) metadata = {} if self.inzip: for archive in self.base_path.iterdir(): if archive.is_dir() or archive.suffix != ".zip": continue with PatchedZipFile(archive, to_extract="context/stats.json") as ar: metadata[archive.stem] = json.loads( ar.read("context/stats.json")) else: for folder in self.base_path.iterdir(): if not folder.is_dir(): continue with (folder / "context/stats.json").open() as fin: metadata[folder.name] = json.load(fin) with open(meta_path, "wb") as fout: msgpack.pack(metadata, fout) with open(meta_path, "rb") as fin: self._metadata = SortedDict() meta_json = msgpack.unpack(fin) for k, v in meta_json.items(): self._metadata[k] = edict(v)
def fetch(bank, key): ''' Fetch information from a msgpack file. bank The name of the directory, inside the configured cache directory, which will hold the data. If slashes are included in the name, then they refer to a nested directory structure. key The name of the file which holds the data. This filename will have ``.p`` appended to it. ''' base = os.path.join(salt.syspaths.CACHE_DIR, bank) if not os.path.isdir(base): log.debug('Cache directory %s does not exist', base) return None outfile = os.path.join(base, '{0}.p'.format(key)) try: with salt.utils.fopen(outfile, 'rb') as fh_: return msgpack.unpack(fh_) except IOError as exc: log.warn('There was an error reading the cache file, {0}: {1}'.format( base, exc)) return None
def test_to_msgpack(self): m_file = os.path.join(tmp_dir, "movie_data") msg_data = _to_msgpack(movie_data) assert b"Rick Moranis" in msg_data _to_msgpack(movie_data, filename=m_file) assert b"Rick Moranis" in open(m_file, "rb").read() assert msgpack.unpack(open(m_file, "rb")) == msgpack.unpackb(msg_data)
def formatGazeData(inputDir): """ - load the pupil_data and timestamps - get the "gaze" fields from pupil data (i.e. the gaze lcoation w/r/t world camera) - sync gaze data with the world_timestamps array """ # load pupil data pupil_data_path = join(inputDir, 'pupil_data') with open(pupil_data_path, 'rb') as fh: try: gc.disable() pupil_data = msgpack.unpack(fh, encoding='utf-8') except Exception as e: print(e) finally: gc.enable() gaze_list = pupil_data['gaze_positions'] # gaze posiiton (world camera) # load timestamps timestamps_path = join(inputDir, 'world_timestamps.npy') frame_timestamps = np.load(timestamps_path) # align gaze with world camera timestamps gaze_by_frame = correlate_data(gaze_list, frame_timestamps) # make frame_timestamps relative to the first data timestamp start_timeStamp = gaze_by_frame[0][0]['timestamp'] frame_timestamps = (frame_timestamps - start_timeStamp) * 1000 # convert to ms return gaze_by_frame, frame_timestamps
def thread_function(): with open(self.mapInfo.filename, 'rb') as f: self.data = msgpack.unpack(f) self._get_landmarks() self._get_trajectory() self.mapInfo.loading = False
def unpack(stream, **kwargs): """Unpack a stream of packed bytes using MessagePack (with extended types support) and return unpacked object.""" return msgpack.unpack(stream, ext_hook=_decode_ext_type, raw=False, **kwargs)
def Deserializer(stream_or_string): if isinstance(stream_or_string, basestring): stream = StringIO(stream_or_string) else: stream = stream_or_string for obj in PythonDeserializer(msgpack.unpack(stream, object_hook=DjangoMsgPackDecoder().decode)): yield obj
def fetch_todays_schedule(self, rooms): """ Fetch the schedules for the given rooms. """ room_ids = self.fetch_room_ids(rooms) inst_id = self.fetch_inst_id() ret = {} now = datetime.datetime.now() day = DAYS[now.isoweekday() - 1] for room_name in room_ids: ret[room_name] = [] events = msgpack.unpack(urllib2.urlopen( "%s/snapshot/head/%s/location/%s?format=msgpack" % ( self.url, inst_id, room_ids[room_name])))['events'] for event in events: starttime = datetime.datetime.strptime( event['starttime'], '%H:%M:%S').time() endtime = datetime.datetime.strptime( event['endtime'], '%H:%M:%S').time() if event['day'] != day: continue ok = False for period in event['eventperiods']: startdate = datetime.datetime.strptime( period['startdate'], '%Y-%m-%d %H:%M:%SZ').date() enddate = datetime.datetime.strptime( period['enddate'], '%Y-%m-%d %H:%M:%SZ').date() if (startdate <= now.date() and now.date() <= enddate): ok = True break if not ok: continue ret[room_name].append((starttime, endtime, event['course']['name'])) return ret
def getPiecemap(self, inner_path): file_info = self.site.content_manager.getFileInfo(inner_path) piecemap_inner_path = helper.getDirname(file_info["content_inner_path"]) + file_info["piecemap"] self.site.needFile(piecemap_inner_path, priority=20) piecemap = msgpack.unpack(self.site.storage.open(piecemap_inner_path))[helper.getFilename(inner_path)] piecemap["piece_size"] = file_info["piece_size"] return piecemap
def readTrainingData(dataName): dataFile = open(dataName, "rb") gc.disable() inputList, outputList = msgpack.unpack(dataFile) gc.enable() dataFile.close() return list(inputList), list(outputList)
def _rpc_all(self, cmd): clients = {} poller = zmq.Poller() ret = [] for k, d in self.workers.items(): socket = self.context.socket(zmq.REQ) socket.setsockopt(zmq.LINGER, 0) socket.connect(d['rpc_address']) poller.register(socket, zmq.POLLIN) socket.send(cmd) clients[socket] = k t = time.time() while time.time() - t < 1.000: socks = dict(poller.poll(50)) # wait 0.05s for socket in socks.keys(): status = unpack(socket.recv()) self.workers[clients[socket]] = status ret.append(status) del clients[socket] if not clients: break # timeouted sockets for k, socket in clients.items(): status = {"status":"no response"} if k in self.workers: status.update( self.workers[k] ) ret.append(status) return ret
def load_latest(self, name='Megavolume', filter='rff2', conversion='EU60', factor=1.): line = open( '/raid1/mh826/Stacking_Algorithm/CCP_stacks/CCP_volumes/' + name + '_' + filter + '_' + conversion + '_' + str(factor) + '/filenames.dat', 'r').readlines()[-1] runnum = int(float(line.split()[0])) volumefile = line.split()[1] print('loading', name, runnum, volumefile) # get last stackfile name # Read in volumes self.VOL.update( msgpack.unpack(open(volumefile, 'rb'), use_list=False, object_hook=m.decode, encoding='utf-8')) # del self.VOL['trackRFs'] return self
def load_latest(self, name='trialarea_1', filter='jgf1', conversion='prem', factor=2.): ''' Loads latest volume ''' print(name) line = open( root + 'CCP_Stack/Volumes/' + name + '_' + filter + '_' + conversion + '_' + str(factor) + '/filenames.dat', 'r').readlines()[-1] runnum = int(float(line.split()[0])) volumefile = line.split()[1] print(runnum, volumefile) # get last stackfile name ####### Read in volumes self.VOL.update( msgpack.unpack(open(volumefile, 'rb'), use_list=False, object_hook=m.decode)) #del self.VOL['trackRFs'] return self
def _init_dataset(): """ Load the bond dataset from MessagePack file. Since loading the database is computationally expensive, this is only done, when the bond database is actually required. """ global _intra_bonds if _intra_bonds is not None: # Database is already initialized return # Bonds are taken from # ftp://ftp.wwpdb.org/pub/pdb/data/monomers/components.cif # (2019/01/27) _info_dir = dirname(realpath(__file__)) with open(join(_info_dir, "intra_bonds.msgpack"), "rb") as file: _intra_bonds_raw = msgpack.unpack(file, use_list=False, raw=False) _intra_bonds = {} for group, group_bonds_raw in _intra_bonds_raw.items(): group_bonds = { frozenset(bond_raw): count for bond_raw, count in group_bonds_raw.items() } _intra_bonds[group] = group_bonds
def __iter__(self): for fname in os.listdir(self.dirname): log('reading file ' + fname) with open(os.path.join(self.dirname, fname), 'rb') as f: sentences = msgpack.unpack(f) for sentence in sentences: yield sentence
def load(self): self.data = {'version' : self._version} if os.path.exists(self.db_path): with open(self.db_path) as f: self.data = msgpack.unpack(f) else: print 'Initializing replay database...' self.update()
def _wait(self, unpacker, sock): while True: data = sock.read() unpacker.feed(data) try: return msgpack.unpack() except StopIteration: pass
def prepare_txn(self, transaction_id, do_cleanup=True): self._active_txn = True if not self.lock.got_exclusive_lock(): if self.exclusive is not None: # self.exclusive is either True or False, thus a new client is active here. # if it is False and we get here, the caller did not use exclusive=True although # it is needed for a write operation. if it is True and we get here, something else # went very wrong, because we should have a exclusive lock, but we don't. raise AssertionError("bug in code, exclusive lock should exist here") # if we are here, this is an old client talking to a new server (expecting lock upgrade). # or we are replaying segments and might need a lock upgrade for that. try: self.lock.upgrade() except (LockError, LockErrorT): # if upgrading the lock to exclusive fails, we do not have an # active transaction. this is important for "serve" mode, where # the repository instance lives on - even if exceptions happened. self._active_txn = False raise if not self.index or transaction_id is None: try: self.index = self.open_index(transaction_id, False) except RuntimeError: self.check_transaction() self.index = self.open_index(transaction_id, False) if transaction_id is None: self.segments = {} # XXX bad name: usage_count_of_segment_x = self.segments[x] self.compact = FreeSpace() # XXX bad name: freeable_space_of_segment_x = self.compact[x] else: if do_cleanup: self.io.cleanup(transaction_id) hints_path = os.path.join(self.path, 'hints.%d' % transaction_id) index_path = os.path.join(self.path, 'index.%d' % transaction_id) try: with open(hints_path, 'rb') as fd: hints = msgpack.unpack(fd) except (msgpack.UnpackException, msgpack.ExtraData, FileNotFoundError) as e: logger.warning('Repository hints file missing or corrupted, trying to recover') if not isinstance(e, FileNotFoundError): os.unlink(hints_path) # index must exist at this point os.unlink(index_path) self.check_transaction() self.prepare_txn(transaction_id) return if hints[b'version'] == 1: logger.debug('Upgrading from v1 hints.%d', transaction_id) self.segments = hints[b'segments'] self.compact = FreeSpace() for segment in sorted(hints[b'compact']): logger.debug('Rebuilding sparse info for segment %d', segment) self._rebuild_sparse(segment) logger.debug('Upgrade to v2 hints complete') elif hints[b'version'] != 2: raise ValueError('Unknown hints file version: %d' % hints[b'version']) else: self.segments = hints[b'segments'] self.compact = FreeSpace(hints[b'compact'])
def load_graph(filename): with open(filename, 'r+b') as outfile: psi, rV, phi, rE = msgpack.unpack(outfile, use_list=False) G, E = {}, {} for u, v, s in zip(rE[::3], rE[1::3], rE[2::3]): E[(u, v)] = bool(s) pg.add_edge(G, u, v) nodes_sign = list(rV) return psi, phi, nodes_sign, G, E
def __iter__(self): while True: try: pos = self.buf.tell() yield unpack(self.buf) except umsgpack.InsufficientDataException: self.buf.seek(pos) self.buf = io.BytesIO(self.buf.read()) raise StopIteration
def load_clips(): """ Load previous clips from DATA_FILE """ try: with open(DATA_FILE, 'r') as f: return msgpack.unpack(f, encoding='utf-8') except IOError: return {}
def fetch_inst_id(self): """ Fetches the institute id of the RU """ try: for d in msgpack.unpack(urllib2.urlopen( "%s/list/institutes?format=msgpack" % self.url)): if d['name'] == 'Radboud Universiteit Nijmegen': return d['id'] except IOError, e: # urllib2 exceptions are a subclass of IOError raise RuusterError(e)
def __init__(self, data): if isinstance(data, bytes): if data[:2] != b'\x1f\x8b': # gzip magic number self._data = msgpack.unpackb(data) return import io, gzip data = gzip.GzipFile(fileobj=io.BytesIO(data)) self._data = msgpack.unpack(data)
def fetch_room_ids(self, names): """ Fetches the ids of the rooms with the given names """ ret = {} names_set = set(names) for d in msgpack.unpack(urllib2.urlopen( "%s/list/locations?format=msgpack" % self.url)): name = d['name'].upper() # normalize: Hg -> HG if name in names_set: ret[name] = d['id'] return ret
def load_from_stream(self, stream, to_container, **opts): """ Load JSON config from given stream `stream`. :param stream: Stream will provide MessagePack-ed config content string :param to_container: callble to make a container object :param opts: keyword options passed to `msgpack.unpack` :return: Dict-like object holding configuration """ return msgpack.unpack(stream, object_hook=to_container, **opts)
def request(self, request): """ processing request {"op":"all_stats"} -> a dict of stats """ response = {} response.update({"status":"ok"}) try: req = unpack(request) except Exception, e: response.update({"status":"error","message":e.message})
def load_msgpack_gc(_file): output = open(_file, 'rb') # disable garbage collector gc.disable() mydict = msgpack.unpack(output) # enable garbage collector again gc.enable() output.close() return mydict
def _init(self, terms_fn=None, bigrams_fn=None, trigrams_fn=None, terms_by_root_form_fn=None, is_use_emoticons=False, is_dump_cls=False, is_load_cached_cls=False): # load dictionaries of terms/bigrams/trigrams (first element) # and sets of all terms/bigrams/trigrams if terms_fn: terms, self._terms_set = self._csv_to_dict(terms_fn) if bigrams_fn: bigrams, self._bigrams_set = self._csv_to_dict(bigrams_fn) if trigrams_fn: trigrams, self._trigrams_set = self._csv_to_dict(trigrams_fn) # try to load cached classifiers if is_load_cached_cls: self.terms_cls = self._load_terms_cls() self.bigrams_cls = self._load_bigrams_cls() self.trigrams_cls = self._load_trigrams_cls() # train classifiers if it didn't work if terms_fn and not self.terms_cls: self.terms_cls = self._train(terms) if is_dump_cls: self._dump_terms_cls() if bigrams_fn and not self.bigrams_cls: self.bigrams_cls = self._train(bigrams) if is_dump_cls: self._dump_bigrams_cls() if trigrams_fn and not self.trigrams_cls: self.trigrams_cls = self._train(trigrams) if is_dump_cls: self._dump_trigrams_cls() # load dictinary for stemming purposes if terms_by_root_form_fn: w = gzip.open(terms_by_root_form_fn) self._terms_by_root_form = msgpack.unpack(w, encoding='utf-8') w.close() # create a set of all terms from 'self._terms_by_root_form' # in 'self._allterms'; this is a temporary solution of increasing # the speed of looking up for terms to stem for aroot in self._terms_by_root_form: self._allterms.add(aroot) for aterm in self._terms_by_root_form[aroot]: self._allterms.add(aterm)
def work(self): time.sleep(random.random()) while True: task = None # check status, sleep longer if not processing, break if finished status = self.status() #print self.name, status if status == 'F': break elif status in ['?']: time.sleep(random.randint(15, 30)) continue try: if self.max_workers is not None and \ conn.llen(self.processing.format(self.name)) >= self.max_workers: time.sleep(1) continue result = conn.lpop(self.tasks.format(self.name)) if result is None: time.sleep(3) continue task = result caller, args, kwargs = unpack(task) conn.rpush(self.processing.format(self.name), task) conn.set(self.updated_at.format(self.name), time.mktime(time.gmtime())) print('work on {}, {}, {}'.format(caller, args[:5], kwargs)) if '.' in caller: module, method = caller.rsplit('.', 1) module = __import__(module, fromlist=[method]) caller = getattr(module, method) else: method = caller caller = sys.modules['__builtin__'].__dict__[method] except: print("can't obtain caller, locals: {}".format(locals())) traceback.print_exc() if task is not None: self.finish_task(task) continue try: caller(*args, **kwargs) except: traceback.print_exc() finally: self.finish_task(task)
def testPiecemapCreate(self, site): inner_path = self.createBigfile(site) content = site.storage.loadJson("content.json") assert "data/optional.any.iso" in content["files_optional"] file_node = content["files_optional"][inner_path] assert file_node["size"] == 10 * 1000 * 1000 assert file_node["sha512"] == "47a72cde3be80b4a829e7674f72b7c6878cf6a70b0c58c6aa6c17d7e9948daf6" assert file_node["piecemap"] == inner_path + ".piecemap.msgpack" piecemap = msgpack.unpack(site.storage.open(file_node["piecemap"], "rb"))["optional.any.iso"] assert len(piecemap["sha512_pieces"]) == 10 assert piecemap["sha512_pieces"][0] != piecemap["sha512_pieces"][1] assert piecemap["sha512_pieces"][0].encode("hex") == "a73abad9992b3d0b672d0c2a292046695d31bebdcb1e150c8410bbe7c972eff3"
def _parse_mirteFile(path): """ Open and parses the mirteFile at <path>. """ cache_path = os.path.join(os.path.dirname(path), CACHE_FILENAME_TEMPLATE % os.path.basename(path)) if (os.path.exists(cache_path) and os.path.getmtime(cache_path) >= os.path.getmtime(path)): with open(cache_path) as f: return msgpack.unpack(f) with open(path) as f: ret = yaml.load(f) with open(cache_path, 'w') as f: msgpack.pack(ret, f) return ret
def prepare_txn(self, transaction_id, do_cleanup=True): self._active_txn = True try: self.lock.upgrade() except (LockError, LockErrorT): # if upgrading the lock to exclusive fails, we do not have an # active transaction. this is important for "serve" mode, where # the repository instance lives on - even if exceptions happened. self._active_txn = False raise if not self.index or transaction_id is None: try: self.index = self.open_index(transaction_id, False) except RuntimeError: self.check_transaction() self.index = self.open_index(transaction_id, False) if transaction_id is None: self.segments = {} # XXX bad name: usage_count_of_segment_x = self.segments[x] self.compact = FreeSpace() # XXX bad name: freeable_space_of_segment_x = self.compact[x] else: if do_cleanup: self.io.cleanup(transaction_id) hints_path = os.path.join(self.path, 'hints.%d' % transaction_id) index_path = os.path.join(self.path, 'index.%d' % transaction_id) try: with open(hints_path, 'rb') as fd: hints = msgpack.unpack(fd) except (msgpack.UnpackException, msgpack.ExtraData, FileNotFoundError) as e: logger.warning('Repository hints file missing or corrupted, trying to recover') if not isinstance(e, FileNotFoundError): os.unlink(hints_path) # index must exist at this point os.unlink(index_path) self.check_transaction() self.prepare_txn(transaction_id) return except OSError as os_error: raise InternalOSError(os_error) from None if hints[b'version'] == 1: logger.debug('Upgrading from v1 hints.%d', transaction_id) self.segments = hints[b'segments'] self.compact = FreeSpace() for segment in sorted(hints[b'compact']): logger.debug('Rebuilding sparse info for segment %d', segment) self._rebuild_sparse(segment) logger.debug('Upgrade to v2 hints complete') elif hints[b'version'] != 2: raise ValueError('Unknown hints file version: %d' % hints[b'version']) else: self.segments = hints[b'segments'] self.compact = FreeSpace(hints[b'compact'])
def __init__(self, dataset, machine_id, all_machines, max_trees=None): with open('{}.random'.format(dataset), 'r+b') as packfile: data = msgpack.unpack(packfile, use_list=False) g_adj, g_ew, gold_signs, phi = load_real_graph(dataset) self.bfs_root = max(g_adj.items(), key=lambda x: len(x[1]))[0] nodes = list((range(len(g_adj)))) self.gold_signs = np.array([gold_signs[u] for u in nodes]) self.gold = np.array([gold_signs[u] for u in nodes]) batch_orders_raw = np.array(data[b'batch_order']) self.batch_orders = {} for bo, tf in zip(batch_orders_raw, train_size): self.batch_orders[int(1000*tf)] = np.array([list(_) for _ in bo]) with open('{}_extra_000.random'.format(dataset), 'r+b') as packfile: pertub_35 = np.array(msgpack.unpack(packfile, use_list=False)) changed_signs = data[b'changed_signs'] self.changed_signs = np.vstack((changed_signs, pertub_35[np.newaxis, :, :])) if max_trees is not None: max_trees *= len(all_machines) mst_raw = data[b'mst'] self.mst_processed = [] for tree_edges in mst_raw[:max_trees]: self.mst_processed.append(process_edges_into_tree(tree_edges, g_ew)) rst_raw = data[b'rst'] self.rst_processed = [] for tree_edges in rst_raw[:max_trees]: self.rst_processed.append(process_edges_into_tree(tree_edges, g_ew)) self.nodes_order = np.array(data[b'nodes_order']) self.mmapping = {v: i for i, v in enumerate(sorted(all_machines))} self.this_machine = self.mmapping[machine_id] self.sorted_training_set = None self.sorted_testing_set = None
def load(paths): for path in sorted(paths): for doc in msgpack.unpack(file(path), encoding='utf-8'): soup=BeautifulSoup(doc['_page']) desc=soup.find(id='jobDescription').text try: date=parse_publication_time(doc['publication_time']) except ValueError, err: warnings.warn(str(err)) continue yield dict(title=doc['name'], desc=desc, url=doc['job_url'], company=doc['employer'], location=doc['location_text'], published=date)
def wrapper(io): http_dict = msgpack.unpack(io) meta = http_dict['meta'] headers = meta['headers'] cookies = meta['cookies'] request = http_dict['request'] environ = { 'wsgi.version': (1, 0), 'wsgi.url_scheme': 'https' if meta['secure'] else 'http', 'wsgi.input': io, 'wsgi.errors': Log(), 'wsgi.multithread': False, 'wsgi.multiprocess': True, 'wsgi.run_once': False, 'SERVER_SOFTWARE': "Cocaine", 'REQUEST_METHOD': meta['method'], 'SCRIPT_NAME': meta.get('script_name', ''), 'PATH_INFO': meta.get('path_info', ''), 'QUERY_STRING': meta.get('query_string', ''), 'CONTENT_TYPE': headers.get('CONTENT-TYPE', ''), 'CONTENT_LENGTH': headers.get('CONTENT_LENGTH', ''), 'REMOTE_ADDR': meta.get('remote_addr', ''), 'REMOTE_PORT': meta.get('remote_port', ''), 'SERVER_NAME': '', 'SERVER_PORT': '', 'SERVER_PROTOCOL': '', 'HTTP_HOST': meta['host'] } for key, value in headers.items(): key = 'HTTP_' + key.upper().replace('-', '_') if key not in ('HTTP_CONTENT_TYPE', 'HTTP_CONTENT_LENGTH'): environ[key] = value def start_response(status, response_headers, exc_info=None): if exc_info: try: raise exc_info[0], exc_info[1], exc_info[2] finally: exc_info = None # Avoid circular ref. pack({'code': int(status.split(' ')[0]), 'headers': response_headers}, io) result = function(environ, start_response) pack(result, io) if hasattr(result, 'close'): result.close()
def load_from_stream(stream, nworkers, use_threads): """ Loads a Safe form a `stream'. If you load from a file, use `open' for that function also handles locking. """ start_time = time.time() l.debug("Unpacking ...") magic = stream.read(len(SAFE_MAGIC)) if magic != SAFE_MAGIC: raise WrongMagicError data = msgpack.unpack(stream, use_list=True) l.debug(" unpacked in %.2fs", time.time() - start_time) if "type" not in data or not isinstance(data["type"], basestring) or data["type"] not in TYPE_MAP: raise SafeFormatError("Invalid `type' attribute") return TYPE_MAP[data["type"]](data, nworkers, use_threads)
def merge_attached_event(mpack_event, data): # Merge msgpack serialized event. if mpack_event.size > MAX_MSGPACK_EVENT_SIZE_BYTES: return try: event = unpack(mpack_event) except (UnpackException, ExtraData) as e: minidumps_logger.exception(e) return for key in event: value = event.get(key) if value is not None: data[key] = value