def dump_data(directory, filter_string, contact_point, keyspace): cluster = Cluster([contact_point], control_connection_timeout=60) session = cluster.connect(keyspace) session.row_factory = dict_factory tables = [] if filter_string is not None: filter_re = re.compile(filter_string) if not os.path.exists(directory): os.makedirs(directory) os.chdir(directory) with open('stream_metadata.mpk', 'wb') as fh: for row in session.execute('select * from stream_metadata', timeout=None): if filter_string is None or any((filter_re.search(x) for x in row.values() if isinstance(x, basestring))): tables.append(row['stream']) fh.write(msgpack.packb(row)) for table in sorted(set(tables)): print 'dumping table: %s' % table with open('%s.mpk' % table, 'wb') as fh: try: for row in session.execute('select * from %s' % table, timeout=None): for k in row: if type(row[k]) == uuid.UUID: row[k] = str(row[k]) fh.write(msgpack.packb(row)) except ReadTimeout: print traceback.format_exc() session.shutdown() cluster.shutdown()
def test_odict(): seq = [(b'one', 1), (b'two', 2), (b'three', 3), (b'four', 4)] od = OrderedDict(seq) assert unpackb(packb(od), use_list=1) == dict(seq) def pair_hook(seq): return list(seq) assert unpackb(packb(od), object_pairs_hook=pair_hook, use_list=1) == seq
def test_call_with_multi_args(self): addr = get_random_ipc_socket() with utils.TestingServer(methods=self.get_methods(), addresses=addr): context = zmq.Context() socket = context.socket(zmq.REQ) socket.connect(addr) socket.send(msgpack.packb({ 'm': 'method3', 'k': {'name': 'WORLD'}, }, encoding='utf-8')) response = msgpack.unpackb(socket.recv(), encoding='utf-8') assert response == {'r': 'Hello, WORLD!'} socket.send(msgpack.packb({ 'm': 'method3', 'k': {'greeting': 'HOWDY'}, }, encoding='utf-8')) response = msgpack.unpackb(socket.recv(), encoding='utf-8') assert response == {'r': 'HOWDY, world!'} socket.send(msgpack.packb({ 'm': 'method3', 'k': {'greeting': 'HOWDY', 'name': 'MAN'}, }, encoding='utf-8')) response = msgpack.unpackb(socket.recv(), encoding='utf-8') assert response == {'r': 'HOWDY, MAN!'} socket.send(msgpack.packb({ 'm': 'method3', 'a': ('hey', 'man'), }, encoding='utf-8')) response = msgpack.unpackb(socket.recv(), encoding='utf-8') assert response == {'r': 'hey, man!'}
def process_request(): frames = socket.recv_multipart() # killed by gsd i = frames.index('') command = frames[i + 2] if command == '\x02': global interrupted interrupted = True return i = frames.index('', 1) sequence, timestamp, expiry = msgpack.unpackb(frames[i+1]) method = frames[i+2] params = msgpack.unpackb(frames[i+3]) try: global converter ret = getattr(converter, method)(*params) except: ret = '' frames = frames[:i+1] now = int(round(time.time() * 1000)) frames.append(msgpack.packb([sequence, now, 200])) frames.append(msgpack.packb(ret)) socket.send_multipart(frames)
def test_msgpack_fifo(self): import msgpack v1 = [1, 2, 3, 4] v2 = [5, 6, 7, 8] v3 = {b"a": 9, b"b": 10, b"c": 11} s1 = msgpack.packb(v1) s2 = msgpack.packb(v2) s3 = msgpack.packb(v3) unpacker = msgpack.Unpacker() unpacker.feed(s1) unpacker.feed(s2) unpacker.feed(s3[:4]) assert next(iter(unpacker)) == v1 assert next(iter(unpacker)) == v2 try: next(iter(unpacker)) except StopIteration: pass else: raise Exception("must fail") unpacker.feed(s3[4:]) assert next(iter(unpacker)) == v3
def test_key_length_msgpacked_items(): key = b'x' * 32 # 31 bytes is the limit for fixstr msgpack type data = {key: b''} item_keys_serialized = [ msgpack.packb(key), ] assert valid_msgpacked_dict(msgpack.packb(data), item_keys_serialized)
def serve(self): # Make stdin non-blocking fl = fcntl.fcntl(sys.stdin.fileno(), fcntl.F_GETFL) fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, fl | os.O_NONBLOCK) # Make stdout blocking fl = fcntl.fcntl(sys.stdout.fileno(), fcntl.F_GETFL) fcntl.fcntl(sys.stdout.fileno(), fcntl.F_SETFL, fl & ~os.O_NONBLOCK) unpacker = msgpack.Unpacker(use_list=False) while True: r, w, es = select.select([sys.stdin], [], [], 10) if r: data = os.read(sys.stdin.fileno(), BUFSIZE) if not data: return unpacker.feed(data) for type, msgid, method, args in unpacker: method = method.decode('ascii') try: try: f = getattr(self, method) except AttributeError: f = getattr(self.repository, method) res = f(*args) except Exception as e: sys.stdout.buffer.write(msgpack.packb((1, msgid, e.__class__.__name__, e.args))) else: sys.stdout.buffer.write(msgpack.packb((1, msgid, None, res))) sys.stdout.flush() if es: return
def proc_luna_running(**kw): """ Fetch test statuses from Redis, if test finisged notify service via API and call reduce job. """ ext = {'test_id': kw.get('redis_value', {}).get('id')} logger = get_logger(**ext) if not ('t_fqdn' in kw and 't_tank_id' in kw): logger.erro('proc_luna_running call. Malformed params:{}'.format(kw)) try: except TankClientError as e: logger.error('Tank API call failed: {}'.format(e)) raise if tank_msg['status_code'] != 'FINISHED': if kw.get('status') and TestStateTr.tank_by_port(kw.get('status')) != tank_msg['status_code']: # test state changes since last check, need to notify port_state = TestStateTr.port_by_tank(tank_msg['status_code']) redis_value = kw['redis_value'] redis_value.update({'status': port_state}) redis.hset(r_adr['monitor_finish'], kw['id'], msgpack.packb(redis_value)) diff = { 'status': port_state, }
def call(self, command, params=None): """ Sends the provided command to Serf for evaluation, with any parameters as the message body. """ if self._socket is None: raise SerfConnectionError('handshake must be made first') header = msgpack.packb({"Seq": self._counter(), "Command": command}) if params is not None: body = msgpack.packb(params) self._socket.sendall(header + body) else: self._socket.sendall(header) unpacker = msgpack.Unpacker() unpacker.feed(self._socket.recv(4096)) response = SerfResult() for item in unpacker: if response.head is None: response.head = item else: response.body = item break return response
def main(app, hosts, timeout): complain = False context = zmq.Context() for host in hosts: request = context.socket(zmq.DEALER) request.setsockopt(zmq.LINGER, 0) request.connect('tcp://%s:5000' % host) # Statistics request.send_multipart([ msgpack.packb(SLOT_INFO), msgpack.packb([]) ]) poller = zmq.Poller() poller.register(request, zmq.POLLIN) sockets = dict(poller.poll(timeout = timeout)) if request in sockets and sockets[request] == zmq.POLLIN: response = msgpack.unpackb(request.recv()) if app not in response["apps"]: print "The '%s' app was not found on '%s'." % (app, host) exit(1) state = response["apps"][app]["state"] print "The '%s' app is %s on '%s'." % (app, state, host) complain |= (state != "running") else: print "ERROR: Host '%s' is not responding." % host complain |= True exit(complain)
def default(obj): # Serialize Bn objects if isinstance(obj, Bn): if obj < 0: neg = b"-" data = (-obj).binary() else: neg = b"+" data = obj.binary() return msgpack.ExtType(0, neg + data) # Serialize EcGroup objects elif isinstance(obj, EcGroup): nid = obj.nid() packed_nid = msgpack.packb(nid) return msgpack.ExtType(1, packed_nid) # Serialize EcPt objects elif isinstance(obj, EcPt): nid = obj.group.nid() data = obj.export() packed_nid = msgpack.packb((nid, data)) return msgpack.ExtType(2, packed_nid) raise TypeError("Unknown type: %r" % (obj,))
def send(self, payload, deprecated=()): """Send a message with topic, payload ` Topic is a unicode string. It will be sent as utf-8 encoded byte array. Payload is a python dict. It will be sent as a msgpack serialized dict. If payload has the key '__raw_data__' we pop if of the payload and send its raw contents as extra frames everything else need to be serializable the contents of the iterable in '__raw_data__' require exposing the pyhton memoryview interface. """ assert deprecated is (), "Depracted use of send()" assert "topic" in payload, "`topic` field required in {}".format(payload) if "__raw_data__" not in payload: # IMPORTANT: serialize first! Else if there is an exception # the next message will have an extra prepended frame serialized_payload = serializer.packb(payload, use_bin_type=True) self.socket.send_string(payload["topic"], flags=zmq.SNDMORE) self.socket.send(serialized_payload) else: extra_frames = payload.pop("__raw_data__") assert isinstance(extra_frames, (list, tuple)) self.socket.send_string(payload["topic"], flags=zmq.SNDMORE) serialized_payload = serializer.packb(payload, use_bin_type=True) self.socket.send(serialized_payload, flags=zmq.SNDMORE) for frame in extra_frames[:-1]: self.socket.send(frame, flags=zmq.SNDMORE, copy=True) self.socket.send(extra_frames[-1], copy=True)
def test_roundtrip_deferred(self): from twisted.internet import reactor from twisted.internet.task import deferLater v = "yaaay!" p_ctx = [] class SomeService(ServiceBase): @rpc(Unicode, _returns=Unicode) def yay(ctx, u): def _cb(): return u p_ctx.append(ctx) return deferLater(reactor, 0.1, _cb) app = Application([SomeService], 'tns', in_protocol=MessagePackDocument(), out_protocol=MessagePackDocument()) prot = self.gen_prot(app) request = msgpack.packb({'yay': [v]}) def _ccb(_): val = prot.transport.value() print repr(val) val = msgpack.unpackb(val) print repr(val) self.assertEquals(val, [0, msgpack.packb(v)]) prot.dataReceived(msgpack.packb([1, request])) return p_ctx[0].out_object[0].addCallback(_ccb)
def encode(self): ret = {'name': self._name, 'time': get_time(), 'epoch_milli': get_milli(), 'host': get_name(), 'data': msgpack.packb(self.run())} return msgpack.packb(ret)
def on_heartbeat(w): global i i += 1 w.write(msgpack.packb([1, 0, []])) w.write(msgpack.packb([3, i, ["echo"]])) w.write(msgpack.packb([4, i, ["echo"]])) w.write(msgpack.packb([6, i, []]))
def write(self, key, data): assert b'\r\n' not in msgpack.packb(data, use_bin_type=True) with open(self.file_loc, 'ab') as fileobj: fileobj.write(key.encode('utf-8')) fileobj.write(b'\t') fileobj.write(msgpack.packb(data, use_bin_type=True)) fileobj.write(b'\r\n')
def main(argv): map_path = argv[1] result_path = argv[2] t = Teemap(map_path) result = {} frontlayer = None # Works thanks to hack in tml for group in t.groups: if group.name == 'Game': for layer in group.layers: if type(layer) == TileLayer and layer.name == 'Front': frontlayer = layer break add(result, gameTiles, t.gamelayer) add(result, frontTiles, frontlayer) # These don't seem to be working #add(result, switchTiles, switchlayer) #add(result, speedupTiles, t.speeduplayer) #add(result, teleTiles, t.telelayer) with open(result_path, 'wb') as out: out.write(msgpack.packb(t.gamelayer.width)) out.write(msgpack.packb(t.gamelayer.height)) out.write(msgpack.packb(result))
def Emit(self, *args): packet = {} extras = {} packet['data'] = args packet['type'] = self.BINARY_EVENT if self._hasBin(args) else self.EVENT # set namespace to packet if 'nsp' in self._flags: packet['nsp'] = self._flags['nsp'] del self._flags['nsp'] else: packet['nsp'] = '/' extras['flags'] = self._flags if len(self._flags) > 0 else '' rooms = self._getRooms() extras['rooms'] = rooms if len(rooms) > 0 else '' self._client.publish(self._key + "#emitter", msgpack.packb([packet, extras])) if extras['rooms']: for room in rooms: chn = "#".join((self._key, packet['nsp'], room, "")) self._client.publish(chn, msgpack.packb([self.uid,packet, extras])) else: chn = "#".join((self._key, packet['nsp'], "")) self._client.publish(chn, msgpack.packb([self.uid,packet, extras])) self._flags = {} self._rooms = []
def add_corpus(self, corpus, name, password, options, tlds=None): now = now_ts() yield self.db()["corpus"].insert_one({ "_id": corpus, "name": name, "password": salt(password), "options": options, "total_webentities": 0, "webentities_in": 0, "webentities_in_untagged": 0, "webentities_in_uncrawled": 0, "webentities_out": 0, "webentities_undecided": 0, "webentities_discovered": 0, "total_crawls": 0, "crawls_pending": 0, "crawls_running": 0, "total_pages": 0, "total_pages_crawled": 0, "total_pages_queued": 0, "total_links_found": 0, "recent_changes": False, "last_index_loop": now, "links_duration": 1, "last_links_loop": 0, "tags": Binary(msgpack.packb({})), "webentities_links": Binary(msgpack.packb({})), "created_at": now, "last_activity": now, "tlds": tlds }) yield self.init_corpus_indexes(corpus)
def test_node_service_bad_on_read(): io = CocaineIO.instance() node = Service("node", host="localhost", port=10053, loop=io) malformed_message = msgpack.packb([-999, 0]) node.on_read(malformed_message) message = msgpack.packb([-999, 0, []]) node.on_read(message)
def test_node_service_bad_on_read(): io = IOLoop.current() node = Service("node", endpoints=[["localhost", 10053]], io_loop=io) malformed_message = msgpack.packb([-999, 0]) node.on_read(malformed_message) message = msgpack.packb([-999, 0, []]) node.on_read(message)
def slave_sync_data(self, port): """ 同步数据 """ import time time.sleep(1) self._server.is_sync = True client = PyCachedClient(self._address[0], port) self.add_slave(client) logger.info('Slave[%s:%d] Sync data start' % (self._address[0], port)) items = {} pos = 0 MAX_SEND = 20000 # 同步数据多少Key为一组 all_len = len(self._server.memory.caches.items()) for key, val in self._server.memory.caches.items(): pos += 1 items[key] = val if pos % MAX_SEND == 0: client.sync_data(pos, all_len, msgpack.packb(items)) items = {} if pos % MAX_SEND != 0: client.sync_data(pos, all_len, msgpack.packb(items)) self._server.is_sync = False client.sync_ok() logger.info('Slave[%s:%d] Sync data success' % (self._address[0], port))
def count(doc): if attr == '__meta__': return len(msgpack.packb(doc.doc)) for i, (name, klass, nelem) in enumerate(doc.stores): if name == attr: return len(msgpack.packb(doc.instances[i])) return 0
def aggregate_host(request, response): raw = yield request.read() TASK = msgpack.unpackb(raw) taskId = TASK['id'] logger = get_logger_adapter(taskId) logger.info("Handle task") cfg = TASK['config'] # config of aggregator token = TASK['token'] prtime = TASK['prevtime'] currtime = TASK['currtime'] dg = MysqlDG.get_service(DATABASEAPP) q = TABLEREGEX.sub(token, cfg['query']) q = TIMEREGEX.sub("1=1", q) logger.debug("QUERY: %s", q) pickled_res = yield dg.enqueue("query", msgpack.packb((token, q))) res = cPickle.loads(pickled_res) logger.debug(str(res)) try: ret = float(res[0][0]) # SELECT COUNT(*) logger.info("Result from DG %s", ret) if cfg.get('rps'): ret = ret / (currtime - prtime) except Exception: ret = 0 logger.info("%s", ret) response.write(msgpack.packb(ret)) response.close()
def handle(self): try: logger.debug("(self:{}) ->".format(self)) while 1: try: buffer = self.read(4) size = struct.unpack("<L", buffer)[0] try: if self.server.serialization == "json": cmd, params = json.loads(self.read(size)) if params is not None: packet = json.dumps(getattr(self.server.instance, cmd)(params)) else: packet = json.dumps(getattr(self.server.instance, cmd)()) else: cmd, params = msgpack.unpackb(self.read(size), encoding='utf-8') packet = msgpack.packb((0, getattr(self.server.instance, cmd)(*params)), use_bin_type=True) except: result = traceback.format_exc() logger.error(result) packet = msgpack.packb((2, result), use_bin_type=True) self.request.sendall(struct.pack("<L", len(packet)) + packet) except self.EndConnection: return except socket.error as e: logger.error(e) return finally: logger.debug("handle(self:{}) <-".format(self))
def send(self, message): if config.debug_socket: self.log("Send: %s, to: %s, site: %s, inner_path: %s, req_id: %s" % (message.get("cmd"), message.get("to"), message.get("params", {}).get("site"), message.get("params", {}).get("inner_path"), message.get("req_id"))) self.last_send_time = time.time() if self.protocol == "zeromq": if self.zmq_sock: # Outgoing connection self.zmq_queue.append(message) if self.zmq_working: self.log("ZeroMQ already working...") return while self.zmq_queue: self.zmq_working = True message = self.zmq_queue.pop(0) self.zmq_sock.send(msgpack.packb(message)) self.handleMessage(msgpack.unpackb(self.zmq_sock.recv())) self.zmq_working = False else: # Incoming request self.server.zmq_sock.send(msgpack.packb(message)) else: # Normal connection data = msgpack.packb(message) self.bytes_sent += len(data) self.server.bytes_sent += len(data) self.sock.sendall(data) self.last_sent_time = time.time() return True
def get_payload(self): # Fill up send_queue with new messages until full. try: for i in xrange(self.send_queue_size - len(self.send_queue)): self.send_queue.append(self.main_queue.get_nowait()) self.main_queue.task_done() except Queue.Empty: pass if not len(self.send_queue): return '', 0 # self.log("info", # "Trying to pack queue with {} messages.".format( # len(self.send_queue) # ) # ) # Pack messages and try to fit into limit. msg = msgpack.packb({'data': self.send_queue}) msg_len = len(self.send_queue) while len(msg) > self.config['max_msg_size_bytes']: self.measr_avg_size = len(msg) / len(self.send_queue) self.update_queue_size() msg = msgpack.packb( {'data': self.send_queue[:self.send_queue_size]} ) msg_len = self.send_queue_size # self.log( # "info", # "Packed {} messages with total size {} bytes. {} messages in" # " the send_queue left.".format(msg_len, len(msg), # len(self.send_queue)) # ) return msg, msg_len
def _serialize_context(context): # Our sending format is made up of two messages. The first has a # quick to unpack set of meta data that our collector is going to # use for routing and stats. This is much faster than having the # collector decode the whole event. We're just going to use python # struct module to make a quick and dirty data structure context_dict = context.to_dict() for key in ("host", "type"): if len(context_dict.get(key, "")) > 64: raise ValueError("Value too long: %r" % key) meta_data = struct.pack( META_STRUCT_FMT, META_STRUCT_VERSION, context_dict["end"], context_dict["host"], context_dict["type"] ) try: context_data = msgpack.packb(context_dict) except TypeError: try: # If we fail to serialize our context, we can try again with an # enhanced packer (it's slower though) context_data = msgpack.packb(context_dict, default=utils.msgpack_encode_default) except TypeError: log.exception("Serialization failure (not fatal, dropping data)") # One last try after dropping the body context_dict["body"] = None context_data = msgpack.packb(context_dict) return meta_data, context_data
def handle_error(self, p_ctx, others, exc): self.spyne_tpt.get_out_string(p_ctx) if isinstance(exc, InternalError): error = self.spyne_tpt.OUT_RESPONSE_SERVER_ERROR else: error = self.spyne_tpt.OUT_RESPONSE_CLIENT_ERROR data = p_ctx.out_document[0] if isinstance(data, dict): data = list(data.values()) out_string = msgpack.packb([ error, msgpack.packb(data), ]) self.enqueue_outresp_data(id(p_ctx), out_string) p_ctx.transport.resp_length = len(out_string) p_ctx.close() try: process_contexts(self, others, p_ctx, error=error) except Exception as e: # Report but ignore any exceptions from auxiliary methods. logger.exception(e)
def handle_data(self, message): payload = msgpack.unpackb(message) if 'connect' in payload: host, port_str = payload['connect'].split('::') port = int(port_str) self._sock = socket.socket() self._sock.connect((host, port)) response = msgpack.packb({'type': 'STATUS', 'value': 'OPEN'}) self._send(response) eventlet.spawn(self._reader) elif 'close' in payload: self._sock.close() status = msgpack.packb({'type': 'STATUS', 'value': 'CLOSED'}) self._send(status) elif 'content' in payload: try: self._sock.send(payload['content']) except Exception as exc: print exc self._sock.close() else: message = 'Unknown payload type: {}\n'.format(payload.type) sys.write(message)
def test_send_msgpack(self, mock): mock.register_uri(requests_mock.ANY, requests_mock.ANY, text='{"result":"OK"}') self.assertTrue(ubirch.API().send(TEST_UUID, msgpack.packb([1, 2, 3])))
async def set(self, key, value, expiration=0): packed = msgpack.packb(value, use_bin_type=True) await self.client.set(key, packed, expiration)
for idy, py in enumerate( range(px - FLAGS.window_size, px) + range(px + 1, px + FLAGS.window_size + 1)): if py < 0 or py >= FLAGS.data_shard_cols: row_weight[px].append(0.0) continue y = data_shared[i, py] if FLAGS.use_PMI and x in cooccur_mat and y in cooccur_mat[x]: if FLAGS.data_shard_rows * FLAGS.data_shard_cols * cooccur_mat[ x][y] < count[x][1] * count[y][ 1] * FLAGS.num_neg_sample: num_filtered_out += 1 filter_out_arr[x] += 1 row_weight[px].append(0.0) continue row_weight[px].append(1.0) slice_cnt += 1 weights.append(row_weight) print("\rFinished %6d / %6d." % (i, FLAGS.data_shard_rows)), sys.stdout.flush() print("\n# filtered out: %d" % num_filtered_out) for i in range(5): filter_rate = filter_out_arr[i] / float( count[i][1] * 2 * FLAGS.window_size) print(str(count[i]) + ' ' + str(filter_rate)) open(FLAGS.output_training_file, 'wb').write( msgpack.packb((data, count, dictionary, reverse_dictionary, weights)))
def test(self): d = StableDict(foo=1, bar=2, boo=3, baz=4) self.assert_equal(list(d.items()), [('bar', 2), ('baz', 4), ('boo', 3), ('foo', 1)]) self.assert_equal(hashlib.md5(msgpack.packb(d)).hexdigest(), 'fc78df42cd60691b3ac3dd2a2b39903f')
def listener(): shutdown = False while not shutdown: msg_str = self._redis.brpoplpush(self.q_name, self.processing_q, timeout=0) if msg_str == b'shutdown': shutdown = True self._redis.lrem(self.processing_q, -1, msg_str) self.loop = None break msg_dict = msgpack.unpackb(msg_str, raw=False) msg_dict['worker'] = self.id msg_dict['processing_started'] = dt.timestamp(dt.utcnow()) msg_dict['progress'] = 0 new_record = msgpack.packb(msg_dict, use_bin_type=False) self._redis.set(self.job_status_stem + str(msg_dict['id']), new_record) all_ok = True short_result = None utils = self._utility_functions() utils['full_message'] = msg_dict utils['update_progress'] = functools.partial( utils['update_progress'], str(msg_dict['id'])) result = None for func in self.funcs_to_execute: if msg_dict[ 'functions'] is None or func.__name__ in msg_dict[ 'functions']: try: result = func(msg_dict['msg'], utils) except Exception as e: all_ok = False logging.error(e) logging.info( "Moving message {} to dead letter queue". format(msg_dict['id'])) if msg_dict['callback']: self.http.request('GET', msg_dict['callback'], fields={ 'success': 0, 'job_id': str(msg_dict['id']), 'short_result': None }) msg_dict['progress'] = -1 msg_dict['result'] = str(e) new_record = msgpack.packb(msg_dict, use_bin_type=False) self._redis.set( self.job_status_stem + str(msg_dict['id']), new_record) self._redis.rpush(self.dead_letter_q, msg_str) if all_ok: logging.info('Completed job {}'.format(str( msg_dict['id']))) msg_dict['worker'] = None if result and type(result) in [tuple, list ] and len(result) > 1: short_result = result[0] result = result[1] else: short_result = result msg_dict['result'] = result msg_dict['short_result'] = short_result self._redis.publish('pub_' + str(msg_dict['id']), short_result) msg_dict['progress'] = 100 msg_dict['processing_finished'] = dt.timestamp(dt.utcnow()) new_record = msgpack.packb(msg_dict, use_bin_type=False) self._redis.set(self.job_status_stem + str(msg_dict['id']), new_record) if msg_dict['callback']: self.http.request('GET', msg_dict['callback'], fields={ 'success': 1, 'job_id': str(msg_dict['id']), 'short_result': short_result }) self._redis.lrem(self.processing_q, -1, msg_str) self._redis.lrem(self.jobs_refs_q, 1, str(msg_dict['id']))
def send_busy(self, client): client.send_bytes(msgpack.packb({"type": MSG_BUSY}, use_bin_type=True))
def test_register_identity_msgpack(self, mock): mock.register_uri(requests_mock.ANY, requests_mock.ANY, text='{"result":"OK"}') self.assertTrue(ubirch.API().register_identity(msgpack.packb([1, 2, 3])))
def dumps(self, obj): return msgpack.packb(obj, use_bin_type=True)
def to_db(data): return sqlite3.Binary(packb(data, use_bin_type=True))
def _dumps(cls, value): return msgpack.packb(value, default=cls._serialize, use_bin_type=True)
def send_event(self, eventname, data=None, target=None): # On the sim side, target is obtained from the currently-parsed stack command target = target or stack.routetosender() or [b'*'] pydata = msgpack.packb(data, default=encode_ndarray, use_bin_type=True) self.event_io.send_multipart(target + [eventname, pydata])
def pack(s): return packb(s, use_bin_type=True)
def process_image(self, frames): # parse frames name = frames[0] metamsg = frames[1] x_comp = frames[2] x_enc = zlib.decompress(x_comp) # decode frames # -- meta meta = decode_larcv1_metamsg(metamsg) # -- array arr = msgpack.unpackb(x_enc, object_hook=m.decode) shape = arr.shape msg_batchsize = shape[0] # prepare numpy array for output # note, we through away the background scores to save egress data # we save results in half-precision. since numbers between [0,1] precision still good to 2^-11 at worse if not self.reply_in_float16: ssnetout = np.zeros( (shape[0], self.NCLASSES - 1, shape[2], shape[3]), dtype=np.float16) else: ssnetout = np.zeros( (shape[0], self.NCLASSES - 1, shape[2], shape[3]), dtype=np.float32) blobshape = (self.BATCHSIZE, 1, shape[2], shape[3]) # run the net for the plane #print "PlaneNetworkWorker[{}]::Plane[{}]: processing {}".format(self._identity,self.PLANEID,name) self.net.blobs['data'].reshape(*blobshape) # process the images for ibatch in range(0, msg_batchsize, self.BATCHSIZE): imgslice = arr[ibatch * self.BATCHSIZE:(ibatch + 1) * self.BATCHSIZE, :] self.net.blobs['data'].data[...] = imgslice tforward = time.time() #print "PlaneNetworkWorker[{}]::Plane[{}]: forward".format(self._identity,self.PLANEID) self.net.forward() tforward = time.time() - tforward #print "PlaneNetworkWorker[{}]::Plane[{}]: network returns after %.03f secs".format(self._identity,self.PLANEID)%(tforward) # copy predictions to ssnetout if (ibatch + 1) * self.BATCHSIZE > msg_batchsize: remaining = msg_batchsize % self.BATCHSIZE start = ibatch * self.BATCHSIZE end = ibatch * self.BATCHSIZE + remaining if self.reply_in_float16: ssnetout[start:end, :] = self.net.blobs['softmax'].data[ 0:remaining, :].astype(np.float16) else: ssnetout[start:end, :] = self.net.blobs['softmax'].data[ 0:remaining, :] else: start = ibatch * self.BATCHSIZE end = (ibatch + 1) * self.BATCHSIZE if self.reply_in_float16: ssnetout[start:end, :] = self.net.blobs['softmax'].data[ 0:self.BATCHSIZE, 1:, :].astype(np.float16) else: ssnetout[start:end, :] = self.net.blobs['softmax'].data[ 0:self.BATCHSIZE, 1:, :] # we threshold score images so compression performs better outslice = ssnetout[start:end, :] for c in range(outslice.shape[1]): chslice = outslice[:, c, :].reshape( (1, 1, imgslice.shape[2], imgslice.shape[3])) chslice[imgslice < 5.0] = 0 # encode x_enc = msgpack.packb(ssnetout, default=m.encode) x_comp = zlib.compress(x_enc, self._compression_level) # make the return message #print "PlaneNetworkWorker[{}]::Plane[{}] processed name=\"{}\" shape={} meta={}".format(self._identity,self.PLANEID,name,ssnetout.shape,meta.dump().strip()) reply = ["plane%d" % (self.PLANEID)] # topic frame reply.append(name.encode('utf-8')) reply.append(meta.dump().strip()) reply.append(x_comp) return reply
def send_stream(self, name, data): self.stream_out.send_multipart( [name + self.node_id, msgpack.packb(data, default=encode_ndarray, use_bin_type=True)])
def pack(obj): buf = msgpack.packb(obj, use_bin_type=True) return buf
def makePickle(self, record): return msgpack.packb(record.__dict__, default=self.msgpack_encoder)
def marshall(to_pack): packed = msgpack.packb(to_pack, default=utils.default, use_bin_type=True) packed_encoded = base64.b64encode(packed) return packed_encoded
def send_action(self, action): dat = msgpack.packb({"command": "".join(map(str, action))}) self.write_message(dat, binary=True)
def gen_forwarder(secret, obj, nonce, options={}): return crypto.encrypt( msgpack.packb((nonce, id(obj), options)), secret )
def pack_msg(msg): # pragma: no cover return msgpack.packb(msg, use_bin_type=True)
def encode(self, buf): buf = np.asanyarray(buf) items = buf.tolist() items.append(buf.dtype.str) return msgpack.packb(items, encoding=self.encoding)
def start(self): context = zmq.Context() logging.info("Connecting to server…") client = context.socket(zmq.REQ) client.connect(SERVER_ENDPOINT + str(self.port)) times = [] startTime = time.time() count = 0 while True: try: item = self.queue.get(block=False) except Exception as e: item = None if (item): seq = str(item.get('seq')).encode() data = item.get('data') self.pendingQueue[seq] = data request = {'seq': seq, 'data': data} encoded = msgpack.packb(request) client.send(encoded) count = count + 1 curTime = time.time() diff = curTime - startTime if (diff > 5): startTime = curTime sequenceDiff = count count = 0 print("got {count} messages in {diff} seconds".format( count=sequenceDiff, diff=diff)) print("thats {mps} messages per second".format( mps=sequenceDiff / diff)) retries_left = REQUEST_RETRIES while True: if (client.poll(REQUEST_TIMEOUT) & zmq.POLLIN) != 0: reply = client.recv() retSeq = int(reply) pendingItem = self.pendingQueue.get(reply) if (pendingItem): self.pendingQueue.pop('reply', None) break else: logging.error("Malformed reply from server: %s", reply) continue retries_left -= 1 logging.warning("No response from server") if retries_left == 0: # Socket is confused. Close and remove it. client.setsockopt(zmq.LINGER, 0) print('close') client.close() logging.error("Server seems to be offline, abandoning") # sys.exit() logging.info("Reconnecting to server…") # Create new connection client = context.socket(zmq.REQ) print('connect') client.connect(SERVER_ENDPOINT + str(self.port)) break
def encode(self, msg): return msgpack.packb(msg, use_bin_type=True)
def RequestResponse(req): data = msgpack.packb(req) socket.send(data) return msgpack.unpackb(socket.recv())
def query_raw(self, obj): raw_bytes = msgpack.packb(obj) raw_bytes = bytes(b ^ ((self.key + i) & 0xFF) for i, b in enumerate(raw_bytes)) self.sock.send(raw_bytes) return msgpack.unpackb(self.sock.recv(BUFFER_SIZE), raw=False)
in_row_indices = list(range(num_rows)) out_row_start_dict = {} cumulative_position = 0 with open(file_path, 'rb') as my_file: mmap_file = mmap.mmap(my_file.fileno(), 0, prot=mmap.PROT_READ) with open(out_file_path, 'wb') as out_file: for row_index in in_row_indices: row_start = row_index * line_length if row_index == in_row_indices[-1]: line = mmap_file[row_start:len(mmap_file)] else: row_end = (row_index + 1) * line_length line = mmap_file[row_start:row_end] compressed_line = eval(compression_code) out_file.write(compressed_line) out_row_start_dict[row_index] = cumulative_position cumulative_position += len(compressed_line) # Serialize and save dictionary that indicates where each row starts with open(out_file_path + ".rowdict", 'wb') as rowdict_file: rowdict_file.write(msgpack.packb(out_row_start_dict, use_bin_type=True)) for file_extension in [".ll", ".mccl", ".cc"]: shutil.copyfile(file_path + file_extension, out_file_path + file_extension)
def msgpack_encode(content): return msgpack.packb(content, use_bin_type=True)
def _mm_serialize(data): return msgpack.packb(data, use_bin_type=True)
def load_data(self, prefix, data): self.storage.set(prefix, compress(packb(data, default=encode)))
def call_many(self, cmd, calls, wait=True, is_preloaded=False): if not calls: return def fetch_from_cache(args): msgid = self.cache[args].pop(0) if not self.cache[args]: del self.cache[args] return msgid calls = list(calls) waiting_for = [] while wait or calls: while waiting_for: try: error, res = self.responses.pop(waiting_for[0]) waiting_for.pop(0) if error: if error == b'DoesNotExist': raise Repository.DoesNotExist(self.location.orig) elif error == b'AlreadyExists': raise Repository.AlreadyExists(self.location.orig) elif error == b'CheckNeeded': raise Repository.CheckNeeded(self.location.orig) elif error == b'IntegrityError': raise IntegrityError(res) elif error == b'PathNotAllowed': raise PathNotAllowed(*res) elif error == b'ObjectNotFound': raise Repository.ObjectNotFound( res[0], self.location.orig) elif error == b'InvalidRPCMethod': raise InvalidRPCMethod(*res) else: raise self.RPCError(res.decode('utf-8')) else: yield res if not waiting_for and not calls: return except KeyError: break if self.to_send or ((calls or self.preload_ids) and len(waiting_for) < MAX_INFLIGHT): w_fds = [self.stdin_fd] else: w_fds = [] r, w, x = select.select(self.r_fds, w_fds, self.x_fds, 1) if x: raise Exception('FD exception occurred') for fd in r: if fd is self.stdout_fd: data = os.read(fd, BUFSIZE) if not data: raise ConnectionClosed() self.unpacker.feed(data) for unpacked in self.unpacker: if not (isinstance(unpacked, tuple) and len(unpacked) == 4): raise Exception("Unexpected RPC data format.") type, msgid, error, res = unpacked if msgid in self.ignore_responses: self.ignore_responses.remove(msgid) else: self.responses[msgid] = error, res elif fd is self.stderr_fd: data = os.read(fd, 32768) if not data: raise ConnectionClosed() data = data.decode('utf-8') for line in data.splitlines(keepends=True): if line.startswith('$LOG '): _, level, msg = line.split(' ', 2) level = getattr(logging, level, logging.CRITICAL) # str -> int if msg.startswith('Remote:'): # server format: '$LOG <level> Remote: <msg>' logging.log(level, msg.rstrip()) else: # server format '$LOG <level> <logname> Remote: <msg>' logname, msg = msg.split(' ', 1) logging.getLogger(logname).log( level, msg.rstrip()) else: sys.stderr.write("Remote: " + line) if w: while not self.to_send and ( calls or self.preload_ids) and len(waiting_for) < MAX_INFLIGHT: if calls: if is_preloaded: if calls[0] in self.cache: waiting_for.append( fetch_from_cache(calls.pop(0))) else: args = calls.pop(0) if cmd == 'get' and args in self.cache: waiting_for.append(fetch_from_cache(args)) else: self.msgid += 1 waiting_for.append(self.msgid) self.to_send = msgpack.packb( (1, self.msgid, cmd, args)) if not self.to_send and self.preload_ids: args = (self.preload_ids.pop(0), ) self.msgid += 1 self.cache.setdefault(args, []).append(self.msgid) self.to_send = msgpack.packb( (1, self.msgid, cmd, args)) if self.to_send: try: self.to_send = self.to_send[ os.write(self.stdin_fd, self.to_send):] except OSError as e: # io.write might raise EAGAIN even though select indicates # that the fd should be writable if e.errno != errno.EAGAIN: raise self.ignore_responses |= set(waiting_for)