def dump_data(directory, filter_string, contact_point, keyspace):
cluster = Cluster([contact_point], control_connection_timeout=60)
session = cluster.connect(keyspace)
session.row_factory = dict_factory
tables = []
if filter_string is not None:
filter_re = re.compile(filter_string)
if not os.path.exists(directory):
os.makedirs(directory)
os.chdir(directory)
with open('stream_metadata.mpk', 'wb') as fh:
for row in session.execute('select * from stream_metadata', timeout=None):
if filter_string is None or any((filter_re.search(x) for x in row.values() if isinstance(x, basestring))):
tables.append(row['stream'])
fh.write(msgpack.packb(row))
for table in sorted(set(tables)):
print 'dumping table: %s' % table
with open('%s.mpk' % table, 'wb') as fh:
try:
for row in session.execute('select * from %s' % table, timeout=None):
for k in row:
if type(row[k]) == uuid.UUID:
row[k] = str(row[k])
fh.write(msgpack.packb(row))
except ReadTimeout:
print traceback.format_exc()
session.shutdown()
cluster.shutdown()
def test_odict():
seq = [(b'one', 1), (b'two', 2), (b'three', 3), (b'four', 4)]
od = OrderedDict(seq)
assert unpackb(packb(od), use_list=1) == dict(seq)
def pair_hook(seq):
return list(seq)
assert unpackb(packb(od), object_pairs_hook=pair_hook, use_list=1) == seq
def test_call_with_multi_args(self):
addr = get_random_ipc_socket()
with utils.TestingServer(methods=self.get_methods(), addresses=addr):
context = zmq.Context()
socket = context.socket(zmq.REQ)
socket.connect(addr)
socket.send(msgpack.packb({
'm': 'method3',
'k': {'name': 'WORLD'},
}, encoding='utf-8'))
response = msgpack.unpackb(socket.recv(), encoding='utf-8')
assert response == {'r': 'Hello, WORLD!'}
socket.send(msgpack.packb({
'm': 'method3',
'k': {'greeting': 'HOWDY'},
}, encoding='utf-8'))
response = msgpack.unpackb(socket.recv(), encoding='utf-8')
assert response == {'r': 'HOWDY, world!'}
socket.send(msgpack.packb({
'm': 'method3',
'k': {'greeting': 'HOWDY', 'name': 'MAN'},
}, encoding='utf-8'))
response = msgpack.unpackb(socket.recv(), encoding='utf-8')
assert response == {'r': 'HOWDY, MAN!'}
socket.send(msgpack.packb({
'm': 'method3',
'a': ('hey', 'man'),
}, encoding='utf-8'))
response = msgpack.unpackb(socket.recv(), encoding='utf-8')
assert response == {'r': 'hey, man!'}
def process_request():
frames = socket.recv_multipart()
# killed by gsd
i = frames.index('')
command = frames[i + 2]
if command == '\x02':
global interrupted
interrupted = True
return
i = frames.index('', 1)
sequence, timestamp, expiry = msgpack.unpackb(frames[i+1])
method = frames[i+2]
params = msgpack.unpackb(frames[i+3])
try:
global converter
ret = getattr(converter, method)(*params)
except:
ret = ''
frames = frames[:i+1]
now = int(round(time.time() * 1000))
frames.append(msgpack.packb([sequence, now, 200]))
frames.append(msgpack.packb(ret))
socket.send_multipart(frames)
def test_msgpack_fifo(self):
import msgpack
v1 = [1, 2, 3, 4]
v2 = [5, 6, 7, 8]
v3 = {b"a": 9, b"b": 10, b"c": 11}
s1 = msgpack.packb(v1)
s2 = msgpack.packb(v2)
s3 = msgpack.packb(v3)
unpacker = msgpack.Unpacker()
unpacker.feed(s1)
unpacker.feed(s2)
unpacker.feed(s3[:4])
assert next(iter(unpacker)) == v1
assert next(iter(unpacker)) == v2
try:
next(iter(unpacker))
except StopIteration:
pass
else:
raise Exception("must fail")
unpacker.feed(s3[4:])
assert next(iter(unpacker)) == v3
def test_key_length_msgpacked_items():
key = b'x' * 32 # 31 bytes is the limit for fixstr msgpack type
data = {key: b''}
item_keys_serialized = [
msgpack.packb(key),
]
assert valid_msgpacked_dict(msgpack.packb(data), item_keys_serialized)
def serve(self):
# Make stdin non-blocking
fl = fcntl.fcntl(sys.stdin.fileno(), fcntl.F_GETFL)
fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, fl | os.O_NONBLOCK)
# Make stdout blocking
fl = fcntl.fcntl(sys.stdout.fileno(), fcntl.F_GETFL)
fcntl.fcntl(sys.stdout.fileno(), fcntl.F_SETFL, fl & ~os.O_NONBLOCK)
unpacker = msgpack.Unpacker(use_list=False)
while True:
r, w, es = select.select([sys.stdin], [], [], 10)
if r:
data = os.read(sys.stdin.fileno(), BUFSIZE)
if not data:
return
unpacker.feed(data)
for type, msgid, method, args in unpacker:
method = method.decode('ascii')
try:
try:
f = getattr(self, method)
except AttributeError:
f = getattr(self.repository, method)
res = f(*args)
except Exception as e:
sys.stdout.buffer.write(msgpack.packb((1, msgid, e.__class__.__name__, e.args)))
else:
sys.stdout.buffer.write(msgpack.packb((1, msgid, None, res)))
sys.stdout.flush()
if es:
return
def proc_luna_running(**kw):
""" Fetch test statuses from Redis, if test finisged notify service via API
and call reduce job.
"""
ext = {'test_id': kw.get('redis_value', {}).get('id')}
logger = get_logger(**ext)
if not ('t_fqdn' in kw and 't_tank_id' in kw):
logger.erro('proc_luna_running call. Malformed params:{}'.format(kw))
try:
except TankClientError as e:
logger.error('Tank API call failed: {}'.format(e))
raise
if tank_msg['status_code'] != 'FINISHED':
if kw.get('status') and TestStateTr.tank_by_port(kw.get('status')) != tank_msg['status_code']:
# test state changes since last check, need to notify
port_state = TestStateTr.port_by_tank(tank_msg['status_code'])
redis_value = kw['redis_value']
redis_value.update({'status': port_state})
redis.hset(r_adr['monitor_finish'], kw['id'],
msgpack.packb(redis_value))
diff = {
'status': port_state,
}
def call(self, command, params=None):
"""
Sends the provided command to Serf for evaluation, with
any parameters as the message body.
"""
if self._socket is None:
raise SerfConnectionError('handshake must be made first')
header = msgpack.packb({"Seq": self._counter(), "Command": command})
if params is not None:
body = msgpack.packb(params)
self._socket.sendall(header + body)
else:
self._socket.sendall(header)
unpacker = msgpack.Unpacker()
unpacker.feed(self._socket.recv(4096))
response = SerfResult()
for item in unpacker:
if response.head is None:
response.head = item
else:
response.body = item
break
return response
def main(app, hosts, timeout):
complain = False
context = zmq.Context()
for host in hosts:
request = context.socket(zmq.DEALER)
request.setsockopt(zmq.LINGER, 0)
request.connect('tcp://%s:5000' % host)
# Statistics
request.send_multipart([
msgpack.packb(SLOT_INFO),
msgpack.packb([])
])
poller = zmq.Poller()
poller.register(request, zmq.POLLIN)
sockets = dict(poller.poll(timeout = timeout))
if request in sockets and sockets[request] == zmq.POLLIN:
response = msgpack.unpackb(request.recv())
if app not in response["apps"]:
print "The '%s' app was not found on '%s'." % (app, host)
exit(1)
state = response["apps"][app]["state"]
print "The '%s' app is %s on '%s'." % (app, state, host)
complain |= (state != "running")
else:
print "ERROR: Host '%s' is not responding." % host
complain |= True
exit(complain)
def default(obj):
# Serialize Bn objects
if isinstance(obj, Bn):
if obj < 0:
neg = b"-"
data = (-obj).binary()
else:
neg = b"+"
data = obj.binary()
return msgpack.ExtType(0, neg + data)
# Serialize EcGroup objects
elif isinstance(obj, EcGroup):
nid = obj.nid()
packed_nid = msgpack.packb(nid)
return msgpack.ExtType(1, packed_nid)
# Serialize EcPt objects
elif isinstance(obj, EcPt):
nid = obj.group.nid()
data = obj.export()
packed_nid = msgpack.packb((nid, data))
return msgpack.ExtType(2, packed_nid)
raise TypeError("Unknown type: %r" % (obj,))
def send(self, payload, deprecated=()):
"""Send a message with topic, payload
`
Topic is a unicode string. It will be sent as utf-8 encoded byte array.
Payload is a python dict. It will be sent as a msgpack serialized dict.
If payload has the key '__raw_data__'
we pop if of the payload and send its raw contents as extra frames
everything else need to be serializable
the contents of the iterable in '__raw_data__'
require exposing the pyhton memoryview interface.
"""
assert deprecated is (), "Depracted use of send()"
assert "topic" in payload, "`topic` field required in {}".format(payload)
if "__raw_data__" not in payload:
# IMPORTANT: serialize first! Else if there is an exception
# the next message will have an extra prepended frame
serialized_payload = serializer.packb(payload, use_bin_type=True)
self.socket.send_string(payload["topic"], flags=zmq.SNDMORE)
self.socket.send(serialized_payload)
else:
extra_frames = payload.pop("__raw_data__")
assert isinstance(extra_frames, (list, tuple))
self.socket.send_string(payload["topic"], flags=zmq.SNDMORE)
serialized_payload = serializer.packb(payload, use_bin_type=True)
self.socket.send(serialized_payload, flags=zmq.SNDMORE)
for frame in extra_frames[:-1]:
self.socket.send(frame, flags=zmq.SNDMORE, copy=True)
self.socket.send(extra_frames[-1], copy=True)
def test_roundtrip_deferred(self):
from twisted.internet import reactor
from twisted.internet.task import deferLater
v = "yaaay!"
p_ctx = []
class SomeService(ServiceBase):
@rpc(Unicode, _returns=Unicode)
def yay(ctx, u):
def _cb():
return u
p_ctx.append(ctx)
return deferLater(reactor, 0.1, _cb)
app = Application([SomeService], 'tns',
in_protocol=MessagePackDocument(),
out_protocol=MessagePackDocument())
prot = self.gen_prot(app)
request = msgpack.packb({'yay': [v]})
def _ccb(_):
val = prot.transport.value()
print repr(val)
val = msgpack.unpackb(val)
print repr(val)
self.assertEquals(val, [0, msgpack.packb(v)])
prot.dataReceived(msgpack.packb([1, request]))
return p_ctx[0].out_object[0].addCallback(_ccb)
def encode(self):
ret = {'name': self._name,
'time': get_time(),
'epoch_milli': get_milli(),
'host': get_name(),
'data': msgpack.packb(self.run())}
return msgpack.packb(ret)
def on_heartbeat(w):
global i
i += 1
w.write(msgpack.packb([1, 0, []]))
w.write(msgpack.packb([3, i, ["echo"]]))
w.write(msgpack.packb([4, i, ["echo"]]))
w.write(msgpack.packb([6, i, []]))
def write(self, key, data):
assert b'\r\n' not in msgpack.packb(data, use_bin_type=True)
with open(self.file_loc, 'ab') as fileobj:
fileobj.write(key.encode('utf-8'))
fileobj.write(b'\t')
fileobj.write(msgpack.packb(data, use_bin_type=True))
fileobj.write(b'\r\n')
def main(argv):
map_path = argv[1]
result_path = argv[2]
t = Teemap(map_path)
result = {}
frontlayer = None # Works thanks to hack in tml
for group in t.groups:
if group.name == 'Game':
for layer in group.layers:
if type(layer) == TileLayer and layer.name == 'Front':
frontlayer = layer
break
add(result, gameTiles, t.gamelayer)
add(result, frontTiles, frontlayer)
# These don't seem to be working
#add(result, switchTiles, switchlayer)
#add(result, speedupTiles, t.speeduplayer)
#add(result, teleTiles, t.telelayer)
with open(result_path, 'wb') as out:
out.write(msgpack.packb(t.gamelayer.width))
out.write(msgpack.packb(t.gamelayer.height))
out.write(msgpack.packb(result))
def Emit(self, *args):
packet = {}
extras = {}
packet['data'] = args
packet['type'] = self.BINARY_EVENT if self._hasBin(args) else self.EVENT
# set namespace to packet
if 'nsp' in self._flags:
packet['nsp'] = self._flags['nsp']
del self._flags['nsp']
else:
packet['nsp'] = '/'
extras['flags'] = self._flags if len(self._flags) > 0 else ''
rooms = self._getRooms()
extras['rooms'] = rooms if len(rooms) > 0 else ''
self._client.publish(self._key + "#emitter", msgpack.packb([packet, extras]))
if extras['rooms']:
for room in rooms:
chn = "#".join((self._key, packet['nsp'], room, ""))
self._client.publish(chn, msgpack.packb([self.uid,packet, extras]))
else:
chn = "#".join((self._key, packet['nsp'], ""))
self._client.publish(chn, msgpack.packb([self.uid,packet, extras]))
self._flags = {}
self._rooms = []
def add_corpus(self, corpus, name, password, options, tlds=None):
now = now_ts()
yield self.db()["corpus"].insert_one({
"_id": corpus,
"name": name,
"password": salt(password),
"options": options,
"total_webentities": 0,
"webentities_in": 0,
"webentities_in_untagged": 0,
"webentities_in_uncrawled": 0,
"webentities_out": 0,
"webentities_undecided": 0,
"webentities_discovered": 0,
"total_crawls": 0,
"crawls_pending": 0,
"crawls_running": 0,
"total_pages": 0,
"total_pages_crawled": 0,
"total_pages_queued": 0,
"total_links_found": 0,
"recent_changes": False,
"last_index_loop": now,
"links_duration": 1,
"last_links_loop": 0,
"tags": Binary(msgpack.packb({})),
"webentities_links": Binary(msgpack.packb({})),
"created_at": now,
"last_activity": now,
"tlds": tlds
})
yield self.init_corpus_indexes(corpus)
def test_node_service_bad_on_read():
io = CocaineIO.instance()
node = Service("node", host="localhost", port=10053, loop=io)
malformed_message = msgpack.packb([-999, 0])
node.on_read(malformed_message)
message = msgpack.packb([-999, 0, []])
node.on_read(message)
def test_node_service_bad_on_read():
io = IOLoop.current()
node = Service("node", endpoints=[["localhost", 10053]], io_loop=io)
malformed_message = msgpack.packb([-999, 0])
node.on_read(malformed_message)
message = msgpack.packb([-999, 0, []])
node.on_read(message)
def slave_sync_data(self, port):
"""
同步数据
"""
import time
time.sleep(1)
self._server.is_sync = True
client = PyCachedClient(self._address[0], port)
self.add_slave(client)
logger.info('Slave[%s:%d] Sync data start' % (self._address[0], port))
items = {}
pos = 0
MAX_SEND = 20000 # 同步数据多少Key为一组
all_len = len(self._server.memory.caches.items())
for key, val in self._server.memory.caches.items():
pos += 1
items[key] = val
if pos % MAX_SEND == 0:
client.sync_data(pos, all_len, msgpack.packb(items))
items = {}
if pos % MAX_SEND != 0:
client.sync_data(pos, all_len, msgpack.packb(items))
self._server.is_sync = False
client.sync_ok()
logger.info('Slave[%s:%d] Sync data success' % (self._address[0], port))
def count(doc):
if attr == '__meta__':
return len(msgpack.packb(doc.doc))
for i, (name, klass, nelem) in enumerate(doc.stores):
if name == attr:
return len(msgpack.packb(doc.instances[i]))
return 0
def aggregate_host(request, response):
raw = yield request.read()
TASK = msgpack.unpackb(raw)
taskId = TASK['id']
logger = get_logger_adapter(taskId)
logger.info("Handle task")
cfg = TASK['config'] # config of aggregator
token = TASK['token']
prtime = TASK['prevtime']
currtime = TASK['currtime']
dg = MysqlDG.get_service(DATABASEAPP)
q = TABLEREGEX.sub(token, cfg['query'])
q = TIMEREGEX.sub("1=1", q)
logger.debug("QUERY: %s", q)
pickled_res = yield dg.enqueue("query",
msgpack.packb((token, q)))
res = cPickle.loads(pickled_res)
logger.debug(str(res))
try:
ret = float(res[0][0]) # SELECT COUNT(*)
logger.info("Result from DG %s", ret)
if cfg.get('rps'):
ret = ret / (currtime - prtime)
except Exception:
ret = 0
logger.info("%s", ret)
response.write(msgpack.packb(ret))
response.close()
def handle(self):
try:
logger.debug("(self:{}) ->".format(self))
while 1:
try:
buffer = self.read(4)
size = struct.unpack("<L", buffer)[0]
try:
if self.server.serialization == "json":
cmd, params = json.loads(self.read(size))
if params is not None:
packet = json.dumps(getattr(self.server.instance, cmd)(params))
else:
packet = json.dumps(getattr(self.server.instance, cmd)())
else:
cmd, params = msgpack.unpackb(self.read(size), encoding='utf-8')
packet = msgpack.packb((0, getattr(self.server.instance, cmd)(*params)), use_bin_type=True)
except:
result = traceback.format_exc()
logger.error(result)
packet = msgpack.packb((2, result), use_bin_type=True)
self.request.sendall(struct.pack("<L", len(packet)) + packet)
except self.EndConnection:
return
except socket.error as e:
logger.error(e)
return
finally:
logger.debug("handle(self:{}) <-".format(self))
def send(self, message):
if config.debug_socket: self.log("Send: %s, to: %s, site: %s, inner_path: %s, req_id: %s" % (message.get("cmd"), message.get("to"), message.get("params", {}).get("site"), message.get("params", {}).get("inner_path"), message.get("req_id")))
self.last_send_time = time.time()
if self.protocol == "zeromq":
if self.zmq_sock: # Outgoing connection
self.zmq_queue.append(message)
if self.zmq_working:
self.log("ZeroMQ already working...")
return
while self.zmq_queue:
self.zmq_working = True
message = self.zmq_queue.pop(0)
self.zmq_sock.send(msgpack.packb(message))
self.handleMessage(msgpack.unpackb(self.zmq_sock.recv()))
self.zmq_working = False
else: # Incoming request
self.server.zmq_sock.send(msgpack.packb(message))
else: # Normal connection
data = msgpack.packb(message)
self.bytes_sent += len(data)
self.server.bytes_sent += len(data)
self.sock.sendall(data)
self.last_sent_time = time.time()
return True
def get_payload(self):
# Fill up send_queue with new messages until full.
try:
for i in xrange(self.send_queue_size - len(self.send_queue)):
self.send_queue.append(self.main_queue.get_nowait())
self.main_queue.task_done()
except Queue.Empty:
pass
if not len(self.send_queue):
return '', 0
# self.log("info",
# "Trying to pack queue with {} messages.".format(
# len(self.send_queue)
# )
# )
# Pack messages and try to fit into limit.
msg = msgpack.packb({'data': self.send_queue})
msg_len = len(self.send_queue)
while len(msg) > self.config['max_msg_size_bytes']:
self.measr_avg_size = len(msg) / len(self.send_queue)
self.update_queue_size()
msg = msgpack.packb(
{'data': self.send_queue[:self.send_queue_size]}
)
msg_len = self.send_queue_size
# self.log(
# "info",
# "Packed {} messages with total size {} bytes. {} messages in"
# " the send_queue left.".format(msg_len, len(msg),
# len(self.send_queue))
# )
return msg, msg_len
def _serialize_context(context):
# Our sending format is made up of two messages. The first has a
# quick to unpack set of meta data that our collector is going to
# use for routing and stats. This is much faster than having the
# collector decode the whole event. We're just going to use python
# struct module to make a quick and dirty data structure
context_dict = context.to_dict()
for key in ("host", "type"):
if len(context_dict.get(key, "")) > 64:
raise ValueError("Value too long: %r" % key)
meta_data = struct.pack(
META_STRUCT_FMT, META_STRUCT_VERSION, context_dict["end"], context_dict["host"], context_dict["type"]
)
try:
context_data = msgpack.packb(context_dict)
except TypeError:
try:
# If we fail to serialize our context, we can try again with an
# enhanced packer (it's slower though)
context_data = msgpack.packb(context_dict, default=utils.msgpack_encode_default)
except TypeError:
log.exception("Serialization failure (not fatal, dropping data)")
# One last try after dropping the body
context_dict["body"] = None
context_data = msgpack.packb(context_dict)
return meta_data, context_data
def handle_error(self, p_ctx, others, exc):
self.spyne_tpt.get_out_string(p_ctx)
if isinstance(exc, InternalError):
error = self.spyne_tpt.OUT_RESPONSE_SERVER_ERROR
else:
error = self.spyne_tpt.OUT_RESPONSE_CLIENT_ERROR
data = p_ctx.out_document[0]
if isinstance(data, dict):
data = list(data.values())
out_string = msgpack.packb([
error, msgpack.packb(data),
])
self.enqueue_outresp_data(id(p_ctx), out_string)
p_ctx.transport.resp_length = len(out_string)
p_ctx.close()
try:
process_contexts(self, others, p_ctx, error=error)
except Exception as e:
# Report but ignore any exceptions from auxiliary methods.
logger.exception(e)
def handle_data(self, message):
payload = msgpack.unpackb(message)
if 'connect' in payload:
host, port_str = payload['connect'].split('::')
port = int(port_str)
self._sock = socket.socket()
self._sock.connect((host, port))
response = msgpack.packb({'type': 'STATUS',
'value': 'OPEN'})
self._send(response)
eventlet.spawn(self._reader)
elif 'close' in payload:
self._sock.close()
status = msgpack.packb({'type': 'STATUS',
'value': 'CLOSED'})
self._send(status)
elif 'content' in payload:
try:
self._sock.send(payload['content'])
except Exception as exc:
print exc
self._sock.close()
else:
message = 'Unknown payload type: {}\n'.format(payload.type)
sys.write(message)
def test_send_msgpack(self, mock):
mock.register_uri(requests_mock.ANY,
requests_mock.ANY,
text='{"result":"OK"}')
self.assertTrue(ubirch.API().send(TEST_UUID, msgpack.packb([1, 2, 3])))
async def set(self, key, value, expiration=0):
packed = msgpack.packb(value, use_bin_type=True)
await self.client.set(key, packed, expiration)
for idy, py in enumerate(
range(px - FLAGS.window_size, px) +
range(px + 1, px + FLAGS.window_size + 1)):
if py < 0 or py >= FLAGS.data_shard_cols:
row_weight[px].append(0.0)
continue
y = data_shared[i, py]
if FLAGS.use_PMI and x in cooccur_mat and y in cooccur_mat[x]:
if FLAGS.data_shard_rows * FLAGS.data_shard_cols * cooccur_mat[
x][y] < count[x][1] * count[y][
1] * FLAGS.num_neg_sample:
num_filtered_out += 1
filter_out_arr[x] += 1
row_weight[px].append(0.0)
continue
row_weight[px].append(1.0)
slice_cnt += 1
weights.append(row_weight)
print("\rFinished %6d / %6d." % (i, FLAGS.data_shard_rows)),
sys.stdout.flush()
print("\n# filtered out: %d" % num_filtered_out)
for i in range(5):
filter_rate = filter_out_arr[i] / float(
count[i][1] * 2 * FLAGS.window_size)
print(str(count[i]) + ' ' + str(filter_rate))
open(FLAGS.output_training_file, 'wb').write(
msgpack.packb((data, count, dictionary, reverse_dictionary, weights)))
def test(self):
d = StableDict(foo=1, bar=2, boo=3, baz=4)
self.assert_equal(list(d.items()), [('bar', 2), ('baz', 4), ('boo', 3), ('foo', 1)])
self.assert_equal(hashlib.md5(msgpack.packb(d)).hexdigest(), 'fc78df42cd60691b3ac3dd2a2b39903f')
def listener():
shutdown = False
while not shutdown:
msg_str = self._redis.brpoplpush(self.q_name,
self.processing_q,
timeout=0)
if msg_str == b'shutdown':
shutdown = True
self._redis.lrem(self.processing_q, -1, msg_str)
self.loop = None
break
msg_dict = msgpack.unpackb(msg_str, raw=False)
msg_dict['worker'] = self.id
msg_dict['processing_started'] = dt.timestamp(dt.utcnow())
msg_dict['progress'] = 0
new_record = msgpack.packb(msg_dict, use_bin_type=False)
self._redis.set(self.job_status_stem + str(msg_dict['id']),
new_record)
all_ok = True
short_result = None
utils = self._utility_functions()
utils['full_message'] = msg_dict
utils['update_progress'] = functools.partial(
utils['update_progress'], str(msg_dict['id']))
result = None
for func in self.funcs_to_execute:
if msg_dict[
'functions'] is None or func.__name__ in msg_dict[
'functions']:
try:
result = func(msg_dict['msg'], utils)
except Exception as e:
all_ok = False
logging.error(e)
logging.info(
"Moving message {} to dead letter queue".
format(msg_dict['id']))
if msg_dict['callback']:
self.http.request('GET',
msg_dict['callback'],
fields={
'success': 0,
'job_id':
str(msg_dict['id']),
'short_result': None
})
msg_dict['progress'] = -1
msg_dict['result'] = str(e)
new_record = msgpack.packb(msg_dict,
use_bin_type=False)
self._redis.set(
self.job_status_stem + str(msg_dict['id']),
new_record)
self._redis.rpush(self.dead_letter_q, msg_str)
if all_ok:
logging.info('Completed job {}'.format(str(
msg_dict['id'])))
msg_dict['worker'] = None
if result and type(result) in [tuple, list
] and len(result) > 1:
short_result = result[0]
result = result[1]
else:
short_result = result
msg_dict['result'] = result
msg_dict['short_result'] = short_result
self._redis.publish('pub_' + str(msg_dict['id']),
short_result)
msg_dict['progress'] = 100
msg_dict['processing_finished'] = dt.timestamp(dt.utcnow())
new_record = msgpack.packb(msg_dict, use_bin_type=False)
self._redis.set(self.job_status_stem + str(msg_dict['id']),
new_record)
if msg_dict['callback']:
self.http.request('GET',
msg_dict['callback'],
fields={
'success': 1,
'job_id': str(msg_dict['id']),
'short_result': short_result
})
self._redis.lrem(self.processing_q, -1, msg_str)
self._redis.lrem(self.jobs_refs_q, 1, str(msg_dict['id']))
def send_busy(self, client):
client.send_bytes(msgpack.packb({"type": MSG_BUSY}, use_bin_type=True))
def test_register_identity_msgpack(self, mock):
mock.register_uri(requests_mock.ANY,
requests_mock.ANY,
text='{"result":"OK"}')
self.assertTrue(ubirch.API().register_identity(msgpack.packb([1, 2,
3])))
def dumps(self, obj):
return msgpack.packb(obj, use_bin_type=True)
def to_db(data):
return sqlite3.Binary(packb(data, use_bin_type=True))
def _dumps(cls, value):
return msgpack.packb(value, default=cls._serialize, use_bin_type=True)
def send_event(self, eventname, data=None, target=None):
# On the sim side, target is obtained from the currently-parsed stack command
target = target or stack.routetosender() or [b'*']
pydata = msgpack.packb(data, default=encode_ndarray, use_bin_type=True)
self.event_io.send_multipart(target + [eventname, pydata])
def pack(s):
return packb(s, use_bin_type=True)
def process_image(self, frames):
# parse frames
name = frames[0]
metamsg = frames[1]
x_comp = frames[2]
x_enc = zlib.decompress(x_comp)
# decode frames
# -- meta
meta = decode_larcv1_metamsg(metamsg)
# -- array
arr = msgpack.unpackb(x_enc, object_hook=m.decode)
shape = arr.shape
msg_batchsize = shape[0]
# prepare numpy array for output
# note, we through away the background scores to save egress data
# we save results in half-precision. since numbers between [0,1] precision still good to 2^-11 at worse
if not self.reply_in_float16:
ssnetout = np.zeros(
(shape[0], self.NCLASSES - 1, shape[2], shape[3]),
dtype=np.float16)
else:
ssnetout = np.zeros(
(shape[0], self.NCLASSES - 1, shape[2], shape[3]),
dtype=np.float32)
blobshape = (self.BATCHSIZE, 1, shape[2], shape[3])
# run the net for the plane
#print "PlaneNetworkWorker[{}]::Plane[{}]: processing {}".format(self._identity,self.PLANEID,name)
self.net.blobs['data'].reshape(*blobshape)
# process the images
for ibatch in range(0, msg_batchsize, self.BATCHSIZE):
imgslice = arr[ibatch * self.BATCHSIZE:(ibatch + 1) *
self.BATCHSIZE, :]
self.net.blobs['data'].data[...] = imgslice
tforward = time.time()
#print "PlaneNetworkWorker[{}]::Plane[{}]: forward".format(self._identity,self.PLANEID)
self.net.forward()
tforward = time.time() - tforward
#print "PlaneNetworkWorker[{}]::Plane[{}]: network returns after %.03f secs".format(self._identity,self.PLANEID)%(tforward)
# copy predictions to ssnetout
if (ibatch + 1) * self.BATCHSIZE > msg_batchsize:
remaining = msg_batchsize % self.BATCHSIZE
start = ibatch * self.BATCHSIZE
end = ibatch * self.BATCHSIZE + remaining
if self.reply_in_float16:
ssnetout[start:end, :] = self.net.blobs['softmax'].data[
0:remaining, :].astype(np.float16)
else:
ssnetout[start:end, :] = self.net.blobs['softmax'].data[
0:remaining, :]
else:
start = ibatch * self.BATCHSIZE
end = (ibatch + 1) * self.BATCHSIZE
if self.reply_in_float16:
ssnetout[start:end, :] = self.net.blobs['softmax'].data[
0:self.BATCHSIZE, 1:, :].astype(np.float16)
else:
ssnetout[start:end, :] = self.net.blobs['softmax'].data[
0:self.BATCHSIZE, 1:, :]
# we threshold score images so compression performs better
outslice = ssnetout[start:end, :]
for c in range(outslice.shape[1]):
chslice = outslice[:, c, :].reshape(
(1, 1, imgslice.shape[2], imgslice.shape[3]))
chslice[imgslice < 5.0] = 0
# encode
x_enc = msgpack.packb(ssnetout, default=m.encode)
x_comp = zlib.compress(x_enc, self._compression_level)
# make the return message
#print "PlaneNetworkWorker[{}]::Plane[{}] processed name=\"{}\" shape={} meta={}".format(self._identity,self.PLANEID,name,ssnetout.shape,meta.dump().strip())
reply = ["plane%d" % (self.PLANEID)] # topic frame
reply.append(name.encode('utf-8'))
reply.append(meta.dump().strip())
reply.append(x_comp)
return reply
def send_stream(self, name, data):
self.stream_out.send_multipart(
[name + self.node_id, msgpack.packb(data, default=encode_ndarray, use_bin_type=True)])
def pack(obj):
buf = msgpack.packb(obj, use_bin_type=True)
return buf
def makePickle(self, record):
return msgpack.packb(record.__dict__, default=self.msgpack_encoder)
def marshall(to_pack):
packed = msgpack.packb(to_pack, default=utils.default, use_bin_type=True)
packed_encoded = base64.b64encode(packed)
return packed_encoded
def send_action(self, action):
dat = msgpack.packb({"command": "".join(map(str, action))})
self.write_message(dat, binary=True)
def gen_forwarder(secret, obj, nonce, options={}):
return crypto.encrypt( msgpack.packb((nonce, id(obj), options)), secret )
def pack_msg(msg): # pragma: no cover
return msgpack.packb(msg, use_bin_type=True)
def encode(self, buf):
buf = np.asanyarray(buf)
items = buf.tolist()
items.append(buf.dtype.str)
return msgpack.packb(items, encoding=self.encoding)
def start(self):
context = zmq.Context()
logging.info("Connecting to server…")
client = context.socket(zmq.REQ)
client.connect(SERVER_ENDPOINT + str(self.port))
times = []
startTime = time.time()
count = 0
while True:
try:
item = self.queue.get(block=False)
except Exception as e:
item = None
if (item):
seq = str(item.get('seq')).encode()
data = item.get('data')
self.pendingQueue[seq] = data
request = {'seq': seq, 'data': data}
encoded = msgpack.packb(request)
client.send(encoded)
count = count + 1
curTime = time.time()
diff = curTime - startTime
if (diff > 5):
startTime = curTime
sequenceDiff = count
count = 0
print("got {count} messages in {diff} seconds".format(
count=sequenceDiff, diff=diff))
print("thats {mps} messages per second".format(
mps=sequenceDiff / diff))
retries_left = REQUEST_RETRIES
while True:
if (client.poll(REQUEST_TIMEOUT) & zmq.POLLIN) != 0:
reply = client.recv()
retSeq = int(reply)
pendingItem = self.pendingQueue.get(reply)
if (pendingItem):
self.pendingQueue.pop('reply', None)
break
else:
logging.error("Malformed reply from server: %s", reply)
continue
retries_left -= 1
logging.warning("No response from server")
if retries_left == 0:
# Socket is confused. Close and remove it.
client.setsockopt(zmq.LINGER, 0)
print('close')
client.close()
logging.error("Server seems to be offline, abandoning")
# sys.exit()
logging.info("Reconnecting to server…")
# Create new connection
client = context.socket(zmq.REQ)
print('connect')
client.connect(SERVER_ENDPOINT + str(self.port))
break
def encode(self, msg):
return msgpack.packb(msg, use_bin_type=True)
def RequestResponse(req):
data = msgpack.packb(req)
socket.send(data)
return msgpack.unpackb(socket.recv())
def query_raw(self, obj):
raw_bytes = msgpack.packb(obj)
raw_bytes = bytes(b ^ ((self.key + i) & 0xFF) for i, b in enumerate(raw_bytes))
self.sock.send(raw_bytes)
return msgpack.unpackb(self.sock.recv(BUFFER_SIZE), raw=False)
in_row_indices = list(range(num_rows))
out_row_start_dict = {}
cumulative_position = 0
with open(file_path, 'rb') as my_file:
mmap_file = mmap.mmap(my_file.fileno(), 0, prot=mmap.PROT_READ)
with open(out_file_path, 'wb') as out_file:
for row_index in in_row_indices:
row_start = row_index * line_length
if row_index == in_row_indices[-1]:
line = mmap_file[row_start:len(mmap_file)]
else:
row_end = (row_index + 1) * line_length
line = mmap_file[row_start:row_end]
compressed_line = eval(compression_code)
out_file.write(compressed_line)
out_row_start_dict[row_index] = cumulative_position
cumulative_position += len(compressed_line)
# Serialize and save dictionary that indicates where each row starts
with open(out_file_path + ".rowdict", 'wb') as rowdict_file:
rowdict_file.write(msgpack.packb(out_row_start_dict, use_bin_type=True))
for file_extension in [".ll", ".mccl", ".cc"]:
shutil.copyfile(file_path + file_extension, out_file_path + file_extension)
def msgpack_encode(content):
return msgpack.packb(content, use_bin_type=True)
def _mm_serialize(data):
return msgpack.packb(data, use_bin_type=True)
def load_data(self, prefix, data):
self.storage.set(prefix, compress(packb(data, default=encode)))
def call_many(self, cmd, calls, wait=True, is_preloaded=False):
if not calls:
return
def fetch_from_cache(args):
msgid = self.cache[args].pop(0)
if not self.cache[args]:
del self.cache[args]
return msgid
calls = list(calls)
waiting_for = []
while wait or calls:
while waiting_for:
try:
error, res = self.responses.pop(waiting_for[0])
waiting_for.pop(0)
if error:
if error == b'DoesNotExist':
raise Repository.DoesNotExist(self.location.orig)
elif error == b'AlreadyExists':
raise Repository.AlreadyExists(self.location.orig)
elif error == b'CheckNeeded':
raise Repository.CheckNeeded(self.location.orig)
elif error == b'IntegrityError':
raise IntegrityError(res)
elif error == b'PathNotAllowed':
raise PathNotAllowed(*res)
elif error == b'ObjectNotFound':
raise Repository.ObjectNotFound(
res[0], self.location.orig)
elif error == b'InvalidRPCMethod':
raise InvalidRPCMethod(*res)
else:
raise self.RPCError(res.decode('utf-8'))
else:
yield res
if not waiting_for and not calls:
return
except KeyError:
break
if self.to_send or ((calls or self.preload_ids)
and len(waiting_for) < MAX_INFLIGHT):
w_fds = [self.stdin_fd]
else:
w_fds = []
r, w, x = select.select(self.r_fds, w_fds, self.x_fds, 1)
if x:
raise Exception('FD exception occurred')
for fd in r:
if fd is self.stdout_fd:
data = os.read(fd, BUFSIZE)
if not data:
raise ConnectionClosed()
self.unpacker.feed(data)
for unpacked in self.unpacker:
if not (isinstance(unpacked, tuple)
and len(unpacked) == 4):
raise Exception("Unexpected RPC data format.")
type, msgid, error, res = unpacked
if msgid in self.ignore_responses:
self.ignore_responses.remove(msgid)
else:
self.responses[msgid] = error, res
elif fd is self.stderr_fd:
data = os.read(fd, 32768)
if not data:
raise ConnectionClosed()
data = data.decode('utf-8')
for line in data.splitlines(keepends=True):
if line.startswith('$LOG '):
_, level, msg = line.split(' ', 2)
level = getattr(logging, level,
logging.CRITICAL) # str -> int
if msg.startswith('Remote:'):
# server format: '$LOG <level> Remote: <msg>'
logging.log(level, msg.rstrip())
else:
# server format '$LOG <level> <logname> Remote: <msg>'
logname, msg = msg.split(' ', 1)
logging.getLogger(logname).log(
level, msg.rstrip())
else:
sys.stderr.write("Remote: " + line)
if w:
while not self.to_send and (
calls or
self.preload_ids) and len(waiting_for) < MAX_INFLIGHT:
if calls:
if is_preloaded:
if calls[0] in self.cache:
waiting_for.append(
fetch_from_cache(calls.pop(0)))
else:
args = calls.pop(0)
if cmd == 'get' and args in self.cache:
waiting_for.append(fetch_from_cache(args))
else:
self.msgid += 1
waiting_for.append(self.msgid)
self.to_send = msgpack.packb(
(1, self.msgid, cmd, args))
if not self.to_send and self.preload_ids:
args = (self.preload_ids.pop(0), )
self.msgid += 1
self.cache.setdefault(args, []).append(self.msgid)
self.to_send = msgpack.packb(
(1, self.msgid, cmd, args))
if self.to_send:
try:
self.to_send = self.to_send[
os.write(self.stdin_fd, self.to_send):]
except OSError as e:
# io.write might raise EAGAIN even though select indicates
# that the fd should be writable
if e.errno != errno.EAGAIN:
raise
self.ignore_responses |= set(waiting_for)
