def on_heartbeat_v0(w): if sys.version_info[0] == 2: packer = msgpack.Packer() else: packer = msgpack.Packer(use_bin_type=True) if s.counter > 6: yield w.write(packer.pack([1, 2, [2, "terminate"]])) s.io_loop.add_callback(s.stop) return req = [METHOD, URI, HTTP_VERSION, HEADERS, BODY] yield w.write(packer.pack([1, 1, []])) yield w.write(packer.pack([s.counter, 3, ["ping"]])) yield w.write(packer.pack([s.counter, 4, ["pong"]])) yield w.write(packer.pack([s.counter, 6, []])) s.counter += 1 yield w.write(packer.pack([s.counter, 3, ["bad_event"]])) s.counter += 1 yield w.write(packer.pack([s.counter, 3, ["http"]])) yield w.write(packer.pack([s.counter, 4, [packer.pack(req)]])) yield w.write(packer.pack([s.counter, 6, []])) s.counter += 1 yield w.write(packer.pack([s.counter, 3, ["http_test"]])) yield w.write(packer.pack([s.counter, 4, [packer.pack(req)]])) yield w.write(packer.pack([s.counter, 6, []])) s.counter += 1 yield w.write(packer.pack([s.counter + 100, 4, ["bad_ping"]])) yield w.write(packer.pack([s.counter, 104, ["bad_ping"]])) s.counter += 1 yield w.write(packer.pack([s.counter, 3, ["bad_ping"]])) yield w.write(packer.pack([s.counter, 4, ["A"]])) yield w.write(packer.pack([s.counter, 6, []])) s.counter += 1 yield w.write(packer.pack([s.counter, 3, ["notclosed"]])) yield w.write(packer.pack([s.counter, 4, ["A"]])) yield w.write(packer.pack([s.counter, 6, []]))
def _start_process(self) -> None: self._stdin: typing.Optional[typing.Any] = None self._queue_id = '' self._queue_in: 'Queue[bytes]' = Queue() self._queue_out: 'Queue[typing.Any]' = Queue() self._queue_err: 'Queue[typing.Any]' = Queue() if msgpack.version < (1, 0, 0): self._packer = msgpack.Packer( encoding='utf-8', unicode_errors='surrogateescape') self._unpacker = msgpack.Unpacker( encoding='utf-8', unicode_errors='surrogateescape') else: self._packer = msgpack.Packer( unicode_errors='surrogateescape') self._unpacker = msgpack.Unpacker( unicode_errors='surrogateescape') self._prev_pos: typing.List[typing.Any] = [] info = None if sys.platform == 'win32': info = subprocess.STARTUPINFO() info.dwFlags |= subprocess.STARTF_USESHOWWINDOW main = str(Path(__file__).parent.parent.parent.parent.joinpath( 'autoload', 'deoplete', '_main.py')) self._hnd = self._vim.loop.create_task( self._vim.loop.subprocess_exec( partial(Process, self), self._get_python_executable(), main, self._vim.vars['deoplete#_serveraddr'], startupinfo=info))
def test_dtype_transform_lamdbas(tmpdir): rt = ResolutionTable(os.path.join(tmpdir, "test/")) # int int_test = 10 assert int_test.to_bytes((int_test.bit_length() + 7) // 8, "big") == rt.to_bytes_func(int)(int_test) assert int_test == rt.from_bytes_func(int)(int_test.to_bytes( (int_test.bit_length() + 7) // 8, "big")) # str str_test = "ten" assert str_test.encode("utf-8") == rt.to_bytes_func(str)(str_test) assert str_test == rt.from_bytes_func(str)(str_test.encode("utf-8")) # else [use msgpack] (list, set, dict, float) other_dtypes = { list: [1, 2, 3], float: 1.23, dict: { "123": 123 }, tuple: (1, 2, 3) } for dtype, value in other_dtypes.items(): assert msgpack.Packer().pack(value) == rt.to_bytes_func(dtype)(value) assert value == dtype( rt.from_bytes_func(dtype)(msgpack.Packer().pack(value)))
def __init__(self, vim: Nvim) -> None: self.name = 'child' self._vim = vim self._filters: typing.Dict[str, typing.Any] = {} self._sources: typing.Dict[str, typing.Any] = {} self._profile_flag = None self._profile_start_time = 0 self._loaded_sources: typing.Dict[str, typing.Any] = {} self._loaded_filters: typing.Dict[str, typing.Any] = {} self._source_errors: typing.Dict[str, int] = defaultdict(int) self._prev_results: typing.Dict[str, Result] = {} if msgpack.version < (1, 0, 0): self._packer = msgpack.Packer( encoding='utf-8', unicode_errors='surrogateescape') self._unpacker = msgpack.Unpacker( encoding='utf-8', unicode_errors='surrogateescape') else: self._unpacker = msgpack.Unpacker( unicode_errors='surrogateescape') self._packer = msgpack.Packer( unicode_errors='surrogateescape') self._ignore_sources: typing.List[typing.Any] = []
def __init__(self, filename='data.bag', buffer_size=500, pack=None): """ filename: either a string containing the desired file name (note that .bag is appended) OR a file-like object from io.Bytes or something buffer_size: number of Bytes, default 10MB """ self.pack = pack self.buffer = [] self.ext_pack = pack self.fd = None self.buffer_size = buffer_size # check to see if this is a file-like object # if (hasattr(filename, 'read') and hasattr(filename, 'write')): # self.fd = filename # self.filename = None # # append .bag to end of filename if necessary # else: if filename.rfind('.bag') < 0: filename = filename + '.bag' self.filename = filename if pack: # print('pack', pack) # must have strict_types for namedtuple to work!!!!!! self.packer = msgpack.Packer(default=pack, use_bin_type=True, strict_types=True) # self.packer = msgpack.Packer(default=pack, use_bin_type=True) else: # self.packer = msgpack.Packer(use_bin_type=True) self.packer = msgpack.Packer(use_bin_type=True, strict_types=True)
def __init__(self): super(MessageEncoder, self).__init__() if msgpack.version >= (1, 0, 0): self._packer = msgpack.Packer() # The strict_map_key=False option is required to use int keys in # maps; it is disabled by default to prevent hash collision denial # of service attacks (hashdos) in scenarios where an attacker can # control the keys to be hashed. self._unpacker = msgpack.Unpacker(strict_map_key=False) else: self._packer = msgpack.Packer(encoding='utf-8', use_bin_type=True) self._unpacker = msgpack.Unpacker(encoding='utf-8') self._next_msgid = 0
def __init__(self, key): self.buffer = BytesIO() self.packer = msgpack.Packer(unicode_errors='surrogateescape') self.chunks = [] self.key = key self.chunker = Chunker(WINDOW_SIZE, CHUNK_MASK, CHUNK_MIN, self.key.chunk_seed)
def __init__(self, *args, **kw): import msgpack super().__init__(*args, **kw) self.buffer = bytearray() self.packer = msgpack.Packer() self.bufmax = 1024 * 25
def _start_process(self) -> None: self._stdin = None self._queue_id = '' self._queue_in: Queue[str] = Queue() self._queue_out: Queue[str] = Queue() self._packer = msgpack.Packer(use_bin_type=True, encoding='utf-8', unicode_errors='surrogateescape') self._unpacker = msgpack.Unpacker(encoding='utf-8', unicode_errors='surrogateescape') startupinfo = None if os.name == 'nt': startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW main = str( Path(__file__).parent.parent.parent.parent.joinpath( 'autoload', 'denite', '_main.py')) self._hnd = self._vim.loop.create_task( self._vim.loop.subprocess_exec( partial(Process, self), self._vim.vars.get('python3_host_prog', 'python3'), main, self._vim.vars['denite#_serveraddr'], stderr=None, startupinfo=startupinfo))
def __init__(self, tag, host='localhost', port=24224, bufmax=1 * 1024 * 1024, timeout=3.0, verbose=False): self.tag = tag self.host = host self.port = port self.bufmax = bufmax self.timeout = timeout self.verbose = verbose self.socket = None self.pendings = None self.packer = msgpack.Packer() self.lock = threading.Lock() try: self._reconnect() except Exception: # will be retried in emit() self._close()
def req(self, data, timeout): """Sends data to the other side and waits for a response. If no response within timeout period (or connection failure) then raises an error. """ # fixme: handle timeout packer = msgpack.Packer() unpacker = msgpack.Unpacker() with self._sock_lock: # fixme: handle some errors here #print "sending data", data self._sock.sendall(packer.pack(data)) while True: amt = self._sock.recv_into(self._req_buffer, 1000000) if not self._req_buffer: raise ValueError("socket closed fixme: raise real error") unpacker.feed(self._req_mv[:amt]) for m in unpacker: # We only expect a single message in response # because of the synchronous pattern from sendall. return m
def obj_2_openvslam_msg(input_obj_file_path, output_msg_file_path): """ generate map.msg file """ print("Generate {}..........".format(output_msg_file_path)) # load_pointcloud_from_obj point_cloud_data = [] obj_file_handle = open(input_obj_file_path, 'r') for line in obj_file_handle: split = line.split() #if blank line, skip if not len(split): continue if split[0] == "v": point_cloud_data.append( [float(split[1]), float(split[2]), float(split[3])]) obj_file_handle.close() # output to msg file # output for msg landmarks = {} for idx in range(len(point_cloud_data)): landmarks[str(idx)] = {"pos_w": point_cloud_data[idx]} data_text = {"landmarks": landmarks} # pack and output msgpack_packer = msgpack.Packer(use_bin_type=True) mapmsg_file = open(output_msg_file_path, 'wb') mapmsg_file.write(msgpack_packer.pack(data_text)) mapmsg_file.close()
def __init__(self, socket, outgoing_msg_sink_iter): super(RpcSession, self).__init__("RpcSession(%s)" % socket) self._packer = msgpack.Packer(encoding='utf-8') self._unpacker = msgpack.Unpacker(encoding='utf-8') self._next_msgid = 0 self._socket = socket self._outgoing_msg_sink_iter = outgoing_msg_sink_iter
def __init__(self, conn, packer=None): self._bytes = None self.conn = conn self._sync = None self._body = '' self.response_class = Response self.packer = packer if self.packer is None: packer_kwargs = dict() # use_bin_type=True is default since msgpack-1.0.0. # # The option controls whether to pack binary (non-unicode) # string values as mp_bin or as mp_str. # # The default behaviour of the connector is to pack both # bytes and Unicode strings as mp_str. # # msgpack-0.5.0 (and only this version) warns when the # option is unset: # # | FutureWarning: use_bin_type option is not specified. # | Default value of the option will be changed in future # | version. # # The option is supported since msgpack-0.4.0, so we can # just always set it for all msgpack versions to get rid # of the warning on msgpack-0.5.0 and to keep our # behaviour on msgpack-1.0.0. packer_kwargs['use_bin_type'] = False self.packer = msgpack.Packer(**packer_kwargs)
def __init__(self, ) -> None: self.msgpack_packer = msgpack.Packer( autoreset=True, default=self.encode_extensions, use_bin_type=True, strict_types=True, )
def __init__(self, conf, timeout): self._conf = conf self._timeout = timeout self._unpacker = msgpack.Unpacker() self._packer = msgpack.Packer() # connection variables self._peers = {} # peer_id -> Peer self._sock_to_peer = {} # socket.connection -> Peer self._peers_lock = threading.Lock() # for _peers and _sock_to_peers # request / response variables self._req_count = 0 self._req_count_lock = threading.Lock() # For reuse of ValueEvent objects by a thread. self._threadlocal = threading.local() self._patch_client_for_gevent() self._bg_thread = threading.Thread( target=self._process_requests_in_background ) self._bg_thread.setDaemon(True) self._bg_thread.start()
def _write_msgpack_stream(self, items, stream): """Write MessagePack stream Parameters ---------- items : list of dict Same format with dataframe.to_dict(orient="records") Examples: ``[{"time": 12345, "col1": "foo"}, {"time": 12345, "col1": "bar"}]`` stream : File like object Target file like object which has `write()` function. This object will be updated in this function. """ with gzip.GzipFile(mode="wb", fileobj=stream) as gz: packer = msgpack.Packer() for item in items: try: mp = packer.pack(item) except (OverflowError, ValueError): packer.reset() mp = packer.pack(normalized_msgpack(item)) gz.write(mp) stream.seek(0) return stream
def dispatcher(): hosts = open('./hosts.conf').read().strip().split('\n') redis_conns = [] for host in hosts: conn = redis.redis_connect(host) redis_conns.append(conn) mongo_conn = mongo.connect('192.168.241.12', 'stream') servernum = len(hosts) packer = msgpack.Packer() tablename = 'general' key = 'stream' maps = {} rawdatas = mongo.find(mongo_conn, tablename, {}, 30000) for raw in rawdatas: raw.pop('_id') #print raw url = raw['url'] num = disp_strategy(url, servernum, 'utf-8') data = packer.pack(raw) redis.add_set_value(redis_conns[num], key, data) if num not in maps: maps.update({num: 1}) else: maps[num] += 1 print maps
def terminateHubProcess(self): """ Send a UDP message to iohub informing it to exit. Use this when force quiting the experiment script process so iohub knows to exit as well. If message is not sent within 1 second, or the iohub server address in incorrect,the issue is logged. """ sock=None try: logging.debug('PsychoPyApp: terminateHubProcess called.') import socket sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) sock.settimeout(1.0) iohub_address='127.0.0.1', 9034 import msgpack tx_data=msgpack.Packer().pack(('STOP_IOHUB_SERVER',)) return sock.sendto(tx_data,iohub_address) except socket.error as e: logging.debug('PsychoPyApp: terminateHubProcess socket.error: %s'%(str(e))) except socket.herror as e: logging.debug('PsychoPyApp: terminateHubProcess socket.herror: %s'%(str(e))) except socket.gaierror as e: logging.debug('PsychoPyApp: terminateHubProcess socket.gaierror: %s'%(str(e))) except socket.timeout as e: logging.debug('PsychoPyApp: terminateHubProcess socket.timeout: %s'%(str(e))) except Exception as e: logging.debug('PsychoPyApp: terminateHubProcess exception: %s'%(str(e))) finally: if sock: sock.close() logging.debug('PsychoPyApp: terminateHubProcess completed.')
def _start_process(self) -> None: self._stdin = None self._queue_id = '' self._queue_in: Queue[str] = Queue() self._queue_out: Queue[str] = Queue() self._packer = msgpack.Packer(unicode_errors='surrogateescape') self._unpacker = msgpack.Unpacker(unicode_errors='surrogateescape') info = None if os.name == 'nt': info = subprocess.STARTUPINFO() # type: ignore info.dwFlags |= subprocess.STARTF_USESHOWWINDOW # type: ignore main = str( Path(__file__).parent.parent.parent.parent.joinpath( 'autoload', 'denite', '_main.py')) command = (self._vim.vars['python3_host_prog'] if 'python3_host_prog' in self._vim.vars else 'python3') self._hnd = self._vim.loop.create_task( self._vim.loop.subprocess_exec( partial(Process, self), command, main, self._vim.vars['denite#_serveraddr'], stderr=None, startupinfo=info))
def _start_process(self) -> None: self._stdin: typing.Optional[typing.Any] = None self._queue_id = '' self._queue_in: Queue = Queue() # type: ignore self._queue_out: Queue = Queue() # type: ignore self._queue_err: Queue = Queue() # type: ignore self._packer = msgpack.Packer( unicode_errors='surrogateescape') self._unpacker = msgpack.Unpacker( unicode_errors='surrogateescape') self._prev_pos: typing.List[typing.Any] = [] info = None if os.name == 'nt': info = subprocess.STARTUPINFO() # type: ignore info.dwFlags |= subprocess.STARTF_USESHOWWINDOW # type: ignore main = str(Path(__file__).parent.parent.parent.parent.joinpath( 'autoload', 'deoplete', '_main.py')) self._hnd = self._vim.loop.create_task( self._vim.loop.subprocess_exec( partial(Process, self), self._get_python_executable(), main, self._vim.vars['deoplete#_serveraddr'], startupinfo=info))
def __init__(self, ws, *, use_list=False): self.ws = ws self._use_list = False self._packer = msgpack.Packer(use_bin_type=True) self._mid = 0 self._tasks = {} asyncio.get_event_loop().create_task(self._run())
def write_msg_to(socket, msg): if not isinstance(msg, Message): debug_print('BAD. attempting to write_msg_to:', msg) else: debug_print('out --> msg', msg, 'to socket', socket) ''' send the given msg into the socket returns True if writing completes ''' my_file = StringIO() packer = msgpack.Packer() my_file.write(packer.pack(msg.serialize())) my_file = StringIO(my_file.getvalue()) # debug_print(vars(my_file)) tot_bytes = len(my_file.buf) sent_now = 1 to_send = tot_bytes while sent_now != 0: # 0 means send done try: sent_now = socket.send(my_file.read(to_send)) to_send -= sent_now if to_send == 0: return True except: return False return True
def __init__(self, endpoint, timeout=None, context=None, connect=True): # TODO # [ ] ask for devices # [ ] ask for keys/channels for device # [ ] server as MDL? # [ ] handles several client connections # [ ] spawn bound device per channel connection (client select shared/copy) self.dumps = msgpack.Packer(use_bin_type=True).pack self.loads = partial(msgpack.loads, raw=False, max_bin_len=0x7fffffff) self.monitored = set() self.ctx = context or zmq.Context() self.request = self.ctx.socket(zmq.PAIR) self.request.SNDTIMEO = 5000 self.request.RCVTIMEO = 5000 self.request.setsockopt(zmq.LINGER, 0) self.request.connect(endpoint) self.data = None self.timeout = timeout self.connected = False self._hb = None if connect: self.connect()
def __init__( self, local_host=None, local_port=None, remote_host=None, remote_port=None, rcvBufferLength=1492, broadcast=False, blocking=0, timeout=0): self._local_port = local_port self._local_host = local_host self._remote_host = remote_host self._remote_port = remote_port self._rcvBufferLength = rcvBufferLength self.lastAddress = None self.sock = None self.initSocket(broadcast, blocking, timeout) self.coder = msgpack self.packer = msgpack.Packer() self.unpacker = msgpack.Unpacker(use_list=True) self.pack = self.packer.pack self.feed = self.unpacker.feed self.unpack = self.unpacker.unpack
def pack(**options): """ Packs system data to portable package file """ # options handling job_ids = options.get('jobs') # preparation package = {'packager': platform.node(), 'timestamp': time.time()} # reading query = FcJob.query.filter_by(deleted=False) if job_ids: query = query.filter(FcJob.id.in_(job_ids)) joblist = query.all() # transforming js = JobSerializer() for job in joblist: js.add(job) package['deps'] = js.dependencies package['jobs'] = js.jobs # packing packer = msgpack.Packer(use_bin_type=True, default=orm_pack) yield packer.pack(package)
def create_msgpack(items): """Create msgpack streaming bytes from list Args: items (list of dict): target list Returns: Converted msgpack streaming (bytes) Examples: >>> t1 = int(time.time()) >>> l1 = [{"a": 1, "b": 2, "time": t1}, {"a":3, "b": 6, "time": t1}] >>> create_msgpack(l1) b'\\x83\\xa1a\\x01\\xa1b\\x02\\xa4time\\xce]\\xa5X\\xa1\\x83\\xa1a\\x03\\xa1b\\x06\\xa4time\\xce]\\xa5X\\xa1' """ stream = io.BytesIO() packer = msgpack.Packer() for item in items: try: mp = packer.pack(item) except (OverflowError, ValueError): packer.reset() mp = packer.pack(normalized_msgpack(item)) stream.write(mp) return stream.getvalue()
def test_exceeding_unpacker_read_size(): dumpf = io.BytesIO() packer = msgpack.Packer() NUMBER_OF_STRINGS = 6 read_size = 16 # 5 ok for read_size=16, while 6 glibc detected *** python: double free or corruption (fasttop): # 20 ok for read_size=256, while 25 segfaults / glibc detected *** python: double free or corruption (!prev) # 40 ok for read_size=1024, while 50 introduces errors # 7000 ok for read_size=1024*1024, while 8000 leads to glibc detected *** python: double free or corruption (!prev): for idx in range(NUMBER_OF_STRINGS): data = gen_binary_data(idx) dumpf.write(packer.pack(data)) f = io.BytesIO(dumpf.getvalue()) dumpf.close() unpacker = msgpack.Unpacker(f, read_size=read_size, use_list=1) read_count = 0 for idx, o in enumerate(unpacker): assert_equal(type(o), bytes) assert_equal(o, gen_binary_data(idx)) read_count += 1 assert_equal(read_count, NUMBER_OF_STRINGS)
def __init__(self, *args, **kwargs): super(ClaimController, self).__init__(*args, **kwargs) self._client = self.driver.connection self._packer = msgpack.Packer(encoding='utf-8', use_bin_type=True).pack self._unpacker = functools.partial(msgpack.unpackb, encoding='utf-8')
async def call(self, *args, command=command): packer = msgpack.Packer(default=default) packed = [pack_msgpack(packer, arg) for arg in args] result = await self.redis.execute(command.command, *packed) r = msgpack.unpackb(result) if command.return_type != None: return parse_obj_as(command.return_type, r)