def on_heartbeat_v0(w):
if sys.version_info[0] == 2:
packer = msgpack.Packer()
else:
packer = msgpack.Packer(use_bin_type=True)
if s.counter > 6:
yield w.write(packer.pack([1, 2, [2, "terminate"]]))
s.io_loop.add_callback(s.stop)
return
req = [METHOD, URI, HTTP_VERSION, HEADERS, BODY]
yield w.write(packer.pack([1, 1, []]))
yield w.write(packer.pack([s.counter, 3, ["ping"]]))
yield w.write(packer.pack([s.counter, 4, ["pong"]]))
yield w.write(packer.pack([s.counter, 6, []]))
s.counter += 1
yield w.write(packer.pack([s.counter, 3, ["bad_event"]]))
s.counter += 1
yield w.write(packer.pack([s.counter, 3, ["http"]]))
yield w.write(packer.pack([s.counter, 4, [packer.pack(req)]]))
yield w.write(packer.pack([s.counter, 6, []]))
s.counter += 1
yield w.write(packer.pack([s.counter, 3, ["http_test"]]))
yield w.write(packer.pack([s.counter, 4, [packer.pack(req)]]))
yield w.write(packer.pack([s.counter, 6, []]))
s.counter += 1
yield w.write(packer.pack([s.counter + 100, 4, ["bad_ping"]]))
yield w.write(packer.pack([s.counter, 104, ["bad_ping"]]))
s.counter += 1
yield w.write(packer.pack([s.counter, 3, ["bad_ping"]]))
yield w.write(packer.pack([s.counter, 4, ["A"]]))
yield w.write(packer.pack([s.counter, 6, []]))
s.counter += 1
yield w.write(packer.pack([s.counter, 3, ["notclosed"]]))
yield w.write(packer.pack([s.counter, 4, ["A"]]))
yield w.write(packer.pack([s.counter, 6, []]))
def _start_process(self) -> None:
self._stdin: typing.Optional[typing.Any] = None
self._queue_id = ''
self._queue_in: 'Queue[bytes]' = Queue()
self._queue_out: 'Queue[typing.Any]' = Queue()
self._queue_err: 'Queue[typing.Any]' = Queue()
if msgpack.version < (1, 0, 0):
self._packer = msgpack.Packer(
encoding='utf-8',
unicode_errors='surrogateescape')
self._unpacker = msgpack.Unpacker(
encoding='utf-8',
unicode_errors='surrogateescape')
else:
self._packer = msgpack.Packer(
unicode_errors='surrogateescape')
self._unpacker = msgpack.Unpacker(
unicode_errors='surrogateescape')
self._prev_pos: typing.List[typing.Any] = []
info = None
if sys.platform == 'win32':
info = subprocess.STARTUPINFO()
info.dwFlags |= subprocess.STARTF_USESHOWWINDOW
main = str(Path(__file__).parent.parent.parent.parent.joinpath(
'autoload', 'deoplete', '_main.py'))
self._hnd = self._vim.loop.create_task(
self._vim.loop.subprocess_exec(
partial(Process, self),
self._get_python_executable(),
main,
self._vim.vars['deoplete#_serveraddr'],
startupinfo=info))
def test_dtype_transform_lamdbas(tmpdir):
rt = ResolutionTable(os.path.join(tmpdir, "test/"))
# int
int_test = 10
assert int_test.to_bytes((int_test.bit_length() + 7) // 8,
"big") == rt.to_bytes_func(int)(int_test)
assert int_test == rt.from_bytes_func(int)(int_test.to_bytes(
(int_test.bit_length() + 7) // 8, "big"))
# str
str_test = "ten"
assert str_test.encode("utf-8") == rt.to_bytes_func(str)(str_test)
assert str_test == rt.from_bytes_func(str)(str_test.encode("utf-8"))
# else [use msgpack] (list, set, dict, float)
other_dtypes = {
list: [1, 2, 3],
float: 1.23,
dict: {
"123": 123
},
tuple: (1, 2, 3)
}
for dtype, value in other_dtypes.items():
assert msgpack.Packer().pack(value) == rt.to_bytes_func(dtype)(value)
assert value == dtype(
rt.from_bytes_func(dtype)(msgpack.Packer().pack(value)))
def __init__(self, vim: Nvim) -> None:
self.name = 'child'
self._vim = vim
self._filters: typing.Dict[str, typing.Any] = {}
self._sources: typing.Dict[str, typing.Any] = {}
self._profile_flag = None
self._profile_start_time = 0
self._loaded_sources: typing.Dict[str, typing.Any] = {}
self._loaded_filters: typing.Dict[str, typing.Any] = {}
self._source_errors: typing.Dict[str, int] = defaultdict(int)
self._prev_results: typing.Dict[str, Result] = {}
if msgpack.version < (1, 0, 0):
self._packer = msgpack.Packer(
encoding='utf-8',
unicode_errors='surrogateescape')
self._unpacker = msgpack.Unpacker(
encoding='utf-8',
unicode_errors='surrogateescape')
else:
self._unpacker = msgpack.Unpacker(
unicode_errors='surrogateescape')
self._packer = msgpack.Packer(
unicode_errors='surrogateescape')
self._ignore_sources: typing.List[typing.Any] = []
def __init__(self, filename='data.bag', buffer_size=500, pack=None):
"""
filename: either a string containing the desired file name (note that
.bag is appended) OR a file-like object from io.Bytes or something
buffer_size: number of Bytes, default 10MB
"""
self.pack = pack
self.buffer = []
self.ext_pack = pack
self.fd = None
self.buffer_size = buffer_size
# check to see if this is a file-like object
# if (hasattr(filename, 'read') and hasattr(filename, 'write')):
# self.fd = filename
# self.filename = None
# # append .bag to end of filename if necessary
# else:
if filename.rfind('.bag') < 0:
filename = filename + '.bag'
self.filename = filename
if pack:
# print('pack', pack)
# must have strict_types for namedtuple to work!!!!!!
self.packer = msgpack.Packer(default=pack, use_bin_type=True, strict_types=True)
# self.packer = msgpack.Packer(default=pack, use_bin_type=True)
else:
# self.packer = msgpack.Packer(use_bin_type=True)
self.packer = msgpack.Packer(use_bin_type=True, strict_types=True)
def __init__(self):
super(MessageEncoder, self).__init__()
if msgpack.version >= (1, 0, 0):
self._packer = msgpack.Packer()
# The strict_map_key=False option is required to use int keys in
# maps; it is disabled by default to prevent hash collision denial
# of service attacks (hashdos) in scenarios where an attacker can
# control the keys to be hashed.
self._unpacker = msgpack.Unpacker(strict_map_key=False)
else:
self._packer = msgpack.Packer(encoding='utf-8', use_bin_type=True)
self._unpacker = msgpack.Unpacker(encoding='utf-8')
self._next_msgid = 0
def __init__(self, key):
self.buffer = BytesIO()
self.packer = msgpack.Packer(unicode_errors='surrogateescape')
self.chunks = []
self.key = key
self.chunker = Chunker(WINDOW_SIZE, CHUNK_MASK, CHUNK_MIN,
self.key.chunk_seed)
def __init__(self, *args, **kw):
import msgpack
super().__init__(*args, **kw)
self.buffer = bytearray()
self.packer = msgpack.Packer()
self.bufmax = 1024 * 25
def _start_process(self) -> None:
self._stdin = None
self._queue_id = ''
self._queue_in: Queue[str] = Queue()
self._queue_out: Queue[str] = Queue()
self._packer = msgpack.Packer(use_bin_type=True,
encoding='utf-8',
unicode_errors='surrogateescape')
self._unpacker = msgpack.Unpacker(encoding='utf-8',
unicode_errors='surrogateescape')
startupinfo = None
if os.name == 'nt':
startupinfo = subprocess.STARTUPINFO()
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
main = str(
Path(__file__).parent.parent.parent.parent.joinpath(
'autoload', 'denite', '_main.py'))
self._hnd = self._vim.loop.create_task(
self._vim.loop.subprocess_exec(
partial(Process, self),
self._vim.vars.get('python3_host_prog', 'python3'),
main,
self._vim.vars['denite#_serveraddr'],
stderr=None,
startupinfo=startupinfo))
def __init__(self,
tag,
host='localhost',
port=24224,
bufmax=1 * 1024 * 1024,
timeout=3.0,
verbose=False):
self.tag = tag
self.host = host
self.port = port
self.bufmax = bufmax
self.timeout = timeout
self.verbose = verbose
self.socket = None
self.pendings = None
self.packer = msgpack.Packer()
self.lock = threading.Lock()
try:
self._reconnect()
except Exception:
# will be retried in emit()
self._close()
def req(self, data, timeout):
"""Sends data to the other side and waits for a response.
If no response within timeout period (or connection failure)
then raises an error.
"""
# fixme: handle timeout
packer = msgpack.Packer()
unpacker = msgpack.Unpacker()
with self._sock_lock:
# fixme: handle some errors here
#print "sending data", data
self._sock.sendall(packer.pack(data))
while True:
amt = self._sock.recv_into(self._req_buffer, 1000000)
if not self._req_buffer:
raise ValueError("socket closed fixme: raise real error")
unpacker.feed(self._req_mv[:amt])
for m in unpacker:
# We only expect a single message in response
# because of the synchronous pattern from sendall.
return m
def obj_2_openvslam_msg(input_obj_file_path, output_msg_file_path):
"""
generate map.msg file
"""
print("Generate {}..........".format(output_msg_file_path))
# load_pointcloud_from_obj
point_cloud_data = []
obj_file_handle = open(input_obj_file_path, 'r')
for line in obj_file_handle:
split = line.split()
#if blank line, skip
if not len(split):
continue
if split[0] == "v":
point_cloud_data.append(
[float(split[1]),
float(split[2]),
float(split[3])])
obj_file_handle.close()
# output to msg file # output for msg
landmarks = {}
for idx in range(len(point_cloud_data)):
landmarks[str(idx)] = {"pos_w": point_cloud_data[idx]}
data_text = {"landmarks": landmarks}
# pack and output
msgpack_packer = msgpack.Packer(use_bin_type=True)
mapmsg_file = open(output_msg_file_path, 'wb')
mapmsg_file.write(msgpack_packer.pack(data_text))
mapmsg_file.close()
def __init__(self, socket, outgoing_msg_sink_iter):
super(RpcSession, self).__init__("RpcSession(%s)" % socket)
self._packer = msgpack.Packer(encoding='utf-8')
self._unpacker = msgpack.Unpacker(encoding='utf-8')
self._next_msgid = 0
self._socket = socket
self._outgoing_msg_sink_iter = outgoing_msg_sink_iter
def __init__(self, conn, packer=None):
self._bytes = None
self.conn = conn
self._sync = None
self._body = ''
self.response_class = Response
self.packer = packer
if self.packer is None:
packer_kwargs = dict()
# use_bin_type=True is default since msgpack-1.0.0.
#
# The option controls whether to pack binary (non-unicode)
# string values as mp_bin or as mp_str.
#
# The default behaviour of the connector is to pack both
# bytes and Unicode strings as mp_str.
#
# msgpack-0.5.0 (and only this version) warns when the
# option is unset:
#
# | FutureWarning: use_bin_type option is not specified.
# | Default value of the option will be changed in future
# | version.
#
# The option is supported since msgpack-0.4.0, so we can
# just always set it for all msgpack versions to get rid
# of the warning on msgpack-0.5.0 and to keep our
# behaviour on msgpack-1.0.0.
packer_kwargs['use_bin_type'] = False
self.packer = msgpack.Packer(**packer_kwargs)
def __init__(self, ) -> None:
self.msgpack_packer = msgpack.Packer(
autoreset=True,
default=self.encode_extensions,
use_bin_type=True,
strict_types=True,
)
def __init__(self, conf, timeout):
self._conf = conf
self._timeout = timeout
self._unpacker = msgpack.Unpacker()
self._packer = msgpack.Packer()
# connection variables
self._peers = {} # peer_id -> Peer
self._sock_to_peer = {} # socket.connection -> Peer
self._peers_lock = threading.Lock() # for _peers and _sock_to_peers
# request / response variables
self._req_count = 0
self._req_count_lock = threading.Lock()
# For reuse of ValueEvent objects by a thread.
self._threadlocal = threading.local()
self._patch_client_for_gevent()
self._bg_thread = threading.Thread(
target=self._process_requests_in_background
)
self._bg_thread.setDaemon(True)
self._bg_thread.start()
def _write_msgpack_stream(self, items, stream):
"""Write MessagePack stream
Parameters
----------
items : list of dict
Same format with dataframe.to_dict(orient="records")
Examples:
``[{"time": 12345, "col1": "foo"}, {"time": 12345, "col1": "bar"}]``
stream : File like object
Target file like object which has `write()` function. This object will be
updated in this function.
"""
with gzip.GzipFile(mode="wb", fileobj=stream) as gz:
packer = msgpack.Packer()
for item in items:
try:
mp = packer.pack(item)
except (OverflowError, ValueError):
packer.reset()
mp = packer.pack(normalized_msgpack(item))
gz.write(mp)
stream.seek(0)
return stream
def dispatcher():
hosts = open('./hosts.conf').read().strip().split('\n')
redis_conns = []
for host in hosts:
conn = redis.redis_connect(host)
redis_conns.append(conn)
mongo_conn = mongo.connect('192.168.241.12', 'stream')
servernum = len(hosts)
packer = msgpack.Packer()
tablename = 'general'
key = 'stream'
maps = {}
rawdatas = mongo.find(mongo_conn, tablename, {}, 30000)
for raw in rawdatas:
raw.pop('_id')
#print raw
url = raw['url']
num = disp_strategy(url, servernum, 'utf-8')
data = packer.pack(raw)
redis.add_set_value(redis_conns[num], key, data)
if num not in maps:
maps.update({num: 1})
else:
maps[num] += 1
print maps
def terminateHubProcess(self):
"""
Send a UDP message to iohub informing it to exit.
Use this when force quiting the experiment script process so iohub
knows to exit as well.
If message is not sent within 1 second, or the iohub server
address in incorrect,the issue is logged.
"""
sock=None
try:
logging.debug('PsychoPyApp: terminateHubProcess called.')
import socket
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.settimeout(1.0)
iohub_address='127.0.0.1', 9034
import msgpack
tx_data=msgpack.Packer().pack(('STOP_IOHUB_SERVER',))
return sock.sendto(tx_data,iohub_address)
except socket.error as e:
logging.debug('PsychoPyApp: terminateHubProcess socket.error: %s'%(str(e)))
except socket.herror as e:
logging.debug('PsychoPyApp: terminateHubProcess socket.herror: %s'%(str(e)))
except socket.gaierror as e:
logging.debug('PsychoPyApp: terminateHubProcess socket.gaierror: %s'%(str(e)))
except socket.timeout as e:
logging.debug('PsychoPyApp: terminateHubProcess socket.timeout: %s'%(str(e)))
except Exception as e:
logging.debug('PsychoPyApp: terminateHubProcess exception: %s'%(str(e)))
finally:
if sock:
sock.close()
logging.debug('PsychoPyApp: terminateHubProcess completed.')
def _start_process(self) -> None:
self._stdin = None
self._queue_id = ''
self._queue_in: Queue[str] = Queue()
self._queue_out: Queue[str] = Queue()
self._packer = msgpack.Packer(unicode_errors='surrogateescape')
self._unpacker = msgpack.Unpacker(unicode_errors='surrogateescape')
info = None
if os.name == 'nt':
info = subprocess.STARTUPINFO() # type: ignore
info.dwFlags |= subprocess.STARTF_USESHOWWINDOW # type: ignore
main = str(
Path(__file__).parent.parent.parent.parent.joinpath(
'autoload', 'denite', '_main.py'))
command = (self._vim.vars['python3_host_prog']
if 'python3_host_prog' in self._vim.vars else 'python3')
self._hnd = self._vim.loop.create_task(
self._vim.loop.subprocess_exec(
partial(Process, self),
command,
main,
self._vim.vars['denite#_serveraddr'],
stderr=None,
startupinfo=info))
def _start_process(self) -> None:
self._stdin: typing.Optional[typing.Any] = None
self._queue_id = ''
self._queue_in: Queue = Queue() # type: ignore
self._queue_out: Queue = Queue() # type: ignore
self._queue_err: Queue = Queue() # type: ignore
self._packer = msgpack.Packer(
unicode_errors='surrogateescape')
self._unpacker = msgpack.Unpacker(
unicode_errors='surrogateescape')
self._prev_pos: typing.List[typing.Any] = []
info = None
if os.name == 'nt':
info = subprocess.STARTUPINFO() # type: ignore
info.dwFlags |= subprocess.STARTF_USESHOWWINDOW # type: ignore
main = str(Path(__file__).parent.parent.parent.parent.joinpath(
'autoload', 'deoplete', '_main.py'))
self._hnd = self._vim.loop.create_task(
self._vim.loop.subprocess_exec(
partial(Process, self),
self._get_python_executable(),
main,
self._vim.vars['deoplete#_serveraddr'],
startupinfo=info))
def __init__(self, ws, *, use_list=False):
self.ws = ws
self._use_list = False
self._packer = msgpack.Packer(use_bin_type=True)
self._mid = 0
self._tasks = {}
asyncio.get_event_loop().create_task(self._run())
def write_msg_to(socket, msg):
if not isinstance(msg, Message):
debug_print('BAD. attempting to write_msg_to:', msg)
else:
debug_print('out --> msg', msg, 'to socket', socket)
'''
send the given msg into the socket
returns True if writing completes
'''
my_file = StringIO()
packer = msgpack.Packer()
my_file.write(packer.pack(msg.serialize()))
my_file = StringIO(my_file.getvalue())
# debug_print(vars(my_file))
tot_bytes = len(my_file.buf)
sent_now = 1
to_send = tot_bytes
while sent_now != 0: # 0 means send done
try:
sent_now = socket.send(my_file.read(to_send))
to_send -= sent_now
if to_send == 0:
return True
except:
return False
return True
def __init__(self, endpoint, timeout=None, context=None, connect=True):
# TODO
# [ ] ask for devices
# [ ] ask for keys/channels for device
# [ ] server as MDL?
# [ ] handles several client connections
# [ ] spawn bound device per channel connection (client select shared/copy)
self.dumps = msgpack.Packer(use_bin_type=True).pack
self.loads = partial(msgpack.loads, raw=False, max_bin_len=0x7fffffff)
self.monitored = set()
self.ctx = context or zmq.Context()
self.request = self.ctx.socket(zmq.PAIR)
self.request.SNDTIMEO = 5000
self.request.RCVTIMEO = 5000
self.request.setsockopt(zmq.LINGER, 0)
self.request.connect(endpoint)
self.data = None
self.timeout = timeout
self.connected = False
self._hb = None
if connect:
self.connect()
def __init__(
self,
local_host=None,
local_port=None,
remote_host=None,
remote_port=None,
rcvBufferLength=1492,
broadcast=False,
blocking=0,
timeout=0):
self._local_port = local_port
self._local_host = local_host
self._remote_host = remote_host
self._remote_port = remote_port
self._rcvBufferLength = rcvBufferLength
self.lastAddress = None
self.sock = None
self.initSocket(broadcast, blocking, timeout)
self.coder = msgpack
self.packer = msgpack.Packer()
self.unpacker = msgpack.Unpacker(use_list=True)
self.pack = self.packer.pack
self.feed = self.unpacker.feed
self.unpack = self.unpacker.unpack
def pack(**options):
"""
Packs system data to portable package file
"""
# options handling
job_ids = options.get('jobs')
# preparation
package = {'packager': platform.node(), 'timestamp': time.time()}
# reading
query = FcJob.query.filter_by(deleted=False)
if job_ids:
query = query.filter(FcJob.id.in_(job_ids))
joblist = query.all()
# transforming
js = JobSerializer()
for job in joblist:
js.add(job)
package['deps'] = js.dependencies
package['jobs'] = js.jobs
# packing
packer = msgpack.Packer(use_bin_type=True, default=orm_pack)
yield packer.pack(package)
def create_msgpack(items):
"""Create msgpack streaming bytes from list
Args:
items (list of dict): target list
Returns:
Converted msgpack streaming (bytes)
Examples:
>>> t1 = int(time.time())
>>> l1 = [{"a": 1, "b": 2, "time": t1}, {"a":3, "b": 6, "time": t1}]
>>> create_msgpack(l1)
b'\\x83\\xa1a\\x01\\xa1b\\x02\\xa4time\\xce]\\xa5X\\xa1\\x83\\xa1a\\x03\\xa1b\\x06\\xa4time\\xce]\\xa5X\\xa1'
"""
stream = io.BytesIO()
packer = msgpack.Packer()
for item in items:
try:
mp = packer.pack(item)
except (OverflowError, ValueError):
packer.reset()
mp = packer.pack(normalized_msgpack(item))
stream.write(mp)
return stream.getvalue()
def test_exceeding_unpacker_read_size():
dumpf = io.BytesIO()
packer = msgpack.Packer()
NUMBER_OF_STRINGS = 6
read_size = 16
# 5 ok for read_size=16, while 6 glibc detected *** python: double free or corruption (fasttop):
# 20 ok for read_size=256, while 25 segfaults / glibc detected *** python: double free or corruption (!prev)
# 40 ok for read_size=1024, while 50 introduces errors
# 7000 ok for read_size=1024*1024, while 8000 leads to glibc detected *** python: double free or corruption (!prev):
for idx in range(NUMBER_OF_STRINGS):
data = gen_binary_data(idx)
dumpf.write(packer.pack(data))
f = io.BytesIO(dumpf.getvalue())
dumpf.close()
unpacker = msgpack.Unpacker(f, read_size=read_size, use_list=1)
read_count = 0
for idx, o in enumerate(unpacker):
assert_equal(type(o), bytes)
assert_equal(o, gen_binary_data(idx))
read_count += 1
assert_equal(read_count, NUMBER_OF_STRINGS)
def __init__(self, *args, **kwargs):
super(ClaimController, self).__init__(*args, **kwargs)
self._client = self.driver.connection
self._packer = msgpack.Packer(encoding='utf-8',
use_bin_type=True).pack
self._unpacker = functools.partial(msgpack.unpackb, encoding='utf-8')
async def call(self, *args, command=command):
packer = msgpack.Packer(default=default)
packed = [pack_msgpack(packer, arg) for arg in args]
result = await self.redis.execute(command.command, *packed)
r = msgpack.unpackb(result)
if command.return_type != None:
return parse_obj_as(command.return_type, r)
