class SReader(): """ Define an asyncio msgpack stream decoder. """ def __init__(self, reader, writer): """ Pass ina stream reader to unmarshall msgpack objects from. """ self.reader = reader self.writer = writer self.decoder = make_decoder() self.unpacker = Unpacker(ext_hook=self.decoder, encoding="utf8") self.obj_buf = [] @asyncio.coroutine def get(self): """ The co-routine providing objects. """ while len(self.obj_buf) == 0: buf = yield from self.reader.read(1000) self.unpacker.feed(buf) for o in self.unpacker: self.obj_buf.append(o) return self.obj_buf.pop(0) def put(self, obj): """ Write an object to the channel. """ self.writer.write(encode(obj))
class MsgpackProtocol(asyncio.Protocol): def __init__(self, routes): self.__routes = routes self.packer = Unpacker() def connection_made(self, transport): peername = transport.get_extra_info('peername') print('Connection from {}'.format(peername)) self.transport = transport self.transport.write(packb([2, 'peername', peername])) def data_received(self, data): self.packer.feed(data) for msg in self.packer: assert_request(msg) self.routing(msg) def routing(self, cmd): assert cmd[2] in self.__routes t = asyncio.ensure_future(response(cmd[1], self.transport, self.__routes[cmd[2]], cmd[3])) def eof_received(self): return True
def test3(): start = 0 end = 10 metric = "marion.channel-0" raw_series = REDIS_CONN.get(settings.FULL_NAMESPACE + metric) if not raw_series: resp = json.dumps({'results': 'Error: No metric by that name'}) return resp, 404 else: unpacker = Unpacker(use_list = False) unpacker.feed(raw_series) timeseries = [] point = {'x':datapoint[0],'y':datapoint[1]} if (start is None) and (end is not None): for datapoint in unpacker: if datapoint[0] < int(end): timeseries.append(point) elif (start is not None) and (end is None): for datapoint in unpacker: if datapoint[0] > int(start): timeseries.append(point) elif (start is not None) and (end is not None): for datapoint in unpacker: if (datapoint[0] > int(start)) and (datapoint[0] < int(end)): timeseries.append(point) elif (start is None) and (end is None): timeseries = [{'x':datapoint[0],'y':datapoint[1]} for datapoint in unpacker] resp = json.dumps({'results': timeseries}) return resp, 200
class ClientProtocol(asyncio.Protocol): def __init__(self): self._cpt = -1 self.packer = Unpacker() self._responses = dict() def connection_made(self, transport): print("connected") self.transport = transport def request(self, name, args, f): print("send request") self._cpt += 1 self._responses[self._cpt] = f self.transport.write(packb([0, self._cpt, name, args])) def data_received(self, data): self.packer.feed(data) for msg in self.packer: if msg[0] == 1: self._responses[msg[1]].set_result(msg) def connection_lost(self, exc): pass
def anomalies(): resp = 'handle_data([])' try: analyzer_key_node = REDIS_BACKENDS.get_node(settings.ANALYZER_ANOMALY_KEY) anomaly_keys = RING.run('smembers', settings.ANALYZER_ANOMALY_KEY) anomalies = {} if not anomaly_keys: logger.info("No anomaly key found!") return resp, 200 for key in list(anomaly_keys): raw_anomalies = RING.run('get',key) if not raw_anomalies: logger.info("Can't get anomalies for key %s, removing it from set" % key) RING.run('srem', settings.ANALYZER_ANOMALY_KEY, key) continue unpacker = Unpacker(use_list = False) unpacker.feed(raw_anomalies) for item in unpacker: anomalies.update(item) anomaly_list = [] for anom, value in anomalies.iteritems(): anomaly_list.append([value, anom]) if len(anomaly_list) > 0: anomaly_list.sort(key=operator.itemgetter(1)) resp = 'handle_data(%s)' % anomaly_list except Exception as e: logger.error("Error getting anomalies: %s" % str(e)) return resp, 200
def data(): metric = request.args.get('metric', None) start = request.args.get('start', None) end = request.args.get('end', None) if metric is None: metrics = ['channel-0', 'channel-1', 'channel-2', 'channel-3', 'channel-4', 'channel-5', 'channel-6', 'channel-7'] else: metrics = [metric] try: all_channels_data = [] for metric in metrics: single_channel_data = {} raw_series = REDIS_CONN.get(settings.FULL_NAMESPACE + metric) if not raw_series: resp = json.dumps({'results': 'Error: No metric by that name'}) return resp, 404 else: unpacker = Unpacker(use_list = False) unpacker.feed(raw_series) timeseries = [] if (start is None) and (end is not None): for datapoint in unpacker: if datapoint[0] < int(end): point = {'x' : datapoint[0], 'y':datapoint[1]} timeseries.append(point) elif (start is not None) and (end is None): for datapoint in unpacker: if datapoint[0] > int(start): point = {'x' : datapoint[0], 'y':datapoint[1]} timeseries.append(point) elif (start is not None) and (end is not None): for datapoint in unpacker: if (datapoint[0] > int(start)) and (datapoint[0] < int(end)): point = {'x' : datapoint[0], 'y':datapoint[1]} timeseries.append(point) elif (start is None) and (end is None): timeseries = [{'x' : datapoint[0], 'y':datapoint[1]} for datapoint in unpacker] single_channel_data['key'] = metric single_channel_data['values'] = timeseries all_channels_data.append(single_channel_data) resp = json.dumps({'results': all_channels_data}) return resp, 200 except Exception as e: error = "Error: " + e resp = json.dumps({'results': error}) return resp, 500 except Exception as e: error = "Error: " + e resp = json.dumps({'results': error}) return resp, 500
def test_incorrect_type_nested_map(): unpacker = Unpacker() unpacker.feed(packb([{"a": "b"}])) try: unpacker.read_map_header() assert 0, "should raise exception" except UnexpectedTypeException: assert 1, "okay"
def test_correct_type_nested_array(): unpacker = Unpacker() unpacker.feed(packb({"a": ["b", "c", "d"]})) try: unpacker.read_array_header() assert 0, "should raise exception" except UnexpectedTypeException: assert 1, "okay"
def mpdecode(iterable): unpacker = Unpacker(encoding='utf8') for chunk in iterable: unpacker.feed(chunk) # Each chunk can have none or many objects, # so here we dispatch any object ready for obj in unpacker: yield obj
def test_incorrect_type_array(): unpacker = Unpacker() unpacker.feed(packb(1)) try: unpacker.read_array_header() assert 0, 'should raise exception' except UnexpectedTypeException: assert 1, 'okay'
def test_correct_type_nested_array(): unpacker = Unpacker() unpacker.feed(packb({'a': ['b', 'c', 'd']})) try: unpacker.read_array_header() assert 0, 'should raise exception' except UnexpectedTypeException: assert 1, 'okay'
def test_incorrect_type_nested_map(): unpacker = Unpacker() unpacker.feed(packb([{'a': 'b'}])) try: unpacker.read_map_header() assert 0, 'should raise exception' except UnexpectedTypeException: assert 1, 'okay'
def __init__(self, cb): self.cb = cb def listhook(obj): return self.cb(obj) self.listhook = listhook Unpacker.__init__(self, list_hook=self.listhook)
def test_auto_max_array_len(): packed = b'\xde\x00\x06zz' with pytest.raises(UnpackValueError): unpackb(packed, raw=False) unpacker = Unpacker(max_buffer_size=5, raw=False) unpacker.feed(packed) with pytest.raises(UnpackValueError): unpacker.unpack()
def setUp(self): address = 0xfa1afe1 device = "LivewareProblem" raw_packet = encode_erase_flash_page(address, device) unpacker = Unpacker() unpacker.feed(raw_packet) self.command = list(unpacker)[1:]
def test_write_bytes_multi_buffer(): long_val = (5) * 100 expected = packb(long_val) unpacker = Unpacker(six.BytesIO(expected), read_size=3, max_buffer_size=3) f = six.BytesIO() unpacked = unpacker.unpack(f.write) assert unpacked == long_val assert f.getvalue() == expected
def unpack_gen(file, size): u = Unpacker() while True: data = file.read(size) if not data: break u.feed(data) for o in u: yield o
def get_edge(self, filenum, offset): if filenum in self.open_file_cache: fileobj = self.open_file_cache[filenum] else: filename = 'part_%02d.msgpack' % filenum fileobj = open(os.path.join(self.edge_dir, filename), 'rb') self.open_file_cache[filenum] = fileobj fileobj.seek(offset) unpacker = Unpacker(fileobj, encoding=encoding) return unpacker.unpack()
def mpack_handler(self, data, sock): unpacker = Unpacker() unpacker.feed(data) while 1: for msg in unpacker: self.on_message(msg) next = sock.recv(1000000) if not next: break unpacker.feed(next)
def test_auto_max_map_len(): # len(packed) == 6 -> max_map_len == 3 packed = b'\xde\x00\x04zzz' with pytest.raises(UnpackValueError): unpackb(packed, raw=False) unpacker = Unpacker(max_buffer_size=6, raw=False) unpacker.feed(packed) with pytest.raises(UnpackValueError): unpacker.unpack()
def test_foobar(): unpacker = Unpacker(read_size=3) unpacker.feed(b'foobar') assert unpacker.unpack() == ord(b'f') assert unpacker.unpack() == ord(b'o') assert unpacker.unpack() == ord(b'o') assert unpacker.unpack() == ord(b'b') assert unpacker.unpack() == ord(b'a') assert unpacker.unpack() == ord(b'r') try: o = unpacker.unpack() print(("Oops!", o)) assert 0 except StopIteration: assert 1 else: assert 0 unpacker.feed(b'foo') unpacker.feed(b'bar') k = 0 for o, e in zip(unpacker, b'foobarbaz'): assert o == e k += 1 assert k == len(b'foobar')
def test_foobar(): unpacker = Unpacker(read_size=3) unpacker.feed('foobar') assert_equal(unpacker.unpack(), ord('f')) assert_equal(unpacker.unpack(), ord('o')) assert_equal(unpacker.unpack(), ord('o')) assert_equal(unpacker.unpack(), ord('b')) assert_equal(unpacker.unpack(), ord('a')) assert_equal(unpacker.unpack(), ord('r')) try: o = unpacker.unpack() print "Oops!", o assert 0 except StopIteration: assert 1 else: assert 0 unpacker.feed('foo') unpacker.feed('bar') k = 0 for o, e in zip(unpacker, 'foobarbaz'): assert o == ord(e) k += 1 assert k == len('foobar')
def test_foobar(): unpacker = Unpacker(read_size=3) unpacker.feed("foobar") assert unpacker.unpack() == ord(b"f") assert unpacker.unpack() == ord(b"o") assert unpacker.unpack() == ord(b"o") assert unpacker.unpack() == ord(b"b") assert unpacker.unpack() == ord(b"a") assert unpacker.unpack() == ord(b"r") try: o = unpacker.unpack() print "Oops!", o assert 0 except StopIteration: assert 1 else: assert 0 unpacker.feed(b"foo") unpacker.feed(b"bar") k = 0 for o, e in zip(unpacker, b"foobarbaz"): assert o == ord(e) k += 1 assert k == len(b"foobar")
def test_has_correct_protocol_version(self): """ Checks that the command encoding function works corectly. """ raw_packet = encode_command(command_code=10) unpacker = Unpacker() unpacker.feed(raw_packet) version, *_ = list(unpacker) self.assertEqual(2, version)
def setUp(self): address = 0xdeadbeef data = bytes(range(4)) device = "dummy" raw_packet = encode_write_flash(data, address, device) unpacker = Unpacker() unpacker.feed(raw_packet) # Discards command set version self.command = list(unpacker)[1:]
class MsgpackStream(object): """Two-way msgpack stream that wraps a event loop byte stream. This wraps the event loop interface for reading/writing bytes and exposes an interface for reading/writing msgpack documents. """ def __init__(self, event_loop): """Wrap `event_loop` on a msgpack-aware interface.""" self.loop = event_loop self._packer = Packer(encoding='utf-8', unicode_errors=unicode_errors_default) self._unpacker = Unpacker() self._message_cb = None def threadsafe_call(self, fn): """Wrapper around `BaseEventLoop.threadsafe_call`.""" self.loop.threadsafe_call(fn) def send(self, msg): """Queue `msg` for sending to Nvim.""" debug('sent %s', msg) self.loop.send(self._packer.pack(msg)) def run(self, message_cb): """Run the event loop to receive messages from Nvim. While the event loop is running, `message_cb` will be called whenever a message has been successfully parsed from the input stream. """ self._message_cb = message_cb self.loop.run(self._on_data) self._message_cb = None def stop(self): """Stop the event loop.""" self.loop.stop() def close(self): """Close the event loop.""" self.loop.close() def _on_data(self, data): self._unpacker.feed(data) while True: try: debug('waiting for message...') msg = next(self._unpacker) debug('received message: %s', msg) self._message_cb(msg) except StopIteration: debug('unpacker needs more data...') break
def testArraySize(sizes=[0, 5, 50, 1000]): bio = BytesIO() packer = Packer() for size in sizes: bio.write(packer.pack_array_header(size)) for i in range(size): bio.write(packer.pack(i)) bio.seek(0) unpacker = Unpacker(bio, use_list=1) for size in sizes: assert unpacker.unpack() == list(range(size))
def mpack_handler(self, data, sock): unpacker = Unpacker() unpacker.feed(data) # default chunk size of memory buffer is 32MB RECV_SIZE = 32*1024*1024 while 1: for msg in unpacker: self.on_message(msg) next_data = sock.recv(RECV_SIZE) if not next_data: break unpacker.feed(next_data)
def call(self, name, args): self._cpt += 1 self._socket.sendall(packb([0, self._cpt, name, args])) u = Unpacker() while True: data = self._socket.recv(2048) u.feed(data) for r in u: if r[0] == 2: self.events.put(r) else: return r
def handle(self): unpacker = Unpacker() dispatch = self.dispatcher.dispatch _send_response = self._send_response while True: data = self.request.recv(4096) if not data: break unpacker.feed(data) for msg in unpacker: dispatch(msg, _send_response)
def testPackUnicode(): test_data = ["", "abcd", ["defgh"], "Русский текст"] for td in test_data: re = unpackb(packb(td), use_list=1, raw=False) assert re == td packer = Packer() data = packer.pack(td) re = Unpacker(BytesIO(data), raw=False, use_list=1).unpack() assert re == td
def test_read_map_header(): unpacker = Unpacker() unpacker.feed(packb({"a": "A"})) assert unpacker.read_map_header() == 1 assert unpacker.unpack() == "a" assert unpacker.unpack() == "A" try: unpacker.unpack() assert 0, "should raise exception" except OutOfData: assert 1, "okay"
def test_issue124(): unpacker = Unpacker() unpacker.feed(b'\xa1?\xa1!') assert tuple(unpacker) == (b'?', b'!') assert tuple(unpacker) == () unpacker.feed(b"\xa1?\xa1") assert tuple(unpacker) == (b'?', ) assert tuple(unpacker) == () unpacker.feed(b"!") assert tuple(unpacker) == (b'!', ) assert tuple(unpacker) == ()
def test_issue124(): unpacker = Unpacker() unpacker.feed(b"\xa1?\xa1!") assert tuple(unpacker) == ("?", "!") assert tuple(unpacker) == () unpacker.feed(b"\xa1?\xa1") assert tuple(unpacker) == ("?", ) assert tuple(unpacker) == () unpacker.feed(b"!") assert tuple(unpacker) == ("!", ) assert tuple(unpacker) == ()
def test_read_map_header(): unpacker = Unpacker() unpacker.feed(packb({'a': 'A'})) assert unpacker.read_map_header() == 1 assert unpacker.unpack() == B'a' assert unpacker.unpack() == B'A' try: unpacker.unpack() assert 0, 'should raise exception' except StopIteration: assert 1, 'okay'
def verify_holt_winters(metric_name): HOLT_CACHE_DURATION = 1800 HOLT_WINTERS_COUNT = 4 CABINET = "/opt/skyline/src/cabinet" full_holt_series = [] known_metrics = {} recent_holt_time = time() - HOLT_CACHE_DURATION db = DB() if not db.open(CABINET + "/" + metric_name + ".kct", DB.OREADER | DB.ONOLOCK): return HOLT_WINTERS_COUNT seen_holt = redis_conn.get('holt_' + metric_name) # We've put a holt_ record in redis for this metric if seen_holt is not None: full_holt_series = unpackb(seen_holt) # The last item in the series was seen > HOLT_CACHE_DURATION ago if full_holt_series[-1][0] < recent_holt_time: full_holt_series = get_holt_from_cabinet(full_holt_series, db) else: full_holt_series = get_holt_from_cabinet(full_holt_series, db) for value in full_holt_series: known_metrics[str(value[0])] = 1 db.close() # Add the last FULL_DURATION to the cabinet data for any missing items raw_metric = redis_conn.mget(metric_name) for i, local_metric in enumerate(raw_metric): unpacker = Unpacker(use_list=False) unpacker.feed(local_metric) potential_new = list(unpacker) for value in potential_new: if not known_metrics.has_key(str(value[0])): full_holt_series.append((float(value[0]), float(value[1]))) redis_conn.set('holt_' + metric_name, packb(full_holt_series)) count = holtWintersDeviants(full_holt_series) return count
def testPackUnicode(): test_data = ["", "abcd", ["defgh"], "Русский текст"] for td in test_data: re = unpackb(packb(td, encoding='utf-8'), use_list=1, encoding='utf-8') assert re == td packer = Packer(encoding='utf-8') data = packer.pack(td) re = Unpacker(BytesIO(data), encoding=str('utf-8'), use_list=1).unpack() assert re == td
def __init__(self, fmt, stream): """Create MessagePack decoder. :param stream: Stream :type stream: file or io.IOBase """ self.format = fmt self.stream = stream self.unpacker = Unpacker(self.stream, encoding='utf-8')
def __init__(self, bns, client): super(BlacknetServerThread, self).__init__() handler = { BlacknetMsgType.HELLO: self.handle_hello, BlacknetMsgType.CLIENT_NAME: self.handle_client_name, BlacknetMsgType.SSH_CREDENTIAL: self.handle_ssh_credential, BlacknetMsgType.SSH_PUBLICKEY: self.handle_ssh_publickey, BlacknetMsgType.PING: self.handle_ping, BlacknetMsgType.GOODBYE: self.handle_goodbye, } self.handler = handler self.started = False self.database = BlacknetDatabase(bns.config, bns.logger) self.__blacklist = bns.blacklist self.__client = None self.__connect_lock = Lock() self.__cursor = None self.__logger = bns.logger self.__mysql_error = 0 self.__session_interval = bns.session_interval self.__unpacker = Unpacker(encoding='utf-8') self.__packer = Packer(encoding='utf-8') self.__dropped_count = 0 self.__attempt_count = 0 self.__atk_cache = {} self.__ses_cache = {} self.__key_cache = {} self.__test_mode = bns.test_mode peer = client.getpeername() self.__peer_ip = peer[0] if peer else "local" self.__use_ssl = (client.family != socket.AF_UNIX) client.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) if self.__use_ssl: client = bns.ssl_context.wrap_socket(client, server_side=True) self.__client = client self.name = self.peername self.log_info("starting session (SSL: %s)" % self.__use_ssl)
def testPackUnicode(): test_data = [ six.u(""), six.u("abcd"), [six.u("defgh")], six.u("Русский текст"), ] for td in test_data: re = unpackb(packb(td, encoding='utf-8'), use_list=1, encoding='utf-8') assert re == td packer = Packer(encoding='utf-8') data = packer.pack(td) re = Unpacker(BytesIO(data), encoding='utf-8', use_list=1).unpack() assert re == td
def testPackUnicode(): test_data = [ six.u(""), six.u("abcd"), (six.u("defgh"),), six.u("Русский текст"), ] for td in test_data: re = unpackb(packb(td, encoding='utf-8'), encoding='utf-8') assert_equal(re, td) packer = Packer(encoding='utf-8') data = packer.pack(td) re = Unpacker(BytesIO(data), encoding='utf-8').unpack() assert_equal(re, td)
def test_unpack_tell(): stream = io.BytesIO() messages = [2**i - 1 for i in range(65)] messages += [-(2**i) for i in range(1, 64)] messages += [ b'hello', b'hello' * 1000, list(range(20)), {i: bytes(i) * i for i in range(10)}, {i: bytes(i) * i for i in range(32)} ] offsets = [] for m in messages: pack(m, stream) offsets.append(stream.tell()) stream.seek(0) unpacker = Unpacker(stream) for m, o in zip(messages, offsets): m2 = next(unpacker) assert m == m2 assert o == unpacker.tell()
def socket_incoming_connection(socket, address): logger.debug('connected %s', address) sockets[address] = socket unpacker = Unpacker(encoding='utf-8') while True: data = socket.recv(4096) if not data: logger.debug('closed connection %s', address) break unpacker.feed(data) for msg in unpacker: receive.put(InMsg(msg, address)) logger.debug('got socket msg: %s', msg) sockets.pop(address)
def test_unpack_array_header_from_file(): f = BytesIO(packb([1, 2, 3, 4])) unpacker = Unpacker(f) assert unpacker.read_array_header() == 4 assert unpacker.unpack() == 1 assert unpacker.unpack() == 2 assert unpacker.unpack() == 3 assert unpacker.unpack() == 4 with raises(OutOfData): unpacker.unpack()
def test_maxbuffersize(): nose.tools.assert_raises(ValueError, Unpacker, read_size=5, max_buffer_size=3) unpacker = Unpacker(read_size=3, max_buffer_size=3) unpacker.feed(b'fo') nose.tools.assert_raises(BufferFull, unpacker.feed, b'ob') unpacker.feed(b'o') assert ord('f') == next(unpacker) unpacker.feed(b'b') assert ord('o') == next(unpacker) assert ord('o') == next(unpacker) assert ord('b') == next(unpacker)
def merge_attached_breadcrumbs(mpack_breadcrumbs, data): """ Merges breadcrumbs attached in the ``__sentry-breadcrumbs`` attachment(s). """ size = mpack_breadcrumbs.size if size == 0 or size > MAX_MSGPACK_BREADCRUMB_SIZE_BYTES: return try: unpacker = Unpacker(mpack_breadcrumbs) breadcrumbs = list(unpacker) except (ValueError, UnpackException, ExtraData) as e: minidumps_logger.exception(e) return if not breadcrumbs: return current_crumbs = data.get("breadcrumbs") if not current_crumbs: data["breadcrumbs"] = breadcrumbs return current_crumb = next( ( c for c in reversed(current_crumbs) if isinstance(c, dict) and c.get("timestamp") is not None ), None, ) new_crumb = next( ( c for c in reversed(breadcrumbs) if isinstance(c, dict) and c.get("timestamp") is not None ), None, ) # cap the breadcrumbs to the highest count of either file cap = max(len(current_crumbs), len(breadcrumbs)) if current_crumb is not None and new_crumb is not None: if dp.parse(current_crumb["timestamp"]) > dp.parse(new_crumb["timestamp"]): data["breadcrumbs"] = breadcrumbs + current_crumbs else: data["breadcrumbs"] = current_crumbs + breadcrumbs else: data["breadcrumbs"] = current_crumbs + breadcrumbs data["breadcrumbs"] = data["breadcrumbs"][-cap:]
def connect(self): self.state = None self.reader, self.writer = yield from asyncio.open_connection( "gimel", 8000) self.writer.write( packb({ "type": "connect", "username": "******", "password": "", "async": True })) print("Connected.") self.writer.write(packb({"type": "activate"})) self.unpacker = Unpacker() #... asyncio. async (self.receiveandunpack()) loop.call_soon(self.run) #self.move("up") #oder down oder halt self.writer.write(packb({"type": "shout", "foo": "bar"})) self.writer.write(packb({"type": "get_state"})) loop.call_later(5, lambda: self.go_to_level(8)) loop.call_later(10, lambda: self.go_to_level(4))
def test_unpacker_hook_refcnt(): result = [] def hook(x): result.append(x) return x basecnt = sys.getrefcount(hook) up = Unpacker(object_hook=hook, list_hook=hook) assert sys.getrefcount(hook) >= basecnt + 2 up.feed(packb([{}])) up.feed(packb([{}])) assert up.unpack() == [{}] assert up.unpack() == [{}] assert result == [{}, [{}], {}, [{}]] del up assert sys.getrefcount(hook) == basecnt
def test_incorrect_type_array(): unpacker = Unpacker() unpacker.feed(packb(1)) try: unpacker.read_array_header() assert 0, "should raise exception" except UnexpectedTypeException: assert 1, "okay"
def test_write_bytes(): unpacker = Unpacker() unpacker.feed(b'abc') f = six.BytesIO() assert unpacker.unpack(f.write) == ord('a') assert f.getvalue() == b'a' f = six.BytesIO() assert unpacker.skip(f.write) is None assert f.getvalue() == b'b' f = six.BytesIO() assert unpacker.skip() is None assert f.getvalue() == b''
def tile_contents(tile, table): """ Generator yielding each item in a gzipped msgpack format file. TODO: This should be generalised? Perhaps move into formatter classes? """ file_name = 'tiles/osm/%s/%d/%d/%d.msgpack.gz' % \ (table, tile.z, tile.x, tile.y) with BufferedReader(gzip.open(file_name, 'rb')) as gz: unpacker = Unpacker(file_like=gz) for obj in unpacker: yield obj
def tile_contents(tile, table, extension): """ Generator yielding each item in a msgpack format file. TODO: This should be generalised? Perhaps move into formatter classes? """ file_name = 'tiles/osm/%s/%d/%d/%d%s' % \ (table, tile.z, tile.x, tile.y, extension) with open(file_name, 'rb') as f: unpacker = Unpacker(file_like=f) for obj in unpacker: yield obj
def test_foobar(): unpacker = Unpacker(read_size=3) unpacker.feed(b'foobar') assert unpacker.unpack() == ord(b'f') assert unpacker.unpack() == ord(b'o') assert unpacker.unpack() == ord(b'o') assert unpacker.unpack() == ord(b'b') assert unpacker.unpack() == ord(b'a') assert unpacker.unpack() == ord(b'r') try: o = unpacker.unpack() assert 0, "should raise exception" except StopIteration: assert 1, "ok" unpacker.feed(b'foo') unpacker.feed(b'bar') k = 0 for o, e in zip(unpacker, 'foobarbaz'): assert o == ord(e) k += 1 assert k == len(b'foobar')
def __iter__(self): """Iterate endlessly over all objects sent by the producer Internally, this method uses a receiving buffer that is lost if interrupted (GeneratorExit). If this buffer was not empty, the queue is left in a inconsistent state and this method can't be called again. So the correct way to split a loop is to first get an iterator explicitly: iq = iter(queue) for x in iq: if ...: break for x in iq: ... """ unpacker = Unpacker(use_list=False, raw=True) feed = unpacker.feed max_size = self._max_size array = self._array pos = self._pos size = self._size lock, get_lock, put_lock = self._locks left = 0 while 1: for data in unpacker: yield data while 1: with lock: p = pos.value s = size.value if s: break get_lock.acquire() e = p + s if e < max_size: feed(array[p:e]) else: feed(array[p:]) e -= max_size feed(array[:e]) with lock: pos.value = e n = size.value size.value = n - s if n == max_size: put_lock.acquire(0) put_lock.release()