def test_read_partition(self): msg = dict(action='open', name='entry1', parameters={}) resp_msg, = self.make_post_request(msg) source_id = resp_msg['source_id'] msg2 = dict(action='read', partition=1, source_id=source_id, accepted_formats=['msgpack']) resp_msgs = self.make_post_request(msg2) self.assertEqual(len(resp_msgs), 1) ser = MsgPackSerializer() part = resp_msgs[0] self.assertEqual(part['format'], 'msgpack') self.assertEqual(part['compression'], 'none') self.assertEqual(part['container'], 'dataframe') data = ser.decode(part['data'], container='dataframe') self.assertEqual(len(data), 4)
def test_read_part_compressed(self): msg = dict(action='open', name='entry1', parameters={}) resp_msg, = self.make_post_request(msg) source_id = resp_msg['source_id'] msg2 = dict(action='read', source_id=source_id, accepted_formats=['msgpack'], accepted_compression=['gzip'], partition=0) resp_msgs = self.make_post_request(msg2) self.assertEqual(len(resp_msgs), 1) ser = MsgPackSerializer() comp = GzipCompressor() for chunk in resp_msgs: self.assertEqual(chunk['format'], 'msgpack') self.assertEqual(chunk['compression'], 'gzip') self.assertEqual(chunk['container'], 'dataframe') data = ser.decode(comp.decompress(chunk['data']), container='dataframe') self.assertEqual(len(data), 4)