def test_reading_numpy_temporals(): BINARY = OrderedDict() with open('tests/QExpressions3.out', 'rb') as f: while True: query = f.readline().strip() binary = f.readline().strip() if not binary: break BINARY[query] = binary print('Deserialization (numpy temporals)') for query, value in iter(NUMPY_TEMPORAL_EXPRESSIONS.items()): buffer_ = BytesIO() binary = binascii.unhexlify(BINARY[query]) buffer_.write(b'\1\0\0\0') buffer_.write(struct.pack('i', len(binary) + 8)) buffer_.write(binary) buffer_.seek(0) sys.stdout.write( ' %-75s' % query ) try: buffer_.seek(0) stream_reader = qreader.QReader(buffer_) result = stream_reader.read(numpy_temporals = True).data assert compare(value, result), 'deserialization failed: %s, expected: %s actual: %s' % (query, value, result) print('.') except QException as e: assert isinstance(value, QException) assert e.args == value.args print('.')
def test_reading_pandas(): print('Deserialization (pandas)') for query, value in iter(PANDAS_EXPRESSIONS.items()): buffer_ = BytesIO() binary = binascii.unhexlify(BINARY[query]) buffer_.write(b'\1\0\0\0') buffer_.write(struct.pack('i', len(binary) + 8)) buffer_.write(binary) buffer_.seek(0) sys.stdout.write(' %-75s' % query) try: buffer_.seek(0) stream_reader = PandasQReader(buffer_) result = stream_reader.read(pandas = True).data if isinstance(value, dict): if 'index' in value: meta = result.meta result = result.reset_index() result._metadata = ["meta"] result.meta = meta if not 'compare_meta' in value or value['compare_meta']: assert value['meta'].as_dict() == result.meta.as_dict(), 'deserialization failed qtype: %s, expected: %s actual: %s' % (query, value['meta'], result.meta) assert compare(value['data'], result), 'deserialization failed: %s, expected: %s actual: %s' % (query, value['data'], result) else: assert compare(value, result), 'deserialization failed: %s, expected: %s actual: %s' % (query, value, result) print('.') except QException as e: assert isinstance(value, QException) assert e.message == value.message print('.')
def test_reading_pandas(): print('Deserialization (pandas)') for query, value in iter(PANDAS_EXPRESSIONS.items()): buffer_ = BytesIO() binary = binascii.unhexlify(BINARY[query]) buffer_.write(b'\1\0\0\0') buffer_.write(struct.pack('i', len(binary) + 8)) buffer_.write(binary) buffer_.seek(0) sys.stdout.write(' %-75s' % query) try: buffer_.seek(0) stream_reader = PandasQReader(buffer_) result = stream_reader.read(pandas = True).data if isinstance(value, dict): if 'index' in value: meta = result.meta result = result.reset_index() result.meta = meta if not 'compare_meta' in value or value['compare_meta']: assert value['meta'].as_dict() == result.meta.as_dict(), 'deserialization failed qtype: %s, expected: %s actual: %s' % (query, value['meta'], result.meta) assert compare(value['data'], result), 'deserialization failed: %s, expected: %s actual: %s' % (query, value['data'], result) else: assert compare(value, result), 'deserialization failed: %s, expected: %s actual: %s' % (query, value, result) print('.') except QException as e: assert isinstance(value, QException) assert e.message == value.message print('.')
class AsyncQWriter(QWriter): async def write(self, data, msg_type, **options): '''Serializes and pushes single data object to a wrapped stream. :Parameters: - `data` - data to be serialized - `msg_type` (one of the constants defined in :class:`.MessageType`) - type of the message :Options: - `single_char_strings` (`boolean`) - if ``True`` single char Python strings are encoded as q strings instead of chars, **Default**: ``False`` :returns: if wraped stream is ``None`` serialized data, otherwise ``None`` ''' self._buffer = BytesIO() self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options)) # header and placeholder for message size self._buffer.write(('%s%s\0\0\0\0\0\0' % (ENDIANESS, chr(msg_type))).encode(self._encoding)) self._write(data) # update message size data_size = self._buffer.tell() self._buffer.seek(4) self._buffer.write(struct.pack('i', data_size)) # write data to socket if self._stream: await self._stream.write(self._buffer.getvalue()) else: return self._buffer.getvalue()
def test_reading_one(query, value): buffer_ = BytesIO() binary = binascii.unhexlify(BINARY[query]) buffer_.write(b'\1\0\0\0') buffer_.write(struct.pack('i', len(binary) + 8)) buffer_.write(binary) buffer_.seek(0) sys.stdout.write(' %-75s' % query) try: header = buffer_reader.read_header(source=buffer_.getvalue()) result = buffer_reader.read_data( message_size=header.size, compression_mode=header.compression_mode, raw=True) assert compare(buffer_.getvalue()[8:], result), 'raw reading failed: %s' % (query) stream_reader = qreader.QReader(buffer_) result = stream_reader.read(raw=True).data assert compare(buffer_.getvalue()[8:], result), 'raw reading failed: %s' % (query) result = buffer_reader.read(source=buffer_.getvalue()).data if type(value) == QTemporalList: assert compare( value, result ), 'deserialization failed: %s, expected: %s actual: %s' % ( query, [x.raw for x in value], [x.raw for x in result]) else: assert compare( value, result ), 'deserialization failed: %s, expected: %s actual: %s' % ( query, value, result) header = buffer_reader.read_header(source=buffer_.getvalue()) result = buffer_reader.read_data( message_size=header.size, compression_mode=header.compression_mode) assert compare(value, result), 'deserialization failed: %s' % (query) buffer_.seek(0) stream_reader = qreader.QReader(buffer_) result = stream_reader.read().data assert compare( value, result ), 'deserialization failed: %s, expected: %s actual: %s' % ( query, value, result) print('.') except QException as e: assert isinstance(value, QException) assert e.args == value.args print('.')
def test_reading(): BINARY = OrderedDict() with open('tests/QExpressions3.out', 'rb') as f: while True: query = f.readline().strip() binary = f.readline().strip() if not binary: break BINARY[query] = binary buffer_reader = qreader.QReader(None) print('Deserialization') for query, value in iter(EXPRESSIONS.items()): buffer_ = BytesIO() binary = binascii.unhexlify(BINARY[query]) buffer_.write(b'\1\0\0\0') buffer_.write(struct.pack('i', len(binary) + 8)) buffer_.write(binary) buffer_.seek(0) sys.stdout.write( ' %-75s' % query ) try: header = buffer_reader.read_header(source = buffer_.getvalue()) result = buffer_reader.read_data(message_size = header.size, is_compressed = header.is_compressed, raw = True) assert compare(buffer_.getvalue()[8:], result), 'raw reading failed: %s' % (query) stream_reader = qreader.QReader(buffer_) result = stream_reader.read(raw = True).data assert compare(buffer_.getvalue()[8:], result), 'raw reading failed: %s' % (query) result = buffer_reader.read(source = buffer_.getvalue()).data assert compare(value, result), 'deserialization failed: %s, expected: %s actual: %s' % (query, value, result) header = buffer_reader.read_header(source = buffer_.getvalue()) result = buffer_reader.read_data(message_size = header.size, is_compressed = header.is_compressed) assert compare(value, result), 'deserialization failed: %s' % (query) buffer_.seek(0) stream_reader = qreader.QReader(buffer_) result = stream_reader.read().data assert compare(value, result), 'deserialization failed: %s, expected: %s actual: %s' % (query, value, result) print('.') except QException as e: assert isinstance(value, QException) assert e.args == value.args print('.')
def test_reading_compressed(): BINARY = OrderedDict() with open(os.path.join(TEST_DATA_DIR, 'QCompressedExpressions3.out'), 'rb') as f: while True: query = f.readline().strip() binary = f.readline().strip() if not binary: break BINARY[query] = binary print('Compressed deserialization') buffer_reader = qreader.QReader(None) for query, value in iter(COMPRESSED_EXPRESSIONS.items()): buffer_ = BytesIO() binary = binascii.unhexlify(BINARY[query]) buffer_.write(b'\1\0\1\0') buffer_.write(struct.pack('i', len(binary) + 8)) buffer_.write(binary) buffer_.seek(0) sys.stdout.write(' %-75s' % query) try: result = buffer_reader.read(source=buffer_.getvalue()).data assert compare(value, result), 'deserialization failed: %s' % (query) header = buffer_reader.read_header(source=buffer_.getvalue()) result = buffer_reader.read_data( message_size=header.size, is_compressed=header.is_compressed) assert compare(value, result), 'deserialization failed: %s' % (query) stream_reader = qreader.QReader(buffer_) result = stream_reader.read().data assert compare(value, result), 'deserialization failed: %s' % (query) print('.') except QException as e: assert isinstance(value, QException) assert e.args == value.args print('.')
def test_reading_pandas(): print("Deserialization (pandas)") for query, value in iter(PANDAS_EXPRESSIONS.items()): buffer_ = BytesIO() binary = binascii.unhexlify(BINARY[query]) buffer_.write(b"\1\0\0\0") buffer_.write(struct.pack("i", len(binary) + 8)) buffer_.write(binary) buffer_.seek(0) sys.stdout.write(" %-75s" % query) try: buffer_.seek(0) stream_reader = qreader.QReader(buffer_) result = stream_reader.read(pandas=True).data if isinstance(value, dict): if "index" in value: meta = result.meta result = result.reset_index() result.meta = meta if not "compare_meta" in value or value["compare_meta"]: assert value["meta"].as_dict() == result.meta.as_dict(), ( "deserialization failed qtype: %s, expected: %s actual: %s" % (query, value["meta"], result.meta) ) assert compare(value["data"], result), "deserialization failed: %s, expected: %s actual: %s" % ( query, value["data"], result, ) else: assert compare(value, result), "deserialization failed: %s, expected: %s actual: %s" % ( query, value, result, ) print(".") except QException as e: assert isinstance(value, QException) assert e.message == value.message print(".")
def test_reading_numpy_temporals_one(query, value): buffer_ = BytesIO() binary = binascii.unhexlify(BINARY[query]) buffer_.write(b'\1\0\0\0') buffer_.write(struct.pack('i', len(binary) + 8)) buffer_.write(binary) buffer_.seek(0) sys.stdout.write(' %-75s' % query) try: buffer_.seek(0) stream_reader = qreader.QReader(buffer_) result = stream_reader.read(numpy_temporals=True).data assert compare( value, result ), 'deserialization failed: %s, expected: %s actual: %s' % ( query, value, result) print('.') except QException as e: assert isinstance(value, QException) assert e.args == value.args print('.')
class QWriter(object): ''' Provides serialization to q IPC protocol. :Parameters: - `stream` (`socket` or `None`) - stream for data serialization - `protocol_version` (`integer`) - version IPC protocol ''' _writer_map = {} serialize = Mapper(_writer_map) def __new__(cls, *args, **kwargs): if cls is QWriter: # try to load optional pandas binding try: from qpython._pandas import PandasQWriter return super(QWriter, cls).__new__(PandasQWriter) except ImportError: return super(QWriter, cls).__new__(QWriter) else: #return super(QWriter, cls).__new__(cls) return super().__new__(cls) #komsit fix def __init__(self, stream, protocol_version): self._stream = stream self._protocol_version = protocol_version def write(self, data, msg_type, **options): '''Serializes and pushes single data object to a wrapped stream. :Parameters: - `data` - data to be serialized - `msg_type` (one of the constants defined in :class:`.MessageType`) - type of the message :Options: - `single_char_strings` (`boolean`) - if ``True`` single char Python strings are encoded as q strings instead of chars, **Default**: ``False`` :returns: if wraped stream is ``None`` serialized data, otherwise ``None`` ''' self._buffer = BytesIO() self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options)) # header and placeholder for message size self._buffer.write(('%s%s\0\0\0\0\0\0' % (ENDIANESS, chr(msg_type))).encode("latin-1")) self._write(data) # update message size data_size = self._buffer.tell() self._buffer.seek(4) self._buffer.write(struct.pack('i', data_size)) # write data to socket if self._stream: self._stream.sendall(self._buffer.getvalue()) else: return self._buffer.getvalue() def _write(self, data): if data is None: self._write_null() else: if isinstance(data, Exception) or (type(data) == type and issubclass(data, Exception)): data_type = Exception else: data_type = type(data) writer = self._writer_map.get(data_type, None) if writer: writer(self, data) else: qtype = Q_TYPE.get(type(data), None) if qtype: self._write_atom(data, qtype) else: raise QWriterException('Unable to serialize type: %s' % data.__class__ if isinstance(data, object) else type(data)) def _write_null(self): self._buffer.write(struct.pack('=bx', QNULL)) @serialize(Exception) def _write_error(self, data): self._buffer.write(struct.pack('b', QERROR)) if isinstance(data, Exception): msg = data.__class__.__name__ if data.args: msg = data.args[0] else: msg = data.__name__ self._buffer.write(msg.encode("latin-1")) self._buffer.write(b'\0') def _write_atom(self, data, qtype): try: self._buffer.write(struct.pack('b', qtype)) fmt = STRUCT_MAP[qtype] self._buffer.write(struct.pack(fmt, data)) except KeyError: raise QWriterException('Unable to serialize type: %s' % data.__class__ if isinstance(data, object) else type(data)) @serialize(tuple, list) def _write_generic_list(self, data): self._buffer.write(struct.pack('=bxi', QGENERAL_LIST, len(data))) for element in data: self._write(element) @serialize(str, bytes) def _write_string(self, data): if not self._options.single_char_strings and len(data) == 1: self._write_atom(ord(data), QCHAR) else: self._buffer.write(struct.pack('=bxi', QSTRING, len(data))) if isinstance(data, str): self._buffer.write(data.encode("latin-1")) else: self._buffer.write(data) @serialize(numpy.string_) def _write_symbol(self, data): self._buffer.write(struct.pack('=b', QSYMBOL)) if data: self._buffer.write(data) self._buffer.write(b'\0') @serialize(uuid.UUID) def _write_guid(self, data): if self._protocol_version < 3: raise QWriterException('kdb+ protocol version violation: Guid not supported pre kdb+ v3.0') self._buffer.write(struct.pack('=b', QGUID)) self._buffer.write(data.bytes) @serialize(QTemporal) def _write_temporal(self, data): try: if self._protocol_version < 1 and (data.meta.qtype == QTIMESPAN or data.meta.qtype == QTIMESTAMP): raise QWriterException('kdb+ protocol version violation: data type %s not supported pre kdb+ v2.6' % hex(data.meta.qtype)) self._buffer.write(struct.pack('=b', data.meta.qtype)) fmt = STRUCT_MAP[data.meta.qtype] self._buffer.write(struct.pack(fmt, to_raw_qtemporal(data.raw, data.meta.qtype))) except KeyError: raise QWriterException('Unable to serialize type: %s' % type(data)) @serialize(numpy.datetime64, numpy.timedelta64) def _write_numpy_temporal(self, data): try: qtype = TEMPORAL_PY_TYPE[str(data.dtype)] if self._protocol_version < 1 and (qtype == QTIMESPAN or qtype == QTIMESTAMP): raise QWriterException('kdb+ protocol version violation: data type %s not supported pre kdb+ v2.6' % hex(qtype)) self._buffer.write(struct.pack('=b', qtype)) fmt = STRUCT_MAP[qtype] self._buffer.write(struct.pack(fmt, to_raw_qtemporal(data, qtype))) except KeyError: raise QWriterException('Unable to serialize type: %s' % data.dtype) @serialize(QLambda) def _write_lambda(self, data): self._buffer.write(struct.pack('=b', QLAMBDA)) self._buffer.write(b'\0') self._write_string(data.expression) @serialize(QProjection) def _write_projection(self, data): self._buffer.write(struct.pack('=bi', QPROJECTION, len(data.parameters))) for parameter in data.parameters: self._write(parameter) @serialize(QDictionary, QKeyedTable) def _write_dictionary(self, data): self._buffer.write(struct.pack('=b', QDICTIONARY)) self._write(data.keys) self._write(data.values) @serialize(QTable) def _write_table(self, data): self._buffer.write(struct.pack('=bxb', QTABLE, QDICTIONARY)) self._write(qlist(numpy.array(data.dtype.names), qtype = QSYMBOL_LIST)) self._buffer.write(struct.pack('=bxi', QGENERAL_LIST, len(data.dtype))) for column in data.dtype.names: self._write_list(data[column], data.meta[column]) @serialize(numpy.ndarray, QList, QTemporalList) def _write_list(self, data, qtype = None): if qtype is not None: qtype = -abs(qtype) if qtype is None: qtype = get_list_qtype(data) if self._protocol_version < 1 and (abs(qtype) == QTIMESPAN_LIST or abs(qtype) == QTIMESTAMP_LIST): raise QWriterException('kdb+ protocol version violation: data type %s not supported pre kdb+ v2.6' % hex(data.meta.qtype)) if qtype == QGENERAL_LIST: self._write_generic_list(data) elif qtype == QCHAR: self._write_string(data.tostring()) else: self._buffer.write(struct.pack('=bxi', -qtype, len(data))) if data.dtype.type in (numpy.datetime64, numpy.timedelta64): # convert numpy temporal to raw q temporal data = array_to_raw_qtemporal(data, qtype = qtype) if qtype == QSYMBOL: for symbol in data: if symbol: self._buffer.write(symbol) self._buffer.write(b'\0') elif qtype == QGUID: if self._protocol_version < 3: raise QWriterException('kdb+ protocol version violation: Guid not supported pre kdb+ v3.0') for guid in data: self._buffer.write(guid.bytes) else: self._buffer.write(data.tostring())
class QWriter(object): ''' Provides serialization to q IPC protocol. :Parameters: - `stream` (`socket` or `None`) - stream for data serialization - `protocol_version` (`integer`) - version IPC protocol - `encoding` (`string`) - encoding for characters serialization :Attrbutes: - `_writer_map` - stores mapping between Python types and functions responsible for serializing into IPC representation ''' _writer_map = {} serialize = Mapper(_writer_map) def __init__(self, stream, protocol_version, encoding='latin-1'): self._stream = stream self._protocol_version = protocol_version self._encoding = encoding def write(self, data, msg_type, **options): '''Serializes and pushes single data object to a wrapped stream. :Parameters: - `data` - data to be serialized - `msg_type` (one of the constants defined in :class:`.MessageType`) - type of the message :Options: - `single_char_strings` (`boolean`) - if ``True`` single char Python strings are encoded as q strings instead of chars, **Default**: ``False`` :returns: if wraped stream is ``None`` serialized data, otherwise ``None`` ''' self._buffer = BytesIO() self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options)) # header and placeholder for message size self._buffer.write(('%s%s\0\0\0\0\0\0' % (ENDIANESS, chr(msg_type))).encode(self._encoding)) self._write(data) # update message size data_size = self._buffer.tell() self._buffer.seek(4) self._buffer.write(struct.pack('i', data_size)) # write data to socket if self._stream: self._stream.sendall(self._buffer.getvalue()) else: return self._buffer.getvalue() def _write(self, data): if data is None: self._write_null() else: if isinstance(data, Exception) or (type(data) == type and issubclass(data, Exception)): data_type = Exception else: data_type = type(data) writer = self._get_writer(data_type) if writer: writer(self, data) else: qtype = Q_TYPE.get(type(data), None) if qtype: self._write_atom(data, qtype) else: raise QWriterException('Unable to serialize type: %s' % data.__class__ if isinstance( data, object) else type(data)) def _get_writer(self, data_type): return self._writer_map.get(data_type, None) def _write_null(self): self._buffer.write(struct.pack('=bx', QNULL)) @serialize(Exception) def _write_error(self, data): self._buffer.write(struct.pack('b', QERROR)) if isinstance(data, Exception): msg = data.__class__.__name__ if data.args: msg = data.args[0] else: msg = data.__name__ self._buffer.write(msg.encode(self._encoding)) self._buffer.write(b'\0') def _write_atom(self, data, qtype): try: self._buffer.write(struct.pack('b', qtype)) fmt = STRUCT_MAP[qtype] self._buffer.write(struct.pack(fmt, data)) except KeyError: raise QWriterException( 'Unable to serialize type: %s' % data.__class__ if isinstance(data, object) else type(data)) @serialize(tuple, list) def _write_generic_list(self, data): self._buffer.write(struct.pack('=bxi', QGENERAL_LIST, len(data))) for element in data: self._write(element) @serialize(str, bytes) def _write_string(self, data): if not self._options.single_char_strings and len(data) == 1: self._write_atom(ord(data), QCHAR) else: self._buffer.write(struct.pack('=bxi', QSTRING, len(data))) if isinstance(data, str): self._buffer.write(data.encode(self._encoding)) else: self._buffer.write(data) @serialize(numpy.string_) def _write_symbol(self, data): self._buffer.write(struct.pack('=b', QSYMBOL)) if data: self._buffer.write(data) self._buffer.write(b'\0') @serialize(uuid.UUID) def _write_guid(self, data): if self._protocol_version < 3: raise QWriterException( 'kdb+ protocol version violation: Guid not supported pre kdb+ v3.0' ) self._buffer.write(struct.pack('=b', QGUID)) self._buffer.write(data.bytes) @serialize(QTemporal) def _write_temporal(self, data): try: if self._protocol_version < 1 and (data.meta.qtype == QTIMESPAN or data.meta.qtype == QTIMESTAMP): raise QWriterException( 'kdb+ protocol version violation: data type %s not supported pre kdb+ v2.6' % hex(data.meta.qtype)) self._buffer.write(struct.pack('=b', data.meta.qtype)) fmt = STRUCT_MAP[data.meta.qtype] self._buffer.write( struct.pack(fmt, to_raw_qtemporal(data.raw, data.meta.qtype))) except KeyError: raise QWriterException('Unable to serialize type: %s' % type(data)) @serialize(numpy.datetime64, numpy.timedelta64) def _write_numpy_temporal(self, data): try: qtype = TEMPORAL_PY_TYPE[str(data.dtype)] if self._protocol_version < 1 and (qtype == QTIMESPAN or qtype == QTIMESTAMP): raise QWriterException( 'kdb+ protocol version violation: data type %s not supported pre kdb+ v2.6' % hex(qtype)) self._buffer.write(struct.pack('=b', qtype)) fmt = STRUCT_MAP[qtype] self._buffer.write(struct.pack(fmt, to_raw_qtemporal(data, qtype))) except KeyError: raise QWriterException('Unable to serialize type: %s' % data.dtype) @serialize(QLambda) def _write_lambda(self, data): self._buffer.write(struct.pack('=b', QLAMBDA)) self._buffer.write(b'\0') self._write_string(data.expression) @serialize(QProjection) def _write_projection(self, data): self._buffer.write( struct.pack('=bi', QPROJECTION, len(data.parameters))) for parameter in data.parameters: self._write(parameter) @serialize(QDictionary, QKeyedTable) def _write_dictionary(self, data): self._buffer.write(struct.pack('=b', QDICTIONARY)) self._write(data.keys) self._write(data.values) @serialize(QTable) def _write_table(self, data): self._buffer.write(struct.pack('=bxb', QTABLE, QDICTIONARY)) self._write(qlist(numpy.array(data.dtype.names), qtype=QSYMBOL_LIST)) self._buffer.write(struct.pack('=bxi', QGENERAL_LIST, len(data.dtype))) for column in data.dtype.names: self._write_list(data[column], data.meta[column]) @serialize(numpy.ndarray, QList, QTemporalList) def _write_list(self, data, qtype=None): if qtype is not None: qtype = -abs(qtype) if qtype is None: qtype = get_list_qtype(data) if self._protocol_version < 1 and (abs(qtype) == QTIMESPAN_LIST or abs(qtype) == QTIMESTAMP_LIST): raise QWriterException( 'kdb+ protocol version violation: data type %s not supported pre kdb+ v2.6' % hex(data.meta.qtype)) if qtype == QGENERAL_LIST: self._write_generic_list(data) elif qtype == QCHAR: self._write_string(data.tostring()) else: self._buffer.write(struct.pack('=bxi', -qtype, len(data))) if data.dtype.type in (numpy.datetime64, numpy.timedelta64): # convert numpy temporal to raw q temporal data = array_to_raw_qtemporal(data, qtype=qtype) if qtype == QSYMBOL: for symbol in data: if symbol: self._buffer.write(symbol) self._buffer.write(b'\0') elif qtype == QGUID: if self._protocol_version < 3: raise QWriterException( 'kdb+ protocol version violation: Guid not supported pre kdb+ v3.0' ) for guid in data: self._buffer.write(guid.bytes) else: self._buffer.write(data.tostring())
# -*- coding: utf-8 -*- # python2 暂时没有找到bytesio模块 from cStringIO import BytesIO f = BytesIO() f.write('中文'.encode('utf-8'))