class AsyncQWriter(QWriter):
async def write(self, data, msg_type, **options):
'''Serializes and pushes single data object to a wrapped stream.
:Parameters:
- `data` - data to be serialized
- `msg_type` (one of the constants defined in :class:`.MessageType`) -
type of the message
:Options:
- `single_char_strings` (`boolean`) - if ``True`` single char Python
strings are encoded as q strings instead of chars,
**Default**: ``False``
:returns: if wraped stream is ``None`` serialized data,
otherwise ``None``
'''
self._buffer = BytesIO()
self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options))
# header and placeholder for message size
self._buffer.write(('%s%s\0\0\0\0\0\0' %
(ENDIANESS, chr(msg_type))).encode(self._encoding))
self._write(data)
# update message size
data_size = self._buffer.tell()
self._buffer.seek(4)
self._buffer.write(struct.pack('i', data_size))
# write data to socket
if self._stream:
await self._stream.write(self._buffer.getvalue())
else:
return self._buffer.getvalue()
class QWriter(object):
'''
Provides serialization to q IPC protocol.
:Parameters:
- `stream` (`socket` or `None`) - stream for data serialization
- `protocol_version` (`integer`) - version IPC protocol
'''
_writer_map = {}
serialize = Mapper(_writer_map)
def __new__(cls, *args, **kwargs):
if cls is QWriter:
# try to load optional pandas binding
try:
from qpython._pandas import PandasQWriter
return super(QWriter, cls).__new__(PandasQWriter)
except ImportError:
return super(QWriter, cls).__new__(QWriter)
else:
#return super(QWriter, cls).__new__(cls)
return super().__new__(cls) #komsit fix
def __init__(self, stream, protocol_version):
self._stream = stream
self._protocol_version = protocol_version
def write(self, data, msg_type, **options):
'''Serializes and pushes single data object to a wrapped stream.
:Parameters:
- `data` - data to be serialized
- `msg_type` (one of the constants defined in :class:`.MessageType`) -
type of the message
:Options:
- `single_char_strings` (`boolean`) - if ``True`` single char Python
strings are encoded as q strings instead of chars,
**Default**: ``False``
:returns: if wraped stream is ``None`` serialized data,
otherwise ``None``
'''
self._buffer = BytesIO()
self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options))
# header and placeholder for message size
self._buffer.write(('%s%s\0\0\0\0\0\0' % (ENDIANESS, chr(msg_type))).encode("latin-1"))
self._write(data)
# update message size
data_size = self._buffer.tell()
self._buffer.seek(4)
self._buffer.write(struct.pack('i', data_size))
# write data to socket
if self._stream:
self._stream.sendall(self._buffer.getvalue())
else:
return self._buffer.getvalue()
def _write(self, data):
if data is None:
self._write_null()
else:
if isinstance(data, Exception) or (type(data) == type and issubclass(data, Exception)):
data_type = Exception
else:
data_type = type(data)
writer = self._writer_map.get(data_type, None)
if writer:
writer(self, data)
else:
qtype = Q_TYPE.get(type(data), None)
if qtype:
self._write_atom(data, qtype)
else:
raise QWriterException('Unable to serialize type: %s' % data.__class__ if isinstance(data, object) else type(data))
def _write_null(self):
self._buffer.write(struct.pack('=bx', QNULL))
@serialize(Exception)
def _write_error(self, data):
self._buffer.write(struct.pack('b', QERROR))
if isinstance(data, Exception):
msg = data.__class__.__name__
if data.args:
msg = data.args[0]
else:
msg = data.__name__
self._buffer.write(msg.encode("latin-1"))
self._buffer.write(b'\0')
def _write_atom(self, data, qtype):
try:
self._buffer.write(struct.pack('b', qtype))
fmt = STRUCT_MAP[qtype]
self._buffer.write(struct.pack(fmt, data))
except KeyError:
raise QWriterException('Unable to serialize type: %s' % data.__class__ if isinstance(data, object) else type(data))
@serialize(tuple, list)
def _write_generic_list(self, data):
self._buffer.write(struct.pack('=bxi', QGENERAL_LIST, len(data)))
for element in data:
self._write(element)
@serialize(str, bytes)
def _write_string(self, data):
if not self._options.single_char_strings and len(data) == 1:
self._write_atom(ord(data), QCHAR)
else:
self._buffer.write(struct.pack('=bxi', QSTRING, len(data)))
if isinstance(data, str):
self._buffer.write(data.encode("latin-1"))
else:
self._buffer.write(data)
@serialize(numpy.string_)
def _write_symbol(self, data):
self._buffer.write(struct.pack('=b', QSYMBOL))
if data:
self._buffer.write(data)
self._buffer.write(b'\0')
@serialize(uuid.UUID)
def _write_guid(self, data):
if self._protocol_version < 3:
raise QWriterException('kdb+ protocol version violation: Guid not supported pre kdb+ v3.0')
self._buffer.write(struct.pack('=b', QGUID))
self._buffer.write(data.bytes)
@serialize(QTemporal)
def _write_temporal(self, data):
try:
if self._protocol_version < 1 and (data.meta.qtype == QTIMESPAN or data.meta.qtype == QTIMESTAMP):
raise QWriterException('kdb+ protocol version violation: data type %s not supported pre kdb+ v2.6' % hex(data.meta.qtype))
self._buffer.write(struct.pack('=b', data.meta.qtype))
fmt = STRUCT_MAP[data.meta.qtype]
self._buffer.write(struct.pack(fmt, to_raw_qtemporal(data.raw, data.meta.qtype)))
except KeyError:
raise QWriterException('Unable to serialize type: %s' % type(data))
@serialize(numpy.datetime64, numpy.timedelta64)
def _write_numpy_temporal(self, data):
try:
qtype = TEMPORAL_PY_TYPE[str(data.dtype)]
if self._protocol_version < 1 and (qtype == QTIMESPAN or qtype == QTIMESTAMP):
raise QWriterException('kdb+ protocol version violation: data type %s not supported pre kdb+ v2.6' % hex(qtype))
self._buffer.write(struct.pack('=b', qtype))
fmt = STRUCT_MAP[qtype]
self._buffer.write(struct.pack(fmt, to_raw_qtemporal(data, qtype)))
except KeyError:
raise QWriterException('Unable to serialize type: %s' % data.dtype)
@serialize(QLambda)
def _write_lambda(self, data):
self._buffer.write(struct.pack('=b', QLAMBDA))
self._buffer.write(b'\0')
self._write_string(data.expression)
@serialize(QProjection)
def _write_projection(self, data):
self._buffer.write(struct.pack('=bi', QPROJECTION, len(data.parameters)))
for parameter in data.parameters:
self._write(parameter)
@serialize(QDictionary, QKeyedTable)
def _write_dictionary(self, data):
self._buffer.write(struct.pack('=b', QDICTIONARY))
self._write(data.keys)
self._write(data.values)
@serialize(QTable)
def _write_table(self, data):
self._buffer.write(struct.pack('=bxb', QTABLE, QDICTIONARY))
self._write(qlist(numpy.array(data.dtype.names), qtype = QSYMBOL_LIST))
self._buffer.write(struct.pack('=bxi', QGENERAL_LIST, len(data.dtype)))
for column in data.dtype.names:
self._write_list(data[column], data.meta[column])
@serialize(numpy.ndarray, QList, QTemporalList)
def _write_list(self, data, qtype = None):
if qtype is not None:
qtype = -abs(qtype)
if qtype is None:
qtype = get_list_qtype(data)
if self._protocol_version < 1 and (abs(qtype) == QTIMESPAN_LIST or abs(qtype) == QTIMESTAMP_LIST):
raise QWriterException('kdb+ protocol version violation: data type %s not supported pre kdb+ v2.6' % hex(data.meta.qtype))
if qtype == QGENERAL_LIST:
self._write_generic_list(data)
elif qtype == QCHAR:
self._write_string(data.tostring())
else:
self._buffer.write(struct.pack('=bxi', -qtype, len(data)))
if data.dtype.type in (numpy.datetime64, numpy.timedelta64):
# convert numpy temporal to raw q temporal
data = array_to_raw_qtemporal(data, qtype = qtype)
if qtype == QSYMBOL:
for symbol in data:
if symbol:
self._buffer.write(symbol)
self._buffer.write(b'\0')
elif qtype == QGUID:
if self._protocol_version < 3:
raise QWriterException('kdb+ protocol version violation: Guid not supported pre kdb+ v3.0')
for guid in data:
self._buffer.write(guid.bytes)
else:
self._buffer.write(data.tostring())
class QWriter(object):
'''
Provides serialization to q IPC protocol.
:Parameters:
- `stream` (`socket` or `None`) - stream for data serialization
- `protocol_version` (`integer`) - version IPC protocol
- `encoding` (`string`) - encoding for characters serialization
:Attrbutes:
- `_writer_map` - stores mapping between Python types and functions
responsible for serializing into IPC representation
'''
_writer_map = {}
serialize = Mapper(_writer_map)
def __init__(self, stream, protocol_version, encoding='latin-1'):
self._stream = stream
self._protocol_version = protocol_version
self._encoding = encoding
def write(self, data, msg_type, **options):
'''Serializes and pushes single data object to a wrapped stream.
:Parameters:
- `data` - data to be serialized
- `msg_type` (one of the constants defined in :class:`.MessageType`) -
type of the message
:Options:
- `single_char_strings` (`boolean`) - if ``True`` single char Python
strings are encoded as q strings instead of chars,
**Default**: ``False``
:returns: if wraped stream is ``None`` serialized data,
otherwise ``None``
'''
self._buffer = BytesIO()
self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options))
# header and placeholder for message size
self._buffer.write(('%s%s\0\0\0\0\0\0' %
(ENDIANESS, chr(msg_type))).encode(self._encoding))
self._write(data)
# update message size
data_size = self._buffer.tell()
self._buffer.seek(4)
self._buffer.write(struct.pack('i', data_size))
# write data to socket
if self._stream:
self._stream.sendall(self._buffer.getvalue())
else:
return self._buffer.getvalue()
def _write(self, data):
if data is None:
self._write_null()
else:
if isinstance(data,
Exception) or (type(data) == type
and issubclass(data, Exception)):
data_type = Exception
else:
data_type = type(data)
writer = self._get_writer(data_type)
if writer:
writer(self, data)
else:
qtype = Q_TYPE.get(type(data), None)
if qtype:
self._write_atom(data, qtype)
else:
raise QWriterException('Unable to serialize type: %s' %
data.__class__ if isinstance(
data, object) else type(data))
def _get_writer(self, data_type):
return self._writer_map.get(data_type, None)
def _write_null(self):
self._buffer.write(struct.pack('=bx', QNULL))
@serialize(Exception)
def _write_error(self, data):
self._buffer.write(struct.pack('b', QERROR))
if isinstance(data, Exception):
msg = data.__class__.__name__
if data.args:
msg = data.args[0]
else:
msg = data.__name__
self._buffer.write(msg.encode(self._encoding))
self._buffer.write(b'\0')
def _write_atom(self, data, qtype):
try:
self._buffer.write(struct.pack('b', qtype))
fmt = STRUCT_MAP[qtype]
self._buffer.write(struct.pack(fmt, data))
except KeyError:
raise QWriterException(
'Unable to serialize type: %s' %
data.__class__ if isinstance(data, object) else type(data))
@serialize(tuple, list)
def _write_generic_list(self, data):
self._buffer.write(struct.pack('=bxi', QGENERAL_LIST, len(data)))
for element in data:
self._write(element)
@serialize(str, bytes)
def _write_string(self, data):
if not self._options.single_char_strings and len(data) == 1:
self._write_atom(ord(data), QCHAR)
else:
self._buffer.write(struct.pack('=bxi', QSTRING, len(data)))
if isinstance(data, str):
self._buffer.write(data.encode(self._encoding))
else:
self._buffer.write(data)
@serialize(numpy.string_)
def _write_symbol(self, data):
self._buffer.write(struct.pack('=b', QSYMBOL))
if data:
self._buffer.write(data)
self._buffer.write(b'\0')
@serialize(uuid.UUID)
def _write_guid(self, data):
if self._protocol_version < 3:
raise QWriterException(
'kdb+ protocol version violation: Guid not supported pre kdb+ v3.0'
)
self._buffer.write(struct.pack('=b', QGUID))
self._buffer.write(data.bytes)
@serialize(QTemporal)
def _write_temporal(self, data):
try:
if self._protocol_version < 1 and (data.meta.qtype == QTIMESPAN or
data.meta.qtype == QTIMESTAMP):
raise QWriterException(
'kdb+ protocol version violation: data type %s not supported pre kdb+ v2.6'
% hex(data.meta.qtype))
self._buffer.write(struct.pack('=b', data.meta.qtype))
fmt = STRUCT_MAP[data.meta.qtype]
self._buffer.write(
struct.pack(fmt, to_raw_qtemporal(data.raw, data.meta.qtype)))
except KeyError:
raise QWriterException('Unable to serialize type: %s' % type(data))
@serialize(numpy.datetime64, numpy.timedelta64)
def _write_numpy_temporal(self, data):
try:
qtype = TEMPORAL_PY_TYPE[str(data.dtype)]
if self._protocol_version < 1 and (qtype == QTIMESPAN
or qtype == QTIMESTAMP):
raise QWriterException(
'kdb+ protocol version violation: data type %s not supported pre kdb+ v2.6'
% hex(qtype))
self._buffer.write(struct.pack('=b', qtype))
fmt = STRUCT_MAP[qtype]
self._buffer.write(struct.pack(fmt, to_raw_qtemporal(data, qtype)))
except KeyError:
raise QWriterException('Unable to serialize type: %s' % data.dtype)
@serialize(QLambda)
def _write_lambda(self, data):
self._buffer.write(struct.pack('=b', QLAMBDA))
self._buffer.write(b'\0')
self._write_string(data.expression)
@serialize(QProjection)
def _write_projection(self, data):
self._buffer.write(
struct.pack('=bi', QPROJECTION, len(data.parameters)))
for parameter in data.parameters:
self._write(parameter)
@serialize(QDictionary, QKeyedTable)
def _write_dictionary(self, data):
self._buffer.write(struct.pack('=b', QDICTIONARY))
self._write(data.keys)
self._write(data.values)
@serialize(QTable)
def _write_table(self, data):
self._buffer.write(struct.pack('=bxb', QTABLE, QDICTIONARY))
self._write(qlist(numpy.array(data.dtype.names), qtype=QSYMBOL_LIST))
self._buffer.write(struct.pack('=bxi', QGENERAL_LIST, len(data.dtype)))
for column in data.dtype.names:
self._write_list(data[column], data.meta[column])
@serialize(numpy.ndarray, QList, QTemporalList)
def _write_list(self, data, qtype=None):
if qtype is not None:
qtype = -abs(qtype)
if qtype is None:
qtype = get_list_qtype(data)
if self._protocol_version < 1 and (abs(qtype) == QTIMESPAN_LIST
or abs(qtype) == QTIMESTAMP_LIST):
raise QWriterException(
'kdb+ protocol version violation: data type %s not supported pre kdb+ v2.6'
% hex(data.meta.qtype))
if qtype == QGENERAL_LIST:
self._write_generic_list(data)
elif qtype == QCHAR:
self._write_string(data.tostring())
else:
self._buffer.write(struct.pack('=bxi', -qtype, len(data)))
if data.dtype.type in (numpy.datetime64, numpy.timedelta64):
# convert numpy temporal to raw q temporal
data = array_to_raw_qtemporal(data, qtype=qtype)
if qtype == QSYMBOL:
for symbol in data:
if symbol:
self._buffer.write(symbol)
self._buffer.write(b'\0')
elif qtype == QGUID:
if self._protocol_version < 3:
raise QWriterException(
'kdb+ protocol version violation: Guid not supported pre kdb+ v3.0'
)
for guid in data:
self._buffer.write(guid.bytes)
else:
self._buffer.write(data.tostring())
