def __init__(self, host, port, username = None, password = None, timeout = None, encoding = 'latin-1', reader_class = None, writer_class = None, **options):
self.host = host
self.port = port
self.username = username
self.password = password
self._connection = None
self._connection_file = None
self._protocol_version = None
self.timeout = timeout
self._encoding = encoding
self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options))
try:
from qpython._pandas import PandasQReader, PandasQWriter
self._reader_class = PandasQReader
self._writer_class = PandasQWriter
except ImportError:
self._reader_class = QReader
self._writer_class = QWriter
if reader_class:
self._reader_class = reader_class
if writer_class:
self._writer_class = writer_class
def _read_list(self, qtype):
if self._options.pandas:
self._options.numpy_temporals = True
qlist = QReader._read_list(self, qtype=qtype)
if self._options.pandas:
atom_qtype = -abs(qtype)
if atom_qtype not in [
QMONTH, QDATE, QDATETIME, QMINUTE, QSECOND, QTIME,
QTIMESTAMP, QTIMESPAN, QSYMBOL, QFLOAT, QDOUBLE, QGUID
]:
null = QNULLMAP[atom_qtype][1]
if atom_qtype in PANDAS_TYPE:
ps = pandas.Series(data=qlist,
dtype=PANDAS_TYPE[atom_qtype]).replace(
null, pandas.NA)
else:
ps = pandas.Series(data=qlist).replace(null, pandas.NA)
else:
ps = pandas.Series(data=qlist)
ps.meta = MetaData(qtype=qtype)
return ps
else:
return qlist
def write(self, data, msg_type, **options):
'''Serializes and pushes single data object to a wrapped stream.
:Parameters:
- `data` - data to be serialized
- `msg_type` (one of the constants defined in :class:`.MessageType`) -
type of the message
:Options:
- `single_char_strings` (`boolean`) - if ``True`` single char Python
strings are encoded as q strings instead of chars,
**Default**: ``False``
:returns: if wraped stream is ``None`` serialized data,
otherwise ``None``
'''
self._buffer = BytesIO()
self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options))
# header and placeholder for message size
self._buffer.write(('%s%s\0\0\0\0\0\0' %
(ENDIANESS, chr(msg_type))).encode(self._encoding))
self._write(data)
# update message size
data_size = self._buffer.tell()
self._buffer.seek(4)
self._buffer.write(struct.pack('i', data_size))
# write data to socket
if self._stream:
self._stream.sendall(self._buffer.getvalue())
else:
return self._buffer.getvalue()
def _read_table(self, qtype=QTABLE, options=READER_CONFIGURATION):
if options.pandas:
self._buffer.skip() # ignore attributes
self._buffer.skip() # ignore dict type stamp
columns = self._read_object(options=options)
data = self._read_object(options=options)
odict = OrderedDict()
meta = MetaData(qtype=QTABLE)
for i in xrange(len(columns)):
if isinstance(data[i], str):
# convert character list (represented as string) to numpy representation
meta[columns[i]] = QSTRING
odict[columns[i]] = pandas.Series(list(data[i]),
dtype=numpy.str).replace(
' ', numpy.nan)
elif isinstance(data[i], (list, tuple)):
meta[columns[i]] = QGENERAL_LIST
tarray = numpy.ndarray(shape=len(data[i]),
dtype=numpy.dtype('O'))
for j in xrange(len(data[i])):
tarray[j] = data[i][j]
odict[columns[i]] = tarray
else:
meta[columns[i]] = data[i].meta.qtype
odict[columns[i]] = data[i]
df = pandas.DataFrame(odict)
df.meta = meta
return df
else:
return QReader._read_table(self, qtype=qtype, options=options)
def run(self, data):
"""Sync data with kdb"""
data.meta = MetaData(**{'qtype': QKEYED_TABLE})
data.reset_index(drop=True)
data.set_index(['timestamp'], inplace=True)
try:
self.q.sync('insert', np.string_('trades'), data)
temp = self.q('trades')
# print(len(temp))
print('KDB updated')
except QException as e:
print(str(e))
def __init__(self, host, port, username = None, password = None, timeout = None, **options):
self.host = host
self.port = port
self.username = username
self.password = password
self._connection = None
self._protocol_version = None
self.timeout = timeout
self._options = MetaData(**READER_CONFIGURATION.union_dict(**options))
def read_data(self, message_size, compression_mode=0, **options):
'''
Reads and optionally parses data part of a message.
.. note:: :func:`.read_header` is required to be called before executing
the :func:`.read_data`
:Parameters:
- `message_size` (`int`) - size of the message to be read
- `compression_mode` (`int`) - indicates whether data is compressed, 1 for <2GB, 2 for larger
:Options:
- `raw` (`boolean`) - indicates whether read data should parsed or
returned in raw byte form
- `numpy_temporals` (`boolean`) - if ``False`` temporal vectors are
backed by raw q representation (:class:`.QTemporalList`,
:class:`.QTemporal`) instances, otherwise are represented as
`numpy datetime64`/`timedelta64` arrays and atoms,
**Default**: ``False``
:returns: read data (parsed or raw byte form)
'''
self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options))
if compression_mode > 0:
comprHeaderLen = 4 if compression_mode == 1 else 8
if self._stream:
self._buffer.wrap(self._read_bytes(comprHeaderLen))
uncompressed_size = -8 + (self._buffer.get_uint(
) if compression_mode == 1 else self._buffer.get_long())
compressed_data = self._read_bytes(message_size - (
8 + comprHeaderLen)) if self._stream else self._buffer.raw(
message_size - (8 + comprHeaderLen))
raw_data = numpy.frombuffer(compressed_data, dtype=numpy.uint8)
if uncompressed_size <= 0:
raise QReaderException('Error while data decompression.')
raw_data = uncompress(raw_data, numpy.int64(uncompressed_size))
raw_data = numpy.ndarray.tobytes(raw_data)
self._buffer.wrap(raw_data)
elif self._stream:
raw_data = self._read_bytes(message_size - 8)
self._buffer.wrap(raw_data)
if not self._stream and self._options.raw:
raw_data = self._buffer.raw(message_size - 8)
return raw_data if self._options.raw else self._read_object()
def _read_list(self, qtype):
if self._options.pandas:
self._options.numpy_temporals = True
qlist = QReader._read_list(self, qtype = qtype)
if self._options.pandas:
if -abs(qtype) not in [QMONTH, QDATE, QDATETIME, QMINUTE, QSECOND, QTIME, QTIMESTAMP, QTIMESPAN, QSYMBOL]:
null = QNULLMAP[-abs(qtype)][1]
ps = pandas.Series(data = qlist).replace(null, numpy.NaN)
else:
ps = pandas.Series(data = qlist)
ps.meta = MetaData(qtype = qtype)
return ps
else:
return qlist
def _read_table(self, qtype=QTABLE):
if self._options.pandas:
self._buffer.skip() # ignore attributes
self._buffer.skip() # ignore dict type stamp
columns = self._read_object()
self._buffer.skip() # ignore generic list type indicator
data = QReader._read_general_list(self, qtype)
odict = OrderedDict()
meta = MetaData(qtype=QTABLE)
for i in range(len(columns)):
column_name = columns[i] if isinstance(
columns[i], str) else columns[i].decode("utf-8")
if isinstance(data[i], str):
# convert character list (represented as string) to numpy representation
meta[column_name] = QSTRING
odict[column_name] = pandas.Series(
list(data[i]),
dtype=numpy.str).replace(b' ', numpy.nan)
elif isinstance(data[i], bytes):
# convert character list (represented as string) to numpy representation
meta[column_name] = QSTRING
odict[column_name] = pandas.Series(list(data[i].decode()),
dtype=str).replace(
b' ', numpy.nan)
elif isinstance(data[i], (list, tuple)):
meta[column_name] = QGENERAL_LIST
tarray = numpy.ndarray(shape=len(data[i]),
dtype=numpy.dtype('O'))
for j in range(len(data[i])):
tarray[j] = data[i][j]
odict[column_name] = tarray
else:
meta[column_name] = data[i].meta.qtype
odict[column_name] = data[i]
df = pandas.DataFrame(odict)
df._metadata = ["meta"]
df.meta = meta
return df
else:
return QReader._read_table(self, qtype=qtype)
def read_data(self, message_size, is_compressed = False, **options):
'''
Reads and optionally parses data part of a message.
.. note:: :func:`.read_header` is required to be called before executing
the :func:`.read_data`
:Parameters:
- `message_size` (`integer`) - size of the message to be read
- `is_compressed` (`boolean`) - indicates whether data is compressed
:Options:
- `raw` (`boolean`) - indicates whether read data should parsed or
returned in raw byte form
- `numpy_temporals` (`boolean`) - if ``False`` temporal vectors are
backed by raw q representation (:class:`.QTemporalList`,
:class:`.QTemporal`) instances, otherwise are represented as
`numpy datetime64`/`timedelta64` arrays and atoms,
**Default**: ``False``
:returns: read data (parsed or raw byte form)
'''
options = MetaData(**READER_CONFIGURATION.union_dict(**options))
if is_compressed:
if self._stream:
self._buffer.wrap(self._read_bytes(4))
uncompressed_size = -8 + self._buffer.get_int()
compressed_data = self._read_bytes(message_size - 12) if self._stream else self._buffer.raw(message_size - 12)
raw_data = numpy.fromstring(compressed_data, dtype = numpy.uint8)
if uncompressed_size <= 0:
raise QReaderException('Error while data decompression.')
raw_data = uncompress(raw_data, numpy.intc(uncompressed_size))
raw_data = numpy.ndarray.tostring(raw_data)
self._buffer.wrap(raw_data)
elif self._stream:
raw_data = self._read_bytes(message_size - 8)
self._buffer.wrap(raw_data)
if not self._stream and options.raw:
raw_data = self._buffer.raw(message_size - 8)
return raw_data if options.raw else self._read_object(options)
async def read_data(self, message_size, is_compressed=False, **options):
self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options))
if is_compressed:
if self._stream:
self._buffer.wrap(await self._read_bytes(4))
uncompressed_size = -8 + self._buffer.get_int()
compressed_data = (await self._read_bytes(message_size - 12)) \
if self._stream else self._buffer.raw(message_size - 12)
raw_data = numpy.fromstring(compressed_data, dtype=numpy.uint8)
if uncompressed_size <= 0:
raise QReaderException('Error while data decompression.')
raw_data = uncompress(raw_data, numpy.intc(uncompressed_size))
raw_data = numpy.ndarray.tostring(raw_data)
self._buffer.wrap(raw_data)
elif self._stream:
raw_data = await self._read_bytes(message_size - 8)
self._buffer.wrap(raw_data)
if not self._stream and self._options.raw:
raw_data = self._buffer.raw(message_size - 8)
return raw_data if self._options.raw else self._read_object()
from qpython.qtemporal import QTemporal
try:
import pandas
BINARY = None
PANDAS_EXPRESSIONS = OrderedDict((
('("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)', {
'data':
pandas.Series(
numpy.array([
uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661'),
numpy.nan
])),
'meta':
MetaData(qtype=QGUID_LIST)
}),
('"quick brown fox jumps over a lazy dog"',
'quick brown fox jumps over a lazy dog'),
('" "', ' '),
('``quick``fox', {
'data':
pandas.Series(
numpy.array([
qnull(QSYMBOL),
numpy.string_('quick'),
qnull(QSYMBOL),
numpy.string_('fox')
])),
'meta':
MetaData(qtype=QSYMBOL_LIST)
def _meta_init(self, **meta):
self.meta = MetaData(**meta)
import sys
from qpython import MetaData
from qpython.qtype import * # @UnusedWildImport
from qpython.qcollection import qlist, QDictionary, qtable, QTable, QKeyedTable
from qpython.qtemporal import qtemporal, from_raw_qtemporal, array_from_raw_qtemporal
try:
from qpython.fastutils import uncompress
except:
from qpython.utils import uncompress
READER_CONFIGURATION = MetaData(raw = False,
numpy_temporals = False,
pandas = False)
class QReaderException(Exception):
'''
Indicates an error raised during data deserialization.
'''
pass
class QMessage(object):
'''
Represents a single message parsed from q protocol.
def __array_finalize__(self, obj):
self.meta = MetaData() if obj is None else getattr(obj, 'meta', MetaData())
def _meta_init(self, **meta):
'''Initialises the meta-information.'''
self.meta = MetaData(**meta)
