def _write_pandas_data_frame(self, data, qtype=None):
data_columns = data.columns.values
if hasattr(data, 'meta') and data.meta.qtype == QKEYED_TABLE:
# data frame represents keyed table
self._buffer.write(struct.pack('=b', QDICTIONARY))
self._buffer.write(struct.pack('=bxb', QTABLE, QDICTIONARY))
index_columns = data.index.names
self._write(qlist(numpy.array(index_columns), qtype=QSYMBOL_LIST))
data.reset_index(inplace=True)
self._buffer.write(
struct.pack('=bxi', QGENERAL_LIST, len(index_columns)))
for column in index_columns:
self._write_pandas_series(
data[column],
qtype=data.meta[column] if hasattr(data, 'meta') else None)
data.set_index(index_columns, inplace=True)
self._buffer.write(struct.pack('=bxb', QTABLE, QDICTIONARY))
self._write(qlist(numpy.array(data_columns), qtype=QSYMBOL_LIST))
self._buffer.write(
struct.pack('=bxi', QGENERAL_LIST, len(data_columns)))
for column in data_columns:
self._write_pandas_series(
data[column],
qtype=data.meta[column] if hasattr(data, 'meta') else None)
def _read_list(self, qtype):
self._buffer.skip() # ignore attributes
length = self._buffer.get_int()
conversion = PY_TYPE.get(-qtype, None)
if qtype == QSYMBOL_LIST:
symbols = self._buffer.get_symbols(length)
data = numpy.array(symbols, dtype=numpy.string_)
return qlist(data, qtype=qtype, adjust_dtype=False)
elif qtype == QGUID_LIST:
data = numpy.array([self._read_guid() for x in range(length)])
return qlist(data, qtype=qtype, adjust_dtype=False)
elif conversion:
raw = self._buffer.raw(length * ATOM_SIZE[qtype])
data = numpy.fromstring(raw, dtype=conversion)
if not self._is_native:
data.byteswap(True)
if qtype >= QTIMESTAMP_LIST and qtype <= QTIME_LIST and self._options.numpy_temporals:
data = array_from_raw_qtemporal(data, qtype)
return qlist(data, qtype=qtype, adjust_dtype=False)
else:
raise QReaderException('Unable to deserialize q type: %s' %
hex(qtype))
def _read_list(self, qtype):
self._buffer.skip() # ignore attributes
length = self._buffer.get_int()
conversion = FROM_Q.get(-qtype, None)
if qtype == QSYMBOL_LIST:
symbols = self._buffer.get_symbols(length)
data = numpy.array(symbols, dtype = numpy.string_)
return qlist(data, qtype = qtype, adjust_dtype = False)
elif qtype >= QTIMESTAMP_LIST and qtype <= QTIME_LIST:
raw = self._buffer.raw(length * ATOM_SIZE[qtype])
data = numpy.fromstring(raw, dtype = conversion)
if not self._is_native:
data.byteswap(True)
return qlist(data, qtype = qtype, adjust_dtype = False)
elif qtype == QGUID_LIST:
data = numpy.array([self._read_guid() for x in xrange(length)])
return qlist(data, qtype = qtype, adjust_dtype = False)
elif conversion:
raw = self._buffer.raw(length * ATOM_SIZE[qtype])
data = numpy.fromstring(raw, dtype = conversion)
if not self._is_native:
data.byteswap(True)
return qlist(data, qtype = qtype, adjust_dtype = False)
else:
raise QReaderException('Unable to deserialize q type: %s' % hex(qtype))
def get_ask_data(self):
c = random.randint(1, 10)
today = numpy.datetime64(datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0))
time = [numpy.timedelta64((numpy.datetime64(datetime.datetime.now()) - today), 'ms') for x in xrange(c)]
instr = ['instr_%d' % random.randint(1, 100) for x in xrange(c)]
src = ['qPython' for x in xrange(c)]
ask = [random.random() * random.randint(1, 100) for x in xrange(c)]
data = [qlist(time, qtype=QTIME_LIST), qlist(instr, qtype=QSYMBOL_LIST), qlist(src, qtype=QSYMBOL_LIST), qlist(ask, qtype=QFLOAT_LIST)]
print data
return data
def get_ask_data(self):
c = random.randint(1, 10)
today = numpy.datetime64(datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0))
time = [numpy.timedelta64((numpy.datetime64(datetime.datetime.now()) - today), 'ms') for x in range(c)]
instr = ['instr_%d' % random.randint(1, 100) for x in range(c)]
src = ['qPython' for x in range(c)]
ask = [random.random() * random.randint(1, 100) for x in range(c)]
data = [qlist(time, qtype=QTIME_LIST), qlist(instr, qtype=QSYMBOL_LIST), qlist(src, qtype=QSYMBOL_LIST), qlist(ask, qtype=QFLOAT_LIST)]
print(data)
return data
def getJsonObject():
f = open('/home/kartik/q/l32/getHistoryCMEClearportEOD.json', 'rU')
data = json.load(f) #Load json object
volume = []
timestamp = []
symbol = []
high = []
tradingDay = []
low = []
close = []
openInterest = []
Open = []
for line in data['results']:
# Use relevant datatypes as mentioned in http://qpython.readthedocs.io/en/latest/type-conversion.html
volume.append(int(line['volume']))
dt1 = gt(str(line['timestamp']))
val = np.datetime64(dt1, 'ns')
timestamp.append(np.datetime64(val, 'ns'))
symbol.append(numpy.string_(line['symbol']))
high.append(float(line['high']))
dt2 = datetime.datetime.strptime(str(line['tradingDay']), "%Y-%m-%d")
tradingDay.append(np.datetime64(dt2, 'D'))
low.append(float(line['low']))
close.append(float(line['close']))
openInterest.append(int(line['openInterest']))
Open.append(float(line['open']))
f.close()
# Create a collection of lists to be inserted into the tickerplant according to the schema specified
dat1 = [
qlist(symbol, QSYMBOL_LIST),
qlist(timestamp, QTIMESTAMP_LIST),
qlist(tradingDay, QDATE_LIST),
qlist(Open, QFLOAT_LIST),
qlist(high, QFLOAT_LIST),
qlist(low, QFLOAT_LIST),
qlist(close, QFLOAT_LIST),
qlist(volume, QINT_LIST),
qlist(openInterest, QINT_LIST)
]
return dat1
def test_write_single_char_string():
w = qwriter.QWriter(None, 3)
for obj in (['one', 'two', '3'], qlist(['one', 'two', '3'], qtype = QSTRING_LIST)):
single_char_strings = False
for query in (b'("one"; "two"; "3")', b'("one"; "two"; enlist "3")'):
serialized = binascii.hexlify(w.write(obj, 1, single_char_strings = single_char_strings ))[16:].lower()
assert serialized == BINARY[query].lower(), 'serialization failed: %s, expected: %s actual: %s' % (query, BINARY[query].lower(), serialized)
single_char_strings = not single_char_strings
def get_trade_data():
my_data = numpy.genfromtxt('trade.csv', delimiter=',', dtype=None)
time = [row[0] for row in my_data]
sym = [row[1] for row in my_data]
size = [row[2] for row in my_data]
price = [row[3] for row in my_data]
exchange = [row[4] for row in my_data]
data = [
time,
qlist(sym, qtype=QSYMBOL_LIST),
qlist(size, qtype=QLONG_LIST),
qlist(price, qtype=QDOUBLE_LIST),
qlist(exchange, qtype=QSYMBOL_LIST)
]
return data
def test_write_single_char_string():
w = qwriter.QWriter(None, 3)
for obj in (['one', 'two', '3'], qlist(['one', 'two', '3'], qtype = QSTRING_LIST)):
single_char_strings = False
for query in ('("one"; "two"; "3")', '("one"; "two"; enlist "3")'):
serialized = binascii.hexlify(w.write(obj, 1, single_char_strings = single_char_strings ))[16:].lower()
assert serialized == BINARY[query].lower(), 'serialization failed: %s, expected: %s actual: %s' % (query, BINARY[query].lower(), serialized)
single_char_strings = not single_char_strings
def _write_pandas_data_frame(self, data, qtype = None):
data_columns = data.columns.values
if hasattr(data, 'meta') and data.meta.qtype == QKEYED_TABLE:
# data frame represents keyed table
self._buffer.write(struct.pack('=b', QDICTIONARY))
self._buffer.write(struct.pack('=bxb', QTABLE, QDICTIONARY))
index_columns = data.index.names
self._write(qlist(numpy.array(index_columns), qtype = QSYMBOL_LIST))
data.reset_index(inplace = True)
self._buffer.write(struct.pack('=bxi', QGENERAL_LIST, len(index_columns)))
for column in index_columns:
self._write_pandas_series(data[column], qtype = data.meta[column] if hasattr(data, 'meta') else None)
data.set_index(index_columns, inplace = True)
self._buffer.write(struct.pack('=bxb', QTABLE, QDICTIONARY))
self._write(qlist(numpy.array(data_columns), qtype = QSYMBOL_LIST))
self._buffer.write(struct.pack('=bxi', QGENERAL_LIST, len(data_columns)))
for column in data_columns:
self._write_pandas_series(data[column], qtype = data.meta[column] if hasattr(data, 'meta') else None)
def export(self, name, columns, points):
"""Export the stats to the kdb+ host/port."""
# Remove non number stats and convert all numbers to float like prometheus
data = {
k: v
for (k, v) in iteritems(dict(zip(columns, points)))
if isinstance(v, Number)
}
# Append all tables name with self.prefix
try:
self.client.sendAsync(
"{if[count z; x insert flip (`time, y)! .z.t, (),/: z]}",
np.string_(f"{self.prefix}{name.capitalize()}"),
qlist(listkeys(data), qtype=QSYMBOL_LIST),
qlist(listvalues(data), qtype=QDOUBLE_LIST),
)
except Exception as e:
logger.error(f"Cannot export stats <{name}> to kdb+ ({e})")
logger.debug(f"Exported <{name}> stats to kdb+")
def _write_pandas_data_frame(self, data, qtype=None):
data_columns = data.columns.values
restore_meta_attr = False
meta = None
if hasattr(data, 'meta'):
meta = data.meta
elif 'meta' in data.attrs:
meta = data.attrs.pop('meta')
restore_meta_attr = True
meta_is_defined = meta is not None
if meta_is_defined and meta.qtype == QKEYED_TABLE:
# data frame represents keyed table
self._buffer.write(struct.pack('=b', QDICTIONARY))
self._buffer.write(struct.pack('=bxb', QTABLE, QDICTIONARY))
index_columns = list(data.index.names)
self._write(qlist(numpy.array(index_columns), qtype=QSYMBOL_LIST))
data.reset_index(inplace=True)
self._buffer.write(
struct.pack('=bxi', QGENERAL_LIST, len(index_columns)))
for column in index_columns:
column_meta = meta[column]
self._write_pandas_series(data[column], qtype=column_meta)
data.set_index(index_columns, inplace=True)
self._buffer.write(struct.pack('=bxb', QTABLE, QDICTIONARY))
self._write(qlist(numpy.array(data_columns), qtype=QSYMBOL_LIST))
self._buffer.write(
struct.pack('=bxi', QGENERAL_LIST, len(data_columns)))
for column in data_columns:
column_meta = meta[column] if meta_is_defined else None
self._write_pandas_series(data[column], qtype=column_meta)
if meta_is_defined and restore_meta_attr:
data.attrs['meta'] = meta
import binascii
import sys
from collections import OrderedDict
from qpython import qwriter
from qpython.qtype import * # @UnusedWildImport
from qpython.qcollection import qlist, QDictionary, qtable, QKeyedTable
from qpython.qtemporal import qtemporal, to_raw_qtemporal, array_to_raw_qtemporal
BINARY = OrderedDict()
EXPRESSIONS = OrderedDict((
('("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)',
qlist(numpy.array(
[uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661'),
qnull(QGUID)]),
qtype=QGUID_LIST)),
('"G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"',
uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661')),
('"G"$"00000000-0000-0000-0000-000000000000"',
uuid.UUID('00000000-0000-0000-0000-000000000000')),
('(2001.01m; 0Nm)', (
qlist(numpy.array([
to_raw_qtemporal(numpy.datetime64('2001-01', 'M'), QMONTH),
qnull(QMONTH)
]),
qtype=QMONTH_LIST),
qlist(numpy.array([12, qnull(QMONTH)]), qtype=QMONTH_LIST),
qlist(array_to_raw_qtemporal(numpy.array(
[numpy.datetime64('2001-01', 'M'),
numpy.datetime64('NaT', 'M')]),
numpy.datetime64(datetime.datetime.strptime(x, f)) - qstarttime, 'ns')
for x in srcTime
]
expiryTime = [
numpy.timedelta64(
numpy.datetime64(datetime.datetime.strptime(x, f)) - qstarttime, 'ns')
for x in expiryTime
]
msgrcv = [
numpy.timedelta64(
numpy.datetime64(datetime.datetime.strptime(x, f)) - qstarttime, 'ns')
for x in msgrcv
]
data = [
qlist(timecol, qtype=QTIMESTAMP_LIST),
qlist(sym, qtype=QSYMBOL_LIST),
qlist(src, qtype=QSYMBOL_LIST),
qlist(bid, qtype=QDOUBLE_LIST),
qlist(ask, qtype=QDOUBLE_LIST),
qlist(bsize, qtype=QINT_LIST),
qlist(asize, qtype=QINT_LIST),
qlist(srcTime, qtype=QTIMESTAMP_LIST),
qlist(cond, qtype=QSYMBOL_LIST),
qlist(layer, qtype=QSYMBOL_LIST),
qlist(expiryTime, qtype=QTIMESTAMP_LIST),
qlist(msgrcv, qtype=QTIMESTAMP_LIST)
]
# create connection object
q = qconnection.QConnection(host='localhost', port=5010)
try:
from cStringIO import BytesIO
except ImportError:
from io import BytesIO
if sys.version > '3':
long = int
from collections import OrderedDict
from qpython import qreader
from qpython.qtype import * # @UnusedWildImport
from qpython.qcollection import qlist, QList, QTemporalList, QDictionary, qtable, QTable, QKeyedTable
from qpython.qtemporal import qtemporal, QTemporal
COMPRESSED_EXPRESSIONS = OrderedDict((
(b'1000#`q', qlist(numpy.array(['q'] * 1000), qtype=QSYMBOL_LIST)),
(b'([] q:1000#`q)',
qtable(qlist(numpy.array(['q']), qtype=QSYMBOL_LIST),
[qlist(numpy.array(['q'] * 1000), qtype=QSYMBOL_LIST)])),
(b'([] a:til 200;b:25+til 200;c:200#`a)',
qtable(qlist(numpy.array(['a', 'b', 'c']), qtype=QSYMBOL_LIST), [
qlist(numpy.arange(200), qtype=QLONG_LIST),
qlist(numpy.arange(200) + 25, qtype=QLONG_LIST),
qlist(numpy.array(['a'] * 200), qtype=QSYMBOL_LIST)
])),
))
def arrays_equal(left, right):
if type(left) != type(right):
return False
#
import binascii
import sys
from collections import OrderedDict
from qpython import qwriter
from qpython.qtype import * # @UnusedWildImport
from qpython.qcollection import qlist, QDictionary, qtable, QKeyedTable
from qpython.qtemporal import qtemporal, to_raw_qtemporal, array_to_raw_qtemporal
BINARY = OrderedDict()
EXPRESSIONS = OrderedDict((
('("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)',
qlist(numpy.array([uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661'), qnull(QGUID)]), qtype=QGUID_LIST)),
('"G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"', uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661')),
('"G"$"00000000-0000-0000-0000-000000000000"', uuid.UUID('00000000-0000-0000-0000-000000000000')),
('(2001.01m; 0Nm)', (qlist(numpy.array([to_raw_qtemporal(numpy.datetime64('2001-01', 'M'), QMONTH), qnull(QMONTH)]), qtype=QMONTH_LIST),
qlist(numpy.array([12, qnull(QMONTH)]), qtype=QMONTH_LIST),
qlist(array_to_raw_qtemporal(numpy.array([numpy.datetime64('2001-01', 'M'), numpy.datetime64('NaT', 'M')]), qtype = QMONTH_LIST), qtype = QMONTH_LIST),
qlist([12, qnull(QMONTH)], qtype=QMONTH_LIST),
qlist(numpy.array([numpy.datetime64('2001-01'), numpy.datetime64('NaT')], dtype='datetime64[M]'), qtype=QMONTH_LIST),
numpy.array([numpy.datetime64('2001-01'), numpy.datetime64('NaT')], dtype='datetime64[M]'),
)),
('2001.01m', (qtemporal(numpy.datetime64('2001-01', 'M'), qtype=QMONTH),
numpy.datetime64('2001-01', 'M'))),
('0Nm', (qtemporal(qnull(QMONTH), qtype=QMONTH),
qtemporal(numpy.datetime64('NaT', 'M'), qtype=QMONTH),
numpy.datetime64('NaT', 'M'))),
('2001.01.01 2000.05.01 0Nd', (qlist(numpy.array([to_raw_qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE), to_raw_qtemporal(numpy.datetime64('2000-05-01', 'D'), qtype=QDATE), qnull(QDATE)]), qtype=QDATE_LIST),
import binascii
import cStringIO
import struct
import sys
from collections import OrderedDict
from qpython import qreader
from qpython.qtype import * # @UnusedWildImport
from qpython.qcollection import qlist, QList, QDictionary, qtable, QKeyedTable
from qpython.qtemporal import qtemporal, QTemporal, qtemporallist, QTemporalList
EXPRESSIONS = OrderedDict((
('("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)',
qlist(numpy.array([uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661'), qnull(QGUID)]), qtype=QGUID_LIST)),
('"G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"', uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661')),
('"G"$"00000000-0000-0000-0000-000000000000"', uuid.UUID('00000000-0000-0000-0000-000000000000')),
('(2001.01m; 0Nm)', qtemporallist(numpy.array([12, qnull(QMONTH)]), qtype=QMONTH_LIST)),
('2001.01m', qtemporal(numpy.datetime64('2001-01', 'M'), qtype=QMONTH)),
('0Nm', qtemporal(qnull(QMONTH), qtype=QMONTH)),
('2001.01.01 2000.05.01 0Nd', qtemporallist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)),
('2001.01.01', qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE)),
('0Nd', qtemporal(qnull(QDATE), qtype=QDATE)),
('2000.01.04T05:36:57.600 0Nz', qtemporallist(numpy.array([3.234, qnull(QDATETIME)]), qtype=QDATETIME_LIST)),
('2000.01.04T05:36:57.600', qtemporal(numpy.datetime64('2000-01-04T05:36:57.600', 'ms'), qtype=QDATETIME)),
('0Nz', qtemporal(qnull(QDATETIME), qtype=QDATETIME)),
('12:01 0Nu', qtemporallist(numpy.array([721, qnull(QMINUTE)]), qtype=QMINUTE_LIST)),
('12:01', qtemporal(numpy.timedelta64(721, 'm'), qtype=QMINUTE)),
('0Nu', qtemporal(qnull(QMINUTE), qtype=QMINUTE)),
('12:05:00 0Nv', qtemporallist(numpy.array([43500, qnull(QSECOND)]), qtype=QSECOND_LIST)),
"meta": MetaData(qtype=QTIMESTAMP_LIST),
},
),
(
b"0D05:36:57.600 0Nn",
{
"data": pandas.Series(
numpy.array([numpy.timedelta64(20217600000000, "ns"), numpy.timedelta64("nat", "ns")])
),
"meta": MetaData(qtype=QTIMESPAN_LIST),
},
),
(
b"1 2!`abc`cdefgh",
QDictionary(
qlist(numpy.array([1, 2], dtype=numpy.int64), qtype=QLONG_LIST),
qlist(numpy.array(["abc", "cdefgh"]), qtype=QSYMBOL_LIST),
),
),
(
b"(0 1; 2 3)!`first`second",
QDictionary(
[
qlist(numpy.array([0, 1], dtype=numpy.int64), qtype=QLONG_LIST),
qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST),
],
qlist(numpy.array(["first", "second"]), qtype=QSYMBOL_LIST),
),
),
(
b'(1;2h;3.234;"4")!(`one;2 3;"456";(7;8 9))',
def _write_table(self, data):
self._buffer.write(struct.pack('=bxb', QTABLE, QDICTIONARY))
self._write(qlist(numpy.array(data.dtype.names), qtype=QSYMBOL_LIST))
self._buffer.write(struct.pack('=bxi', QGENERAL_LIST, len(data.dtype)))
for column in data.dtype.names:
self._write_list(data[column], data.meta[column])
if sys.version > '3':
long = int
from collections import OrderedDict
from qpython import qreader
from qpython.qtype import * # @UnusedWildImport
from qpython.qcollection import qlist, QList, QTemporalList, QDictionary, qtable, QKeyedTable, QTable
from qpython.qtemporal import qtemporal, QTemporal
TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), 'test_data')
EXPRESSIONS = OrderedDict((
(b'0Nm', qtemporal(numpy.datetime64('NaT', 'M'), qtype=QMONTH)),
(b'("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)',
qlist(numpy.array(
[uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661'),
qnull(QGUID)]),
qtype=QGUID_LIST)),
(b'"G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"',
uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661')),
(b'"G"$"00000000-0000-0000-0000-000000000000"',
uuid.UUID('00000000-0000-0000-0000-000000000000')),
(b'(2001.01m; 0Nm)',
qlist(numpy.array([12, qnull(QMONTH)]), qtype=QMONTH_LIST)),
(b'2001.01m', qtemporal(numpy.datetime64('2001-01', 'M'), qtype=QMONTH)),
(b'0Nm', qtemporal(numpy.datetime64('NaT', 'M'), qtype=QMONTH)),
(b'2001.01.01 2000.05.01 0Nd',
qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)),
(b'2001.01.01', qtemporal(numpy.datetime64('2001-01-01', 'D'),
qtype=QDATE)),
(b'0Nd', qtemporal(numpy.datetime64('NaT', 'D'), qtype=QDATE)),
(b'2000.01.04T05:36:57.600 0Nz',
def _write_table(self, data):
self._buffer.write(struct.pack('=bxb', QTABLE, QDICTIONARY))
self._write(qlist(numpy.array(data.dtype.names), qtype = QSYMBOL_LIST))
self._buffer.write(struct.pack('=bxi', QGENERAL_LIST, len(data.dtype)))
for column in data.dtype.names:
self._write_list(data[column], data.meta[column])
('2001.01.01 2000.05.01 0Nd', {'data': pandas.Series(numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]')),
'meta': MetaData(qtype = QDATE_LIST) }),
('2000.01.04T05:36:57.600 0Nz', {'data': pandas.Series(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')])),
'meta': MetaData(qtype = QDATETIME_LIST) }),
('12:01 0Nu', {'data': pandas.Series(numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')])),
'meta': MetaData(qtype = QMINUTE_LIST) }),
('12:05:00 0Nv', {'data': pandas.Series(numpy.array([numpy.timedelta64(43500, 's'), numpy.timedelta64('nat', 's')])),
'meta': MetaData(qtype = QSECOND_LIST) }),
('12:04:59.123 0Nt', {'data': pandas.Series(numpy.array([numpy.timedelta64(43499123, 'ms'), numpy.timedelta64('nat', 'ms')])),
'meta': MetaData(qtype = QTIME_LIST) }),
('2000.01.04D05:36:57.600 0Np', {'data': pandas.Series(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), numpy.datetime64('nat', 'ns')])),
'meta': MetaData(qtype = QTIMESTAMP_LIST) }),
('0D05:36:57.600 0Nn', {'data': pandas.Series(numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')])),
'meta': MetaData(qtype = QTIMESPAN_LIST) }),
('1 2!`abc`cdefgh', QDictionary(qlist(numpy.array([1, 2], dtype=numpy.int64), qtype=QLONG_LIST),
qlist(numpy.array(['abc', 'cdefgh']), qtype = QSYMBOL_LIST))),
('(0 1; 2 3)!`first`second', QDictionary([qlist(numpy.array([0, 1], dtype=numpy.int64), qtype=QLONG_LIST), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST)],
qlist(numpy.array(['first', 'second']), qtype = QSYMBOL_LIST))),
('(1;2h;3.234;"4")!(`one;2 3;"456";(7;8 9))', QDictionary([numpy.int64(1), numpy.int16(2), numpy.float64(3.234), '4'],
[numpy.string_('one'), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST), '456', [numpy.int64(7), qlist(numpy.array([8, 9], dtype=numpy.int64), qtype=QLONG_LIST)]])),
('`A`B`C!((1;3.234;3);(`x`y!(`a;2));5.5e)', QDictionary(qlist(numpy.array(['A', 'B', 'C']), qtype = QSYMBOL_LIST),
[[numpy.int64(1), numpy.float64(3.234), numpy.int64(3)], QDictionary(qlist(numpy.array(['x', 'y']), qtype = QSYMBOL_LIST), [numpy.string_('a'), numpy.int64(2)]), numpy.float32(5.5)])),
('flip `abc`def!(1 2 3; 4 5 6)', {'data': pandas.DataFrame(OrderedDict((('abc', pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int64))),
('def', pandas.Series(numpy.array([4, 5, 6], dtype = numpy.int64)))))
),
'meta': MetaData(**{'qtype': QTABLE, 'abc': QLONG_LIST, 'def': QLONG_LIST}) }),
('flip `name`iq!(`Dent`Beeblebrox`Prefect;98 42 126)',
{'data': pandas.DataFrame(OrderedDict((('name', pandas.Series(['Dent', 'Beeblebrox', 'Prefect'])),
('iq', pandas.Series(numpy.array([98, 42, 126], dtype = numpy.int64)))))
def test_reading():
BINARY = OrderedDict()
with open('tests/QExpressions3.out', 'rb') as f:
while True:
query = f.readline().strip()
binary = f.readline().strip()
if not binary:
break
BINARY[query] = binary
buffer_reader = qreader.QReader(None)
print('Deserialization')
def test_reading_one(query, value):
buffer_ = BytesIO()
binary = binascii.unhexlify(BINARY[query])
buffer_.write(b'\1\0\0\0')
buffer_.write(struct.pack('i', len(binary) + 8))
buffer_.write(binary)
buffer_.seek(0)
sys.stdout.write(' %-75s' % query)
try:
header = buffer_reader.read_header(source=buffer_.getvalue())
result = buffer_reader.read_data(
message_size=header.size,
compression_mode=header.compression_mode,
raw=True)
assert compare(buffer_.getvalue()[8:],
result), 'raw reading failed: %s' % (query)
stream_reader = qreader.QReader(buffer_)
result = stream_reader.read(raw=True).data
assert compare(buffer_.getvalue()[8:],
result), 'raw reading failed: %s' % (query)
result = buffer_reader.read(source=buffer_.getvalue()).data
if type(value) == QTemporalList:
assert compare(
value, result
), 'deserialization failed: %s, expected: %s actual: %s' % (
query, [x.raw for x in value], [x.raw for x in result])
else:
assert compare(
value, result
), 'deserialization failed: %s, expected: %s actual: %s' % (
query, value, result)
header = buffer_reader.read_header(source=buffer_.getvalue())
result = buffer_reader.read_data(
message_size=header.size,
compression_mode=header.compression_mode)
assert compare(value,
result), 'deserialization failed: %s' % (query)
buffer_.seek(0)
stream_reader = qreader.QReader(buffer_)
result = stream_reader.read().data
assert compare(
value, result
), 'deserialization failed: %s, expected: %s actual: %s' % (
query, value, result)
print('.')
except QException as e:
assert isinstance(value, QException)
assert e.args == value.args
print('.')
test_reading_one(
b'("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)',
qlist(numpy.array(
[uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661'),
qnull(QGUID)]),
qtype=QGUID_LIST))
test_reading_one(b'"G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"',
uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661'))
test_reading_one(b'"G"$"00000000-0000-0000-0000-000000000000"',
uuid.UUID('00000000-0000-0000-0000-000000000000'))
test_reading_one(
b'(2001.01m; 0Nm)',
qlist(numpy.array([12, qnull(QMONTH)]), qtype=QMONTH_LIST))
test_reading_one(b'2001.01m',
qtemporal(numpy.datetime64('2001-01', 'M'), qtype=QMONTH))
test_reading_one(b'0Nm',
qtemporal(numpy.datetime64('NaT', 'M'), qtype=QMONTH))
test_reading_one(
b'2001.01.01 2000.05.01 0Nd',
qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST))
test_reading_one(
b'2001.01.01',
qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE))
test_reading_one(b'0Nd',
qtemporal(numpy.datetime64('NaT', 'D'), qtype=QDATE))
test_reading_one(
b'2000.01.04T05:36:57.600 0Nz',
qlist(numpy.array([3.234, qnull(QDATETIME)]), qtype=QDATETIME_LIST))
test_reading_one(
b'2000.01.04T05:36:57.600',
qtemporal(numpy.datetime64('2000-01-04T05:36:57.600', 'ms'),
qtype=QDATETIME))
test_reading_one(b'0Nz',
qtemporal(numpy.datetime64('NaT', 'ms'), qtype=QDATETIME))
test_reading_one(
b'12:01 0Nu',
qlist(numpy.array([721, qnull(QMINUTE)]), qtype=QMINUTE_LIST))
test_reading_one(b'12:01',
qtemporal(numpy.timedelta64(721, 'm'), qtype=QMINUTE))
test_reading_one(b'0Nu',
qtemporal(numpy.timedelta64('NaT', 'm'), qtype=QMINUTE))
test_reading_one(
b'12:05:00 0Nv',
qlist(numpy.array([43500, qnull(QSECOND)]), qtype=QSECOND_LIST))
test_reading_one(b'12:05:00',
qtemporal(numpy.timedelta64(43500, 's'), qtype=QSECOND))
test_reading_one(b'0Nv',
qtemporal(numpy.timedelta64('NaT', 's'), qtype=QSECOND))
test_reading_one(
b'12:04:59.123 0Nt',
qlist(numpy.array([43499123, qnull(QTIME)]), qtype=QTIME_LIST))
test_reading_one(b'12:04:59.123',
qtemporal(numpy.timedelta64(43499123, 'ms'), qtype=QTIME))
test_reading_one(b'0Nt',
qtemporal(numpy.timedelta64('NaT', 'ms'), qtype=QTIME))
test_reading_one(
b'2000.01.04D05:36:57.600 0Np',
qlist(numpy.array([long(279417600000000),
qnull(QTIMESTAMP)]),
qtype=QTIMESTAMP_LIST))
test_reading_one(
b'2000.01.04D05:36:57.600',
qtemporal(numpy.datetime64('2000-01-04T05:36:57.600', 'ns'),
qtype=QTIMESTAMP))
test_reading_one(
b'0Np', qtemporal(numpy.datetime64('NaT', 'ns'), qtype=QTIMESTAMP))
test_reading_one(
b'0D05:36:57.600 0Nn',
qlist(numpy.array([long(20217600000000),
qnull(QTIMESPAN)]),
qtype=QTIMESPAN_LIST))
test_reading_one(
b'0D05:36:57.600',
qtemporal(numpy.timedelta64(20217600000000, 'ns'), qtype=QTIMESPAN))
test_reading_one(
b'0Nn', qtemporal(numpy.timedelta64('NaT', 'ns'), qtype=QTIMESPAN))
test_reading_one(b'::', None)
test_reading_one(b'1+`', QException(b'type'))
test_reading_one(b'1', numpy.int64(1))
test_reading_one(b'1i', numpy.int32(1))
test_reading_one(b'-234h', numpy.int16(-234))
test_reading_one(b'0b', numpy.bool_(False))
test_reading_one(b'1b', numpy.bool_(True))
test_reading_one(b'0x2a', numpy.byte(0x2a))
test_reading_one(b'89421099511627575j',
numpy.int64(long(89421099511627575)))
test_reading_one(b'5.5e', numpy.float32(5.5))
test_reading_one(b'3.234', numpy.float64(3.234))
test_reading_one(b'"0"', b'0')
test_reading_one(b'"abc"', b'abc')
test_reading_one(b'"quick brown fox jumps over a lazy dog"',
b'quick brown fox jumps over a lazy dog')
test_reading_one(b'`abc', numpy.string_('abc'))
test_reading_one(b'`quickbrownfoxjumpsoveralazydog',
numpy.string_('quickbrownfoxjumpsoveralazydog'))
test_reading_one(b'0Nh', qnull(QSHORT))
test_reading_one(b'0N', qnull(QLONG))
test_reading_one(b'0Ni', qnull(QINT))
test_reading_one(b'0Nj', qnull(QLONG))
test_reading_one(b'0Ne', qnull(QFLOAT))
test_reading_one(b'0n', qnull(QDOUBLE))
test_reading_one(b'" "', qnull(QSTRING))
test_reading_one(b'`', qnull(QSYMBOL))
test_reading_one(b'0Ng', qnull(QGUID))
test_reading_one(b'()', []),
test_reading_one(
b'(0b;1b;0b)',
qlist(numpy.array([False, True, False], dtype=numpy.bool_),
qtype=QBOOL_LIST))
test_reading_one(
b'(0x01;0x02;0xff)',
qlist(numpy.array([0x01, 0x02, 0xff], dtype=numpy.byte),
qtype=QBYTE_LIST))
test_reading_one(
b'(1h;2h;3h)',
qlist(numpy.array([1, 2, 3], dtype=numpy.int16), qtype=QSHORT_LIST))
test_reading_one(
b'(1h;0Nh;3h)',
qlist(numpy.array([1, qnull(QSHORT), 3], dtype=numpy.int16),
qtype=QSHORT_LIST))
test_reading_one(
b'1 2 3',
qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype=QLONG_LIST))
test_reading_one(
b'1 0N 3',
qlist(numpy.array([1, qnull(QLONG), 3], dtype=numpy.int64),
qtype=QLONG_LIST))
test_reading_one(
b'(1i;2i;3i)',
qlist(numpy.array([1, 2, 3], dtype=numpy.int32), qtype=QINT_LIST))
test_reading_one(
b'(1i;0Ni;3i)',
qlist(numpy.array([1, qnull(QINT), 3], dtype=numpy.int32),
qtype=QINT_LIST))
test_reading_one(
b'(1j;2j;3j)',
qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype=QLONG_LIST))
test_reading_one(
b'(1j;0Nj;3j)',
qlist(numpy.array([1, qnull(QLONG), 3], dtype=numpy.int64),
qtype=QLONG_LIST))
test_reading_one(
b'(5.5e; 8.5e)',
qlist(numpy.array([5.5, 8.5], dtype=numpy.float32), qtype=QFLOAT_LIST))
test_reading_one(
b'(5.5e; 0Ne)',
qlist(numpy.array([5.5, qnull(QFLOAT)], dtype=numpy.float32),
qtype=QFLOAT_LIST))
test_reading_one(
b'3.23 6.46',
qlist(numpy.array([3.23, 6.46], dtype=numpy.float64),
qtype=QDOUBLE_LIST))
test_reading_one(
b'3.23 0n',
qlist(numpy.array([3.23, qnull(QDOUBLE)], dtype=numpy.float64),
qtype=QDOUBLE_LIST))
test_reading_one(
b'(1;`bcd;"0bc";5.5e)',
[numpy.int64(1),
numpy.string_('bcd'), b'0bc',
numpy.float32(5.5)])
test_reading_one(
b'(42;::;`foo)',
[numpy.int64(42), None, numpy.string_('foo')])
test_reading_one(
b'`the`quick`brown`fox',
qlist(numpy.array([
numpy.string_('the'),
numpy.string_('quick'),
numpy.string_('brown'),
numpy.string_('fox')
],
dtype=object),
qtype=QSYMBOL_LIST))
test_reading_one(
b'``quick``fox',
qlist(numpy.array([
qnull(QSYMBOL),
numpy.string_('quick'),
qnull(QSYMBOL),
numpy.string_('fox')
],
dtype=object),
qtype=QSYMBOL_LIST))
test_reading_one(
b'``',
qlist(numpy.array([qnull(QSYMBOL), qnull(QSYMBOL)], dtype=object),
qtype=QSYMBOL_LIST))
test_reading_one(
b'("quick"; "brown"; "fox"; "jumps"; "over"; "a lazy"; "dog")',
[b'quick', b'brown', b'fox', b'jumps', b'over', b'a lazy', b'dog'])
test_reading_one(
b'("quick"; " "; "fox"; "jumps"; "over"; "a lazy"; "dog")',
[b'quick', b' ', b'fox', b'jumps', b'over', b'a lazy', b'dog'])
test_reading_one(b'{x+y}', QLambda('{x+y}')),
test_reading_one(b'{x+y}[3]',
QProjection([QLambda('{x+y}'),
numpy.int64(3)]))
test_reading_one(b'insert [1]', QProjection([QFunction(0),
numpy.int64(1)]))
test_reading_one(b'xbar', QLambda('k){x*y div x:$[16h=abs[@x];"j"$x;x]}'))
test_reading_one(b'not', QFunction(0))
test_reading_one(b'and', QFunction(0))
test_reading_one(b'md5', QProjection([QFunction(0), numpy.int64(-15)]))
test_reading_one(b'any', QFunction(0))
test_reading_one(b'save', QFunction(0))
test_reading_one(b'raze', QFunction(0))
test_reading_one(b'sums', QFunction(0))
test_reading_one(b'prev', QFunction(0))
test_reading_one(
b'(enlist `a)!(enlist 1)',
QDictionary(
qlist(numpy.array(['a']), qtype=QSYMBOL_LIST),
qlist(numpy.array([1], dtype=numpy.int64), qtype=QLONG_LIST)))
test_reading_one(
b'1 2!`abc`cdefgh',
QDictionary(
qlist(numpy.array([1, 2], dtype=numpy.int64), qtype=QLONG_LIST),
qlist(numpy.array(['abc', 'cdefgh']), qtype=QSYMBOL_LIST)))
test_reading_one(
b'`abc`def`gh!([] one: 1 2 3; two: 4 5 6)',
QDictionary(
qlist(numpy.array(['abc', 'def', 'gh']), qtype=QSYMBOL_LIST),
qtable(qlist(numpy.array(['one', 'two']), qtype=QSYMBOL_LIST), [
qlist(numpy.array([1, 2, 3]), qtype=QLONG_LIST),
qlist(numpy.array([4, 5, 6]), qtype=QLONG_LIST)
])))
test_reading_one(
b'(0 1; 2 3)!`first`second',
QDictionary([
qlist(numpy.array([0, 1], dtype=numpy.int64), qtype=QLONG_LIST),
qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST)
], qlist(numpy.array(['first', 'second']), qtype=QSYMBOL_LIST)))
test_reading_one(
b'(1;2h;3.234;"4")!(`one;2 3;"456";(7;8 9))',
QDictionary(
[numpy.int64(1),
numpy.int16(2),
numpy.float64(3.234), b'4'], [
numpy.string_('one'),
qlist(numpy.array([2, 3], dtype=numpy.int64),
qtype=QLONG_LIST), b'456',
[
numpy.int64(7),
qlist(numpy.array([8, 9], dtype=numpy.int64),
qtype=QLONG_LIST)
]
]))
test_reading_one(
b'`A`B`C!((1;3.234;3);(`x`y!(`a;2));5.5e)',
QDictionary(
qlist(numpy.array(['A', 'B', 'C']), qtype=QSYMBOL_LIST),
[[numpy.int64(1),
numpy.float64(3.234),
numpy.int64(3)],
QDictionary(qlist(numpy.array(['x', 'y']), qtype=QSYMBOL_LIST),
[b'a', numpy.int64(2)]),
numpy.float32(5.5)]))
test_reading_one(
b'flip `abc`def!(1 2 3; 4 5 6)',
qtable(qlist(numpy.array(['abc', 'def']), qtype=QSYMBOL_LIST), [
qlist(numpy.array([1, 2, 3]), qtype=QLONG_LIST),
qlist(numpy.array([4, 5, 6]), qtype=QLONG_LIST)
]))
test_reading_one(
b'flip `name`iq!(`Dent`Beeblebrox`Prefect;98 42 126)',
qtable(qlist(numpy.array(['name', 'iq']), qtype=QSYMBOL_LIST), [
qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']),
qtype=QSYMBOL_LIST),
qlist(numpy.array([98, 42, 126]), qtype=QLONG_LIST)
]))
test_reading_one(
b'flip `name`iq`grade!(`Dent`Beeblebrox`Prefect;98 42 126;"a c")',
qtable(qlist(numpy.array(['name', 'iq', 'grade']), qtype=QSYMBOL_LIST),
[
qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']),
qtype=QSYMBOL_LIST),
qlist(numpy.array([98, 42, 126]), qtype=QLONG_LIST), b"a c"
]))
test_reading_one(
b'flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; "Zaphod Beeblebrox"; "Ford Prefect"))',
qtable(
qlist(numpy.array(['name', 'iq', 'fullname']), qtype=QSYMBOL_LIST),
[
qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']),
qtype=QSYMBOL_LIST),
qlist(numpy.array([98, 42, 126]), qtype=QLONG_LIST),
[b"Arthur Dent", b"Zaphod Beeblebrox", b"Ford Prefect"]
]))
test_reading_one(
b'flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; " "; "Ford Prefect"))',
qtable(
qlist(numpy.array(['name', 'iq', 'fullname']), qtype=QSYMBOL_LIST),
[
qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']),
qtype=QSYMBOL_LIST),
qlist(numpy.array([98, 42, 126]), qtype=QLONG_LIST),
[b"Arthur Dent", b" ", b"Ford Prefect"]
]))
test_reading_one(
b'([] sc:1 2 3; nsc:(1 2; 3 4; 5 6 7))',
qtable(qlist(numpy.array(['sc', 'nsc']), qtype=QSYMBOL_LIST), [
qlist(numpy.array([1, 2, 3]), qtype=QLONG_LIST),
[
qlist(numpy.array([1, 2]), qtype=QLONG_LIST),
qlist(numpy.array([3, 4]), qtype=QLONG_LIST),
qlist(numpy.array([5, 6, 7]), qtype=QLONG_LIST)
]
]))
test_reading_one(
b'([] sc:1 2 3; nsc:(1 2; 3 4; 5 6))',
qtable(qlist(numpy.array(['sc', 'nsc']), qtype=QSYMBOL_LIST), [
qlist(numpy.array([1, 2, 3]), qtype=QLONG_LIST),
[
qlist(numpy.array([1, 2]), qtype=QLONG_LIST),
qlist(numpy.array([3, 4]), qtype=QLONG_LIST),
qlist(numpy.array([5, 6]), qtype=QLONG_LIST)
]
]))
test_reading_one(
b'1#([] sym:`x`x`x;str:" a")',
qtable(qlist(numpy.array(['sym', 'str']), qtype=QSYMBOL_LIST), [
qlist(numpy.array(['x'], dtype=numpy.string_), qtype=QSYMBOL_LIST),
b" "
]))
test_reading_one(
b'-1#([] sym:`x`x`x;str:" a")',
qtable(qlist(numpy.array(['sym', 'str']), qtype=QSYMBOL_LIST), [
qlist(numpy.array(['x'], dtype=numpy.string_), qtype=QSYMBOL_LIST),
b"a"
]))
test_reading_one(
b'2#([] sym:`x`x`x`x;str:" aa")',
qtable(qlist(numpy.array(['sym', 'str']), qtype=QSYMBOL_LIST), [
qlist(numpy.array(['x', 'x'], dtype=numpy.string_),
qtype=QSYMBOL_LIST), b" "
]))
test_reading_one(
b'-2#([] sym:`x`x`x`x;str:" aa")',
qtable(qlist(numpy.array(['sym', 'str']), qtype=QSYMBOL_LIST), [
qlist(numpy.array(['x', 'x'], dtype=numpy.string_),
qtype=QSYMBOL_LIST), b"aa"
]))
test_reading_one(
b'([] name:`symbol$(); iq:`int$())',
qtable(qlist(numpy.array(['name', 'iq']), qtype=QSYMBOL_LIST), [
qlist(numpy.array([], dtype=numpy.string_), qtype=QSYMBOL_LIST),
qlist(numpy.array([]), qtype=QINT_LIST)
]))
test_reading_one(
b'([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))',
qtable(qlist(numpy.array(['pos', 'dates']), qtype=QSYMBOL_LIST), [
qlist(numpy.array(['d1', 'd2', 'd3']), qtype=QSYMBOL_LIST),
qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)
]))
test_reading_one(
b'([eid:1001 1002 1003] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))',
QKeyedTable(
qtable(qlist(numpy.array(['eid']), qtype=QSYMBOL_LIST),
[qlist(numpy.array([1001, 1002, 1003]), qtype=QLONG_LIST)]),
qtable(qlist(numpy.array(['pos', 'dates']), qtype=QSYMBOL_LIST), [
qlist(numpy.array(['d1', 'd2', 'd3']), qtype=QSYMBOL_LIST),
qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)
])))
def test_writing():
w = qwriter.QWriter(None, 3)
def test_writing_one(query, value):
sys.stdout.write( '%-75s' % query )
if isinstance(value, tuple):
for obj in value:
sys.stdout.write( '.' )
serialized = binascii.hexlify(w.write(obj, 1))[16:].lower()
assert serialized == BINARY[query].lower(), 'serialization failed: %s, expected: %s actual: %s' % (query, BINARY[query].lower(), serialized)
elif isinstance(value, dict):
sys.stdout.write( '.' )
single_char_strings = value['single_char_strings'] if 'single_char_strings' in value else False
serialized = binascii.hexlify(w.write(value['data'], 1, single_char_strings = single_char_strings))[16:].lower()
assert serialized == BINARY[query].lower(), 'serialization failed: %s, expected: %s actual: %s' % (query, BINARY[query].lower(), serialized)
else:
sys.stdout.write( '.' )
serialized = binascii.hexlify(w.write(value, 1))[16:].lower()
assert serialized == BINARY[query].lower(), 'serialization failed: %s, expected: %s actual: %s' % (query, BINARY[query].lower(), serialized)
print('')
test_writing_one(b'("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)',
qlist(numpy.array([uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661'), qnull(QGUID)]), qtype=QGUID_LIST))
test_writing_one(b'"G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"', uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661'))
test_writing_one(b'"G"$"00000000-0000-0000-0000-000000000000"', uuid.UUID('00000000-0000-0000-0000-000000000000'))
test_writing_one (b'(2001.01m; 0Nm)', (qlist(numpy.array([to_raw_qtemporal(numpy.datetime64('2001-01', 'M'), QMONTH), qnull(QMONTH)]), qtype=QMONTH_LIST),
qlist(numpy.array([12, qnull(QMONTH)]), qtype=QMONTH_LIST),
qlist(array_to_raw_qtemporal(numpy.array([numpy.datetime64('2001-01', 'M'), numpy.datetime64('NaT', 'M')]), qtype = QMONTH_LIST), qtype = QMONTH_LIST),
qlist([12, qnull(QMONTH)], qtype=QMONTH_LIST),
qlist(numpy.array([numpy.datetime64('2001-01'), numpy.datetime64('NaT')], dtype='datetime64[M]'), qtype=QMONTH_LIST),
numpy.array([numpy.datetime64('2001-01'), numpy.datetime64('NaT')], dtype='datetime64[M]'),
))
test_writing_one(b'2001.01m', (qtemporal(numpy.datetime64('2001-01', 'M'), qtype=QMONTH),
numpy.datetime64('2001-01', 'M')))
test_writing_one(b'0Nm', (qtemporal(qnull(QMONTH), qtype=QMONTH),
qtemporal(numpy.datetime64('NaT', 'M'), qtype=QMONTH),
numpy.datetime64('NaT', 'M')))
test_writing_one(b'2001.01.01 2000.05.01 0Nd', (qlist(numpy.array([to_raw_qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE), to_raw_qtemporal(numpy.datetime64('2000-05-01', 'D'), qtype=QDATE), qnull(QDATE)]), qtype=QDATE_LIST),
qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST),
qlist(array_to_raw_qtemporal(numpy.array([numpy.datetime64('2001-01-01', 'D'), numpy.datetime64('2000-05-01', 'D'), numpy.datetime64('NaT', 'D')]), qtype = QDATE_LIST), qtype = QDATE_LIST),
qlist([366, 121, qnull(QDATE)], qtype=QDATE_LIST),
qlist(numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]'), qtype=QDATE_LIST),
numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]'),
))
test_writing_one(b'2001.01.01', (qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE),
numpy.datetime64('2001-01-01', 'D')))
test_writing_one(b'0Nd', (qtemporal(qnull(QDATE), qtype=QDATE),
qtemporal(numpy.datetime64('NaT', 'D'), qtype=QDATE),
numpy.datetime64('NaT', 'D')))
test_writing_one(b'2000.01.04T05:36:57.600 0Nz', (qlist(numpy.array([3.234, qnull(QDATETIME)]), qtype=QDATETIME_LIST),
qlist(array_to_raw_qtemporal(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ms'), numpy.datetime64('nat', 'ms')]), qtype=QDATETIME_LIST), qtype=QDATETIME_LIST),
qlist([3.234, qnull(QDATETIME)], qtype=QDATETIME_LIST),
qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ms'), numpy.datetime64('nat', 'ms')]), qtype = QDATETIME_LIST),
numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ms'), numpy.datetime64('nat', 'ms')])
))
test_writing_one(b'2000.01.04T05:36:57.600', (qtemporal(numpy.datetime64('2000-01-04T05:36:57.600', 'ms'), qtype=QDATETIME),
numpy.datetime64('2000-01-04T05:36:57.600', 'ms')))
test_writing_one(b'0Nz', (qtemporal(qnull(QDATETIME), qtype=QDATETIME),
qtemporal(numpy.datetime64('NaT', 'ms'), qtype=QDATETIME),
numpy.datetime64('NaT', 'ms')))
test_writing_one(b'12:01 0Nu', (qlist(numpy.array([721, qnull(QMINUTE)]), qtype=QMINUTE_LIST),
qlist(array_to_raw_qtemporal(numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')]), qtype=QMINUTE_LIST), qtype=QMINUTE_LIST),
qlist([721, qnull(QMINUTE)], qtype=QMINUTE_LIST),
qlist(numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')]), qtype = QMINUTE),
numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')]),
))
test_writing_one(b'12:01', (qtemporal(numpy.timedelta64(721, 'm'), qtype=QMINUTE),
numpy.timedelta64(721, 'm')))
test_writing_one(b'0Nu', (qtemporal(qnull(QMINUTE), qtype=QMINUTE),
qtemporal(numpy.timedelta64('NaT', 'm'), qtype=QMINUTE),
numpy.timedelta64('NaT', 'm')))
test_writing_one(b'12:05:00 0Nv', (qlist(numpy.array([43500, qnull(QSECOND)]), qtype=QSECOND_LIST),
qlist(array_to_raw_qtemporal(numpy.array([numpy.timedelta64(43500, 's'), numpy.timedelta64('nat', 's')]), qtype=QSECOND_LIST), qtype=QSECOND_LIST),
qlist([43500, qnull(QSECOND)], qtype=QSECOND_LIST),
qlist(numpy.array([numpy.timedelta64(43500, 's'), numpy.timedelta64('nat', 's')]), qtype = QSECOND),
numpy.array([numpy.timedelta64(43500, 's'), numpy.timedelta64('nat', 's')])
))
test_writing_one(b'12:05:00', (qtemporal(numpy.timedelta64(43500, 's'), qtype=QSECOND),
numpy.timedelta64(43500, 's')))
test_writing_one(b'0Nv', (qtemporal(qnull(QSECOND), qtype=QSECOND),
qtemporal(numpy.timedelta64('nat', 's'), qtype=QSECOND),
numpy.timedelta64('nat', 's')))
test_writing_one(b'12:04:59.123 0Nt', (qlist(numpy.array([43499123, qnull(QTIME)]), qtype=QTIME_LIST),
qlist([43499123, qnull(QTIME)], qtype=QTIME_LIST),
qlist(numpy.array([numpy.timedelta64(43499123, 'ms'), numpy.timedelta64('nat', 'ms')]), qtype = QTIME_LIST),
numpy.array([numpy.timedelta64(43499123, 'ms'), numpy.timedelta64('nat', 'ms')])
))
test_writing_one(b'12:04:59.123', (qtemporal(numpy.timedelta64(43499123, 'ms'), qtype=QTIME),
numpy.timedelta64(43499123, 'ms')))
test_writing_one(b'0Nt', (qtemporal(qnull(QTIME), qtype=QTIME),
qtemporal(numpy.timedelta64('NaT', 'ms'), qtype=QTIME),
numpy.timedelta64('NaT', 'ms')))
test_writing_one(b'2000.01.04D05:36:57.600 0Np', (qlist(numpy.array([long(279417600000000), qnull(QTIMESTAMP)]), qtype=QTIMESTAMP_LIST),
qlist(array_to_raw_qtemporal(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ns'), numpy.datetime64('nat', 'ns')]), qtype=QTIMESTAMP_LIST), qtype=QTIMESTAMP_LIST),
qlist([long(279417600000000), qnull(QTIMESTAMP)], qtype=QTIMESTAMP_LIST),
qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ns'), numpy.datetime64('nat', 'ns')]), qtype = QTIMESTAMP_LIST),
numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ns'), numpy.datetime64('nat', 'ns')])
))
test_writing_one(b'2000.01.04D05:36:57.600', (qtemporal(numpy.datetime64('2000-01-04T05:36:57.600', 'ns'), qtype=QTIMESTAMP),
numpy.datetime64('2000-01-04T05:36:57.600', 'ns')))
test_writing_one(b'0Np', (qtemporal(qnull(QTIMESTAMP), qtype=QTIMESTAMP),
qtemporal(numpy.datetime64('NaT', 'ns'), qtype=QTIMESTAMP),
numpy.datetime64('NaT', 'ns')))
test_writing_one(b'0D05:36:57.600 0Nn', (qlist(numpy.array([long(20217600000000), qnull(QTIMESPAN)]), qtype=QTIMESPAN_LIST),
qlist(array_to_raw_qtemporal(numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')]), qtype=QTIMESPAN_LIST), qtype=QTIMESPAN_LIST),
qlist([long(20217600000000), qnull(QTIMESPAN)], qtype=QTIMESPAN_LIST),
qlist(numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')]), qtype = QTIMESPAN_LIST),
numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')])
))
test_writing_one(b'0D05:36:57.600', (qtemporal(numpy.timedelta64(20217600000000, 'ns'), qtype=QTIMESPAN),
numpy.timedelta64(20217600000000, 'ns')))
test_writing_one(b'0Nn', (qtemporal(qnull(QTIMESPAN), qtype=QTIMESPAN),
qtemporal(numpy.timedelta64('NaT', 'ns'), qtype=QTIMESPAN),
numpy.timedelta64('NaT', 'ns')))
test_writing_one(b'::', None)
test_writing_one(b'1+`', QException('type'))
test_writing_one(b'1', numpy.int64(1))
test_writing_one(b'1i', numpy.int32(1))
test_writing_one(b'-234h', numpy.int16(-234))
test_writing_one(b'0b', numpy.bool_(False))
test_writing_one(b'1b', numpy.bool_(True))
test_writing_one(b'0x2a', numpy.byte(0x2a))
test_writing_one(b'89421099511627575j', numpy.int64(long(89421099511627575)))
test_writing_one(b'5.5e', numpy.float32(5.5))
test_writing_one(b'3.234', numpy.float64(3.234))
test_writing_one(b'"0"', '0')
test_writing_one(b'"abc"', ('abc',
numpy.array(list('abc'), dtype='S')))
test_writing_one(b'"quick brown fox jumps over a lazy dog"', 'quick brown fox jumps over a lazy dog')
test_writing_one(b'`abc', numpy.string_('abc'))
test_writing_one(b'`quickbrownfoxjumpsoveralazydog', numpy.string_('quickbrownfoxjumpsoveralazydog'))
test_writing_one(b'0Nh', qnull(QSHORT))
test_writing_one(b'0N', qnull(QLONG))
test_writing_one(b'0Ni', qnull(QINT))
test_writing_one(b'0Nj', qnull(QLONG))
test_writing_one(b'0Ne', qnull(QFLOAT))
test_writing_one(b'0n', qnull(QDOUBLE))
test_writing_one(b'" "', qnull(QSTRING))
test_writing_one(b'`', qnull(QSYMBOL))
test_writing_one(b'0Ng', qnull(QGUID))
test_writing_one(b'()', [])
test_writing_one(b'(0b;1b;0b)', (numpy.array([False, True, False], dtype=numpy.bool_),
qlist(numpy.array([False, True, False]), qtype = QBOOL_LIST),
qlist([False, True, False], qtype = QBOOL_LIST)))
test_writing_one(b'(0x01;0x02;0xff)', (numpy.array([0x01, 0x02, 0xff], dtype=numpy.byte),
qlist(numpy.array([0x01, 0x02, 0xff], dtype=numpy.byte), qtype = QBYTE_LIST),
qlist(numpy.array([0x01, 0x02, 0xff]), qtype = QBYTE_LIST),
qlist([0x01, 0x02, 0xff], qtype = QBYTE_LIST)))
test_writing_one(b'(1h;2h;3h)', (numpy.array([1, 2, 3], dtype=numpy.int16),
qlist(numpy.array([1, 2, 3], dtype=numpy.int16), qtype = QSHORT_LIST),
qlist(numpy.array([1, 2, 3]), qtype = QSHORT_LIST),
qlist([1, 2, 3], qtype = QSHORT_LIST)))
test_writing_one(b'(1h;0Nh;3h)', qlist(numpy.array([1, qnull(QSHORT), 3], dtype=numpy.int16), qtype=QSHORT_LIST))
test_writing_one(b'1 2 3', (numpy.array([1, 2, 3], dtype=numpy.int64),
qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype = QLONG_LIST),
qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST),
qlist([1, 2, 3], qtype = QLONG_LIST)))
test_writing_one(b'1 0N 3', qlist(numpy.array([1, qnull(QLONG), 3], dtype=numpy.int64), qtype=QLONG_LIST))
test_writing_one(b'(1i;2i;3i)', (numpy.array([1, 2, 3], dtype=numpy.int32),
qlist(numpy.array([1, 2, 3], dtype=numpy.int32), qtype = QINT_LIST),
qlist(numpy.array([1, 2, 3]), qtype = QINT_LIST),
qlist([1, 2, 3], qtype = QINT_LIST)))
test_writing_one(b'(1i;0Ni;3i)', qlist(numpy.array([1, qnull(QINT), 3], dtype=numpy.int32), qtype=QINT_LIST))
test_writing_one(b'(1j;2j;3j)', (numpy.array([1, 2, 3], dtype=numpy.int64),
qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype = QLONG_LIST),
qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST),
qlist([1, 2, 3], qtype = QLONG_LIST)))
test_writing_one(b'(1j;0Nj;3j)', qlist(numpy.array([1, qnull(QLONG), 3], dtype=numpy.int64), qtype=QLONG_LIST))
test_writing_one(b'(5.5e; 8.5e)', (numpy.array([5.5, 8.5], dtype=numpy.float32),
qlist(numpy.array([5.5, 8.5], dtype=numpy.float32), qtype = QFLOAT_LIST),
qlist(numpy.array([5.5, 8.5]), qtype = QFLOAT_LIST),
qlist([5.5, 8.5], qtype = QFLOAT_LIST)))
test_writing_one(b'(5.5e; 0Ne)', qlist(numpy.array([5.5, qnull(QFLOAT)], dtype=numpy.float32), qtype=QFLOAT_LIST))
test_writing_one(b'3.23 6.46', (numpy.array([3.23, 6.46], dtype=numpy.float64),
qlist(numpy.array([3.23, 6.46], dtype=numpy.float64), qtype = QDOUBLE_LIST),
qlist(numpy.array([3.23, 6.46]), qtype = QDOUBLE_LIST),
qlist([3.23, 6.46], qtype = QDOUBLE_LIST)))
test_writing_one(b'3.23 0n', qlist(numpy.array([3.23, qnull(QDOUBLE)], dtype=numpy.float64), qtype=QDOUBLE_LIST))
test_writing_one(b'(1;`bcd;"0bc";5.5e)', [numpy.int64(1), numpy.string_('bcd'), '0bc', numpy.float32(5.5)])
test_writing_one(b'(42;::;`foo)', [numpy.int64(42), None, numpy.string_('foo')])
test_writing_one(b'(1;2h;3.234;"4")', [numpy.int64(1), numpy.int16(2), numpy.float64(3.234), '4'])
test_writing_one(b'(`one;2 3;"456";(7;8 9))', [numpy.string_('one'), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST), '456', [numpy.int64(7), qlist(numpy.array([8, 9], dtype=numpy.int64), qtype=QLONG_LIST)]])
test_writing_one(b'`jumps`over`a`lazy`dog', (numpy.array(['jumps', 'over', 'a', 'lazy', 'dog'], dtype=numpy.string_),
qlist(numpy.array(['jumps', 'over', 'a', 'lazy', 'dog']), qtype = QSYMBOL_LIST),
qlist(['jumps', 'over', 'a', 'lazy', 'dog'], qtype = QSYMBOL_LIST)))
test_writing_one(b'`the`quick`brown`fox', numpy.array([numpy.string_('the'), numpy.string_('quick'), numpy.string_('brown'), numpy.string_('fox')], dtype=object))
test_writing_one(b'``quick``fox', qlist(numpy.array([qnull(QSYMBOL), numpy.string_('quick'), qnull(QSYMBOL), numpy.string_('fox')], dtype=object), qtype=QSYMBOL_LIST))
test_writing_one(b'``', qlist(numpy.array([qnull(QSYMBOL), qnull(QSYMBOL)], dtype=object), qtype=QSYMBOL_LIST))
test_writing_one(b'("quick"; "brown"; "fox"; "jumps"; "over"; "a lazy"; "dog")',
(['quick', 'brown', 'fox', 'jumps', 'over', 'a lazy', 'dog'],
qlist(numpy.array(['quick', 'brown', 'fox', 'jumps', 'over', 'a lazy', 'dog']), qtype = QSTRING_LIST),
qlist(['quick', 'brown', 'fox', 'jumps', 'over', 'a lazy', 'dog'], qtype = QSTRING_LIST)))
test_writing_one(b'{x+y}', QLambda('{x+y}'))
test_writing_one(b'{x+y}[3]', QProjection([QLambda('{x+y}'), numpy.int64(3)]))
test_writing_one(b'(enlist `a)!(enlist 1)', (QDictionary(qlist(numpy.array(['a']), qtype = QSYMBOL_LIST),
qlist(numpy.array([1], dtype=numpy.int64), qtype=QLONG_LIST)),
QDictionary(qlist(numpy.array(['a']), qtype = QSYMBOL_LIST),
qlist(numpy.array([1]), qtype=QLONG_LIST))))
test_writing_one(b'1 2!`abc`cdefgh', QDictionary(qlist(numpy.array([1, 2], dtype=numpy.int64), qtype=QLONG_LIST),
qlist(numpy.array(['abc', 'cdefgh']), qtype = QSYMBOL_LIST)))
test_writing_one(b'`abc`def`gh!([] one: 1 2 3; two: 4 5 6)', QDictionary(qlist(numpy.array(['abc', 'def', 'gh']), qtype = QSYMBOL_LIST),
qtable(qlist(numpy.array(['one', 'two']), qtype = QSYMBOL_LIST),
[qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST),
qlist(numpy.array([4, 5, 6]), qtype = QLONG_LIST)])))
test_writing_one(b'(`x`y!(`a;2))', QDictionary(qlist(numpy.array(['x', 'y']), qtype = QSYMBOL_LIST),
[numpy.string_('a'), numpy.int64(2)]))
test_writing_one(b'(0 1; 2 3)!`first`second', QDictionary([qlist(numpy.array([0, 1], dtype=numpy.int64), qtype=QLONG_LIST), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST)],
qlist(numpy.array(['first', 'second']), qtype = QSYMBOL_LIST)))
test_writing_one(b'(1;2h;3.234;"4")!(`one;2 3;"456";(7;8 9))', QDictionary([numpy.int64(1), numpy.int16(2), numpy.float64(3.234), '4'],
[numpy.string_('one'), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST), '456', [numpy.int64(7), qlist(numpy.array([8, 9], dtype=numpy.int64), qtype=QLONG_LIST)]]))
test_writing_one(b'`A`B`C!((1;3.234;3);(`x`y!(`a;2));5.5e)', QDictionary(qlist(numpy.array(['A', 'B', 'C']), qtype = QSYMBOL_LIST),
[[numpy.int64(1), numpy.float64(3.234), numpy.int64(3)], QDictionary(qlist(numpy.array(['x', 'y']), qtype = QSYMBOL_LIST), [numpy.string_('a'), numpy.int64(2)]), numpy.float32(5.5)]))
test_writing_one(b'flip `abc`def!(1 2 3; 4 5 6)', (qtable(qlist(numpy.array(['abc', 'def']), qtype = QSYMBOL_LIST),
[qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype=QLONG_LIST),
qlist(numpy.array([4, 5, 6], dtype=numpy.int64), qtype=QLONG_LIST)],
qtype=QTABLE),
qtable(qlist(numpy.array(['abc', 'def']), qtype = QSYMBOL_LIST),
[qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST),
qlist(numpy.array([4, 5, 6]), qtype = QLONG_LIST)]),
qtable(qlist(['abc', 'def'], qtype = QSYMBOL_LIST),
[qlist([1, 2, 3], qtype = QLONG_LIST),
qlist([4, 5, 6], qtype = QLONG_LIST)]),
qtable(qlist(['abc', 'def'], qtype = QSYMBOL_LIST),
[qlist([1, 2, 3]), qlist([4, 5, 6])],
**{'abc': QLONG_LIST, 'def': QLONG_LIST}),
qtable(['abc', 'def'],
[[1, 2, 3], [4, 5, 6]],
**{'abc': QLONG, 'def': QLONG})))
test_writing_one(b'flip `name`iq!(`Dent`Beeblebrox`Prefect;98 42 126)',
(qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST),
[qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST),
qlist(numpy.array([98, 42, 126], dtype=numpy.int64), qtype = QLONG_LIST)]),
qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST),
[qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST),
qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST)]),
qtable(qlist(['name', 'iq'], qtype = QSYMBOL_LIST),
[qlist(['Dent', 'Beeblebrox', 'Prefect'], qtype = QSYMBOL_LIST),
qlist([98, 42, 126], qtype = QLONG_LIST)]),
qtable(qlist(['name', 'iq'], qtype = QSYMBOL_LIST),
[qlist(['Dent', 'Beeblebrox', 'Prefect']),
qlist([98, 42, 126])],
name = QSYMBOL, iq = QLONG),
qtable(['name', 'iq'],
[['Dent', 'Beeblebrox', 'Prefect'],
[98, 42, 126]],
name = QSYMBOL, iq = QLONG),
qtable(['name', 'iq'],
[['Dent', 'Beeblebrox', 'Prefect'],
[98, 42, 126]],
**{'name': QSYMBOL, 'iq': QLONG})))
test_writing_one(b'flip `name`iq`grade!(`Dent`Beeblebrox`Prefect;98 42 126;"a c")',
qtable(qlist(numpy.array(['name', 'iq', 'grade']), qtype = QSYMBOL_LIST),
[qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST),
qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST),
"a c"]))
test_writing_one(b'flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; "Zaphod Beeblebrox"; "Ford Prefect"))',
qtable(qlist(numpy.array(['name', 'iq', 'fullname']), qtype = QSYMBOL_LIST),
[qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST),
qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST),
qlist(numpy.array(["Arthur Dent", "Zaphod Beeblebrox", "Ford Prefect"]), qtype = QSTRING_LIST)]))
test_writing_one(b'flip `name`iq`misc!(`Dent`Beeblebrox`Prefect;98 42 126;("The Hitch Hiker\'s Guide to the Galaxy"; 160; 1979.10.12))',
qtable(qlist(numpy.array(['name', 'iq', 'misc']), qtype = QSYMBOL_LIST),
[qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST),
qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST),
qlist(numpy.array(["The Hitch Hiker\'s Guide to the Galaxy", long(160), qtemporal(numpy.datetime64('1979-10-12', 'D'), qtype=QDATE)]), qtype = QGENERAL_LIST)]))
test_writing_one(b'([] sc:1 2 3; nsc:(1 2; 3 4; 5 6 7))', (qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST),
[qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype = QLONG_LIST),
[qlist(numpy.array([1, 2], dtype=numpy.int64), qtype = QLONG_LIST),
qlist(numpy.array([3, 4], dtype=numpy.int64), qtype = QLONG_LIST),
qlist(numpy.array([5, 6, 7], dtype=numpy.int64), qtype = QLONG_LIST)]]),
qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST),
[qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST),
[qlist(numpy.array([1, 2]), qtype = QLONG_LIST),
qlist(numpy.array([3, 4]), qtype = QLONG_LIST),
qlist(numpy.array([5, 6, 7]), qtype = QLONG_LIST)]]),
qtable(qlist(['sc', 'nsc'], qtype = QSYMBOL_LIST),
[qlist([1, 2, 3], qtype = QLONG_LIST),
[qlist([1, 2], qtype = QLONG_LIST),
qlist([3, 4], qtype = QLONG_LIST),
qlist([5, 6, 7], qtype = QLONG_LIST)]])))
test_writing_one(b'([] sc:1 2 3; nsc:(1 2; 3 4; 5 6))', qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST),
[qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST),
[qlist(numpy.array([1, 2]), qtype = QLONG_LIST),
qlist(numpy.array([3, 4]), qtype = QLONG_LIST),
qlist(numpy.array([5, 6]), qtype = QLONG_LIST)]]))
test_writing_one(b'1#([] sym:`x`x`x;str:" a")', {'data': qtable(qlist(numpy.array(['sym', 'str']), qtype = QSYMBOL_LIST),
[qlist(numpy.array(['x'], dtype=numpy.string_), qtype = QSYMBOL_LIST),
b" "]),
'single_char_strings': True
})
test_writing_one(b'-1#([] sym:`x`x`x;str:" a")', {'data': qtable(qlist(numpy.array(['sym', 'str']), qtype = QSYMBOL_LIST),
[qlist(numpy.array(['x'], dtype=numpy.string_), qtype = QSYMBOL_LIST),
b"a"]),
'single_char_strings': True
})
test_writing_one(b'2#([] sym:`x`x`x`x;str:" aa")', qtable(qlist(numpy.array(['sym', 'str']), qtype = QSYMBOL_LIST),
[qlist(numpy.array(['x', 'x'], dtype=numpy.string_), qtype = QSYMBOL_LIST),
b" "]))
test_writing_one(b'-2#([] sym:`x`x`x`x;str:" aa")', qtable(qlist(numpy.array(['sym', 'str']), qtype = QSYMBOL_LIST),
[qlist(numpy.array(['x', 'x'], dtype=numpy.string_), qtype = QSYMBOL_LIST),
b"aa"]))
test_writing_one(b'([] name:`symbol$(); iq:`int$())', (qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST),
[qlist(numpy.array([], dtype=numpy.string_), qtype = QSYMBOL_LIST),
qlist(numpy.array([], dtype=numpy.int32), qtype = QINT_LIST)]),
qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST),
[qlist(numpy.array([]), qtype = QSYMBOL_LIST),
qlist(numpy.array([]), qtype = QINT_LIST)]),
qtable(qlist(['name', 'iq'], qtype = QSYMBOL_LIST),
[qlist([], qtype = QSYMBOL_LIST),
qlist([], qtype = QINT_LIST)])))
test_writing_one(b'([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))',
(qtable(qlist(numpy.array(['pos', 'dates']), qtype = QSYMBOL_LIST),
[qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST),
qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)]),
qtable(['pos', 'dates'],
[qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST),
numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]')])
))
test_writing_one(b'([eid:1001 1002 1003] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))',
QKeyedTable(qtable(qlist(numpy.array(['eid']), qtype = QSYMBOL_LIST),
[qlist(numpy.array([1001, 1002, 1003]), qtype = QLONG_LIST)]),
qtable(qlist(numpy.array(['pos', 'dates']), qtype = QSYMBOL_LIST),
[qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST),
qlist(numpy.array([366, 121, qnull(QDATE)]), qtype = QDATE_LIST)]))
)
def test_reading_numpy_temporals():
BINARY = OrderedDict()
with open('tests/QExpressions3.out', 'rb') as f:
while True:
query = f.readline().strip()
binary = f.readline().strip()
if not binary:
break
BINARY[query] = binary
def test_reading_numpy_temporals_one(query, value):
buffer_ = BytesIO()
binary = binascii.unhexlify(BINARY[query])
buffer_.write(b'\1\0\0\0')
buffer_.write(struct.pack('i', len(binary) + 8))
buffer_.write(binary)
buffer_.seek(0)
sys.stdout.write(' %-75s' % query)
try:
buffer_.seek(0)
stream_reader = qreader.QReader(buffer_)
result = stream_reader.read(numpy_temporals=True).data
assert compare(
value, result
), 'deserialization failed: %s, expected: %s actual: %s' % (
query, value, result)
print('.')
except QException as e:
assert isinstance(value, QException)
assert e.args == value.args
print('.')
print('Deserialization (numpy temporals)')
test_reading_numpy_temporals_one(
b'(2001.01m; 0Nm)',
qlist(numpy.array(
[numpy.datetime64('2001-01'),
numpy.datetime64('NaT')],
dtype='datetime64[M]'),
qtype=QMONTH_LIST))
test_reading_numpy_temporals_one(b'2001.01m',
numpy.datetime64('2001-01', 'M'))
test_reading_numpy_temporals_one(b'0Nm', numpy.datetime64('NaT', 'M'))
test_reading_numpy_temporals_one(
b'2001.01.01 2000.05.01 0Nd',
qlist(numpy.array([
numpy.datetime64('2001-01-01'),
numpy.datetime64('2000-05-01'),
numpy.datetime64('NaT')
],
dtype='datetime64[D]'),
qtype=QDATE_LIST))
test_reading_numpy_temporals_one(b'2001.01.01',
numpy.datetime64('2001-01-01', 'D'))
test_reading_numpy_temporals_one(b'0Nd', numpy.datetime64('NaT', 'D'))
test_reading_numpy_temporals_one(
b'2000.01.04T05:36:57.600 0Nz',
qlist(numpy.array([
numpy.datetime64('2000-01-04T05:36:57.600', 'ms'),
numpy.datetime64('nat', 'ms')
]),
qtype=QDATETIME_LIST))
test_reading_numpy_temporals_one(
b'2000.01.04T05:36:57.600',
numpy.datetime64('2000-01-04T05:36:57.600', 'ms'))
test_reading_numpy_temporals_one(b'0Nz', numpy.datetime64('NaT', 'ms'))
test_reading_numpy_temporals_one(
b'12:01 0Nu',
qlist(numpy.array(
[numpy.timedelta64(721, 'm'),
numpy.timedelta64('nat', 'm')]),
qtype=QMINUTE))
test_reading_numpy_temporals_one(b'12:01', numpy.timedelta64(721, 'm'))
test_reading_numpy_temporals_one(b'0Nu', numpy.timedelta64('NaT', 'm'))
test_reading_numpy_temporals_one(
b'12:05:00 0Nv',
qlist(numpy.array(
[numpy.timedelta64(43500, 's'),
numpy.timedelta64('nat', 's')]),
qtype=QSECOND))
test_reading_numpy_temporals_one(b'12:05:00',
numpy.timedelta64(43500, 's'))
test_reading_numpy_temporals_one(b'0Nv', numpy.timedelta64('nat', 's'))
test_reading_numpy_temporals_one(
b'12:04:59.123 0Nt',
qlist(numpy.array([
numpy.timedelta64(43499123, 'ms'),
numpy.timedelta64('nat', 'ms')
]),
qtype=QTIME_LIST))
test_reading_numpy_temporals_one(b'12:04:59.123',
numpy.timedelta64(43499123, 'ms'))
test_reading_numpy_temporals_one(b'0Nt', numpy.timedelta64('NaT', 'ms'))
test_reading_numpy_temporals_one(
b'2000.01.04D05:36:57.600 0Np',
qlist(numpy.array([
numpy.datetime64('2000-01-04T05:36:57.600', 'ns'),
numpy.datetime64('nat', 'ns')
]),
qtype=QTIMESTAMP_LIST))
test_reading_numpy_temporals_one(
b'2000.01.04D05:36:57.600',
numpy.datetime64('2000-01-04T05:36:57.600', 'ns'))
test_reading_numpy_temporals_one(b'0Np', numpy.datetime64('NaT', 'ns'))
test_reading_numpy_temporals_one(
b'0D05:36:57.600 0Nn',
qlist(numpy.array([
numpy.timedelta64(20217600000000, 'ns'),
numpy.timedelta64('nat', 'ns')
]),
qtype=QTIMESPAN_LIST))
test_reading_numpy_temporals_one(b'0D05:36:57.600',
numpy.timedelta64(20217600000000, 'ns'))
test_reading_numpy_temporals_one(b'0Nn', numpy.timedelta64('NaT', 'ns'))
test_reading_numpy_temporals_one(
b'([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))',
qtable(['pos', 'dates'], [
qlist(numpy.array(['d1', 'd2', 'd3']), qtype=QSYMBOL_LIST),
numpy.array([
numpy.datetime64('2001-01-01'),
numpy.datetime64('2000-05-01'),
numpy.datetime64('NaT')
],
dtype='datetime64[D]')
]))
('2001.01.01 2000.05.01 0Nd', {'data': pandas.Series(numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]')),
'meta': MetaData(qtype = QDATE_LIST) }),
('2000.01.04T05:36:57.600 0Nz', {'data': pandas.Series(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')])),
'meta': MetaData(qtype = QDATETIME_LIST) }),
('12:01 0Nu', {'data': pandas.Series(numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')])),
'meta': MetaData(qtype = QMINUTE_LIST) }),
('12:05:00 0Nv', {'data': pandas.Series(numpy.array([numpy.timedelta64(43500, 's'), numpy.timedelta64('nat', 's')])),
'meta': MetaData(qtype = QSECOND_LIST) }),
('12:04:59.123 0Nt', {'data': pandas.Series(numpy.array([numpy.timedelta64(43499123, 'ms'), numpy.timedelta64('nat', 'ms')])),
'meta': MetaData(qtype = QTIME_LIST) }),
('2000.01.04D05:36:57.600 0Np', {'data': pandas.Series(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), numpy.datetime64('nat', 'ns')])),
'meta': MetaData(qtype = QTIMESTAMP_LIST) }),
('0D05:36:57.600 0Nn', {'data': pandas.Series(numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')])),
'meta': MetaData(qtype = QTIMESPAN_LIST) }),
('1 2!`abc`cdefgh', QDictionary(qlist(numpy.array([1, 2], dtype=numpy.int64), qtype=QLONG_LIST),
qlist(numpy.array(['abc', 'cdefgh']), qtype = QSYMBOL_LIST))),
('(0 1; 2 3)!`first`second', QDictionary([qlist(numpy.array([0, 1], dtype=numpy.int64), qtype=QLONG_LIST), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST)],
qlist(numpy.array(['first', 'second']), qtype = QSYMBOL_LIST))),
('(1;2h;3.234;"4")!(`one;2 3;"456";(7;8 9))', QDictionary([numpy.int64(1), numpy.int16(2), numpy.float64(3.234), '4'],
[numpy.string_('one'), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST), '456', [numpy.int64(7), qlist(numpy.array([8, 9], dtype=numpy.int64), qtype=QLONG_LIST)]])),
('`A`B`C!((1;3.234;3);(`x`y!(`a;2));5.5e)', QDictionary(qlist(numpy.array(['A', 'B', 'C']), qtype = QSYMBOL_LIST),
[[numpy.int64(1), numpy.float64(3.234), numpy.int64(3)], QDictionary(qlist(numpy.array(['x', 'y']), qtype = QSYMBOL_LIST), [numpy.string_('a'), numpy.int64(2)]), numpy.float32(5.5)])),
('flip `abc`def!(1 2 3; 4 5 6)', {'data': pandas.DataFrame(OrderedDict((('abc', pandas.Series(numpy.array([1, 2, 3], dtype = numpy.int64))),
('def', pandas.Series(numpy.array([4, 5, 6], dtype = numpy.int64)))))
),
'meta': MetaData(**{'qtype': QTABLE, 'abc': QLONG_LIST, 'def': QLONG_LIST}) }),
('flip `name`iq!(`Dent`Beeblebrox`Prefect;98 42 126)',
{'data': pandas.DataFrame(OrderedDict((('name', pandas.Series(['Dent', 'Beeblebrox', 'Prefect'])),
('iq', pandas.Series(numpy.array([98, 42, 126], dtype = numpy.int64)))))
from io import BytesIO
if sys.version > '3':
long = int
from collections import OrderedDict
from qpython import qreader
from qpython.qtype import * # @UnusedWildImport
from qpython.qcollection import qlist, QList, QTemporalList, QDictionary, qtable, QKeyedTable
from qpython.qtemporal import qtemporal, QTemporal
EXPRESSIONS = OrderedDict((
(b'("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)',
qlist(numpy.array([uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661'), qnull(QGUID)]), qtype=QGUID_LIST)),
(b'"G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"', uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661')),
(b'"G"$"00000000-0000-0000-0000-000000000000"', uuid.UUID('00000000-0000-0000-0000-000000000000')),
(b'(2001.01m; 0Nm)', qlist(numpy.array([12, qnull(QMONTH)]), qtype=QMONTH_LIST)),
(b'2001.01m', qtemporal(numpy.datetime64('2001-01', 'M'), qtype=QMONTH)),
(b'0Nm', qtemporal(numpy.datetime64('NaT', 'M'), qtype=QMONTH)),
(b'2001.01.01 2000.05.01 0Nd', qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)),
(b'2001.01.01', qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE)),
(b'0Nd', qtemporal(numpy.datetime64('NaT', 'D'), qtype=QDATE)),
(b'2000.01.04T05:36:57.600 0Nz', qlist(numpy.array([3.234, qnull(QDATETIME)]), qtype=QDATETIME_LIST)),
(b'2000.01.04T05:36:57.600', qtemporal(numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), qtype=QDATETIME)),
(b'0Nz', qtemporal(numpy.datetime64('NaT', 'ms'), qtype=QDATETIME)),
(b'12:01 0Nu', qlist(numpy.array([721, qnull(QMINUTE)]), qtype=QMINUTE_LIST)),
(b'12:01', qtemporal(numpy.timedelta64(721, 'm'), qtype=QMINUTE)),
(b'0Nu', qtemporal(numpy.timedelta64('NaT', 'm'), qtype=QMINUTE)),
(b'12:05:00 0Nv', qlist(numpy.array([43500, qnull(QSECOND)]), qtype=QSECOND_LIST)),
'meta':
MetaData(qtype=QTIMESTAMP_LIST)
}),
('0D05:36:57.600 0Nn', {
'data':
pandas.Series(
numpy.array([
numpy.timedelta64(20217600000000, 'ns'),
numpy.timedelta64('nat', 'ns')
])),
'meta':
MetaData(qtype=QTIMESPAN_LIST)
}),
('1 2!`abc`cdefgh',
QDictionary(
qlist(numpy.array([1, 2], dtype=numpy.int64), qtype=QLONG_LIST),
qlist(numpy.array(['abc', 'cdefgh']), qtype=QSYMBOL_LIST))),
('(0 1; 2 3)!`first`second',
QDictionary([
qlist(numpy.array([0, 1], dtype=numpy.int64), qtype=QLONG_LIST),
qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST)
], qlist(numpy.array(['first', 'second']), qtype=QSYMBOL_LIST))),
('(1;2h;3.234;"4")!(`one;2 3;"456";(7;8 9))',
QDictionary(
[numpy.int64(1),
numpy.int16(2),
numpy.float64(3.234), '4'], [
numpy.string_('one'),
qlist(numpy.array([2, 3], dtype=numpy.int64),
qtype=QLONG_LIST), '456',
[
#
import binascii
import sys
from collections import OrderedDict
from qpython import qwriter
from qpython.qtype import * # @UnusedWildImport
from qpython.qcollection import qlist, QDictionary, qtable, QKeyedTable
from qpython.qtemporal import qtemporal, qtemporallist, to_raw_qtemporal
BINARY = OrderedDict()
EXPRESSIONS = OrderedDict((
('("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)',
qlist(numpy.array([uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661'), qnull(QGUID)]), qtype=QGUID_LIST)),
('"G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"', uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661')),
('"G"$"00000000-0000-0000-0000-000000000000"', uuid.UUID('00000000-0000-0000-0000-000000000000')),
('(2001.01m; 0Nm)', (qtemporallist(numpy.array([to_raw_qtemporal(numpy.datetime64('2001-01', 'M'), QMONTH), qnull(QMONTH)]), qtype=QMONTH_LIST),
qtemporallist(numpy.array([12, qnull(QMONTH)]), qtype=QMONTH_LIST))),
('2001.01m', qtemporal(numpy.datetime64('2001-01', 'M'), qtype=QMONTH)),
('0Nm', qtemporal(qnull(QMONTH), qtype=QMONTH)),
('2001.01.01 2000.05.01 0Nd', (qtemporallist(numpy.array([to_raw_qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE), to_raw_qtemporal(numpy.datetime64('2000-05-01', 'D'), qtype=QDATE), qnull(QDATE)]), qtype=QDATE_LIST),
qtemporallist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST))),
('2001.01.01', qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE)),
('0Nd', qtemporal(qnull(QDATE), qtype=QDATE)),
('2000.01.04T05:36:57.600 0Nz', qtemporallist(numpy.array([3.234, qnull(QDATETIME)]), qtype=QDATETIME_LIST)),
('2000.01.04T05:36:57.600', qtemporal(numpy.datetime64('2000-01-04T05:36:57.600', 'ms'), qtype=QDATETIME)),
('0Nz', qtemporal(qnull(QDATETIME), qtype=QDATETIME)),
('12:01 0Nu', qtemporallist(numpy.array([721, qnull(QMINUTE)]), qtype=QMINUTE_LIST)),
('12:01', qtemporal(numpy.timedelta64(721, 'm'), qtype=QMINUTE)),
data = csv.reader(f)
#go through each line in file and read in the date
for row in data:
newT = numpy.datetime64(
datetime.combine(
date.today(),
datetime.strptime(row[0][2:-3], '%H:%M:%S.%f').time()))
newT = numpy.timedelta64((numpy.datetime64(newT) - today), 'ns')
time.append(newT)
sym = numpy.append(sym, row[1])
price = numpy.append(price, float(row[2]))
size = numpy.append(size, int(float(row[3])))
exc = numpy.append(exc, row[4])
#Convert each python dataType to their kdb+ equivalent
tme = qlist(time, qtype=QTIMESPAN_LIST)
sm = qlist(sym, qtype=QSYMBOL_LIST)
prc = qlist(price, qtype=QDOUBLE_LIST)
sze = qlist(size, qtype=QLONG_LIST)
ex = qlist(exc, qtype=QSYMBOL_LIST)
data = [tme, sm, prc, sze, ex]
#Open connection to tp
with qconnection.QConnection(host='localhost', port=5000) as q:
print(q)
print('IPC version: %s. Is connected: %s' %
(q.protocol_version, q.is_connected()))
q.sendSync('.u.upd', numpy.string_('bid'), data)
#call upd function on tp
