def _read_dictionary(self,
qtype=QDICTIONARY,
options=READER_CONFIGURATION):
if options.pandas:
keys = self._read_object(options=options)
values = self._read_object(options=options)
if isinstance(keys, pandas.DataFrame):
if not isinstance(values, pandas.DataFrame):
raise QReaderException(
'Keyed table creation: values are expected to be of type pandas.DataFrame. Actual: %s'
% type(values))
indices = keys.columns
table = keys
table.meta = keys.meta
table.meta.qtype = QKEYED_TABLE
for column in values.columns:
table[column] = values[column]
table.meta[column] = values.meta[column]
table.set_index([column for column in indices], inplace=True)
return table
else:
keys = keys if not isinstance(
keys, pandas.Series) else keys.as_matrix()
values = values if not isinstance(
values, pandas.Series) else values.as_matrix()
return QDictionary(keys, values)
else:
return QReader._read_dictionary(self, qtype=qtype, options=options)
def _read_list(self, qtype):
if self._options.pandas:
self._options.numpy_temporals = True
qlist = QReader._read_list(self, qtype=qtype)
if self._options.pandas:
atom_qtype = -abs(qtype)
if atom_qtype not in [
QMONTH, QDATE, QDATETIME, QMINUTE, QSECOND, QTIME,
QTIMESTAMP, QTIMESPAN, QSYMBOL, QFLOAT, QDOUBLE, QGUID
]:
null = QNULLMAP[atom_qtype][1]
if atom_qtype in PANDAS_TYPE:
ps = pandas.Series(data=qlist,
dtype=PANDAS_TYPE[atom_qtype]).replace(
null, pandas.NA)
else:
ps = pandas.Series(data=qlist).replace(null, pandas.NA)
else:
ps = pandas.Series(data=qlist)
ps.meta = MetaData(qtype=qtype)
return ps
else:
return qlist
def _read_dictionary(self, qtype = QDICTIONARY, options = READER_CONFIGURATION):
if options.pandas:
keys = self._read_object(options = options)
values = self._read_object(options = options)
if isinstance(keys, pandas.DataFrame):
if not isinstance(values, pandas.DataFrame):
raise QReaderException('Keyed table creation: values are expected to be of type pandas.DataFrame. Actual: %s' % type(values))
indices = keys.columns
table = keys
table.meta = keys.meta
table.meta.qtype = QKEYED_TABLE
for column in values.columns:
table[column] = values[column]
table.meta[column] = values.meta[column]
table.set_index([column for column in indices], inplace = True)
return table
else:
keys = keys if not isinstance(keys, pandas.Series) else keys.as_matrix()
values = values if not isinstance(values, pandas.Series) else values.as_matrix()
return QDictionary(keys, values)
else:
return QReader._read_dictionary(self, qtype = qtype, options = options)
def _read_table(self, qtype = QTABLE):
if self._options.pandas:
self._buffer.skip() # ignore attributes
self._buffer.skip() # ignore dict type stamp
columns = self._read_object()
data = self._read_object()
odict = OrderedDict()
meta = MetaData(qtype = QTABLE)
for i in xrange(len(columns)):
if isinstance(data[i], str):
# convert character list (represented as string) to numpy representation
meta[columns[i]] = QSTRING
odict[columns[i]] = pandas.Series(list(data[i]), dtype = numpy.str).replace(' ', numpy.nan)
elif isinstance(data[i], (list, tuple)):
meta[columns[i]] = QGENERAL_LIST
tarray = numpy.ndarray(shape = len(data[i]), dtype = numpy.dtype('O'))
for j in xrange(len(data[i])):
tarray[j] = data[i][j]
odict[columns[i]] = tarray
else:
meta[columns[i]] = data[i].meta.qtype
odict[columns[i]] = data[i]
df = pandas.DataFrame(odict)
df.meta = meta
return df
else:
return QReader._read_table(self, qtype = qtype)
def _read_table(self, qtype=QTABLE, options=READER_CONFIGURATION):
if options.pandas:
self._buffer.skip() # ignore attributes
self._buffer.skip() # ignore dict type stamp
columns = self._read_object(options=options)
data = self._read_object(options=options)
odict = OrderedDict()
meta = MetaData(qtype=QTABLE)
for i in xrange(len(columns)):
if isinstance(data[i], str):
# convert character list (represented as string) to numpy representation
meta[columns[i]] = QSTRING
odict[columns[i]] = pandas.Series(list(data[i]),
dtype=numpy.str).replace(
' ', numpy.nan)
elif isinstance(data[i], (list, tuple)):
meta[columns[i]] = QGENERAL_LIST
tarray = numpy.ndarray(shape=len(data[i]),
dtype=numpy.dtype('O'))
for j in xrange(len(data[i])):
tarray[j] = data[i][j]
odict[columns[i]] = tarray
else:
meta[columns[i]] = data[i].meta.qtype
odict[columns[i]] = data[i]
df = pandas.DataFrame(odict)
df.meta = meta
return df
else:
return QReader._read_table(self, qtype=qtype, options=options)
def _read_table(self, qtype = QTABLE, options = READER_CONFIGURATION):
if options.pandas:
self._buffer.skip() # ignore attributes
self._buffer.skip() # ignore dict type stamp
columns = self._read_object(options = options)
data = self._read_object(options = options)
odict = OrderedDict()
meta = MetaData(qtype = QTABLE)
for i in xrange(len(columns)):
if isinstance(data[i], str):
# convert character list (represented as string) to numpy representation
meta[columns[i]] = QSTRING
odict[columns[i]] = numpy.array(list(data[i]), dtype = numpy.str)
elif isinstance(data[i], (list, tuple)):
# convert character list (represented as string) to numpy representation
meta[columns[i]] = QGENERAL_LIST
odict[columns[i]] = numpy.array(list(data[i]))
else:
meta[columns[i]] = data[i].meta.qtype
odict[columns[i]] = data[i]
df = pandas.DataFrame(odict)
df.meta = meta
return df
else:
return QReader._read_table(self, qtype = qtype, options = options)
def _read_table(self, qtype = QTABLE, options = READER_CONFIGURATION):
if options.pandas:
self._buffer.skip() # ignore attributes
self._buffer.skip() # ignore dict type stamp
columns = self._read_object(options = options)
data = self._read_object(options = options)
odict = OrderedDict()
meta = MetaData(qtype = QTABLE)
strcols = []
for i in xrange(len(columns)):
if isinstance(data[i], str):
# convert character list (represented as string) to numpy representation
meta[columns[i]] = QSTRING
odict[columns[i]] = pandas.Series(list(data[i]), dtype = numpy.str).replace(' ', numpy.nan)
elif isinstance(data[i], (list, tuple)):
meta[columns[i]] = QGENERAL_LIST
tarray = numpy.ndarray(shape = len(data[i]), dtype = numpy.dtype('O'))
for j in xrange(len(data[i])):
tarray[j] = data[i][j]
odict[columns[i]] = tarray
else:
meta[columns[i]] = data[i].meta.qtype
odict[columns[i]] = data[i]
df = pandas.DataFrame(odict)
df.meta = meta
for column in strcols:
# q uses the space character as the NULL value for strings
df[column] = df[column].replace([' ', ''], numpy.nan)
return df
else:
return QReader._read_table(self, qtype = qtype, options = options)
def _init(self, data):
self.state = IPCProtocol.State.CONNECTED
self.protocol_version = min(struct.unpack('B', data)[0], 3)
self._writer = QWriter(stream=None,
protocol_version=self.protocol_version)
self._reader = QReader(stream=None)
self.factory.clientReady(self)
def _read_list(self, qtype):
if qtype == QSYMBOL_LIST:
self._buffer.skip()
length = self._buffer.get_int()
symbols = self._buffer.get_symbols(length)
return [s.decode(self._encoding) for s in symbols]
else:
return QReader._read_list(self, qtype=qtype)
def _read_list(self, qtype):
if qtype == QSYMBOL_LIST:
self._buffer.skip()
length = self._buffer.get_int()
symbols = self._buffer.get_symbols(length)
return [s.decode(self._encoding) for s in symbols]
else:
return QReader._read_list(self, qtype = qtype)
def open(self):
'''Initialises connection to q service.
If the connection hasn't been initialised yet, invoking the
:func:`.open` creates a new socket and performs a handshake with a q
service.
:raises: :class:`.QConnectionException`, :class:`.QAuthenticationException`
'''
if not self._connection:
if not self.host:
raise QConnectionException('Host cannot be None')
self._init_socket()
self._initialize()
self._writer = QWriter(self._connection, protocol_version = self._protocol_version)
self._reader = QReader(self._connection.makefile())
def _read_general_list(self, qtype=QGENERAL_LIST):
qlist = QReader._read_general_list(self, qtype)
if self._options.pandas:
return [
numpy.nan if isinstance(element, basestring)
and element == b' ' else element for element in qlist
]
else:
return qlist
def _read_table(self, qtype=QTABLE):
if self._options.pandas:
self._buffer.skip() # ignore attributes
self._buffer.skip() # ignore dict type stamp
columns = self._read_object()
self._buffer.skip() # ignore generic list type indicator
data = QReader._read_general_list(self, qtype)
odict = OrderedDict()
meta = MetaData(qtype=QTABLE)
for i in range(len(columns)):
column_name = columns[i] if isinstance(
columns[i], str) else columns[i].decode("utf-8")
if isinstance(data[i], str):
# convert character list (represented as string) to numpy representation
meta[column_name] = QSTRING
odict[column_name] = pandas.Series(
list(data[i]),
dtype=numpy.str).replace(b' ', numpy.nan)
elif isinstance(data[i], bytes):
# convert character list (represented as string) to numpy representation
meta[column_name] = QSTRING
odict[column_name] = pandas.Series(list(data[i].decode()),
dtype=str).replace(
b' ', numpy.nan)
elif isinstance(data[i], (list, tuple)):
meta[column_name] = QGENERAL_LIST
tarray = numpy.ndarray(shape=len(data[i]),
dtype=numpy.dtype('O'))
for j in range(len(data[i])):
tarray[j] = data[i][j]
odict[column_name] = tarray
else:
meta[column_name] = data[i].meta.qtype
odict[column_name] = data[i]
df = pandas.DataFrame(odict)
df._metadata = ["meta"]
df.meta = meta
return df
else:
return QReader._read_table(self, qtype=qtype)
def _read_general_list(self,
qtype=QGENERAL_LIST,
options=READER_CONFIGURATION):
list = QReader._read_general_list(self, qtype, options)
if options.pandas:
return [
numpy.nan if isinstance(element, basestring) and element == ' '
else element for element in list
]
else:
return list
def _read_list(self, qtype, options):
if options.pandas:
options.numpy_temporals = True
list = QReader._read_list(self, qtype = qtype, options = options)
if options.pandas:
if -abs(qtype) not in [QMONTH, QDATE, QDATETIME, QMINUTE, QSECOND, QTIME, QTIMESTAMP, QTIMESPAN, QSYMBOL]:
null = QNULLMAP[-abs(qtype)][1]
ps = pandas.Series(data = list).replace(null, numpy.NaN)
else:
ps = pandas.Series(data = list)
ps.meta = MetaData(qtype = qtype)
return ps
else:
return list
def _read_list(self, qtype):
if self._options.pandas:
self._options.numpy_temporals = True
qlist = QReader._read_list(self, qtype = qtype)
if self._options.pandas:
if -abs(qtype) not in [QMONTH, QDATE, QDATETIME, QMINUTE, QSECOND, QTIME, QTIMESTAMP, QTIMESPAN, QSYMBOL]:
null = QNULLMAP[-abs(qtype)][1]
ps = pandas.Series(data = qlist).replace(null, numpy.NaN)
else:
ps = pandas.Series(data = qlist)
ps.meta = MetaData(qtype = qtype)
return ps
else:
return qlist
def _read_dictionary(self, qtype=QDICTIONARY):
if self._options.pandas:
keys = self._read_object()
values = self._read_object()
if isinstance(keys, pandas.DataFrame):
if not isinstance(values, pandas.DataFrame):
raise QReaderException(
'Keyed table creation: values are expected to be of type pandas.DataFrame. Actual: %s'
% type(values))
indices = keys.columns
table = keys
# meta attribute is obsolete, use attrs['meta'] instead
if hasattr(keys, 'meta'):
table.attrs['meta'] = keys.meta
warnings.warn(
"Usage of meta attribute is deprecated, please use attrs['meta'] instead",
DeprecationWarning)
else:
table.attrs['meta'] = keys.attrs['meta']
table.attrs['meta'].qtype = QKEYED_TABLE
if hasattr(values, 'meta'):
warnings.warn(
"Usage of meta attribute is deprecated, please use attrs['meta'] instead",
DeprecationWarning)
for column in values.columns:
table[column] = values[column]
table.attrs['meta'][column] = values.meta[column]
else:
for column in values.columns:
table[column] = values[column]
table.attrs['meta'][column] = values.attrs['meta'][
column]
table.set_index([column for column in indices], inplace=True)
return table
else:
keys = keys if not isinstance(keys,
pandas.Series) else keys.values
values = values if not isinstance(
values, pandas.Series) else values.values
return QDictionary(keys, values)
else:
return QReader._read_dictionary(self, qtype=qtype)
def _read_general_list(self, qtype = QGENERAL_LIST, options = READER_CONFIGURATION):
list = QReader._read_general_list(self, qtype, options)
if options.pandas:
return [numpy.nan if isinstance(element, basestring) and element == ' ' else element for element in list]
else:
return list
def _read_general_list(self, qtype = QGENERAL_LIST):
list = QReader._read_general_list(self, qtype)
if self._options.pandas:
return [numpy.nan if isinstance(element, basestring) and element == ' ' else element for element in list]
else:
return list
class QConnection(object):
'''Connector class for interfacing with the q service.
Provides methods for synchronous and asynchronous interaction.
The :class:`.QConnection` class provides a context manager API and can be
used with a ``with`` statement::
with qconnection.QConnection(host = 'localhost', port = 5000) as q:
print q
print q('{`int$ til x}', 10)
:Parameters:
- `host` (`string`) - q service hostname
- `port` (`integer`) - q service port
- `username` (`string` or `None`) - username for q authentication/authorization
- `password` (`string` or `None`) - password for q authentication/authorization
- `timeout` (`nonnegative float` or `None`) - set a timeout on blocking socket operations
:Options:
- `raw` (`boolean`) - if ``True`` returns raw data chunk instead of parsed
data, **Default**: ``False``
- `numpy_temporals` (`boolean`) - if ``False`` temporal vectors are
backed by raw q representation (:class:`.QTemporalList`,
:class:`.QTemporal`) instances, otherwise are represented as
`numpy datetime64`/`timedelta64` arrays and atoms,
**Default**: ``False``
'''
def __init__(self,
host,
port,
username=None,
password=None,
timeout=None,
**options):
self.host = host
self.port = port
self.username = username
self.password = password
self._connection = None
self._protocol_version = None
self.timeout = timeout
self._options = MetaData(**READER_CONFIGURATION.union_dict(**options))
def __enter__(self):
self.open()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
@property
def protocol_version(self):
'''Retrieves established version of the IPC protocol.
:returns: `integer` -- version of the IPC protocol
'''
return self._protocol_version
def open(self):
'''Initialises connection to q service.
If the connection hasn't been initialised yet, invoking the
:func:`.open` creates a new socket and performs a handshake with a q
service.
:raises: :class:`.QConnectionException`, :class:`.QAuthenticationException`
'''
if not self._connection:
if not self.host:
raise QConnectionException('Host cannot be None')
self._init_socket()
self._initialize()
self._writer = QWriter(self._connection,
protocol_version=self._protocol_version)
self._reader = QReader(self._connection.makefile())
def _init_socket(self):
'''Initialises the socket used for communicating with a q service,'''
try:
self._connection = socket.socket(socket.AF_INET,
socket.SOCK_STREAM)
self._connection.connect((self.host, self.port))
self._connection.settimeout(self.timeout)
except:
self._connection = None
raise
def close(self):
'''Closes connection with the q service.'''
if self._connection:
self._connection.close()
self._connection = None
def is_connected(self):
'''Checks whether connection with a q service has been established.
Connection is considered inactive when:
- it has not been initialised,
- it has been closed.
:returns: `boolean` -- ``True`` if connection has been established,
``False`` otherwise
'''
return True if self._connection else False
def _initialize(self):
'''Performs a IPC protocol handshake.'''
credentials = self.username + ':' + self.password if self.password else ''
self._connection.send(credentials + '\3\0')
response = self._connection.recv(1)
if len(response) != 1:
self.close()
self._init_socket()
self._connection.send(credentials + '\0')
response = self._connection.recv(1)
if len(response) != 1:
self.close()
raise QAuthenticationException('Connection denied.')
self._protocol_version = min(struct.unpack('B', response)[0], 3)
def __str__(self):
return '%s@:%s:%s' % (self.username, self.host,
self.port) if self.username else ':%s:%s' % (
self.host, self.port)
def query(self, msg_type, query, *parameters):
'''Performs a query against a q service.
In typical use case, `query` is the name of the function to call and
`parameters` are its parameters. When `parameters` list is empty, the
query can be an arbitrary q expression (e.g. ``0 +/ til 100``).
Calls a anonymous function with a single parameter:
>>> q.query(qconnection.MessageType.SYNC,'{til x}', 10)
Executes a q expression:
>>> q.query(qconnection.MessageType.SYNC,'til 10')
:Parameters:
- `msg_type` (one of the constants defined in :class:`.MessageType`) -
type of the query to be executed
- `query` (`string`) - query to be executed
- `parameters` (`list` or `None`) - parameters for the query
:raises: :class:`.QConnectionException`, :class:`.QWriterException`
'''
if not self._connection:
raise QConnectionException('Connection is not established.')
if parameters and len(parameters) > 8:
raise QWriterException('Too many parameters.')
if not parameters or len(parameters) == 0:
self._writer.write(query, msg_type)
else:
self._writer.write([query] + list(parameters), msg_type)
def sync(self, query, *parameters, **options):
'''Performs a synchronous query against a q service and returns parsed
data.
In typical use case, `query` is the name of the function to call and
`parameters` are its parameters. When `parameters` list is empty, the
query can be an arbitrary q expression (e.g. ``0 +/ til 100``).
Executes a q expression:
>>> print q.sync('til 10')
[0 1 2 3 4 5 6 7 8 9]
Executes an anonymous q function with a single parameter:
>>> print q.sync('{til x}', 10)
[0 1 2 3 4 5 6 7 8 9]
Executes an anonymous q function with two parameters:
>>> print q.sync('{y + til x}', 10, 1)
[ 1 2 3 4 5 6 7 8 9 10]
>>> print q.sync('{y + til x}', *[10, 1])
[ 1 2 3 4 5 6 7 8 9 10]
The :func:`.sync` is called from the overloaded :func:`.__call__`
function. This allows :class:`.QConnection` instance to be called as
a function:
>>> print q('{y + til x}', 10, 1)
[ 1 2 3 4 5 6 7 8 9 10]
:Parameters:
- `query` (`string`) - query to be executed
- `parameters` (`list` or `None`) - parameters for the query
:Options:
- `raw` (`boolean`) - if ``True`` returns raw data chunk instead of
parsed data, **Default**: ``False``
- `numpy_temporals` (`boolean`) - if ``False`` temporal vectors are
backed by raw q representation (:class:`.QTemporalList`,
:class:`.QTemporal`) instances, otherwise are represented as
`numpy datetime64`/`timedelta64` arrays and atoms,
**Default**: ``False``
:returns: query result parsed to Python data structures
:raises: :class:`.QConnectionException`, :class:`.QWriterException`,
:class:`.QReaderException`
'''
self.query(MessageType.SYNC, query, *parameters)
response = self.receive(data_only=False, **options)
if response.type == MessageType.RESPONSE:
return response.data
else:
self._writer.write(
QException('nyi: qPython expected response message'),
MessageType.ASYNC if response.type == MessageType.ASYNC else
MessageType.RESPONSE)
raise QReaderException(
'Received message of type: %s where response was expected')
def async (self, query, *parameters):
'''Performs an asynchronous query and returns **without** retrieving of
the response.
In typical use case, `query` is the name of the function to call and
`parameters` are its parameters. When `parameters` list is empty, the
query can be an arbitrary q expression (e.g. ``0 +/ til 100``).
Calls a anonymous function with a single parameter:
>>> q.async('{til x}', 10)
Executes a q expression:
>>> q.async('til 10')
:Parameters:
- `query` (`string`) - query to be executed
- `parameters` (`list` or `None`) - parameters for the query
:raises: :class:`.QConnectionException`, :class:`.QWriterException`
'''
self.query(MessageType.ASYNC, query, *parameters)
def receive(self, data_only=True, **options):
'''Reads and (optionally) parses the response from a q service.
Retrieves query result along with meta-information:
>>> q.query(qconnection.MessageType.SYNC,'{x}', 10)
>>> print q.receive(data_only = False, raw = False)
QMessage: message type: 2, data size: 13, is_compressed: False, data: 10
Retrieves parsed query result:
>>> q.query(qconnection.MessageType.SYNC,'{x}', 10)
>>> print q.receive(data_only = True, raw = False)
10
Retrieves not-parsed (raw) query result:
>>> from binascii import hexlify
>>> q.query(qconnection.MessageType.SYNC,'{x}', 10)
>>> print hexlify(q.receive(data_only = True, raw = True))
fa0a000000
:Parameters:
- `data_only` (`boolean`) - if ``True`` returns only data part of the
message, otherwise returns data and message meta-information
encapsulated in :class:`.QMessage` instance
:Options:
- `raw` (`boolean`) - if ``True`` returns raw data chunk instead of
parsed data, **Default**: ``False``
- `numpy_temporals` (`boolean`) - if ``False`` temporal vectors are
backed by raw q representation (:class:`.QTemporalList`,
:class:`.QTemporal`) instances, otherwise are represented as
`numpy datetime64`/`timedelta64` arrays and atoms,
**Default**: ``False``
:returns: depending on parameter flags: :class:`.QMessage` instance,
parsed message, raw data
:raises: :class:`.QReaderException`
'''
result = self._reader.read(**self._options.union_dict(**options))
return result.data if data_only else result
def __call__(self, *parameters, **options):
return self.sync(parameters[0], *parameters[1:], **options)
