def _read_dictionary(self, qtype=QDICTIONARY, options=READER_CONFIGURATION): if options.pandas: keys = self._read_object(options=options) values = self._read_object(options=options) if isinstance(keys, pandas.DataFrame): if not isinstance(values, pandas.DataFrame): raise QReaderException( 'Keyed table creation: values are expected to be of type pandas.DataFrame. Actual: %s' % type(values)) indices = keys.columns table = keys table.meta = keys.meta table.meta.qtype = QKEYED_TABLE for column in values.columns: table[column] = values[column] table.meta[column] = values.meta[column] table.set_index([column for column in indices], inplace=True) return table else: keys = keys if not isinstance( keys, pandas.Series) else keys.as_matrix() values = values if not isinstance( values, pandas.Series) else values.as_matrix() return QDictionary(keys, values) else: return QReader._read_dictionary(self, qtype=qtype, options=options)
def _read_list(self, qtype): if self._options.pandas: self._options.numpy_temporals = True qlist = QReader._read_list(self, qtype=qtype) if self._options.pandas: atom_qtype = -abs(qtype) if atom_qtype not in [ QMONTH, QDATE, QDATETIME, QMINUTE, QSECOND, QTIME, QTIMESTAMP, QTIMESPAN, QSYMBOL, QFLOAT, QDOUBLE, QGUID ]: null = QNULLMAP[atom_qtype][1] if atom_qtype in PANDAS_TYPE: ps = pandas.Series(data=qlist, dtype=PANDAS_TYPE[atom_qtype]).replace( null, pandas.NA) else: ps = pandas.Series(data=qlist).replace(null, pandas.NA) else: ps = pandas.Series(data=qlist) ps.meta = MetaData(qtype=qtype) return ps else: return qlist
def _read_dictionary(self, qtype = QDICTIONARY, options = READER_CONFIGURATION): if options.pandas: keys = self._read_object(options = options) values = self._read_object(options = options) if isinstance(keys, pandas.DataFrame): if not isinstance(values, pandas.DataFrame): raise QReaderException('Keyed table creation: values are expected to be of type pandas.DataFrame. Actual: %s' % type(values)) indices = keys.columns table = keys table.meta = keys.meta table.meta.qtype = QKEYED_TABLE for column in values.columns: table[column] = values[column] table.meta[column] = values.meta[column] table.set_index([column for column in indices], inplace = True) return table else: keys = keys if not isinstance(keys, pandas.Series) else keys.as_matrix() values = values if not isinstance(values, pandas.Series) else values.as_matrix() return QDictionary(keys, values) else: return QReader._read_dictionary(self, qtype = qtype, options = options)
def _read_table(self, qtype = QTABLE): if self._options.pandas: self._buffer.skip() # ignore attributes self._buffer.skip() # ignore dict type stamp columns = self._read_object() data = self._read_object() odict = OrderedDict() meta = MetaData(qtype = QTABLE) for i in xrange(len(columns)): if isinstance(data[i], str): # convert character list (represented as string) to numpy representation meta[columns[i]] = QSTRING odict[columns[i]] = pandas.Series(list(data[i]), dtype = numpy.str).replace(' ', numpy.nan) elif isinstance(data[i], (list, tuple)): meta[columns[i]] = QGENERAL_LIST tarray = numpy.ndarray(shape = len(data[i]), dtype = numpy.dtype('O')) for j in xrange(len(data[i])): tarray[j] = data[i][j] odict[columns[i]] = tarray else: meta[columns[i]] = data[i].meta.qtype odict[columns[i]] = data[i] df = pandas.DataFrame(odict) df.meta = meta return df else: return QReader._read_table(self, qtype = qtype)
def _read_table(self, qtype=QTABLE, options=READER_CONFIGURATION): if options.pandas: self._buffer.skip() # ignore attributes self._buffer.skip() # ignore dict type stamp columns = self._read_object(options=options) data = self._read_object(options=options) odict = OrderedDict() meta = MetaData(qtype=QTABLE) for i in xrange(len(columns)): if isinstance(data[i], str): # convert character list (represented as string) to numpy representation meta[columns[i]] = QSTRING odict[columns[i]] = pandas.Series(list(data[i]), dtype=numpy.str).replace( ' ', numpy.nan) elif isinstance(data[i], (list, tuple)): meta[columns[i]] = QGENERAL_LIST tarray = numpy.ndarray(shape=len(data[i]), dtype=numpy.dtype('O')) for j in xrange(len(data[i])): tarray[j] = data[i][j] odict[columns[i]] = tarray else: meta[columns[i]] = data[i].meta.qtype odict[columns[i]] = data[i] df = pandas.DataFrame(odict) df.meta = meta return df else: return QReader._read_table(self, qtype=qtype, options=options)
def _read_table(self, qtype = QTABLE, options = READER_CONFIGURATION): if options.pandas: self._buffer.skip() # ignore attributes self._buffer.skip() # ignore dict type stamp columns = self._read_object(options = options) data = self._read_object(options = options) odict = OrderedDict() meta = MetaData(qtype = QTABLE) for i in xrange(len(columns)): if isinstance(data[i], str): # convert character list (represented as string) to numpy representation meta[columns[i]] = QSTRING odict[columns[i]] = numpy.array(list(data[i]), dtype = numpy.str) elif isinstance(data[i], (list, tuple)): # convert character list (represented as string) to numpy representation meta[columns[i]] = QGENERAL_LIST odict[columns[i]] = numpy.array(list(data[i])) else: meta[columns[i]] = data[i].meta.qtype odict[columns[i]] = data[i] df = pandas.DataFrame(odict) df.meta = meta return df else: return QReader._read_table(self, qtype = qtype, options = options)
def _read_table(self, qtype = QTABLE, options = READER_CONFIGURATION): if options.pandas: self._buffer.skip() # ignore attributes self._buffer.skip() # ignore dict type stamp columns = self._read_object(options = options) data = self._read_object(options = options) odict = OrderedDict() meta = MetaData(qtype = QTABLE) strcols = [] for i in xrange(len(columns)): if isinstance(data[i], str): # convert character list (represented as string) to numpy representation meta[columns[i]] = QSTRING odict[columns[i]] = pandas.Series(list(data[i]), dtype = numpy.str).replace(' ', numpy.nan) elif isinstance(data[i], (list, tuple)): meta[columns[i]] = QGENERAL_LIST tarray = numpy.ndarray(shape = len(data[i]), dtype = numpy.dtype('O')) for j in xrange(len(data[i])): tarray[j] = data[i][j] odict[columns[i]] = tarray else: meta[columns[i]] = data[i].meta.qtype odict[columns[i]] = data[i] df = pandas.DataFrame(odict) df.meta = meta for column in strcols: # q uses the space character as the NULL value for strings df[column] = df[column].replace([' ', ''], numpy.nan) return df else: return QReader._read_table(self, qtype = qtype, options = options)
def _init(self, data): self.state = IPCProtocol.State.CONNECTED self.protocol_version = min(struct.unpack('B', data)[0], 3) self._writer = QWriter(stream=None, protocol_version=self.protocol_version) self._reader = QReader(stream=None) self.factory.clientReady(self)
def _read_list(self, qtype): if qtype == QSYMBOL_LIST: self._buffer.skip() length = self._buffer.get_int() symbols = self._buffer.get_symbols(length) return [s.decode(self._encoding) for s in symbols] else: return QReader._read_list(self, qtype=qtype)
def _read_list(self, qtype): if qtype == QSYMBOL_LIST: self._buffer.skip() length = self._buffer.get_int() symbols = self._buffer.get_symbols(length) return [s.decode(self._encoding) for s in symbols] else: return QReader._read_list(self, qtype = qtype)
def open(self): '''Initialises connection to q service. If the connection hasn't been initialised yet, invoking the :func:`.open` creates a new socket and performs a handshake with a q service. :raises: :class:`.QConnectionException`, :class:`.QAuthenticationException` ''' if not self._connection: if not self.host: raise QConnectionException('Host cannot be None') self._init_socket() self._initialize() self._writer = QWriter(self._connection, protocol_version = self._protocol_version) self._reader = QReader(self._connection.makefile())
def _read_general_list(self, qtype=QGENERAL_LIST): qlist = QReader._read_general_list(self, qtype) if self._options.pandas: return [ numpy.nan if isinstance(element, basestring) and element == b' ' else element for element in qlist ] else: return qlist
def _read_table(self, qtype=QTABLE): if self._options.pandas: self._buffer.skip() # ignore attributes self._buffer.skip() # ignore dict type stamp columns = self._read_object() self._buffer.skip() # ignore generic list type indicator data = QReader._read_general_list(self, qtype) odict = OrderedDict() meta = MetaData(qtype=QTABLE) for i in range(len(columns)): column_name = columns[i] if isinstance( columns[i], str) else columns[i].decode("utf-8") if isinstance(data[i], str): # convert character list (represented as string) to numpy representation meta[column_name] = QSTRING odict[column_name] = pandas.Series( list(data[i]), dtype=numpy.str).replace(b' ', numpy.nan) elif isinstance(data[i], bytes): # convert character list (represented as string) to numpy representation meta[column_name] = QSTRING odict[column_name] = pandas.Series(list(data[i].decode()), dtype=str).replace( b' ', numpy.nan) elif isinstance(data[i], (list, tuple)): meta[column_name] = QGENERAL_LIST tarray = numpy.ndarray(shape=len(data[i]), dtype=numpy.dtype('O')) for j in range(len(data[i])): tarray[j] = data[i][j] odict[column_name] = tarray else: meta[column_name] = data[i].meta.qtype odict[column_name] = data[i] df = pandas.DataFrame(odict) df._metadata = ["meta"] df.meta = meta return df else: return QReader._read_table(self, qtype=qtype)
def _read_general_list(self, qtype=QGENERAL_LIST, options=READER_CONFIGURATION): list = QReader._read_general_list(self, qtype, options) if options.pandas: return [ numpy.nan if isinstance(element, basestring) and element == ' ' else element for element in list ] else: return list
def _read_list(self, qtype, options): if options.pandas: options.numpy_temporals = True list = QReader._read_list(self, qtype = qtype, options = options) if options.pandas: if -abs(qtype) not in [QMONTH, QDATE, QDATETIME, QMINUTE, QSECOND, QTIME, QTIMESTAMP, QTIMESPAN, QSYMBOL]: null = QNULLMAP[-abs(qtype)][1] ps = pandas.Series(data = list).replace(null, numpy.NaN) else: ps = pandas.Series(data = list) ps.meta = MetaData(qtype = qtype) return ps else: return list
def _read_list(self, qtype): if self._options.pandas: self._options.numpy_temporals = True qlist = QReader._read_list(self, qtype = qtype) if self._options.pandas: if -abs(qtype) not in [QMONTH, QDATE, QDATETIME, QMINUTE, QSECOND, QTIME, QTIMESTAMP, QTIMESPAN, QSYMBOL]: null = QNULLMAP[-abs(qtype)][1] ps = pandas.Series(data = qlist).replace(null, numpy.NaN) else: ps = pandas.Series(data = qlist) ps.meta = MetaData(qtype = qtype) return ps else: return qlist
def _read_dictionary(self, qtype=QDICTIONARY): if self._options.pandas: keys = self._read_object() values = self._read_object() if isinstance(keys, pandas.DataFrame): if not isinstance(values, pandas.DataFrame): raise QReaderException( 'Keyed table creation: values are expected to be of type pandas.DataFrame. Actual: %s' % type(values)) indices = keys.columns table = keys # meta attribute is obsolete, use attrs['meta'] instead if hasattr(keys, 'meta'): table.attrs['meta'] = keys.meta warnings.warn( "Usage of meta attribute is deprecated, please use attrs['meta'] instead", DeprecationWarning) else: table.attrs['meta'] = keys.attrs['meta'] table.attrs['meta'].qtype = QKEYED_TABLE if hasattr(values, 'meta'): warnings.warn( "Usage of meta attribute is deprecated, please use attrs['meta'] instead", DeprecationWarning) for column in values.columns: table[column] = values[column] table.attrs['meta'][column] = values.meta[column] else: for column in values.columns: table[column] = values[column] table.attrs['meta'][column] = values.attrs['meta'][ column] table.set_index([column for column in indices], inplace=True) return table else: keys = keys if not isinstance(keys, pandas.Series) else keys.values values = values if not isinstance( values, pandas.Series) else values.values return QDictionary(keys, values) else: return QReader._read_dictionary(self, qtype=qtype)
def _read_general_list(self, qtype = QGENERAL_LIST, options = READER_CONFIGURATION): list = QReader._read_general_list(self, qtype, options) if options.pandas: return [numpy.nan if isinstance(element, basestring) and element == ' ' else element for element in list] else: return list
def _read_general_list(self, qtype = QGENERAL_LIST): list = QReader._read_general_list(self, qtype) if self._options.pandas: return [numpy.nan if isinstance(element, basestring) and element == ' ' else element for element in list] else: return list
class QConnection(object): '''Connector class for interfacing with the q service. Provides methods for synchronous and asynchronous interaction. The :class:`.QConnection` class provides a context manager API and can be used with a ``with`` statement:: with qconnection.QConnection(host = 'localhost', port = 5000) as q: print q print q('{`int$ til x}', 10) :Parameters: - `host` (`string`) - q service hostname - `port` (`integer`) - q service port - `username` (`string` or `None`) - username for q authentication/authorization - `password` (`string` or `None`) - password for q authentication/authorization - `timeout` (`nonnegative float` or `None`) - set a timeout on blocking socket operations :Options: - `raw` (`boolean`) - if ``True`` returns raw data chunk instead of parsed data, **Default**: ``False`` - `numpy_temporals` (`boolean`) - if ``False`` temporal vectors are backed by raw q representation (:class:`.QTemporalList`, :class:`.QTemporal`) instances, otherwise are represented as `numpy datetime64`/`timedelta64` arrays and atoms, **Default**: ``False`` ''' def __init__(self, host, port, username=None, password=None, timeout=None, **options): self.host = host self.port = port self.username = username self.password = password self._connection = None self._protocol_version = None self.timeout = timeout self._options = MetaData(**READER_CONFIGURATION.union_dict(**options)) def __enter__(self): self.open() return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() @property def protocol_version(self): '''Retrieves established version of the IPC protocol. :returns: `integer` -- version of the IPC protocol ''' return self._protocol_version def open(self): '''Initialises connection to q service. If the connection hasn't been initialised yet, invoking the :func:`.open` creates a new socket and performs a handshake with a q service. :raises: :class:`.QConnectionException`, :class:`.QAuthenticationException` ''' if not self._connection: if not self.host: raise QConnectionException('Host cannot be None') self._init_socket() self._initialize() self._writer = QWriter(self._connection, protocol_version=self._protocol_version) self._reader = QReader(self._connection.makefile()) def _init_socket(self): '''Initialises the socket used for communicating with a q service,''' try: self._connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self._connection.connect((self.host, self.port)) self._connection.settimeout(self.timeout) except: self._connection = None raise def close(self): '''Closes connection with the q service.''' if self._connection: self._connection.close() self._connection = None def is_connected(self): '''Checks whether connection with a q service has been established. Connection is considered inactive when: - it has not been initialised, - it has been closed. :returns: `boolean` -- ``True`` if connection has been established, ``False`` otherwise ''' return True if self._connection else False def _initialize(self): '''Performs a IPC protocol handshake.''' credentials = self.username + ':' + self.password if self.password else '' self._connection.send(credentials + '\3\0') response = self._connection.recv(1) if len(response) != 1: self.close() self._init_socket() self._connection.send(credentials + '\0') response = self._connection.recv(1) if len(response) != 1: self.close() raise QAuthenticationException('Connection denied.') self._protocol_version = min(struct.unpack('B', response)[0], 3) def __str__(self): return '%s@:%s:%s' % (self.username, self.host, self.port) if self.username else ':%s:%s' % ( self.host, self.port) def query(self, msg_type, query, *parameters): '''Performs a query against a q service. In typical use case, `query` is the name of the function to call and `parameters` are its parameters. When `parameters` list is empty, the query can be an arbitrary q expression (e.g. ``0 +/ til 100``). Calls a anonymous function with a single parameter: >>> q.query(qconnection.MessageType.SYNC,'{til x}', 10) Executes a q expression: >>> q.query(qconnection.MessageType.SYNC,'til 10') :Parameters: - `msg_type` (one of the constants defined in :class:`.MessageType`) - type of the query to be executed - `query` (`string`) - query to be executed - `parameters` (`list` or `None`) - parameters for the query :raises: :class:`.QConnectionException`, :class:`.QWriterException` ''' if not self._connection: raise QConnectionException('Connection is not established.') if parameters and len(parameters) > 8: raise QWriterException('Too many parameters.') if not parameters or len(parameters) == 0: self._writer.write(query, msg_type) else: self._writer.write([query] + list(parameters), msg_type) def sync(self, query, *parameters, **options): '''Performs a synchronous query against a q service and returns parsed data. In typical use case, `query` is the name of the function to call and `parameters` are its parameters. When `parameters` list is empty, the query can be an arbitrary q expression (e.g. ``0 +/ til 100``). Executes a q expression: >>> print q.sync('til 10') [0 1 2 3 4 5 6 7 8 9] Executes an anonymous q function with a single parameter: >>> print q.sync('{til x}', 10) [0 1 2 3 4 5 6 7 8 9] Executes an anonymous q function with two parameters: >>> print q.sync('{y + til x}', 10, 1) [ 1 2 3 4 5 6 7 8 9 10] >>> print q.sync('{y + til x}', *[10, 1]) [ 1 2 3 4 5 6 7 8 9 10] The :func:`.sync` is called from the overloaded :func:`.__call__` function. This allows :class:`.QConnection` instance to be called as a function: >>> print q('{y + til x}', 10, 1) [ 1 2 3 4 5 6 7 8 9 10] :Parameters: - `query` (`string`) - query to be executed - `parameters` (`list` or `None`) - parameters for the query :Options: - `raw` (`boolean`) - if ``True`` returns raw data chunk instead of parsed data, **Default**: ``False`` - `numpy_temporals` (`boolean`) - if ``False`` temporal vectors are backed by raw q representation (:class:`.QTemporalList`, :class:`.QTemporal`) instances, otherwise are represented as `numpy datetime64`/`timedelta64` arrays and atoms, **Default**: ``False`` :returns: query result parsed to Python data structures :raises: :class:`.QConnectionException`, :class:`.QWriterException`, :class:`.QReaderException` ''' self.query(MessageType.SYNC, query, *parameters) response = self.receive(data_only=False, **options) if response.type == MessageType.RESPONSE: return response.data else: self._writer.write( QException('nyi: qPython expected response message'), MessageType.ASYNC if response.type == MessageType.ASYNC else MessageType.RESPONSE) raise QReaderException( 'Received message of type: %s where response was expected') def async (self, query, *parameters): '''Performs an asynchronous query and returns **without** retrieving of the response. In typical use case, `query` is the name of the function to call and `parameters` are its parameters. When `parameters` list is empty, the query can be an arbitrary q expression (e.g. ``0 +/ til 100``). Calls a anonymous function with a single parameter: >>> q.async('{til x}', 10) Executes a q expression: >>> q.async('til 10') :Parameters: - `query` (`string`) - query to be executed - `parameters` (`list` or `None`) - parameters for the query :raises: :class:`.QConnectionException`, :class:`.QWriterException` ''' self.query(MessageType.ASYNC, query, *parameters) def receive(self, data_only=True, **options): '''Reads and (optionally) parses the response from a q service. Retrieves query result along with meta-information: >>> q.query(qconnection.MessageType.SYNC,'{x}', 10) >>> print q.receive(data_only = False, raw = False) QMessage: message type: 2, data size: 13, is_compressed: False, data: 10 Retrieves parsed query result: >>> q.query(qconnection.MessageType.SYNC,'{x}', 10) >>> print q.receive(data_only = True, raw = False) 10 Retrieves not-parsed (raw) query result: >>> from binascii import hexlify >>> q.query(qconnection.MessageType.SYNC,'{x}', 10) >>> print hexlify(q.receive(data_only = True, raw = True)) fa0a000000 :Parameters: - `data_only` (`boolean`) - if ``True`` returns only data part of the message, otherwise returns data and message meta-information encapsulated in :class:`.QMessage` instance :Options: - `raw` (`boolean`) - if ``True`` returns raw data chunk instead of parsed data, **Default**: ``False`` - `numpy_temporals` (`boolean`) - if ``False`` temporal vectors are backed by raw q representation (:class:`.QTemporalList`, :class:`.QTemporal`) instances, otherwise are represented as `numpy datetime64`/`timedelta64` arrays and atoms, **Default**: ``False`` :returns: depending on parameter flags: :class:`.QMessage` instance, parsed message, raw data :raises: :class:`.QReaderException` ''' result = self._reader.read(**self._options.union_dict(**options)) return result.data if data_only else result def __call__(self, *parameters, **options): return self.sync(parameters[0], *parameters[1:], **options)