def __init__(self, host, port, username = None, password = None, timeout = None, encoding = 'latin-1', reader_class = None, writer_class = None, **options): self.host = host self.port = port self.username = username self.password = password self._connection = None self._connection_file = None self._protocol_version = None self.timeout = timeout self._encoding = encoding self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options)) try: from qpython._pandas import PandasQReader, PandasQWriter self._reader_class = PandasQReader self._writer_class = PandasQWriter except ImportError: self._reader_class = QReader self._writer_class = QWriter if reader_class: self._reader_class = reader_class if writer_class: self._writer_class = writer_class
def __init__(self, host, port, username = None, password = None, timeout = None, **options): self.host = host self.port = port self.username = username self.password = password self._connection = None self._protocol_version = None self.timeout = timeout self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options))
def write(self, data, msg_type, **options): '''Serializes and pushes single data object to a wrapped stream. :Parameters: - `data` - data to be serialized - `msg_type` (one of the constants defined in :class:`.MessageType`) - type of the message :Options: - `single_char_strings` (`boolean`) - if ``True`` single char Python strings are encoded as q strings instead of chars, **Default**: ``False`` :returns: if wraped stream is ``None`` serialized data, otherwise ``None`` ''' self._buffer = BytesIO() self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options)) # header and placeholder for message size self._buffer.write( ('%s%s\0\0\0\0\0\0' % (ENDIANNESS, chr(msg_type))).encode( self._encoding)) self._write(data) # update message size data_size = self._buffer.tell() self._buffer.seek(4) self._buffer.write(struct.pack('i', data_size)) # write data to socket if self._stream: self._stream.sendall(self._buffer.getvalue()) else: return self._buffer.getvalue()
class QConnection(object): '''Connector class for interfacing with the q service. Provides methods for synchronous and asynchronous interaction. The :class:`.QConnection` class provides a context manager API and can be used with a ``with`` statement:: with qconnection.QConnection(host = 'localhost', port = 5000) as q: print(q) print(q('{`int$ til x}', 10)) :Parameters: - `host` (`string`) - q service hostname - `port` (`integer`) - q service port - `username` (`string` or `None`) - username for q authentication/authorization - `password` (`string` or `None`) - password for q authentication/authorization - `timeout` (`nonnegative float` or `None`) - set a timeout on blocking socket operations - `encoding` (`string`) - string encoding for data deserialization - `reader_class` (subclass of `QReader`) - data deserializer - `writer_class` (subclass of `QWriter`) - data serializer :Options: - `raw` (`boolean`) - if ``True`` returns raw data chunk instead of parsed data, **Default**: ``False`` - `numpy_temporals` (`boolean`) - if ``False`` temporal vectors are backed by raw q representation (:class:`.QTemporalList`, :class:`.QTemporal`) instances, otherwise are represented as `numpy datetime64`/`timedelta64` arrays and atoms, **Default**: ``False`` - `single_char_strings` (`boolean`) - if ``True`` single char Python strings are encoded as q strings instead of chars, **Default**: ``False`` ''' MAX_PROTOCOL_VERSION = 6 def __init__(self, host, port, username = None, password = None, timeout = None, encoding = 'latin-1', reader_class = None, writer_class = None, **options): self.host = host self.port = port self.username = username self.password = password self._connection = None self._connection_file = None self._protocol_version = None self.timeout = timeout self._encoding = encoding self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options)) try: from qpython._pandas import PandasQReader, PandasQWriter self._reader_class = PandasQReader self._writer_class = PandasQWriter except ImportError: self._reader_class = QReader self._writer_class = QWriter if reader_class: self._reader_class = reader_class if writer_class: self._writer_class = writer_class def __enter__(self): self.open() return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() @property def protocol_version(self): '''Retrieves established version of the IPC protocol. :returns: `integer` -- version of the IPC protocol ''' return self._protocol_version def open(self): '''Initialises connection to q service. If the connection hasn't been initialised yet, invoking the :func:`.open` creates a new socket and performs a handshake with a q service. :raises: :class:`.QConnectionException`, :class:`.QAuthenticationException` ''' if not self._connection: if not self.host: raise QConnectionException('Host cannot be None') self._init_socket() self._initialize() self._writer = self._writer_class(self._connection, protocol_version = self._protocol_version, encoding = self._encoding) self._reader = self._reader_class(self._connection_file, encoding = self._encoding) def _init_socket(self): '''Initialises the socket used for communicating with a q service,''' try: self._connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self._connection.connect((self.host, self.port)) self._connection.settimeout(self.timeout) self._connection_file = self._connection.makefile('b') except: self._connection = None self._connection_file = None raise def close(self): '''Closes connection with the q service.''' if self._connection: self._connection_file.close() self._connection_file = None self._connection.close() self._connection = None def is_connected(self): '''Checks whether connection with a q service has been established. Connection is considered inactive when: - it has not been initialised, - it has been closed. :returns: `boolean` -- ``True`` if connection has been established, ``False`` otherwise ''' return True if self._connection else False def _initialize(self): '''Performs a IPC protocol handshake.''' credentials = (self.username if self.username else '') + ':' + (self.password if self.password else '') credentials = credentials.encode(self._encoding) self._connection.send(credentials + bytes([self.MAX_PROTOCOL_VERSION, 0])) response = self._connection.recv(1) if len(response) != 1: self.close() self._init_socket() self._connection.send(credentials + b'\0') response = self._connection.recv(1) if len(response) != 1: self.close() raise QAuthenticationException('Connection denied.') self._protocol_version = min(struct.unpack('B', response)[0], self.MAX_PROTOCOL_VERSION) def __str__(self): return '%s@:%s:%s' % (self.username, self.host, self.port) if self.username else ':%s:%s' % (self.host, self.port) def query(self, msg_type, query, *parameters, **options): '''Performs a query against a q service. In typical use case, `query` is the name of the function to call and `parameters` are its parameters. When `parameters` list is empty, the query can be an arbitrary q expression (e.g. ``0 +/ til 100``). Calls a anonymous function with a single parameter: >>> q.query(qconnection.MessageType.SYNC,'{til x}', 10) Executes a q expression: >>> q.query(qconnection.MessageType.SYNC,'til 10') :Parameters: - `msg_type` (one of the constants defined in :class:`.MessageType`) - type of the query to be executed - `query` (`string`) - query to be executed - `parameters` (`list` or `None`) - parameters for the query :Options: - `single_char_strings` (`boolean`) - if ``True`` single char Python strings are encoded as q strings instead of chars, **Default**: ``False`` :raises: :class:`.QConnectionException`, :class:`.QWriterException` ''' if not self._connection: raise QConnectionException('Connection is not established.') if parameters and len(parameters) > 8: raise QWriterException('Too many parameters.') if not parameters or len(parameters) == 0: self._writer.write(query, msg_type, **self._options.union_dict(**options)) else: self._writer.write([query] + list(parameters), msg_type, **self._options.union_dict(**options)) def sendSync(self, query, *parameters, **options): '''Performs a synchronous query against a q service and returns parsed data. In typical use case, `query` is the name of the function to call and `parameters` are its parameters. When `parameters` list is empty, the query can be an arbitrary q expression (e.g. ``0 +/ til 100``). Executes a q expression: >>> print(q.sendSync('til 10')) [0 1 2 3 4 5 6 7 8 9] Executes an anonymous q function with a single parameter: >>> print(q.sendSync('{til x}', 10)) [0 1 2 3 4 5 6 7 8 9] Executes an anonymous q function with two parameters: >>> print(q.sendSync('{y + til x}', 10, 1)) [ 1 2 3 4 5 6 7 8 9 10] >>> print(q.sendSync('{y + til x}', *[10, 1])) [ 1 2 3 4 5 6 7 8 9 10] The :func:`.sendSync` is called from the overloaded :func:`.__call__` function. This allows :class:`.QConnection` instance to be called as a function: >>> print(q('{y + til x}', 10, 1)) [ 1 2 3 4 5 6 7 8 9 10] :Parameters: - `query` (`string`) - query to be executed - `parameters` (`list` or `None`) - parameters for the query :Options: - `raw` (`boolean`) - if ``True`` returns raw data chunk instead of parsed data, **Default**: ``False`` - `numpy_temporals` (`boolean`) - if ``False`` temporal vectors are backed by raw q representation (:class:`.QTemporalList`, :class:`.QTemporal`) instances, otherwise are represented as `numpy datetime64`/`timedelta64` arrays and atoms, **Default**: ``False`` - `single_char_strings` (`boolean`) - if ``True`` single char Python strings are encoded as q strings instead of chars, **Default**: ``False`` :returns: query result parsed to Python data structures :raises: :class:`.QConnectionException`, :class:`.QWriterException`, :class:`.QReaderException` ''' self.query(MessageType.SYNC, query, *parameters, **options) response = self.receive(data_only = False, **options) if response.type == MessageType.RESPONSE: return response.data else: self._writer.write(QException('nyi: qPython expected response message'), MessageType.ASYNC if response.type == MessageType.ASYNC else MessageType.RESPONSE) raise QReaderException('Received message of type: %s where response was expected') def sendAsync(self, query, *parameters, **options): '''Performs an asynchronous query and returns **without** retrieving of the response. In typical use case, `query` is the name of the function to call and `parameters` are its parameters. When `parameters` list is empty, the query can be an arbitrary q expression (e.g. ``0 +/ til 100``). Calls a anonymous function with a single parameter: >>> q.sendAsync('{til x}', 10) Executes a q expression: >>> q.sendAsync('til 10') :Parameters: - `query` (`string`) - query to be executed - `parameters` (`list` or `None`) - parameters for the query :Options: - `single_char_strings` (`boolean`) - if ``True`` single char Python strings are encoded as q strings instead of chars, **Default**: ``False`` :raises: :class:`.QConnectionException`, :class:`.QWriterException` ''' self.query(MessageType.ASYNC, query, *parameters, **options) def receive(self, data_only = True, **options): '''Reads and (optionally) parses the response from a q service. Retrieves query result along with meta-information: >>> q.query(qconnection.MessageType.SYNC,'{x}', 10) >>> print(q.receive(data_only = False, raw = False)) QMessage: message type: 2, data size: 13, is_compressed: False, data: 10 Retrieves parsed query result: >>> q.query(qconnection.MessageType.SYNC,'{x}', 10) >>> print(q.receive(data_only = True, raw = False)) 10 Retrieves not-parsed (raw) query result: >>> from binascii import hexlify >>> q.query(qconnection.MessageType.SYNC,'{x}', 10) >>> print(hexlify(q.receive(data_only = True, raw = True))) fa0a000000 :Parameters: - `data_only` (`boolean`) - if ``True`` returns only data part of the message, otherwise returns data and message meta-information encapsulated in :class:`.QMessage` instance :Options: - `raw` (`boolean`) - if ``True`` returns raw data chunk instead of parsed data, **Default**: ``False`` - `numpy_temporals` (`boolean`) - if ``False`` temporal vectors are backed by raw q representation (:class:`.QTemporalList`, :class:`.QTemporal`) instances, otherwise are represented as `numpy datetime64`/`timedelta64` arrays and atoms, **Default**: ``False`` :returns: depending on parameter flags: :class:`.QMessage` instance, parsed message, raw data :raises: :class:`.QReaderException` ''' result = self._reader.read(**self._options.union_dict(**options)) return result.data if data_only else result def __call__(self, *parameters, **options): return self.sendSync(parameters[0], *parameters[1:], **options)
from qpython.qtemporal import QTemporal try: import pandas BINARY = None PANDAS_EXPRESSIONS = OrderedDict(( ('("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)', { 'data': pandas.Series( numpy.array([ uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661'), numpy.nan ])), 'meta': MetaData(qtype=QGUID_LIST) }), ('"quick brown fox jumps over a lazy dog"', 'quick brown fox jumps over a lazy dog'), ('" "', ' '), ('``quick``fox', { 'data': pandas.Series( numpy.array([ qnull(QSYMBOL), numpy.string_('quick'), qnull(QSYMBOL), numpy.string_('fox') ])), 'meta': MetaData(qtype=QSYMBOL_LIST)
def _meta_init(self, **meta): self.meta = MetaData(**meta)
class QConnection(object): '''Connector class for interfacing with the q service. Provides methods for synchronous and asynchronous interaction. The :class:`.QConnection` class provides a context manager API and can be used with a ``with`` statement:: with qconnection.QConnection(host = 'localhost', port = 5000) as q: print(q) print(q('{`int$ til x}', 10)) :Parameters: - `host` (`string`) - q service hostname - `port` (`integer`) - q service port - `username` (`string` or `None`) - username for q authentication/authorization - `password` (`string` or `None`) - password for q authentication/authorization - `timeout` (`nonnegative float` or `None`) - set a timeout on blocking socket operations - `encoding` (`string`) - string encoding for data deserialization - `reader_class` (subclass of `QReader`) - data deserializer - `writer_class` (subclass of `QWriter`) - data serializer :Options: - `raw` (`boolean`) - if ``True`` returns raw data chunk instead of parsed data, **Default**: ``False`` - `numpy_temporals` (`boolean`) - if ``False`` temporal vectors are backed by raw q representation (:class:`.QTemporalList`, :class:`.QTemporal`) instances, otherwise are represented as `numpy datetime64`/`timedelta64` arrays and atoms, **Default**: ``False`` - `single_char_strings` (`boolean`) - if ``True`` single char Python strings are encoded as q strings instead of chars, **Default**: ``False`` ''' def __init__(self, host, port, username = None, password = None, timeout = None, encoding = 'latin-1', reader_class = None, writer_class = None, **options): self.host = host self.port = port self.username = username self.password = password self._connection = None self._connection_file = None self._protocol_version = None self.timeout = timeout self._encoding = encoding self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options)) try: from qpython._pandas import PandasQReader, PandasQWriter self._reader_class = PandasQReader self._writer_class = PandasQWriter except ImportError: self._reader_class = QReader self._writer_class = QWriter if reader_class: self._reader_class = reader_class if writer_class: self._writer_class = writer_class def __enter__(self): self.open() return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() @property def protocol_version(self): '''Retrieves established version of the IPC protocol. :returns: `integer` -- version of the IPC protocol ''' return self._protocol_version def open(self): '''Initialises connection to q service. If the connection hasn't been initialised yet, invoking the :func:`.open` creates a new socket and performs a handshake with a q service. :raises: :class:`.QConnectionException`, :class:`.QAuthenticationException` ''' if not self._connection: if not self.host: raise QConnectionException('Host cannot be None') self._init_socket() self._initialize() self._writer = self._writer_class(self._connection, protocol_version = self._protocol_version, encoding = self._encoding) self._reader = self._reader_class(self._connection_file, encoding = self._encoding) def _init_socket(self): '''Initialises the socket used for communicating with a q service,''' try: self._connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self._connection.connect((self.host, self.port)) self._connection.settimeout(self.timeout) self._connection_file = self._connection.makefile('b') except: self._connection = None self._connection_file = None raise def close(self): '''Closes connection with the q service.''' if self._connection: self._connection_file.close() self._connection_file = None self._connection.close() self._connection = None def is_connected(self): '''Checks whether connection with a q service has been established. Connection is considered inactive when: - it has not been initialised, - it has been closed. :returns: `boolean` -- ``True`` if connection has been established, ``False`` otherwise ''' return True if self._connection else False def _initialize(self): '''Performs a IPC protocol handshake.''' credentials = (self.username if self.username else '') + ':' + (self.password if self.password else '') credentials = credentials.encode(self._encoding) self._connection.send(credentials + b'\3\0') response = self._connection.recv(1) if len(response) != 1: self.close() self._init_socket() self._connection.send(credentials + b'\0') response = self._connection.recv(1) if len(response) != 1: self.close() raise QAuthenticationException('Connection denied.') self._protocol_version = min(struct.unpack('B', response)[0], 3) def __str__(self): return '%s@:%s:%s' % (self.username, self.host, self.port) if self.username else ':%s:%s' % (self.host, self.port) def query(self, msg_type, query, *parameters, **options): '''Performs a query against a q service. In typical use case, `query` is the name of the function to call and `parameters` are its parameters. When `parameters` list is empty, the query can be an arbitrary q expression (e.g. ``0 +/ til 100``). Calls a anonymous function with a single parameter: >>> q.query(qconnection.MessageType.SYNC,'{til x}', 10) Executes a q expression: >>> q.query(qconnection.MessageType.SYNC,'til 10') :Parameters: - `msg_type` (one of the constants defined in :class:`.MessageType`) - type of the query to be executed - `query` (`string`) - query to be executed - `parameters` (`list` or `None`) - parameters for the query :Options: - `single_char_strings` (`boolean`) - if ``True`` single char Python strings are encoded as q strings instead of chars, **Default**: ``False`` :raises: :class:`.QConnectionException`, :class:`.QWriterException` ''' if not self._connection: raise QConnectionException('Connection is not established.') if parameters and len(parameters) > 8: raise QWriterException('Too many parameters.') if not parameters or len(parameters) == 0: self._writer.write(query, msg_type, **self._options.union_dict(**options)) else: self._writer.write([query] + list(parameters), msg_type, **self._options.union_dict(**options)) def sync(self, query, *parameters, **options): '''Performs a synchronous query against a q service and returns parsed data. In typical use case, `query` is the name of the function to call and `parameters` are its parameters. When `parameters` list is empty, the query can be an arbitrary q expression (e.g. ``0 +/ til 100``). Executes a q expression: >>> print(q.sync('til 10')) [0 1 2 3 4 5 6 7 8 9] Executes an anonymous q function with a single parameter: >>> print(q.sync('{til x}', 10)) [0 1 2 3 4 5 6 7 8 9] Executes an anonymous q function with two parameters: >>> print(q.sync('{y + til x}', 10, 1)) [ 1 2 3 4 5 6 7 8 9 10] >>> print(q.sync('{y + til x}', *[10, 1])) [ 1 2 3 4 5 6 7 8 9 10] The :func:`.sync` is called from the overloaded :func:`.__call__` function. This allows :class:`.QConnection` instance to be called as a function: >>> print(q('{y + til x}', 10, 1)) [ 1 2 3 4 5 6 7 8 9 10] :Parameters: - `query` (`string`) - query to be executed - `parameters` (`list` or `None`) - parameters for the query :Options: - `raw` (`boolean`) - if ``True`` returns raw data chunk instead of parsed data, **Default**: ``False`` - `numpy_temporals` (`boolean`) - if ``False`` temporal vectors are backed by raw q representation (:class:`.QTemporalList`, :class:`.QTemporal`) instances, otherwise are represented as `numpy datetime64`/`timedelta64` arrays and atoms, **Default**: ``False`` - `single_char_strings` (`boolean`) - if ``True`` single char Python strings are encoded as q strings instead of chars, **Default**: ``False`` :returns: query result parsed to Python data structures :raises: :class:`.QConnectionException`, :class:`.QWriterException`, :class:`.QReaderException` ''' self.query(MessageType.SYNC, query, *parameters, **options) response = self.receive(data_only = False, **options) if response.type == MessageType.RESPONSE: return response.data else: self._writer.write(QException('nyi: qPython expected response message'), MessageType.ASYNC if response.type == MessageType.ASYNC else MessageType.RESPONSE) raise QReaderException('Received message of type: %s where response was expected') def async(self, query, *parameters, **options): '''Performs an asynchronous query and returns **without** retrieving of the response. In typical use case, `query` is the name of the function to call and `parameters` are its parameters. When `parameters` list is empty, the query can be an arbitrary q expression (e.g. ``0 +/ til 100``). Calls a anonymous function with a single parameter: >>> q.async('{til x}', 10) Executes a q expression: >>> q.async('til 10') :Parameters: - `query` (`string`) - query to be executed - `parameters` (`list` or `None`) - parameters for the query :Options: - `single_char_strings` (`boolean`) - if ``True`` single char Python strings are encoded as q strings instead of chars, **Default**: ``False`` :raises: :class:`.QConnectionException`, :class:`.QWriterException` ''' self.query(MessageType.ASYNC, query, *parameters, **options)
import sys from qpython import MetaData from qpython.qtype import * # @UnusedWildImport from qpython.qcollection import qlist, QDictionary, qtable, QTable, QKeyedTable from qpython.qtemporal import qtemporal, from_raw_qtemporal, array_from_raw_qtemporal try: from qpython.fastutils import uncompress except: from qpython.utils import uncompress READER_CONFIGURATION = MetaData(raw = False, numpy_temporals = False, pandas = False) class QReaderException(Exception): ''' Indicates an error raised during data deserialization. ''' pass class QMessage(object): ''' Represents a single message parsed from q protocol.
def __array_finalize__(self, obj): self.meta = MetaData() if obj is None else getattr(obj, 'meta', MetaData())
def _meta_init(self, **meta): '''Initialises the meta-information.''' self.meta = MetaData(**meta)