def __init__(self, host, port, username = None, password = None, timeout = None, encoding = 'latin-1', reader_class = None, writer_class = None, **options): self.host = host self.port = port self.username = username self.password = password self._connection = None self._connection_file = None self._protocol_version = None self.timeout = timeout self._encoding = encoding self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options)) try: from qpython._pandas import PandasQReader, PandasQWriter self._reader_class = PandasQReader self._writer_class = PandasQWriter except ImportError: self._reader_class = QReader self._writer_class = QWriter if reader_class: self._reader_class = reader_class if writer_class: self._writer_class = writer_class
def write(self, data, msg_type, **options): '''Serializes and pushes single data object to a wrapped stream. :Parameters: - `data` - data to be serialized - `msg_type` (one of the constants defined in :class:`.MessageType`) - type of the message :Options: - `single_char_strings` (`boolean`) - if ``True`` single char Python strings are encoded as q strings instead of chars, **Default**: ``False`` :returns: if wraped stream is ``None`` serialized data, otherwise ``None`` ''' self._buffer = BytesIO() self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options)) # header and placeholder for message size self._buffer.write(('%s%s\0\0\0\0\0\0' % (ENDIANESS, chr(msg_type))).encode(self._encoding)) self._write(data) # update message size data_size = self._buffer.tell() self._buffer.seek(4) self._buffer.write(struct.pack('i', data_size)) # write data to socket if self._stream: self._stream.sendall(self._buffer.getvalue()) else: return self._buffer.getvalue()
def write(self, data, msg_type, **options): '''Serializes and pushes single data object to a wrapped stream. :Parameters: - `data` - data to be serialized - `msg_type` (one of the constants defined in :class:`.MessageType`) - type of the message :Options: - `single_char_strings` (`boolean`) - if ``True`` single char Python strings are encoded as q strings instead of chars, **Default**: ``False`` :returns: if wraped stream is ``None`` serialized data, otherwise ``None`` ''' self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options)) self._buffer = cStringIO.StringIO() # header and placeholder for message size self._buffer.write('%s%s\0\0\0\0\0\0' % (ENDIANESS, chr(msg_type))) self._write(data) # update message size data_size = self._buffer.tell() self._buffer.seek(4) self._buffer.write(struct.pack('i', data_size)) # write data to socket if self._stream: self._stream.sendall(self._buffer.getvalue()) else: return self._buffer.getvalue()
def __init__(self, host, port, username = None, password = None, timeout = None, **options): self.host = host self.port = port self.username = username self.password = password self._connection = None self._protocol_version = None self.timeout = timeout self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options))
def read_data(self, message_size, compression_mode=0, **options): ''' Reads and optionally parses data part of a message. .. note:: :func:`.read_header` is required to be called before executing the :func:`.read_data` :Parameters: - `message_size` (`int`) - size of the message to be read - `compression_mode` (`int`) - indicates whether data is compressed, 1 for <2GB, 2 for larger :Options: - `raw` (`boolean`) - indicates whether read data should parsed or returned in raw byte form - `numpy_temporals` (`boolean`) - if ``False`` temporal vectors are backed by raw q representation (:class:`.QTemporalList`, :class:`.QTemporal`) instances, otherwise are represented as `numpy datetime64`/`timedelta64` arrays and atoms, **Default**: ``False`` :returns: read data (parsed or raw byte form) ''' self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options)) if compression_mode > 0: comprHeaderLen = 4 if compression_mode == 1 else 8 if self._stream: self._buffer.wrap(self._read_bytes(comprHeaderLen)) uncompressed_size = -8 + (self._buffer.get_uint( ) if compression_mode == 1 else self._buffer.get_long()) compressed_data = self._read_bytes(message_size - ( 8 + comprHeaderLen)) if self._stream else self._buffer.raw( message_size - (8 + comprHeaderLen)) raw_data = numpy.frombuffer(compressed_data, dtype=numpy.uint8) if uncompressed_size <= 0: raise QReaderException('Error while data decompression.') raw_data = uncompress(raw_data, numpy.int64(uncompressed_size)) raw_data = numpy.ndarray.tobytes(raw_data) self._buffer.wrap(raw_data) elif self._stream: raw_data = self._read_bytes(message_size - 8) self._buffer.wrap(raw_data) if not self._stream and self._options.raw: raw_data = self._buffer.raw(message_size - 8) return raw_data if self._options.raw else self._read_object()
def __init__(self, host, port, username=None, password=None, timeout=None, **options): self.host = host self.port = port self.username = username self.password = password self._connection = None self._protocol_version = None self.timeout = timeout self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options))
def read_data(self, message_size, is_compressed = False, **options): ''' Reads and optionally parses data part of a message. .. note:: :func:`.read_header` is required to be called before executing the :func:`.read_data` :Parameters: - `message_size` (`integer`) - size of the message to be read - `is_compressed` (`boolean`) - indicates whether data is compressed :Options: - `raw` (`boolean`) - indicates whether read data should parsed or returned in raw byte form - `numpy_temporals` (`boolean`) - if ``False`` temporal vectors are backed by raw q representation (:class:`.QTemporalList`, :class:`.QTemporal`) instances, otherwise are represented as `numpy datetime64`/`timedelta64` arrays and atoms, **Default**: ``False`` :returns: read data (parsed or raw byte form) ''' self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options)) if is_compressed: if self._stream: self._buffer.wrap(self._read_bytes(4)) uncompressed_size = -8 + self._buffer.get_int() compressed_data = self._read_bytes(message_size - 12) if self._stream else self._buffer.raw(message_size - 12) raw_data = numpy.fromstring(compressed_data, dtype = numpy.uint8) if uncompressed_size <= 0: raise QReaderException('Error while data decompression.') raw_data = uncompress(raw_data, numpy.intc(uncompressed_size)) raw_data = numpy.ndarray.tostring(raw_data) self._buffer.wrap(raw_data) elif self._stream: raw_data = self._read_bytes(message_size - 8) self._buffer.wrap(raw_data) if not self._stream and self._options.raw: raw_data = self._buffer.raw(message_size - 8) return raw_data if self._options.raw else self._read_object()
async def read_data(self, message_size, is_compressed=False, **options): self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options)) if is_compressed: if self._stream: self._buffer.wrap(await self._read_bytes(4)) uncompressed_size = -8 + self._buffer.get_int() compressed_data = (await self._read_bytes(message_size - 12)) \ if self._stream else self._buffer.raw(message_size - 12) raw_data = numpy.fromstring(compressed_data, dtype=numpy.uint8) if uncompressed_size <= 0: raise QReaderException('Error while data decompression.') raw_data = uncompress(raw_data, numpy.intc(uncompressed_size)) raw_data = numpy.ndarray.tostring(raw_data) self._buffer.wrap(raw_data) elif self._stream: raw_data = await self._read_bytes(message_size - 8) self._buffer.wrap(raw_data) if not self._stream and self._options.raw: raw_data = self._buffer.raw(message_size - 8) return raw_data if self._options.raw else self._read_object()