def read_data(self, message_size, raw = False, is_compressed = False): ''' Reads and optionally parses data part of a message. .. note:: :func:`.read_header` is required to be called before executing the :func:`.read_data` :Parameters: - `message_size` (`integer`) - size of the message to be read - `raw` (`boolean`) - indicates whether read data should parsed or returned in raw byte form - `is_compressed` (`boolean`) - indicates whether data is compressed :returns: read data (parsed or raw byte form) ''' if is_compressed: if self._stream: self._buffer.wrap(self._read_bytes(4)) uncompressed_size = -8 + self._buffer.get_int() compressed_data = self._read_bytes(message_size - 12) if self._stream else self._buffer.raw(message_size - 12) raw_data = numpy.fromstring(compressed_data, dtype = numpy.uint8) if uncompressed_size <= 0: raise QReaderException('Error while data decompression.') raw_data = uncompress(raw_data, numpy.intc(uncompressed_size)) raw_data = numpy.ndarray.tostring(raw_data) self._buffer.wrap(raw_data) elif self._stream: raw_data = self._read_bytes(message_size - 8) self._buffer.wrap(raw_data) if not self._stream and raw: raw_data = self._buffer.raw(message_size - 8) return raw_data if raw else self._read_object()
def read_data(self, message_size, compression_mode=0, **options): ''' Reads and optionally parses data part of a message. .. note:: :func:`.read_header` is required to be called before executing the :func:`.read_data` :Parameters: - `message_size` (`int`) - size of the message to be read - `compression_mode` (`int`) - indicates whether data is compressed, 1 for <2GB, 2 for larger :Options: - `raw` (`boolean`) - indicates whether read data should parsed or returned in raw byte form - `numpy_temporals` (`boolean`) - if ``False`` temporal vectors are backed by raw q representation (:class:`.QTemporalList`, :class:`.QTemporal`) instances, otherwise are represented as `numpy datetime64`/`timedelta64` arrays and atoms, **Default**: ``False`` :returns: read data (parsed or raw byte form) ''' self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options)) if compression_mode > 0: comprHeaderLen = 4 if compression_mode == 1 else 8 if self._stream: self._buffer.wrap(self._read_bytes(comprHeaderLen)) uncompressed_size = -8 + (self._buffer.get_uint( ) if compression_mode == 1 else self._buffer.get_long()) compressed_data = self._read_bytes(message_size - ( 8 + comprHeaderLen)) if self._stream else self._buffer.raw( message_size - (8 + comprHeaderLen)) raw_data = numpy.frombuffer(compressed_data, dtype=numpy.uint8) if uncompressed_size <= 0: raise QReaderException('Error while data decompression.') raw_data = uncompress(raw_data, numpy.int64(uncompressed_size)) raw_data = numpy.ndarray.tobytes(raw_data) self._buffer.wrap(raw_data) elif self._stream: raw_data = self._read_bytes(message_size - 8) self._buffer.wrap(raw_data) if not self._stream and self._options.raw: raw_data = self._buffer.raw(message_size - 8) return raw_data if self._options.raw else self._read_object()
def read_data(self, message_size, is_compressed = False, **options): ''' Reads and optionally parses data part of a message. .. note:: :func:`.read_header` is required to be called before executing the :func:`.read_data` :Parameters: - `message_size` (`integer`) - size of the message to be read - `is_compressed` (`boolean`) - indicates whether data is compressed :Options: - `raw` (`boolean`) - indicates whether read data should parsed or returned in raw byte form - `numpy_temporals` (`boolean`) - if ``False`` temporal vectors are backed by raw q representation (:class:`.QTemporalList`, :class:`.QTemporal`) instances, otherwise are represented as `numpy datetime64`/`timedelta64` arrays and atoms, **Default**: ``False`` :returns: read data (parsed or raw byte form) ''' self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options)) if is_compressed: if self._stream: self._buffer.wrap(self._read_bytes(4)) uncompressed_size = -8 + self._buffer.get_int() compressed_data = self._read_bytes(message_size - 12) if self._stream else self._buffer.raw(message_size - 12) raw_data = numpy.fromstring(compressed_data, dtype = numpy.uint8) if uncompressed_size <= 0: raise QReaderException('Error while data decompression.') raw_data = uncompress(raw_data, numpy.intc(uncompressed_size)) raw_data = numpy.ndarray.tostring(raw_data) self._buffer.wrap(raw_data) elif self._stream: raw_data = self._read_bytes(message_size - 8) self._buffer.wrap(raw_data) if not self._stream and self._options.raw: raw_data = self._buffer.raw(message_size - 8) return raw_data if self._options.raw else self._read_object()
def read_data(self, message_size, raw = False, is_compressed = False): if is_compressed: if self._stream: self._buffer.wrap(self._read_bytes(4)) uncompressed_size = -8 + self._buffer.get_int() compressed_data = self._read_bytes(message_size - 12) if self._stream else self._buffer.raw(message_size - 12) raw_data = numpy.fromstring(compressed_data, dtype = numpy.uint8) if uncompressed_size <= 0: raise QReaderException('Error while data decompression.') raw_data = uncompress(raw_data, numpy.intc(uncompressed_size)) raw_data = numpy.ndarray.tostring(raw_data) self._buffer.wrap(raw_data) elif self._stream: raw_data = self._read_bytes(message_size - 8) self._buffer.wrap(raw_data) return raw_data if raw else self._read_object()
def read_data(self, message_size, is_compressed = False, **options): ''' Reads and optionally parses data part of a message. .. note:: :func:`.read_header` is required to be called before executing the :func:`.read_data` :Parameters: - `message_size` (`integer`) - size of the message to be read - `is_compressed` (`boolean`) - indicates whether data is compressed :Options: - `raw` (`boolean`) - indicates whether read data should parsed or returned in raw byte form - `numpy_temporals` (`boolean`) - if ``False`` temporal vectors are backed by raw q representation (:class:`.QTemporalList`, :class:`.QTemporal`) instances, otherwise are represented as `numpy datetime64`/`timedelta64` arrays and atoms, **Default**: ``False`` :returns: read data (parsed or raw byte form) ''' options = MetaData(**READER_CONFIGURATION.union_dict(**options)) if is_compressed: if self._stream: self._buffer.wrap(self._read_bytes(4)) uncompressed_size = -8 + self._buffer.get_int() compressed_data = self._read_bytes(message_size - 12) if self._stream else self._buffer.raw(message_size - 12) raw_data = numpy.fromstring(compressed_data, dtype = numpy.uint8) if uncompressed_size <= 0: raise QReaderException('Error while data decompression.') raw_data = uncompress(raw_data, numpy.intc(uncompressed_size)) raw_data = numpy.ndarray.tostring(raw_data) self._buffer.wrap(raw_data) elif self._stream: raw_data = self._read_bytes(message_size - 8) self._buffer.wrap(raw_data) if not self._stream and options.raw: raw_data = self._buffer.raw(message_size - 8) return raw_data if options.raw else self._read_object(options)
async def read_data(self, message_size, is_compressed=False, **options): self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options)) if is_compressed: if self._stream: self._buffer.wrap(await self._read_bytes(4)) uncompressed_size = -8 + self._buffer.get_int() compressed_data = (await self._read_bytes(message_size - 12)) \ if self._stream else self._buffer.raw(message_size - 12) raw_data = numpy.fromstring(compressed_data, dtype=numpy.uint8) if uncompressed_size <= 0: raise QReaderException('Error while data decompression.') raw_data = uncompress(raw_data, numpy.intc(uncompressed_size)) raw_data = numpy.ndarray.tostring(raw_data) self._buffer.wrap(raw_data) elif self._stream: raw_data = await self._read_bytes(message_size - 8) self._buffer.wrap(raw_data) if not self._stream and self._options.raw: raw_data = self._buffer.raw(message_size - 8) return raw_data if self._options.raw else self._read_object()