def read_data(self, message_size, raw = False, is_compressed = False):
'''
Reads and optionally parses data part of a message.
.. note:: :func:`.read_header` is required to be called before executing
the :func:`.read_data`
:Parameters:
- `message_size` (`integer`) - size of the message to be read
- `raw` (`boolean`) - indicates whether read data should parsed or
returned in raw byte form
- `is_compressed` (`boolean`) - indicates whether data is compressed
:returns: read data (parsed or raw byte form)
'''
if is_compressed:
if self._stream:
self._buffer.wrap(self._read_bytes(4))
uncompressed_size = -8 + self._buffer.get_int()
compressed_data = self._read_bytes(message_size - 12) if self._stream else self._buffer.raw(message_size - 12)
raw_data = numpy.fromstring(compressed_data, dtype = numpy.uint8)
if uncompressed_size <= 0:
raise QReaderException('Error while data decompression.')
raw_data = uncompress(raw_data, numpy.intc(uncompressed_size))
raw_data = numpy.ndarray.tostring(raw_data)
self._buffer.wrap(raw_data)
elif self._stream:
raw_data = self._read_bytes(message_size - 8)
self._buffer.wrap(raw_data)
if not self._stream and raw:
raw_data = self._buffer.raw(message_size - 8)
return raw_data if raw else self._read_object()
def read_data(self, message_size, compression_mode=0, **options):
'''
Reads and optionally parses data part of a message.
.. note:: :func:`.read_header` is required to be called before executing
the :func:`.read_data`
:Parameters:
- `message_size` (`int`) - size of the message to be read
- `compression_mode` (`int`) - indicates whether data is compressed, 1 for <2GB, 2 for larger
:Options:
- `raw` (`boolean`) - indicates whether read data should parsed or
returned in raw byte form
- `numpy_temporals` (`boolean`) - if ``False`` temporal vectors are
backed by raw q representation (:class:`.QTemporalList`,
:class:`.QTemporal`) instances, otherwise are represented as
`numpy datetime64`/`timedelta64` arrays and atoms,
**Default**: ``False``
:returns: read data (parsed or raw byte form)
'''
self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options))
if compression_mode > 0:
comprHeaderLen = 4 if compression_mode == 1 else 8
if self._stream:
self._buffer.wrap(self._read_bytes(comprHeaderLen))
uncompressed_size = -8 + (self._buffer.get_uint(
) if compression_mode == 1 else self._buffer.get_long())
compressed_data = self._read_bytes(message_size - (
8 + comprHeaderLen)) if self._stream else self._buffer.raw(
message_size - (8 + comprHeaderLen))
raw_data = numpy.frombuffer(compressed_data, dtype=numpy.uint8)
if uncompressed_size <= 0:
raise QReaderException('Error while data decompression.')
raw_data = uncompress(raw_data, numpy.int64(uncompressed_size))
raw_data = numpy.ndarray.tobytes(raw_data)
self._buffer.wrap(raw_data)
elif self._stream:
raw_data = self._read_bytes(message_size - 8)
self._buffer.wrap(raw_data)
if not self._stream and self._options.raw:
raw_data = self._buffer.raw(message_size - 8)
return raw_data if self._options.raw else self._read_object()
def read_data(self, message_size, is_compressed = False, **options):
'''
Reads and optionally parses data part of a message.
.. note:: :func:`.read_header` is required to be called before executing
the :func:`.read_data`
:Parameters:
- `message_size` (`integer`) - size of the message to be read
- `is_compressed` (`boolean`) - indicates whether data is compressed
:Options:
- `raw` (`boolean`) - indicates whether read data should parsed or
returned in raw byte form
- `numpy_temporals` (`boolean`) - if ``False`` temporal vectors are
backed by raw q representation (:class:`.QTemporalList`,
:class:`.QTemporal`) instances, otherwise are represented as
`numpy datetime64`/`timedelta64` arrays and atoms,
**Default**: ``False``
:returns: read data (parsed or raw byte form)
'''
self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options))
if is_compressed:
if self._stream:
self._buffer.wrap(self._read_bytes(4))
uncompressed_size = -8 + self._buffer.get_int()
compressed_data = self._read_bytes(message_size - 12) if self._stream else self._buffer.raw(message_size - 12)
raw_data = numpy.fromstring(compressed_data, dtype = numpy.uint8)
if uncompressed_size <= 0:
raise QReaderException('Error while data decompression.')
raw_data = uncompress(raw_data, numpy.intc(uncompressed_size))
raw_data = numpy.ndarray.tostring(raw_data)
self._buffer.wrap(raw_data)
elif self._stream:
raw_data = self._read_bytes(message_size - 8)
self._buffer.wrap(raw_data)
if not self._stream and self._options.raw:
raw_data = self._buffer.raw(message_size - 8)
return raw_data if self._options.raw else self._read_object()
def read_data(self, message_size, raw = False, is_compressed = False):
if is_compressed:
if self._stream:
self._buffer.wrap(self._read_bytes(4))
uncompressed_size = -8 + self._buffer.get_int()
compressed_data = self._read_bytes(message_size - 12) if self._stream else self._buffer.raw(message_size - 12)
raw_data = numpy.fromstring(compressed_data, dtype = numpy.uint8)
if uncompressed_size <= 0:
raise QReaderException('Error while data decompression.')
raw_data = uncompress(raw_data, numpy.intc(uncompressed_size))
raw_data = numpy.ndarray.tostring(raw_data)
self._buffer.wrap(raw_data)
elif self._stream:
raw_data = self._read_bytes(message_size - 8)
self._buffer.wrap(raw_data)
return raw_data if raw else self._read_object()
def read_data(self, message_size, is_compressed = False, **options):
'''
Reads and optionally parses data part of a message.
.. note:: :func:`.read_header` is required to be called before executing
the :func:`.read_data`
:Parameters:
- `message_size` (`integer`) - size of the message to be read
- `is_compressed` (`boolean`) - indicates whether data is compressed
:Options:
- `raw` (`boolean`) - indicates whether read data should parsed or
returned in raw byte form
- `numpy_temporals` (`boolean`) - if ``False`` temporal vectors are
backed by raw q representation (:class:`.QTemporalList`,
:class:`.QTemporal`) instances, otherwise are represented as
`numpy datetime64`/`timedelta64` arrays and atoms,
**Default**: ``False``
:returns: read data (parsed or raw byte form)
'''
options = MetaData(**READER_CONFIGURATION.union_dict(**options))
if is_compressed:
if self._stream:
self._buffer.wrap(self._read_bytes(4))
uncompressed_size = -8 + self._buffer.get_int()
compressed_data = self._read_bytes(message_size - 12) if self._stream else self._buffer.raw(message_size - 12)
raw_data = numpy.fromstring(compressed_data, dtype = numpy.uint8)
if uncompressed_size <= 0:
raise QReaderException('Error while data decompression.')
raw_data = uncompress(raw_data, numpy.intc(uncompressed_size))
raw_data = numpy.ndarray.tostring(raw_data)
self._buffer.wrap(raw_data)
elif self._stream:
raw_data = self._read_bytes(message_size - 8)
self._buffer.wrap(raw_data)
if not self._stream and options.raw:
raw_data = self._buffer.raw(message_size - 8)
return raw_data if options.raw else self._read_object(options)
async def read_data(self, message_size, is_compressed=False, **options):
self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options))
if is_compressed:
if self._stream:
self._buffer.wrap(await self._read_bytes(4))
uncompressed_size = -8 + self._buffer.get_int()
compressed_data = (await self._read_bytes(message_size - 12)) \
if self._stream else self._buffer.raw(message_size - 12)
raw_data = numpy.fromstring(compressed_data, dtype=numpy.uint8)
if uncompressed_size <= 0:
raise QReaderException('Error while data decompression.')
raw_data = uncompress(raw_data, numpy.intc(uncompressed_size))
raw_data = numpy.ndarray.tostring(raw_data)
self._buffer.wrap(raw_data)
elif self._stream:
raw_data = await self._read_bytes(message_size - 8)
self._buffer.wrap(raw_data)
if not self._stream and self._options.raw:
raw_data = self._buffer.raw(message_size - 8)
return raw_data if self._options.raw else self._read_object()