def __inner_test_gzip(data_: bytes) -> None:
gzipped_data = gzip(data_)
assert len(gzipped_data) > 0
assert isinstance(gzipped_data, bytes)
assert gzipped_data != data_
gunzipped_data = gunzip(gzipped_data)
assert gunzipped_data == data_
def __inner_test_gzip(data_: bytes) -> None:
gzipped_data = gzip(data_)
assert len(gzipped_data) > 0
assert isinstance(gzipped_data, bytes)
assert gzipped_data != data_
gunzipped_data = gunzip(gzipped_data)
assert gunzipped_data == data_
def ungzipped_response_content(response: Response) -> str:
"""Return HTTP response's decoded content, gunzip it if neccessary."""
if __response_is_gzipped_data(response):
gzipped_data = response.raw_data()
try:
data = gunzip(gzipped_data).decode('utf-8', errors='replace')
except McGunzipException as ex:
log.error("Unable to gunzip response {}: {}".format(response, ex))
data = response.decoded_content()
else:
data = response.decoded_content()
assert isinstance(data, str)
return data
def test_gzip_bad_input():
with pytest.raises(McGzipException):
# noinspection PyTypeChecker
gzip(None)
with pytest.raises(McGunzipException):
# noinspection PyTypeChecker
gunzip(None)
with pytest.raises(McGunzipException):
gunzip(b'')
with pytest.raises(McGunzipException):
gunzip(b'No way this is valid Gzip data')
def test_gzip_bad_input():
with pytest.raises(McGzipException):
# noinspection PyTypeChecker
gzip(None)
with pytest.raises(McGunzipException):
# noinspection PyTypeChecker
gunzip(None)
with pytest.raises(McGunzipException):
gunzip(b'')
with pytest.raises(McGunzipException):
gunzip(b'No way this is valid Gzip data')
def _uncompress_data_for_method(data: bytes, compression_method: Compression) -> bytes:
"""Uncompress data."""
if data is None:
raise McKeyValueStoreCompressionException("Data is None.")
if not isinstance(data, bytes):
raise McKeyValueStoreCompressionException("Compressed data is not str or bytes: %s" % str(data))
if compression_method == KeyValueStore.Compression.NONE:
pass
elif compression_method == KeyValueStore.Compression.GZIP:
data = gunzip(data)
elif compression_method == KeyValueStore.Compression.BZIP2:
data = bunzip2(data)
else:
raise McKeyValueStoreCompressionException("Invalid compression method: %s" % compression_method)
return data
def _uncompress_data_for_method(data: bytes,
compression_method: Compression) -> bytes:
"""Uncompress data."""
if data is None:
raise McKeyValueStoreCompressionException("Data is None.")
if not isinstance(data, bytes):
raise McKeyValueStoreCompressionException(
"Compressed data is not str or bytes: %s" % str(data))
if compression_method == KeyValueStore.Compression.NONE:
pass
elif compression_method == KeyValueStore.Compression.GZIP:
data = gunzip(data)
elif compression_method == KeyValueStore.Compression.BZIP2:
data = bunzip2(data)
else:
raise McKeyValueStoreCompressionException(
"Invalid compression method: %s" % compression_method)
return data
def read_model(self, models_id: int) -> bytes:
model = self.__db.select(
table='snap.word2vec_models',
what_to_select='raw_data',
condition_hash={
'topics_id': self.__topics_id,
'snapshots_id': self.__snapshots_id,
'snap_word2vec_models_id': models_id,
}
).hash()
if not model:
raise McWord2vecException(
f"Model {models_id} for topic {self.__topics_id}, snapshot {self.__snapshots_id} was not found"
)
compressed_model_data = model['raw_data']
if isinstance(compressed_model_data, memoryview):
compressed_model_data = compressed_model_data.tobytes()
model_data = gunzip(compressed_model_data)
return model_data
def __inner_test_wrong_algorithm(data_: bytes) -> None:
with pytest.raises(McBunzip2Exception):
bunzip2(gzip(data_))
with pytest.raises(McGunzipException):
gunzip(bzip2(data_))
def __inner_test_wrong_algorithm(data_: bytes) -> None:
with pytest.raises(McBunzip2Exception):
bunzip2(gzip(data_))
with pytest.raises(McGunzipException):
gunzip(bzip2(data_))