def test_compress_data_three(self):
data = np.array([2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048,
4096, 8192, 16384, 32768, 65536], dtype=np.float64)
compression_result = compress_array(data, 'e')
c_arr = compression_result.numpy_array
self.assertEqual(2, compression_result.reference_value)
self.assertEqual(2, c_arr.itemsize)
self.assertEqual(15, c_arr.size)
self.assertEqual(2, c_arr[0])
self.assertEqual(4, c_arr[1])
self.assertEqual(8, c_arr[2])
self.assertEqual(16, c_arr[3])
self.assertEqual(32, c_arr[4])
self.assertEqual(64, c_arr[5])
self.assertEqual(128, c_arr[6])
self.assertEqual(256, c_arr[7])
self.assertEqual(512, c_arr[8])
self.assertEqual(1024, c_arr[9])
self.assertEqual(2048, c_arr[10])
self.assertEqual(4096, c_arr[11])
self.assertEqual(8192, c_arr[12])
self.assertEqual(16384, c_arr[13])
self.assertEqual(32768, c_arr[14])
compression_result = compress_array(data, 'm')
c_arr = compression_result.numpy_array
self.assertEqual(2, compression_result.reference_value)
self.assertEqual(4, c_arr.itemsize)
self.assertEqual(16, c_arr.size)
self.assertEqual(0, c_arr[0])
self.assertEqual(2, c_arr[1])
self.assertEqual(6, c_arr[2])
self.assertEqual(65536 - 2, c_arr[15])
return
def test_compress_tiny_arrays(self):
self.assertEqual(
1,
compress_array(np.array([1], dtype=np.uint8), 'm').itemsize)
self.assertEqual(
1,
compress_array(np.array([1], dtype=np.int8), 'm').itemsize)
self.assertEqual(
2,
compress_array(np.array([1], dtype=np.float16), 'm').itemsize)
self.assertEqual(
2,
compress_array(np.array([1], dtype=np.uint16), 'e').itemsize)
return
def test_compress_data_three(self):
data = np.array([
2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384,
32768, 65536
],
dtype=np.uint64)
compression_result = compress_array(data, 'e')
dec_array = decompress_array(compression_result.numpy_array, 'e',
compression_result.reference_value)
self.assertEqual(16, dec_array.size)
self.assertEqual(2, dec_array[0])
self.assertEqual(4, dec_array[1])
self.assertEqual(8, dec_array[2])
self.assertEqual(16, dec_array[3])
self.assertEqual(32, dec_array[4])
self.assertEqual(64, dec_array[5])
self.assertEqual(128, dec_array[6])
self.assertEqual(256, dec_array[7])
self.assertEqual(512, dec_array[8])
self.assertEqual(1024, dec_array[9])
self.assertEqual(2048, dec_array[10])
self.assertEqual(4096, dec_array[11])
self.assertEqual(8192, dec_array[12])
self.assertEqual(16384, dec_array[13])
self.assertEqual(32768, dec_array[14])
self.assertEqual(65536, dec_array[15])
compression_result = compress_array(data, 'm')
dec_array = decompress_array(compression_result.numpy_array, 'm',
compression_result.reference_value)
self.assertEqual(16, dec_array.size)
self.assertEqual(2, dec_array[0])
self.assertEqual(4, dec_array[1])
self.assertEqual(8, dec_array[2])
self.assertEqual(16, dec_array[3])
self.assertEqual(32, dec_array[4])
self.assertEqual(64, dec_array[5])
self.assertEqual(128, dec_array[6])
self.assertEqual(256, dec_array[7])
self.assertEqual(512, dec_array[8])
self.assertEqual(1024, dec_array[9])
self.assertEqual(2048, dec_array[10])
self.assertEqual(4096, dec_array[11])
self.assertEqual(8192, dec_array[12])
self.assertEqual(16384, dec_array[13])
self.assertEqual(32768, dec_array[14])
self.assertEqual(65536, dec_array[15])
return
def _encode_data(self):
"""
Performs compression and alteration on data to produce data-set
that will be written to binary form
:return: None
"""
if self._encoded_data is not None:
return
self._encoded_data = self._data
# TODO datetimes should be handled here
if self._use_floating_point_rounding:
self._encoded_data = round_array_returning_integers(
self._encoded_data,
self._floating_point_rounding_num_decimals
)
if self._use_compression:
self._compression_reference_value_dtype = self._encoded_data.dtype
mode = 'm' if self._compression_mode is None else self._compression_mode
compression_result = compress_array(self._encoded_data, mode)
self._compression_mode = mode
self._compression_dtype = compression_result.numpy_array.dtype
self._encoded_data = compression_result.numpy_array
self._compression_reference_value = compression_result.reference_value
return
def test_compress_data_two(self):
data = np.array([5.2, 0.8, 3.1415, 8], dtype=np.float64)
compression_result = compress_array(data, 'm')
dec_array = decompress_array(compression_result.numpy_array, 'm',
compression_result.reference_value)
self.assertEqual(4, dec_array.size)
self.assertEqual(5.2, dec_array[0])
self.assertEqual(0.8, dec_array[1])
self.assertAlmostEqual(3.1415, dec_array[2])
self.assertEqual(8, dec_array[3])
return
def test_decompress_data_one(self):
data = np.array([-4, -2, 0, 2000], dtype=np.int16)
compression_result = compress_array(data, 'e')
dec_array = decompress_array(compression_result.numpy_array, 'e',
compression_result.reference_value)
self.assertEqual(4, dec_array.size)
self.assertEqual(-4, dec_array[0])
self.assertEqual(-2, dec_array[1])
self.assertEqual(0, dec_array[2])
self.assertEqual(2000, dec_array[3])
compression_result = compress_array(data, 'm')
dec_array = decompress_array(compression_result.numpy_array, 'm',
compression_result.reference_value)
self.assertEqual(4, dec_array.size)
self.assertEqual(-4, dec_array[0])
self.assertEqual(-2, dec_array[1])
self.assertEqual(0, dec_array[2])
self.assertEqual(2000, dec_array[3])
return
def test_decompress_data_zero(self):
data = np.array([1, 2, 3, 4], dtype=np.uint32)
compression_result = compress_array(data, 'e')
dec_array = decompress_array(compression_result.numpy_array, 'e',
compression_result.reference_value)
self.assertEqual(4, dec_array.size)
self.assertEqual(1, dec_array[0])
self.assertEqual(2, dec_array[1])
self.assertEqual(3, dec_array[2])
self.assertEqual(4, dec_array[3])
compression_result = compress_array(data, 'm')
dec_array = decompress_array(compression_result.numpy_array, 'm',
compression_result.reference_value)
self.assertEqual(4, dec_array.size)
self.assertEqual(1, dec_array[0])
self.assertEqual(2, dec_array[1])
self.assertEqual(3, dec_array[2])
self.assertEqual(4, dec_array[3])
return
def test_compress_data_two(self):
data = np.array([5.2, 0.8, 3.1415, 8], dtype=np.float64)
compression_result = compress_array(data, 'm')
c_arr = compression_result.numpy_array
self.assertEqual(0.8, compression_result.reference_value)
self.assertEqual(8, c_arr.itemsize)
self.assertEqual(4, c_arr.size)
self.assertEqual(4.4, c_arr[0])
self.assertEqual(0, c_arr[1])
self.assertEqual(2.3415, c_arr[2])
self.assertEqual(7.2, c_arr[3])
return
def test_compress_data_one(self):
data = np.array([-4, -2, 0, 2000], dtype=np.int16)
compression_result = compress_array(data, 'e')
c_arr = compression_result.numpy_array
self.assertEqual(-4, compression_result.reference_value)
self.assertEqual(2, c_arr.itemsize)
self.assertEqual(3, c_arr.size)
self.assertEqual(2, c_arr[0])
self.assertEqual(2, c_arr[1])
self.assertEqual(2000, c_arr[2])
compression_result = compress_array(data, 'm')
c_arr = compression_result.numpy_array
self.assertEqual(-4, compression_result.reference_value)
self.assertEqual(2, c_arr.itemsize)
self.assertEqual(4, c_arr.size)
self.assertEqual(0, c_arr[0])
self.assertEqual(2, c_arr[1])
self.assertEqual(4, c_arr[2])
self.assertEqual(2004, c_arr[3])
return
def test_compress_data_zero(self):
data = np.array([1, 2, 3, 4], dtype=np.uint32)
compression_result = compress_array(data, 'e')
c_arr = compression_result.numpy_array
self.assertEqual(1, compression_result.reference_value)
self.assertEqual(1, c_arr.itemsize)
self.assertEqual(3, c_arr.size)
self.assertEqual(1, c_arr[0])
self.assertEqual(1, c_arr[1])
self.assertEqual(1, c_arr[2])
compression_result = compress_array(data, 'm')
c_arr = compression_result[0]
self.assertEqual(1, compression_result[1])
self.assertEqual(1, c_arr.itemsize)
self.assertEqual(4, c_arr.size)
self.assertEqual(0, c_arr[0])
self.assertEqual(1, c_arr[1])
self.assertEqual(2, c_arr[2])
self.assertEqual(3, c_arr[3])
return
def test_compress_array_errors(self):
data = np.array([1, -2, 3, 4], dtype=np.int32)
with self.assertRaises(CompressionModeInvalidError):
compress_array(data, 'bad_mode')
return
def test_unable_to_compress_type(self):
with self.assertRaises(CompressionError):
compress_array(np.array(['x'], dtype='U1'), 'e')
return