def _convert_data(data: (list, tuple), coldtype: str):
if re.match(r'c', coldtype):
data = [msgpack_dumps(x, compress=True) for x in data]
elif re.match(r'o', coldtype):
data = [msgpack_dumps(x, compress=False) for x in data]
data = np.array(data)
if re.search(r'U', str(data.dtype)):
data = np.core.defchararray.encode(data, 'utf-8')
assert isinstance(data, (list, tuple, np.ndarray))
return data
def test_msgpack_dump_load_list(self):
d = ['abc', b'def', [1, 2, 3.4], {'hello': [1, 'world']}]
comp = True
r = msgpack_loads(msgpack_dumps(d, compress=comp),
compress=comp,
use_list=True)
self.assertListEqual(d, r)
comp = False
r = msgpack_loads(msgpack_dumps(d, compress=comp),
compress=comp,
use_list=True)
self.assertListEqual(d, r)
def test_msgpack_dump_load_tuple(self):
dresult = ('abc', b'def', (1, 2, 3.4), {'hello': (1, 'world')})
d = ['abc', b'def', [1, 2, 3.4], {'hello': [1, 'world']}]
comp = True
r = msgpack_loads(msgpack_dumps(d, compress=comp),
compress=comp,
use_list=False)
self.assertTupleEqual(dresult, r)
comp = False
r = msgpack_loads(msgpack_dumps(d, compress=comp),
compress=comp,
use_list=False)
self.assertTupleEqual(dresult, r)
def _create_column_from_data(self, h5: tb.File, table_path: str, col_name: str,
data: (list, tuple, np.ndarray)) -> str:
if isinstance(data[0], str):
objdt = 's'
elif isinstance(data[0], (list, tuple, dict, bytes)):
data = [msgpack_dumps(x, compress=False) for x in data]
objdt = 'o'
elif not isinstance(data[0], (int, float, np.int, np.float, np.ndarray)):
raise Exception(f"Unknown type in col: {col_name} type:{type(data[0])} in {self._h5file}")
earray_col = self._create_column(h5, self._path(table_path, col_name), data=data)
if re.match(r'[if]', str(earray_col.dtype)):
m = re.match(r'([if])', str(earray_col.dtype))
dtype = m.group(1)
elif re.search(r'S(\d+)', str(earray_col.dtype)):
m = re.search(r'S(\d+)', str(earray_col.dtype))
dtype = f'{objdt}{m.group(1)}'
else:
raise Exception(f"Corruption possible: Unallowed datatype in created column "
f"{table_path}/{col_name}: {str(earray_col.dtype)} in {self._h5file}")
return dtype
def _write_attrs(self, h5, table_path, attrs_name, attrs_value):
table = self._get_node(h5, table_path)
try:
attr_bytes = table._v_attrs[ATTR_CTABLE]
simpah5_attrs = msgpack_loads(attr_bytes, use_list=True)
except KeyError:
simpah5_attrs = {}
simpah5_attrs['_version'] = VERSION
simpah5_attrs[attrs_name] = attrs_value
table._v_attrs[ATTR_CTABLE] = msgpack_dumps(simpah5_attrs)