def test_unit_chunk(self):
dimensions = (16384, 16384 * 4)
dtype_bytesize = 4
unit_chunks = (3, 7)
ret_val = anc_build_utils.calc_chunks(dimensions,
dtype_bytesize,
unit_chunks=unit_chunks)
self.assertTrue(np.allclose(ret_val, (27, 98)))
def test_shape_mismatch(self):
dimensions = (16384, 16384 * 4)
dtype_bytesize = 4
unit_chunks = (1, 5, 9)
with self.assertRaises(ValueError):
_ = anc_build_utils.calc_chunks(dimensions,
dtype_bytesize,
unit_chunks=unit_chunks)
def test_unit_not_iterable(self):
dimensions = (16384, 16384 * 4)
dtype_bytesize = 4
unit_chunks = 4
with self.assertRaises(TypeError):
_ = anc_build_utils.calc_chunks(dimensions,
dtype_bytesize,
unit_chunks=unit_chunks)
def test_unit_chunk_max_mem(self):
dimensions = (16384, 16384 * 4)
dtype_bytesize = 4
unit_chunks = (3, 7)
max_mem = 50000
ret_val = anc_build_utils.calc_chunks(dimensions,
dtype_bytesize,
unit_chunks=unit_chunks,
max_chunk_mem=max_mem)
self.assertTrue(np.allclose(ret_val, (57, 224)))
def _write_results_chunk(self):
"""
Writes the provided SVD results to file
Parameters
----------
"""
comp_dim = Dimension('Principal Component', 'a. u.', len(self.__s))
h5_svd_group = create_results_group(
self.h5_main,
self.process_name,
h5_parent_group=self._h5_target_group)
self.h5_results_grp = h5_svd_group
self._write_source_dset_provenance()
write_simple_attrs(h5_svd_group, self.parms_dict)
write_simple_attrs(h5_svd_group, {'svd_method': 'sklearn-randomized'})
h5_u = write_main_dataset(h5_svd_group,
np.float32(self.__u),
'U',
'Abundance',
'a.u.',
None,
comp_dim,
h5_pos_inds=self.h5_main.h5_pos_inds,
h5_pos_vals=self.h5_main.h5_pos_vals,
dtype=np.float32,
chunks=calc_chunks(self.__u.shape,
np.float32(0).itemsize))
# print(get_attr(self.h5_main, 'quantity')[0])
h5_v = write_main_dataset(h5_svd_group,
self.__v,
'V',
get_attr(self.h5_main, 'quantity')[0],
'a.u.',
comp_dim,
None,
h5_spec_inds=self.h5_main.h5_spec_inds,
h5_spec_vals=self.h5_main.h5_spec_vals,
chunks=calc_chunks(
self.__v.shape,
self.h5_main.dtype.itemsize))
# No point making this 1D dataset a main dataset
h5_s = h5_svd_group.create_dataset('S', data=np.float32(self.__s))
'''
Check h5_main for plot group references.
Copy them into V if they exist
'''
for key in self.h5_main.attrs.keys():
if '_Plot_Group' not in key:
continue
ref_inds = get_indices_for_region_ref(self.h5_main,
self.h5_main.attrs[key],
return_method='corners')
ref_inds = ref_inds.reshape([-1, 2, 2])
ref_inds[:, 1, 0] = h5_v.shape[0] - 1
svd_ref = create_region_reference(h5_v, ref_inds)
h5_v.attrs[key] = svd_ref
# Marking completion:
self._status_dset_name = 'completed_positions'
self._h5_status_dset = h5_svd_group.create_dataset(
self._status_dset_name,
data=np.ones(self.h5_main.shape[0], dtype=np.uint8))
# keeping legacy option:
h5_svd_group.attrs['last_pixel'] = self.h5_main.shape[0]
def test_invalid_types(self):
with self.assertRaises(TypeError):
_ = anc_build_utils.calc_chunks("Fdfd", 14)
with self.assertRaises(TypeError):
_ = anc_build_utils.calc_chunks((16384, 16384 * 4), 2.124)