def test_unit_chunk(self): dimensions = (16384, 16384 * 4) dtype_bytesize = 4 unit_chunks = (3, 7) ret_val = anc_build_utils.calc_chunks(dimensions, dtype_bytesize, unit_chunks=unit_chunks) self.assertTrue(np.allclose(ret_val, (27, 98)))
def test_shape_mismatch(self): dimensions = (16384, 16384 * 4) dtype_bytesize = 4 unit_chunks = (1, 5, 9) with self.assertRaises(ValueError): _ = anc_build_utils.calc_chunks(dimensions, dtype_bytesize, unit_chunks=unit_chunks)
def test_unit_not_iterable(self): dimensions = (16384, 16384 * 4) dtype_bytesize = 4 unit_chunks = 4 with self.assertRaises(TypeError): _ = anc_build_utils.calc_chunks(dimensions, dtype_bytesize, unit_chunks=unit_chunks)
def test_unit_chunk_max_mem(self): dimensions = (16384, 16384 * 4) dtype_bytesize = 4 unit_chunks = (3, 7) max_mem = 50000 ret_val = anc_build_utils.calc_chunks(dimensions, dtype_bytesize, unit_chunks=unit_chunks, max_chunk_mem=max_mem) self.assertTrue(np.allclose(ret_val, (57, 224)))
def _write_results_chunk(self): """ Writes the provided SVD results to file Parameters ---------- """ comp_dim = Dimension('Principal Component', 'a. u.', len(self.__s)) h5_svd_group = create_results_group( self.h5_main, self.process_name, h5_parent_group=self._h5_target_group) self.h5_results_grp = h5_svd_group self._write_source_dset_provenance() write_simple_attrs(h5_svd_group, self.parms_dict) write_simple_attrs(h5_svd_group, {'svd_method': 'sklearn-randomized'}) h5_u = write_main_dataset(h5_svd_group, np.float32(self.__u), 'U', 'Abundance', 'a.u.', None, comp_dim, h5_pos_inds=self.h5_main.h5_pos_inds, h5_pos_vals=self.h5_main.h5_pos_vals, dtype=np.float32, chunks=calc_chunks(self.__u.shape, np.float32(0).itemsize)) # print(get_attr(self.h5_main, 'quantity')[0]) h5_v = write_main_dataset(h5_svd_group, self.__v, 'V', get_attr(self.h5_main, 'quantity')[0], 'a.u.', comp_dim, None, h5_spec_inds=self.h5_main.h5_spec_inds, h5_spec_vals=self.h5_main.h5_spec_vals, chunks=calc_chunks( self.__v.shape, self.h5_main.dtype.itemsize)) # No point making this 1D dataset a main dataset h5_s = h5_svd_group.create_dataset('S', data=np.float32(self.__s)) ''' Check h5_main for plot group references. Copy them into V if they exist ''' for key in self.h5_main.attrs.keys(): if '_Plot_Group' not in key: continue ref_inds = get_indices_for_region_ref(self.h5_main, self.h5_main.attrs[key], return_method='corners') ref_inds = ref_inds.reshape([-1, 2, 2]) ref_inds[:, 1, 0] = h5_v.shape[0] - 1 svd_ref = create_region_reference(h5_v, ref_inds) h5_v.attrs[key] = svd_ref # Marking completion: self._status_dset_name = 'completed_positions' self._h5_status_dset = h5_svd_group.create_dataset( self._status_dset_name, data=np.ones(self.h5_main.shape[0], dtype=np.uint8)) # keeping legacy option: h5_svd_group.attrs['last_pixel'] = self.h5_main.shape[0]
def test_invalid_types(self): with self.assertRaises(TypeError): _ = anc_build_utils.calc_chunks("Fdfd", 14) with self.assertRaises(TypeError): _ = anc_build_utils.calc_chunks((16384, 16384 * 4), 2.124)