def test_illegal_key(self):
with h5py.File(data_utils.std_beps_path, mode='r') as h5_f:
h5_inds = h5_f['/Raw_Measurement/Spectroscopic_Indices']
h5_vals = h5_f['/Raw_Measurement/Spectroscopic_Values']
with self.assertRaises(KeyError):
_ = hdf_utils.get_unit_values(
h5_inds, h5_vals, dim_names=['Cycle', 'Does not exist'])
def test_illegal_dset(self):
with h5py.File(data_utils.std_beps_path, mode='r') as h5_f:
h5_inds = h5_f['/Raw_Measurement/Spectroscopic_Indices']
h5_vals = h5_f['/Raw_Measurement/Ancillary']
with self.assertRaises(ValueError):
_ = hdf_utils.get_unit_values(h5_inds,
h5_vals,
dim_names=['Cycle', 'Bias'])
def test_all_dim_names_not_provided(self):
with h5py.File(data_utils.std_beps_path, mode='r') as h5_f:
h5_inds = h5_f['/Raw_Measurement/Position_Indices'][()]
h5_vals = h5_f['/Raw_Measurement/Position_Values'][()]
with self.assertRaises(TypeError):
_ = hdf_utils.get_unit_values(h5_inds,
h5_vals,
dim_names=['Y'])
def test_sparse_samp_w_attr(self):
# What should the user expect this function to do? throw an error.
with h5py.File(data_utils.sparse_sampling_path, mode='r') as h5_f:
h5_inds = h5_f['/Measurement_000/Channel_001/Position_Indices']
h5_vals = h5_f['/Measurement_000/Channel_001/Position_Values']
with self.assertRaises(ValueError):
_ = hdf_utils.get_unit_values(h5_inds,
h5_vals,
dim_names=['Y'])
def test_dependent_dim(self):
with h5py.File(data_utils.relaxation_path, mode='r') as h5_f:
h5_inds = h5_f[
'/Measurement_000/Channel_000/Spectroscopic_Indices']
h5_vals = h5_f['/Measurement_000/Channel_000/Spectroscopic_Values']
spec_dim_names = hdf_utils.get_attr(h5_inds, 'labels')
ret_dict = hdf_utils.get_unit_values(h5_inds, h5_vals)
for dim_ind, dim_name in enumerate(spec_dim_names):
exp_val = hdf_utils.get_attr(h5_inds,
'unit_vals_dim_' + str(dim_ind))
act_val = ret_dict[dim_name]
self.assertTrue(np.allclose(exp_val, act_val))
def test_source_pos_single(self):
with h5py.File(data_utils.std_beps_path, mode='r') as h5_f:
h5_inds = h5_f['/Raw_Measurement/Position_Indices']
h5_vals = h5_f['/Raw_Measurement/Position_Values']
num_rows = 3
expected = {'Y': np.float16(np.arange(num_rows) * 1.25)}
ret_val = hdf_utils.get_unit_values(h5_inds,
h5_vals,
dim_names='Y')
self.assertEqual(len(expected), len(ret_val))
for key, exp in expected.items():
self.assertTrue(np.allclose(exp, ret_val[key]))
def test_incomp_dim_no_attr(self):
# What should the user expect this function to do? throw an error.
# Given that the unit values for each tile are different, it should throw a ValueError for X.
# Even though we know Y is incomplete, it won't know since it wasn't looking at X.
# However, now this function will automatically find unit values for ALL dimensions just to catch such scenarios
with h5py.File(data_utils.incomplete_measurement_path,
mode='r') as h5_f:
h5_inds = h5_f['/Measurement_000/Channel_000/Position_Indices']
h5_vals = h5_f['/Measurement_000/Channel_000/Position_Values']
with self.assertRaises(ValueError):
_ = hdf_utils.get_unit_values(h5_inds, h5_vals)
with self.assertRaises(ValueError):
_ = hdf_utils.get_unit_values(h5_inds,
h5_vals,
dim_names=['X'])
with self.assertRaises(ValueError):
_ = hdf_utils.get_unit_values(h5_inds,
h5_vals,
dim_names=['Y'])
def test_sparse_samp_no_attr(self):
# What should the user expect this function to do? throw an error.
# Without the attribute, this function will have no idea that it is looking at a sparse sampling case
# it will return the first and second columns of vals blindly
with h5py.File(data_utils.sparse_sampling_path, mode='r') as h5_f:
h5_inds = h5_f['/Measurement_000/Channel_000/Position_Indices']
h5_vals = h5_f['/Measurement_000/Channel_000/Position_Values']
dim_names = hdf_utils.get_attr(h5_inds, 'labels')
ret_dict = hdf_utils.get_unit_values(h5_inds, h5_vals)
for dim_ind, dim_name in enumerate(dim_names):
exp_val = h5_vals[:, dim_ind]
act_val = ret_dict[dim_name]
self.assertTrue(np.allclose(exp_val, act_val))
def test_source_spec_all(self):
with h5py.File(data_utils.std_beps_path, mode='r') as h5_f:
h5_inds = h5_f['/Raw_Measurement/Spectroscopic_Indices']
h5_vals = h5_f['/Raw_Measurement/Spectroscopic_Values']
num_cycle_pts = 7
expected = {
'Bias':
np.float32(2.5 * np.sin(
np.linspace(0, np.pi, num_cycle_pts, endpoint=False))),
'Cycle': [0., 1.]
}
ret_val = hdf_utils.get_unit_values(h5_inds, h5_vals)
self.assertEqual(len(expected), len(ret_val))
for key, exp in expected.items():
self.assertTrue(np.allclose(exp, ret_val[key]))
def _read_data_chunk(self):
"""
Returns the next chunk of data for the guess or the fit
"""
# The Process class should take care of all the basic reading
super(BELoopProjector, self)._read_data_chunk()
if self.data is None:
# Nothing we can do at this point
return
if self.verbose and self.mpi_rank == 0:
print('BELoopProjector got raw data of shape {} from super'
'.'.format(self.data.shape))
"""
Now self.data contains data for N pixels.
The challenge is that this may contain M FORC cycles
Each FORC cycle needs its own V DC vector
So, we can't blindly use the inherited unit_compute.
Our variables now are Position, Vdc, FORC, all others
We want M lists of [VDC x all other variables]
The challenge is that VDC and FORC are inner dimensions -
neither the fastest nor the slowest (guaranteed)
"""
spec_dim_order_s2f = get_sort_order(self.h5_main.h5_spec_inds)[::-1]
# order_to_s2f = list(pos_dim_order_s2f) + list( len(pos_dim_order_s2f) + spec_dim_order_s2f)
order_to_s2f = [0] + list(1 + spec_dim_order_s2f)
print('Order for reshaping to S2F: {}'.format(order_to_s2f))
self._dim_labels_s2f = list(['Positions']) + list(
np.array(self.h5_main.spec_dim_labels)[spec_dim_order_s2f])
print(self._dim_labels_s2f, order_to_s2f)
self._num_forcs = int(
any([
targ in self.h5_main.spec_dim_labels
for targ in ['FORC', 'FORC_Cycle']
]))
if self._num_forcs:
forc_pos = self.h5_main.spec_dim_labels.index(self._forc_dim_name)
self._num_forcs = self.h5_main.spec_dim_sizes[forc_pos]
print('Num FORCS: {}'.format(self._num_forcs))
all_but_forc_rows = []
for ind, dim_name in enumerate(self.h5_main.spec_dim_labels):
if dim_name not in ['FORC', 'FORC_Cycle', 'FORC_repeat']:
all_but_forc_rows.append(ind)
print('All but FORC rows: {}'.format(all_but_forc_rows))
dc_mats = []
forc_mats = []
num_reps = 1 if self._num_forcs == 0 else self._num_forcs
for forc_ind in range(num_reps):
print('')
print('Working on FORC #{}'.format(forc_ind))
if self._num_forcs:
this_forc_spec_inds = \
np.where(self.h5_main.h5_spec_inds[forc_pos] == forc_ind)[0]
else:
this_forc_spec_inds = np.ones(
shape=self.h5_main.h5_spec_inds.shape[1], dtype=np.bool)
if self._num_forcs:
this_forc_dc_vec = get_unit_values(
self.h5_main.h5_spec_inds[all_but_forc_rows]
[:, this_forc_spec_inds],
self.h5_main.h5_spec_vals[all_but_forc_rows]
[:, this_forc_spec_inds],
all_dim_names=list(
np.array(
self.h5_main.spec_dim_labels)[all_but_forc_rows]),
dim_names=self._fit_dim_name)
else:
this_forc_dc_vec = get_unit_values(
self.h5_main.h5_spec_inds,
self.h5_main.h5_spec_vals,
dim_names=self._fit_dim_name)
this_forc_dc_vec = this_forc_dc_vec[self._fit_dim_name]
dc_mats.append(this_forc_dc_vec)
this_forc_2d = self.h5_main[:, this_forc_spec_inds]
print('2D slice shape for this FORC: {}'.format(
this_forc_2d.shape))
"""
this_forc_nd, success = reshape_to_n_dims(this_forc_2d,
h5_pos=self.h5_main.h5_pos_inds[:,:], # THis line will need to change
h5_spec=self.h5_main.h5_spec_inds[:, this_forc_spec_inds])
"""
this_forc_nd, success = reshape_to_n_dims(
this_forc_2d,
h5_pos=None,
# THis line will need to change
h5_spec=self.h5_main.h5_spec_inds[:, this_forc_spec_inds])
print(this_forc_nd.shape)
this_forc_nd_s2f = this_forc_nd.transpose(
order_to_s2f).squeeze() # squeeze out FORC
dim_names_s2f = self._dim_labels_s2f.copy()
if self._num_forcs > 0:
dim_names_s2f.remove(
self._forc_dim_name
) # because it was never there in the first place.
print('Reordered to S2F: {}, {}'.format(this_forc_nd_s2f.shape,
dim_names_s2f))
rest_dc_order = list(range(len(dim_names_s2f)))
_dc_ind = dim_names_s2f.index(self._fit_dim_name)
rest_dc_order.remove(_dc_ind)
rest_dc_order = rest_dc_order + [_dc_ind]
print('Transpose for reordering to rest, DC: {}'.format(
rest_dc_order))
rest_dc_nd = this_forc_nd_s2f.transpose(rest_dc_order)
rest_dc_names = list(np.array(dim_names_s2f)[rest_dc_order])
self._pre_flattening_shape = list(rest_dc_nd.shape)
self._pre_flattening_dim_name_order = list(rest_dc_names)
print('After reodering: {}, {}'.format(rest_dc_nd.shape,
rest_dc_names))
dc_rest_2d = rest_dc_nd.reshape(np.prod(rest_dc_nd.shape[:-1]),
np.prod(rest_dc_nd.shape[-1]))
print('Shape after flattening to 2D: {}'.format(dc_rest_2d.shape))
forc_mats.append(dc_rest_2d)
self.data = forc_mats, dc_mats
def _get_dc_offsets(h5_spec_inds,
h5_spec_vals,
fit_dim_name,
forc_dim_name,
verbose=False):
# FORC is the decider whether or not DC_Offset changes.
# FORC_Repeats etc. should not matter
spec_unit_vals = get_unit_values(h5_spec_inds,
h5_spec_vals,
verbose=False)
if forc_dim_name not in spec_unit_vals.keys():
if verbose:
print(
'This is not a FORC dataset. Just taking unit values for DC Offset'
)
dc_val_mat = np.expand_dims(spec_unit_vals[fit_dim_name], axis=0)
else:
# Reshape the Spec values matrix into an N dimensional array
if verbose:
print(
'This is a FORC dataset. Reshaping Spectroscopic Values to N dimensions'
)
ret_vals = reshape_to_n_dims(h5_spec_vals,
np.expand_dims(np.arange(
h5_spec_vals.shape[0]),
axis=1),
h5_spec_inds,
get_labels=True)
spec_vals_nd, success, spec_nd_labels = ret_vals
if success != True:
raise ValueError(
'Unable to reshape Spectroscopic values to get DC offsets for each FORC'
)
# We will be using "in" quite a bit. So convert to list
spec_nd_labels = list(spec_nd_labels)
if verbose:
print('Reshaped Spectroscopic Values to: {}'.format(
spec_vals_nd.shape))
print(
'Spectroscopic dimension names: {}'.format(spec_nd_labels))
# Note the indices of all other dimensions
all_other_dims = set(range(len(spec_nd_labels))) - \
set([spec_nd_labels.index(fit_dim_name),
spec_nd_labels.index(forc_dim_name)])
# Set up a new order where FORC is at 0 and DC is at 1 and all
# other dimensions (useless) follow
new_order = [
spec_nd_labels.index(forc_dim_name),
spec_nd_labels.index(fit_dim_name)
] + list(all_other_dims)
if verbose:
print('Will transpose this N-dim matrix as: {}'.format(
new_order))
# Apply this new order to the matrix and the labels
spec_vals_nd = spec_vals_nd.transpose(new_order)
spec_nd_labels = np.array(spec_nd_labels)[new_order]
if verbose:
print('After transpose shape and names:\n\t{}\n\t{}'.format(
spec_vals_nd.shape, spec_nd_labels))
# Now remove all other dimensions using a list of slices:
keep_list = [slice(None), slice(None)
] + [slice(0, 1) for _ in range(len(all_other_dims))]
# Don't forget to remove singular dimensions using squeeze
dc_val_mat = spec_vals_nd[keep_list].squeeze()
# Unnecessary but let's keep track of dimension names anyway
spec_nd_labels = spec_nd_labels[:2]
if verbose:
print(
'After removing all other dimensions. Shape is: {} and dimensions are: {}'
.format(dc_val_mat.shape, spec_nd_labels))
return dc_val_mat
