def _write_results_chunk(self):
"""
Writes the labels and mean response to the h5 file
Returns
---------
h5_group : HDF5 Group reference
Reference to the group that contains the decomposition results
"""
h5_decomp_group = create_results_group(self.h5_main, self.process_name)
write_simple_attrs(h5_decomp_group, self.parms_dict)
write_simple_attrs(h5_decomp_group, {'n_components': self.__components.shape[0],
'n_samples': self.h5_main.shape[0], 'last_pixel': self.h5_main.shape[0]})
decomp_desc = Dimension('Endmember', 'a. u.', self.__components.shape[0])
# equivalent to V - compound / complex
h5_components = write_main_dataset(h5_decomp_group, self.__components, 'Components',
get_attr(self.h5_main, 'quantity')[0], 'a.u.', decomp_desc,
None,
h5_spec_inds=self.h5_main.h5_spec_inds,
h5_spec_vals=self.h5_main.h5_spec_vals)
# equivalent of U - real
h5_projections = write_main_dataset(h5_decomp_group, np.float32(self.__projection), 'Projection', 'abundance',
'a.u.', None, decomp_desc, dtype=np.float32,
h5_pos_inds=self.h5_main.h5_pos_inds, h5_pos_vals=self.h5_main.h5_pos_vals)
# return the h5 group object
self.h5_results_grp = h5_decomp_group
return self.h5_results_grp
def _create_guess_datasets(self):
"""
Creates the h5 group, guess dataset, corresponding spectroscopic datasets and also
links the guess dataset to the spectroscopic datasets.
"""
h5_group = create_results_group(self.h5_main, 'SHO_Fit')
write_simple_attrs(h5_group, {'SHO_guess_method': "pycroscopy BESHO"})
h5_sho_inds, h5_sho_vals = write_reduced_spec_dsets(
h5_group, self.h5_main.h5_spec_inds, self.h5_main.h5_spec_vals,
self._fit_dim_name)
self.h5_guess = write_main_dataset(
h5_group, (self.h5_main.shape[0], self.num_udvs_steps),
'Guess',
'SHO',
'compound',
None,
None,
h5_pos_inds=self.h5_main.h5_pos_inds,
h5_pos_vals=self.h5_main.h5_pos_vals,
h5_spec_inds=h5_sho_inds,
h5_spec_vals=h5_sho_vals,
chunks=(1, self.num_udvs_steps),
dtype=sho32,
main_dset_attrs=self._parms_dict,
verbose=self._verbose)
write_simple_attrs(self.h5_guess, {
'SHO_guess_method': "pycroscopy BESHO",
'last_pixel': 0
})
copy_region_refs(self.h5_main, self.h5_guess)
def reshape_from_lines_to_pixels(h5_main, pts_per_cycle, scan_step_x_m=None):
"""
Breaks up the provided raw G-mode dataset into lines and pixels (from just lines)
Parameters
----------
h5_main : h5py.Dataset object
Reference to the main dataset that contains the raw data that is only broken up by lines
pts_per_cycle : unsigned int
Number of points in a single pixel
scan_step_x_m : float
Step in meters for pixels
Returns
-------
h5_resh : h5py.Dataset object
Reference to the main dataset that contains the reshaped data
"""
if not check_if_main(h5_main):
raise TypeError('h5_main is not a Main dataset')
h5_main = USIDataset(h5_main)
if pts_per_cycle % 1 != 0 or pts_per_cycle < 1:
raise TypeError('pts_per_cycle should be a positive integer')
if scan_step_x_m is not None:
if not isinstance(scan_step_x_m, Number):
raise TypeError('scan_step_x_m should be a real number')
else:
scan_step_x_m = 1
if h5_main.shape[1] % pts_per_cycle != 0:
warn('Error in reshaping the provided dataset to pixels. Check points per pixel')
raise ValueError
num_cols = int(h5_main.shape[1] / pts_per_cycle)
# TODO: DO NOT assume simple 1 spectral dimension!
single_ao = np.squeeze(h5_main.h5_spec_vals[:, :pts_per_cycle])
spec_dims = Dimension(get_attr(h5_main.h5_spec_vals, 'labels')[0],
get_attr(h5_main.h5_spec_vals, 'units')[0], single_ao)
# TODO: DO NOT assume simple 1D in positions!
pos_dims = [Dimension('X', 'm', np.linspace(0, scan_step_x_m, num_cols)),
Dimension('Y', 'm', np.linspace(0, h5_main.h5_pos_vals[1, 0], h5_main.shape[0]))]
h5_group = create_results_group(h5_main, 'Reshape')
# TODO: Create empty datasets and then write for very large datasets
h5_resh = write_main_dataset(h5_group, (num_cols * h5_main.shape[0], pts_per_cycle), 'Reshaped_Data',
get_attr(h5_main, 'quantity')[0], get_attr(h5_main, 'units')[0], pos_dims, spec_dims,
chunks=(10, pts_per_cycle), dtype=h5_main.dtype, compression=h5_main.compression)
# TODO: DON'T write in one shot assuming small datasets fit in memory!
print('Starting to reshape G-mode line data. Please be patient')
h5_resh[()] = np.reshape(h5_main[()], (-1, pts_per_cycle))
print('Finished reshaping G-mode line data to rows and columns')
return USIDataset(h5_resh)
def _write_results_chunk(self):
"""
Writes the labels and mean response to the h5 file
Returns
---------
h5_group : HDF5 Group reference
Reference to the group that contains the clustering results
"""
print('Writing clustering results to file.')
num_clusters = self.__mean_resp.shape[0]
h5_cluster_group = create_results_group(self.h5_main, self.process_name)
write_simple_attrs(h5_cluster_group, self.parms_dict)
h5_cluster_group.attrs['last_pixel'] = self.h5_main.shape[0]
h5_labels = write_main_dataset(h5_cluster_group, np.uint32(self.__labels.reshape([-1, 1])), 'Labels',
'Cluster ID', 'a. u.', None, Dimension('Cluster', 'ID', 1),
h5_pos_inds=self.h5_main.h5_pos_inds, h5_pos_vals=self.h5_main.h5_pos_vals,
aux_spec_prefix='Cluster_', dtype=np.uint32)
if self.num_comps != self.h5_main.shape[1]:
'''
Setup the Spectroscopic Indices and Values for the Mean Response if we didn't use all components
Note that a sliced spectroscopic matrix may not be contiguous. Let's just lose the spectroscopic data
for now until a better method is figured out
'''
"""
if isinstance(self.data_slice[1], np.ndarray):
centroid_vals_mat = h5_centroids.h5_spec_vals[self.data_slice[1].tolist()]
else:
centroid_vals_mat = h5_centroids.h5_spec_vals[self.data_slice[1]]
ds_centroid_values.data[0, :] = centroid_vals_mat
"""
if isinstance(self.data_slice[1], np.ndarray):
vals_slice = self.data_slice[1].tolist()
else:
vals_slice = self.data_slice[1]
vals = self.h5_main.h5_spec_vals[:, vals_slice].squeeze()
new_spec = Dimension('Original_Spectral_Index', 'a.u.', vals)
h5_inds, h5_vals = write_ind_val_dsets(h5_cluster_group, new_spec, is_spectral=True)
else:
h5_inds = self.h5_main.h5_spec_inds
h5_vals = self.h5_main.h5_spec_vals
# For now, link centroids with default spectroscopic indices and values.
h5_centroids = write_main_dataset(h5_cluster_group, self.__mean_resp, 'Mean_Response',
get_attr(self.h5_main, 'quantity')[0], get_attr(self.h5_main, 'units')[0],
Dimension('Cluster', 'a. u.', np.arange(num_clusters)), None,
h5_spec_inds=h5_inds, aux_pos_prefix='Mean_Resp_Pos_',
h5_spec_vals=h5_vals)
return h5_cluster_group
def _write_results_chunk(self):
"""
Writes the provided SVD results to file
Parameters
----------
"""
comp_dim = Dimension('Principal Component', 'a. u.', len(self.__s))
h5_svd_group = create_results_group(self.h5_main, self.process_name,
h5_parent_group=self._h5_target_group)
self.h5_results_grp = h5_svd_group
self._write_source_dset_provenance()
write_simple_attrs(h5_svd_group, self.parms_dict)
write_simple_attrs(h5_svd_group, {'svd_method': 'sklearn-randomized'})
h5_u = write_main_dataset(h5_svd_group, np.float32(self.__u), 'U', 'Abundance', 'a.u.', None, comp_dim,
h5_pos_inds=self.h5_main.h5_pos_inds, h5_pos_vals=self.h5_main.h5_pos_vals,
dtype=np.float32, chunks=calc_chunks(self.__u.shape, np.float32(0).itemsize))
# print(get_attr(self.h5_main, 'quantity')[0])
h5_v = write_main_dataset(h5_svd_group, self.__v, 'V', get_attr(self.h5_main, 'quantity')[0],
'a.u.', comp_dim, None, h5_spec_inds=self.h5_main.h5_spec_inds,
h5_spec_vals=self.h5_main.h5_spec_vals,
chunks=calc_chunks(self.__v.shape, self.h5_main.dtype.itemsize))
# No point making this 1D dataset a main dataset
h5_s = h5_svd_group.create_dataset('S', data=np.float32(self.__s))
'''
Check h5_main for plot group references.
Copy them into V if they exist
'''
for key in self.h5_main.attrs.keys():
if '_Plot_Group' not in key:
continue
ref_inds = get_indices_for_region_ref(self.h5_main, self.h5_main.attrs[key], return_method='corners')
ref_inds = ref_inds.reshape([-1, 2, 2])
ref_inds[:, 1, 0] = h5_v.shape[0] - 1
svd_ref = create_region_reference(h5_v, ref_inds)
h5_v.attrs[key] = svd_ref
# Marking completion:
self._status_dset_name = 'completed_positions'
self._h5_status_dset = h5_svd_group.create_dataset(self._status_dset_name,
data=np.ones(self.h5_main.shape[0], dtype=np.uint8))
# keeping legacy option:
h5_svd_group.attrs['last_pixel'] = self.h5_main.shape[0]
def _create_results_datasets(self):
self.h5_results_grp = hdf_utils.create_results_group(
self.h5_main, self.process_name)
assert isinstance(self.h5_results_grp, h5py.Group)
if self.verbose: print('Results group created.')
self.results = hdf_utils.create_empty_dataset(
self.h5_main,
self.h5_main.dtype,
'Filtered_Data',
h5_group=self.h5_results_grp)
#self.results = hdf_utils.write_main_dataset(self.h5_results_grp, (self.h5_main.shape[0], 1), "Results", "Results", "Units", None,
#usid.io.write_utils.Dimension('arb', '', [1]), h5_pos_inds=self.h5_main.h5_pos_inds, h5_pos_vals=self.h5_main.h5_pos_vals, dtype=np.float32)
if self.verbose: print('Empty main dataset for results written')
def _create_guess_datasets(self):
"""
Creates the h5 group, guess dataset, corresponding spectroscopic datasets and also
links the guess dataset to the spectroscopic datasets.
"""
self.h5_results_grp = create_results_group(
self.h5_main,
self.process_name,
h5_parent_group=self._h5_target_group)
write_simple_attrs(self.h5_results_grp, self.parms_dict)
# If writing to a new HDF5 file:
# Add back the data_type attribute - still being used in the visualizer
if self.h5_results_grp.file != self.h5_main.file:
write_simple_attrs(
self.h5_results_grp.file,
{'data_type': get_attr(self.h5_main.file, 'data_type')})
ret_vals = write_reduced_anc_dsets(self.h5_results_grp,
self.h5_main.h5_spec_inds,
self.h5_main.h5_spec_vals,
self._fit_dim_name,
verbose=self.verbose)
h5_sho_inds, h5_sho_vals = ret_vals
self._h5_guess = write_main_dataset(
self.h5_results_grp, (self.h5_main.shape[0], self.num_udvs_steps),
'Guess',
'SHO',
'compound',
None,
None,
h5_pos_inds=self.h5_main.h5_pos_inds,
h5_pos_vals=self.h5_main.h5_pos_vals,
h5_spec_inds=h5_sho_inds,
h5_spec_vals=h5_sho_vals,
chunks=(1, self.num_udvs_steps),
dtype=sho32,
main_dset_attrs=self.parms_dict,
verbose=self.verbose)
# Does not make sense to propagate region refs - nobody uses them
# copy_region_refs(self.h5_main, self._h5_guess)
self._h5_guess.file.flush()
if self.verbose and self.mpi_rank == 0:
print('Finished creating Guess dataset')
def _write_results_chunk(self):
"""
Writes the provided SVD results to file
Parameters
----------
"""
comp_dim = Dimension('Principal Component', 'a. u.', len(self.__s))
h5_svd_group = create_results_group(self.h5_main, self.process_name)
self.h5_results_grp = h5_svd_group
write_simple_attrs(h5_svd_group, self.parms_dict)
write_simple_attrs(h5_svd_group, {'svd_method': 'sklearn-randomized'})
h5_u = write_main_dataset(h5_svd_group, np.float32(self.__u), 'U', 'Abundance', 'a.u.', None, comp_dim,
h5_pos_inds=self.h5_main.h5_pos_inds, h5_pos_vals=self.h5_main.h5_pos_vals,
dtype=np.float32, chunks=calc_chunks(self.__u.shape, np.float32(0).itemsize))
# print(get_attr(self.h5_main, 'quantity')[0])
h5_v = write_main_dataset(h5_svd_group, self.__v, 'V', get_attr(self.h5_main, 'quantity')[0],
'a.u.', comp_dim, None, h5_spec_inds=self.h5_main.h5_spec_inds,
h5_spec_vals=self.h5_main.h5_spec_vals,
chunks=calc_chunks(self.__v.shape, self.h5_main.dtype.itemsize))
# No point making this 1D dataset a main dataset
h5_s = h5_svd_group.create_dataset('S', data=np.float32(self.__s))
'''
Check h5_main for plot group references.
Copy them into V if they exist
'''
for key in self.h5_main.attrs.keys():
if '_Plot_Group' not in key:
continue
ref_inds = get_indices_for_region_ref(self.h5_main, self.h5_main.attrs[key], return_method='corners')
ref_inds = ref_inds.reshape([-1, 2, 2])
ref_inds[:, 1, 0] = h5_v.shape[0] - 1
svd_ref = create_region_reference(h5_v, ref_inds)
h5_v.attrs[key] = svd_ref
# Marking completion:
self._status_dset_name = 'completed_positions'
self._h5_status_dset = h5_svd_group.create_dataset(self._status_dset_name,
data=np.ones(self.h5_main.shape[0], dtype=np.uint8))
# keeping legacy option:
h5_svd_group.attrs['last_pixel'] = self.h5_main.shape[0]
def _create_projection_datasets(self):
"""
Setup the Loop_Fit Group and the loop projection datasets
"""
# First grab the spectroscopic indices and values and position indices
self._sho_spec_inds = self.h5_main.h5_spec_inds
self._sho_spec_vals = self.h5_main.h5_spec_vals
self._sho_pos_inds = self.h5_main.h5_pos_inds
fit_dim_ind = self.h5_main.spec_dim_labels.index(self._fit_dim_name)
self._fit_spec_index = fit_dim_ind
self._fit_offset_index = 1 + fit_dim_ind
# Calculate the number of loops per position
cycle_start_inds = np.argwhere(self._sho_spec_inds[fit_dim_ind, :] == 0).flatten()
tot_cycles = cycle_start_inds.size
# Make the results group
self._h5_group = create_results_group(self.h5_main, 'Loop_Fit')
write_simple_attrs(self._h5_group, {'projection_method': 'pycroscopy BE loop model'})
# Write datasets
self.h5_projected_loops = create_empty_dataset(self.h5_main, np.float32, 'Projected_Loops',
h5_group=self._h5_group)
h5_loop_met_spec_inds, h5_loop_met_spec_vals = write_reduced_spec_dsets(self._h5_group, self._sho_spec_inds,
self._sho_spec_vals, self._fit_dim_name,
basename='Loop_Metrics')
self.h5_loop_metrics = write_main_dataset(self._h5_group, (self.h5_main.shape[0], tot_cycles), 'Loop_Metrics',
'Metrics', 'compound', None, None, dtype=loop_metrics32,
h5_pos_inds=self.h5_main.h5_pos_inds,
h5_pos_vals=self.h5_main.h5_pos_vals,
h5_spec_inds=h5_loop_met_spec_inds,
h5_spec_vals=h5_loop_met_spec_vals)
# Copy region reference:
copy_region_refs(self.h5_main, self.h5_projected_loops)
copy_region_refs(self.h5_main, self.h5_loop_metrics)
self.h5_main.file.flush()
self._met_spec_inds = self.h5_loop_metrics.h5_spec_inds
return
def _write_results_chunk(self):
"""
Writes the labels and mean response to the h5 file
Returns
---------
h5_group : HDF5 Group reference
Reference to the group that contains the decomposition results
"""
h5_decomp_group = create_results_group(self.h5_main, self.process_name,
h5_parent_group=self._h5_target_group)
self._write_source_dset_provenance()
write_simple_attrs(h5_decomp_group, self.parms_dict)
write_simple_attrs(h5_decomp_group, {'n_components': self.__components.shape[0],
'n_samples': self.h5_main.shape[0]})
decomp_desc = Dimension('Endmember', 'a. u.', self.__components.shape[0])
# equivalent to V - compound / complex
h5_components = write_main_dataset(h5_decomp_group, self.__components, 'Components',
get_attr(self.h5_main, 'quantity')[0], 'a.u.', decomp_desc,
None,
h5_spec_inds=self.h5_main.h5_spec_inds,
h5_spec_vals=self.h5_main.h5_spec_vals)
# equivalent of U - real
h5_projections = write_main_dataset(h5_decomp_group, np.float32(self.__projection), 'Projection', 'abundance',
'a.u.', None, decomp_desc, dtype=np.float32,
h5_pos_inds=self.h5_main.h5_pos_inds, h5_pos_vals=self.h5_main.h5_pos_vals)
# return the h5 group object
self.h5_results_grp = h5_decomp_group
# Marking completion:
self._status_dset_name = 'completed_positions'
self._h5_status_dset = h5_decomp_group.create_dataset(self._status_dset_name,
data=np.ones(self.h5_main.shape[0], dtype=np.uint8))
# keeping legacy option:
h5_decomp_group.attrs['last_pixel'] = self.h5_main.shape[0]
return self.h5_results_grp
def _create_guess_datasets(self):
"""
Creates the h5 group, guess dataset, corresponding spectroscopic datasets and also
links the guess dataset to the spectroscopic datasets.
"""
h5_group = create_results_group(self.h5_main, 'SHO_Fit')
write_simple_attrs(h5_group, {'SHO_guess_method': "pycroscopy BESHO"})
h5_sho_inds, h5_sho_vals = write_reduced_spec_dsets(h5_group, self.h5_main.h5_spec_inds,
self.h5_main.h5_spec_vals, self._fit_dim_name)
self.h5_guess = write_main_dataset(h5_group, (self.h5_main.shape[0], self.num_udvs_steps), 'Guess', 'SHO',
'compound', None, None, h5_pos_inds=self.h5_main.h5_pos_inds,
h5_pos_vals=self.h5_main.h5_pos_vals, h5_spec_inds=h5_sho_inds,
h5_spec_vals=h5_sho_vals, chunks=(1, self.num_udvs_steps), dtype=sho32,
main_dset_attrs=self._parms_dict, verbose=self._verbose)
write_simple_attrs(self.h5_guess, {'SHO_guess_method': "pycroscopy BESHO", 'last_pixel': 0})
copy_region_refs(self.h5_main, self.h5_guess)
def _write_results_chunk(self):
"""
Writes the labels and mean response to the h5 file
Returns
---------
h5_group : HDF5 Group reference
Reference to the group that contains the decomposition results
"""
h5_decomp_group = create_results_group(self.h5_main, self.process_name)
write_simple_attrs(h5_decomp_group, self.parms_dict)
write_simple_attrs(h5_decomp_group, {'n_components': self.__components.shape[0],
'n_samples': self.h5_main.shape[0]})
decomp_desc = Dimension('Endmember', 'a. u.', self.__components.shape[0])
# equivalent to V - compound / complex
h5_components = write_main_dataset(h5_decomp_group, self.__components, 'Components',
get_attr(self.h5_main, 'quantity')[0], 'a.u.', decomp_desc,
None,
h5_spec_inds=self.h5_main.h5_spec_inds,
h5_spec_vals=self.h5_main.h5_spec_vals)
# equivalent of U - real
h5_projections = write_main_dataset(h5_decomp_group, np.float32(self.__projection), 'Projection', 'abundance',
'a.u.', None, decomp_desc, dtype=np.float32,
h5_pos_inds=self.h5_main.h5_pos_inds, h5_pos_vals=self.h5_main.h5_pos_vals)
# return the h5 group object
self.h5_results_grp = h5_decomp_group
# Marking completion:
self._status_dset_name = 'completed_positions'
self._h5_status_dset = h5_decomp_group.create_dataset(self._status_dset_name,
data=np.ones(self.h5_main.shape[0], dtype=np.uint8))
# keeping legacy option:
h5_decomp_group.attrs['last_pixel'] = self.h5_main.shape[0]
return self.h5_results_grp
def _create_guess_datasets(self):
"""
Creates the h5 group, guess dataset, corresponding spectroscopic datasets and also
links the guess dataset to the spectroscopic datasets.
"""
self.h5_results_grp = create_results_group(self.h5_main, self.process_name)
write_simple_attrs(self.h5_results_grp, self.parms_dict)
h5_sho_inds, h5_sho_vals = write_reduced_anc_dsets(self.h5_results_grp, self.h5_main.h5_spec_inds,
self.h5_main.h5_spec_vals, self._fit_dim_name)
self._h5_guess = write_main_dataset(self.h5_results_grp, (self.h5_main.shape[0], self.num_udvs_steps), 'Guess', 'SHO',
'compound', None, None, h5_pos_inds=self.h5_main.h5_pos_inds,
h5_pos_vals=self.h5_main.h5_pos_vals, h5_spec_inds=h5_sho_inds,
h5_spec_vals=h5_sho_vals, chunks=(1, self.num_udvs_steps), dtype=sho32,
main_dset_attrs=self.parms_dict, verbose=self.verbose)
copy_region_refs(self.h5_main, self._h5_guess)
self._h5_guess.file.flush()
if self.verbose and self.mpi_rank == 0:
print('Finished creating Guess dataset')
def reshape_from_lines_to_pixels(h5_main, pts_per_cycle, scan_step_x_m=None):
"""
Breaks up the provided raw G-mode dataset into lines and pixels (from just lines)
Parameters
----------
h5_main : h5py.Dataset object
Reference to the main dataset that contains the raw data that is only broken up by lines
pts_per_cycle : unsigned int
Number of points in a single pixel
scan_step_x_m : float
Step in meters for pixels
Returns
-------
h5_resh : h5py.Dataset object
Reference to the main dataset that contains the reshaped data
"""
if not check_if_main(h5_main):
raise TypeError('h5_main is not a Main dataset')
h5_main = USIDataset(h5_main)
if pts_per_cycle % 1 != 0 or pts_per_cycle < 1:
raise TypeError('pts_per_cycle should be a positive integer')
if scan_step_x_m is not None:
if not isinstance(scan_step_x_m, Number):
raise TypeError('scan_step_x_m should be a real number')
else:
scan_step_x_m = 1
if h5_main.shape[1] % pts_per_cycle != 0:
warn(
'Error in reshaping the provided dataset to pixels. Check points per pixel'
)
raise ValueError
num_cols = int(h5_main.shape[1] / pts_per_cycle)
# TODO: DO NOT assume simple 1 spectral dimension!
single_ao = np.squeeze(h5_main.h5_spec_vals[:, :pts_per_cycle])
spec_dims = Dimension(
get_attr(h5_main.h5_spec_vals, 'labels')[0],
get_attr(h5_main.h5_spec_vals, 'units')[0], single_ao)
# TODO: DO NOT assume simple 1D in positions!
pos_dims = [
Dimension('X', 'm', np.linspace(0, scan_step_x_m, num_cols)),
Dimension('Y', 'm',
np.linspace(0, h5_main.h5_pos_vals[1, 0], h5_main.shape[0]))
]
h5_group = create_results_group(h5_main, 'Reshape')
# TODO: Create empty datasets and then write for very large datasets
h5_resh = write_main_dataset(h5_group,
(num_cols * h5_main.shape[0], pts_per_cycle),
'Reshaped_Data',
get_attr(h5_main, 'quantity')[0],
get_attr(h5_main, 'units')[0],
pos_dims,
spec_dims,
chunks=(10, pts_per_cycle),
dtype=h5_main.dtype,
compression=h5_main.compression)
# TODO: DON'T write in one shot assuming small datasets fit in memory!
print('Starting to reshape G-mode line data. Please be patient')
h5_resh[()] = np.reshape(h5_main[()], (-1, pts_per_cycle))
print('Finished reshaping G-mode line data to rows and columns')
return USIDataset(h5_resh)
def _write_results_chunk(self):
"""
Writes the labels and mean response to the h5 file
Returns
---------
h5_group : HDF5 Group reference
Reference to the group that contains the clustering results
"""
print('Writing clustering results to file.')
num_clusters = self.__mean_resp.shape[0]
h5_cluster_group = create_results_group(self.h5_main, self.process_name)
write_simple_attrs(h5_cluster_group, self.parms_dict)
h5_labels = write_main_dataset(h5_cluster_group, np.uint32(self.__labels.reshape([-1, 1])), 'Labels',
'Cluster ID', 'a. u.', None, Dimension('Cluster', 'ID', 1),
h5_pos_inds=self.h5_main.h5_pos_inds, h5_pos_vals=self.h5_main.h5_pos_vals,
aux_spec_prefix='Cluster_', dtype=np.uint32)
if self.num_comps != self.h5_main.shape[1]:
'''
Setup the Spectroscopic Indices and Values for the Mean Response if we didn't use all components
Note that a sliced spectroscopic matrix may not be contiguous. Let's just lose the spectroscopic data
for now until a better method is figured out
'''
"""
if isinstance(self.data_slice[1], np.ndarray):
centroid_vals_mat = h5_centroids.h5_spec_vals[self.data_slice[1].tolist()]
else:
centroid_vals_mat = h5_centroids.h5_spec_vals[self.data_slice[1]]
ds_centroid_values.data[0, :] = centroid_vals_mat
"""
if isinstance(self.data_slice[1], np.ndarray):
vals_slice = self.data_slice[1].tolist()
else:
vals_slice = self.data_slice[1]
vals = self.h5_main.h5_spec_vals[:, vals_slice].squeeze()
new_spec = Dimension('Original_Spectral_Index', 'a.u.', vals)
h5_inds, h5_vals = write_ind_val_dsets(h5_cluster_group, new_spec, is_spectral=True)
else:
h5_inds = self.h5_main.h5_spec_inds
h5_vals = self.h5_main.h5_spec_vals
# For now, link centroids with default spectroscopic indices and values.
h5_centroids = write_main_dataset(h5_cluster_group, self.__mean_resp, 'Mean_Response',
get_attr(self.h5_main, 'quantity')[0], get_attr(self.h5_main, 'units')[0],
Dimension('Cluster', 'a. u.', np.arange(num_clusters)), None,
h5_spec_inds=h5_inds, aux_pos_prefix='Mean_Resp_Pos_',
h5_spec_vals=h5_vals)
# Marking completion:
self._status_dset_name = 'completed_positions'
self._h5_status_dset = h5_cluster_group.create_dataset(self._status_dset_name,
data=np.ones(self.h5_main.shape[0], dtype=np.uint8))
# keeping legacy option:
h5_cluster_group.attrs['last_pixel'] = self.h5_main.shape[0]
return h5_cluster_group
def _create_results_datasets(self):
"""
Creates all the datasets necessary for holding all parameters + data.
"""
self.h5_results_grp = create_results_group(self.h5_main, self.process_name)
self.parms_dict.update({'last_pixel': 0, 'algorithm': 'pycroscopy_SignalFilter'})
write_simple_attrs(self.h5_results_grp, self.parms_dict)
assert isinstance(self.h5_results_grp, h5py.Group)
if isinstance(self.composite_filter, np.ndarray):
h5_comp_filt = self.h5_results_grp.create_dataset('Composite_Filter',
data=np.float32(self.composite_filter))
if self.verbose and self.mpi_rank == 0:
print('Rank {} - Finished creating the Composite_Filter dataset'.format(self.mpi_rank))
# First create the position datsets if the new indices are smaller...
if self.num_effective_pix != self.h5_main.shape[0]:
# TODO: Do this part correctly. See past solution:
"""
# need to make new position datasets by taking every n'th index / value:
new_pos_vals = np.atleast_2d(h5_pos_vals[slice(0, None, self.num_effective_pix), :])
pos_descriptor = []
for name, units, leng in zip(h5_pos_inds.attrs['labels'], h5_pos_inds.attrs['units'],
[int(np.unique(h5_pos_inds[:, dim_ind]).size / self.num_effective_pix)
for dim_ind in range(h5_pos_inds.shape[1])]):
pos_descriptor.append(Dimension(name, units, np.arange(leng)))
ds_pos_inds, ds_pos_vals = build_ind_val_dsets(pos_descriptor, is_spectral=False, verbose=self.verbose)
h5_pos_vals.data = np.atleast_2d(new_pos_vals) # The data generated above varies linearly. Override.
"""
h5_pos_inds_new, h5_pos_vals_new = write_ind_val_dsets(self.h5_results_grp,
Dimension('pixel', 'a.u.', self.num_effective_pix),
is_spectral=False, verbose=self.verbose and self.mpi_rank==0)
if self.verbose and self.mpi_rank == 0:
print('Rank {} - Created the new position ancillary dataset'.format(self.mpi_rank))
else:
h5_pos_inds_new = self.h5_main.h5_pos_inds
h5_pos_vals_new = self.h5_main.h5_pos_vals
if self.verbose and self.mpi_rank == 0:
print('Rank {} - Reusing source datasets position datasets'.format(self.mpi_rank))
if self.noise_threshold is not None:
self.h5_noise_floors = write_main_dataset(self.h5_results_grp, (self.num_effective_pix, 1), 'Noise_Floors',
'Noise', 'a.u.', None, Dimension('arb', '', [1]),
dtype=np.float32, aux_spec_prefix='Noise_Spec_',
h5_pos_inds=h5_pos_inds_new, h5_pos_vals=h5_pos_vals_new,
verbose=self.verbose and self.mpi_rank == 0)
if self.verbose and self.mpi_rank == 0:
print('Rank {} - Finished creating the Noise_Floors dataset'.format(self.mpi_rank))
if self.write_filtered:
# Filtered data is identical to Main_Data in every way - just a duplicate
self.h5_filtered = create_empty_dataset(self.h5_main, self.h5_main.dtype, 'Filtered_Data',
h5_group=self.h5_results_grp)
if self.verbose and self.mpi_rank == 0:
print('Rank {} - Finished creating the Filtered dataset'.format(self.mpi_rank))
self.hot_inds = None
if self.write_condensed:
self.hot_inds = np.where(self.composite_filter > 0)[0]
self.hot_inds = np.uint(self.hot_inds[int(0.5 * len(self.hot_inds)):]) # only need to keep half the data
condensed_spec = Dimension('hot_frequencies', '', int(0.5 * len(self.hot_inds)))
self.h5_condensed = write_main_dataset(self.h5_results_grp, (self.num_effective_pix, len(self.hot_inds)),
'Condensed_Data', 'Complex', 'a. u.', None, condensed_spec,
h5_pos_inds=h5_pos_inds_new, h5_pos_vals=h5_pos_vals_new,
dtype=np.complex, verbose=self.verbose and self.mpi_rank == 0)
if self.verbose and self.mpi_rank == 0:
print('Rank {} - Finished creating the Condensed dataset'.format(self.mpi_rank))
if self.mpi_size > 1:
self.mpi_comm.Barrier()
self.h5_main.file.flush()
def _setup_h5(self, data_gen_parms):
"""
Setups up the hdf5 file structure before doing the actual generation
Parameters
----------
data_gen_parms : dict
Dictionary containing the parameters to write to the Measurement Group as attributes
Returns
-------
"""
'''
Build the group structure down to the channel group
'''
# Set up the basic group structure
root_parms = dict()
root_parms['translator'] = 'FAKEBEPS'
root_parms['data_type'] = data_gen_parms['data_type']
# Write the file
self.h5_f = h5py.File(self.h5_path, 'w')
write_simple_attrs(self.h5_f, root_parms)
meas_grp = create_indexed_group(self.h5_f, 'Measurement')
chan_grp = create_indexed_group(meas_grp, 'Channel')
write_simple_attrs(meas_grp, data_gen_parms)
# Create the Position and Spectroscopic datasets for the Raw Data
h5_pos_dims, h5_spec_dims = self._build_ancillary_datasets()
h5_raw_data = write_main_dataset(chan_grp,
(self.n_pixels, self.n_spec_bins),
'Raw_Data',
'Deflection',
'Volts',
h5_pos_dims,
h5_spec_dims,
slow_to_fast=True,
dtype=np.complex64,
verbose=True)
'''
Build the SHO Group
'''
sho_grp = create_results_group(h5_raw_data, 'SHO_Fit')
# Build the Spectroscopic datasets for the SHO Guess and Fit
h5_sho_spec_inds, h5_sho_spec_vals = write_reduced_anc_dsets(
sho_grp,
h5_raw_data.h5_spec_inds,
h5_raw_data.h5_spec_vals,
'Frequency',
is_spec=True)
h5_sho_fit = write_main_dataset(
sho_grp, (self.n_pixels, int(self.n_spec_bins // self.n_bins)),
'Fit',
'SHO Parameters',
'a.u.',
None,
None,
h5_pos_inds=h5_raw_data.h5_pos_inds,
h5_pos_vals=h5_raw_data.h5_pos_vals,
h5_spec_inds=h5_sho_spec_inds,
h5_spec_vals=h5_sho_spec_vals,
slow_to_fast=True,
dtype=sho32)
h5_sho_guess = copy_dataset(h5_sho_fit, sho_grp, alias='Guess')
'''
Build the loop group
'''
loop_grp = create_results_group(h5_sho_fit, 'Loop_Fit')
# Build the Spectroscopic datasets for the loops
h5_loop_spec_inds, h5_loop_spec_vals = write_reduced_anc_dsets(
loop_grp,
h5_sho_fit.h5_spec_inds,
h5_sho_fit.h5_spec_vals,
'DC_Offset',
is_spec=True)
h5_loop_fit = write_main_dataset(loop_grp,
(self.n_pixels, self.n_loops),
'Fit',
'Loop Fitting Parameters',
'a.u.',
None,
None,
h5_pos_inds=h5_raw_data.h5_pos_inds,
h5_pos_vals=h5_raw_data.h5_pos_vals,
h5_spec_inds=h5_loop_spec_inds,
h5_spec_vals=h5_loop_spec_vals,
slow_to_fast=True,
dtype=loop_fit32)
h5_loop_guess = copy_dataset(h5_loop_fit, loop_grp, alias='Guess')
copy_all_region_refs(h5_loop_guess, h5_loop_fit)
self.h5_raw = h5_raw_data
self.h5_sho_guess = h5_sho_guess
self.h5_sho_fit = h5_sho_fit
self.h5_loop_guess = h5_loop_guess
self.h5_loop_fit = h5_loop_fit
self.h5_spec_vals = h5_raw_data.h5_spec_vals
self.h5_spec_inds = h5_raw_data.h5_spec_inds
self.h5_sho_spec_inds = h5_sho_fit.h5_spec_inds
self.h5_sho_spec_vals = h5_sho_fit.h5_spec_vals
self.h5_loop_spec_inds = h5_loop_fit.h5_spec_inds
self.h5_loop_spec_vals = h5_loop_fit.h5_spec_vals
self.h5_file = h5_raw_data.file
return
def _create_results_datasets(self):
"""
Creates hdf5 datasets and datagroups to hold the resutls
"""
# create all h5 datasets here:
num_pos = self.h5_main.shape[0]
if self.verbose and self.mpi_rank == 0:
print('Now creating the datasets')
self.h5_results_grp = create_results_group(self.h5_main,
self.process_name)
write_simple_attrs(self.h5_results_grp, {
'algorithm_author': 'Kody J. Law',
'last_pixel': 0
})
write_simple_attrs(self.h5_results_grp, self.parms_dict)
if self.verbose and self.mpi_rank == 0:
print('created group: {} with attributes:'.format(
self.h5_results_grp.name))
print(get_attributes(self.h5_results_grp))
# One of those rare instances when the result is exactly the same as the source
self.h5_i_corrected = create_empty_dataset(
self.h5_main,
np.float32,
'Corrected_Current',
h5_group=self.h5_results_grp)
if self.verbose and self.mpi_rank == 0:
print('Created I Corrected')
# print_tree(self.h5_results_grp)
# For some reason, we cannot specify chunks or compression!
# The resistance dataset requires the creation of a new spectroscopic dimension
self.h5_resistance = write_main_dataset(
self.h5_results_grp,
(num_pos, self.num_x_steps),
'Resistance',
'Resistance',
'GOhms',
None,
Dimension('Bias', 'V', self.num_x_steps),
dtype=np.
float32, # chunks=(1, self.num_x_steps), #compression='gzip',
h5_pos_inds=self.h5_main.h5_pos_inds,
h5_pos_vals=self.h5_main.h5_pos_vals)
if self.verbose and self.mpi_rank == 0:
print('Created Resistance')
# print_tree(self.h5_results_grp)
assert isinstance(self.h5_resistance,
USIDataset) # only here for PyCharm
self.h5_new_spec_vals = self.h5_resistance.h5_spec_vals
# The variance is identical to the resistance dataset
self.h5_variance = create_empty_dataset(self.h5_resistance, np.float32,
'R_variance')
if self.verbose and self.mpi_rank == 0:
print('Created Variance')
# print_tree(self.h5_results_grp)
# The capacitance dataset requires new spectroscopic dimensions as well
self.h5_cap = write_main_dataset(
self.h5_results_grp,
(num_pos, 1),
'Capacitance',
'Capacitance',
'pF',
None,
Dimension('Direction', '', [1]),
h5_pos_inds=self.h5_main.h5_pos_inds,
h5_pos_vals=self.h5_main.h5_pos_vals,
dtype=cap_dtype, #compression='gzip',
aux_spec_prefix='Cap_Spec_')
if self.verbose and self.mpi_rank == 0:
print('Created Capacitance')
# print_tree(self.h5_results_grp)
print('Done creating all results datasets!')
if self.mpi_size > 1:
self.mpi_comm.Barrier()
self.h5_main.file.flush()
def _create_results_datasets(self):
"""
Setup the Loop_Fit Group and the loop projection datasets
"""
# First grab the spectroscopic indices and values and position indices
# TODO: Avoid unnecessary namespace pollution
# self._sho_spec_inds = self.h5_main.h5_spec_inds
# self._sho_spec_vals = self.h5_main.h5_spec_vals
# self._sho_pos_inds = self.h5_main.h5_pos_inds
# Which row in the spec datasets is DC offset?
self._fit_spec_index = self.h5_main.spec_dim_labels.index(
self._fit_dim_name)
# TODO: Unkown usage of variable. Waste either way
# self._fit_offset_index = 1 + self._fit_spec_index
# Calculate the number of loops per position
cycle_start_inds = np.argwhere(
self.h5_main.h5_spec_inds[self._fit_spec_index, :] == 0).flatten()
tot_cycles = cycle_start_inds.size
if self.verbose:
print('Found {} cycles starting at indices: {}'.format(
tot_cycles, cycle_start_inds))
# Make the results group
self.h5_results_grp = create_results_group(self.h5_main,
self.process_name)
write_simple_attrs(self.h5_results_grp, self.parms_dict)
# Write datasets
self.h5_projected_loops = create_empty_dataset(
self.h5_main,
np.float32,
'Projected_Loops',
h5_group=self.h5_results_grp)
h5_loop_met_spec_inds, h5_loop_met_spec_vals = write_reduced_anc_dsets(
self.h5_results_grp,
self.h5_main.h5_spec_inds,
self.h5_main.h5_spec_vals,
self._fit_dim_name,
basename='Loop_Metrics',
verbose=self.verbose)
self.h5_loop_metrics = write_main_dataset(
self.h5_results_grp, (self.h5_main.shape[0], tot_cycles),
'Loop_Metrics',
'Metrics',
'compound',
None,
None,
dtype=loop_metrics32,
h5_pos_inds=self.h5_main.h5_pos_inds,
h5_pos_vals=self.h5_main.h5_pos_vals,
h5_spec_inds=h5_loop_met_spec_inds,
h5_spec_vals=h5_loop_met_spec_vals)
# Copy region reference:
# copy_region_refs(self.h5_main, self.h5_projected_loops)
# copy_region_refs(self.h5_main, self.h5_loop_metrics)
self.h5_main.file.flush()
self._met_spec_inds = self.h5_loop_metrics.h5_spec_inds
if self.verbose and self.mpi_rank == 0:
print('Finished creating Guess dataset')
def _create_results_datasets(self):
'''
Creates the datasets an Groups necessary to store the results.
Parameters
----------
h5_if : 'Inst_Freq' h5 Dataset
Contains the Instantaneous Frequencies
tfp : 'tfp' h5 Dataset
Contains the time-to-first-peak data as a 1D matrix
shift : 'shift' h5 Dataset
Contains the frequency shift data as a 1D matrix
'''
print('Creating results datasets')
# Get relevant parameters
num_rows = self.parm_dict['num_rows']
num_cols = self.parm_dict['num_cols']
pnts_per_avg = self.parm_dict['pnts_per_avg']
ds_shape = [num_rows * num_cols, pnts_per_avg]
self.h5_results_grp = create_results_group(self.h5_main,
self.process_name)
copy_attributes(self.h5_main.parent, self.h5_results_grp)
# Create dimensions
pos_desc = [
Dimension('X', 'm',
np.linspace(0, self.parm_dict['FastScanSize'],
num_cols)),
Dimension('Y', 'm',
np.linspace(0, self.parm_dict['SlowScanSize'], num_rows))
]
# ds_pos_ind, ds_pos_val = build_ind_val_matrices(pos_desc, is_spectral=False)
spec_desc = [
Dimension(
'Time', 's',
np.linspace(0, self.parm_dict['total_time'], pnts_per_avg))
]
# ds_spec_inds, ds_spec_vals = build_ind_val_matrices(spec_desc, is_spectral=True)
# Writes main dataset
self.h5_if = write_main_dataset(
self.h5_results_grp,
ds_shape,
'Inst_Freq', # Name of main dataset
'Frequency', # Physical quantity contained in Main dataset
'Hz', # Units for the physical quantity
pos_desc, # Position dimensions
spec_desc, # Spectroscopic dimensions
dtype=np.float32, # data type / precision
main_dset_attrs=self.parm_dict)
self.h5_amp = write_main_dataset(
self.h5_results_grp,
ds_shape,
'Amplitude', # Name of main dataset
'Amplitude', # Physical quantity contained in Main dataset
'nm', # Units for the physical quantity
None, # Position dimensions
None, # Spectroscopic dimensions
h5_pos_inds=self.h5_main.h5_pos_inds, # Copy Pos Dimensions
h5_pos_vals=self.h5_main.h5_pos_vals,
h5_spec_inds=self.h5_main.h5_spec_inds,
# Copy Spectroscopy Dimensions
h5_spec_vals=self.h5_main.h5_spec_vals,
dtype=np.float32, # data type / precision
main_dset_attrs=self.parm_dict)
self.h5_phase = write_main_dataset(
self.h5_results_grp,
ds_shape,
'Phase', # Name of main dataset
'Phase', # Physical quantity contained in Main dataset
'degrees', # Units for the physical quantity
None, # Position dimensions
None, # Spectroscopic dimensions
h5_pos_inds=self.h5_main.h5_pos_inds, # Copy Pos Dimensions
h5_pos_vals=self.h5_main.h5_pos_vals,
h5_spec_inds=self.h5_main.h5_spec_inds,
# Copy Spectroscopy Dimensions
h5_spec_vals=self.h5_main.h5_spec_vals,
dtype=np.float32, # data type / precision
main_dset_attrs=self.parm_dict)
self.h5_pwrdis = write_main_dataset(
self.h5_results_grp,
ds_shape,
'PowerDissipation', # Name of main dataset
'Power', # Physical quantity contained in Main dataset
'W', # Units for the physical quantity
None, # Position dimensions
None, # Spectroscopic dimensions
h5_pos_inds=self.h5_main.h5_pos_inds, # Copy Pos Dimensions
h5_pos_vals=self.h5_main.h5_pos_vals,
h5_spec_inds=self.h5_main.h5_spec_inds,
# Copy Spectroscopy Dimensions
h5_spec_vals=self.h5_main.h5_spec_vals,
dtype=np.float32, # data type / precision
main_dset_attrs=self.parm_dict)
_arr = np.zeros([num_rows * num_cols, 1])
self.h5_tfp = self.h5_results_grp.create_dataset('tfp',
data=_arr,
dtype=np.float32)
self.h5_shift = self.h5_results_grp.create_dataset('shift',
data=_arr,
dtype=np.float32)
self.h5_if.file.flush()
return
def _create_results_datasets(self):
"""
Creates hdf5 datasets and datagroups to hold the resutls
"""
# create all h5 datasets here:
num_pos = self.h5_main.shape[0]
if self.verbose and self.mpi_rank == 0:
print('Now creating the datasets')
self.h5_results_grp = create_results_group(self.h5_main, self.process_name)
write_simple_attrs(self.h5_results_grp, {'algorithm_author': 'Kody J. Law', 'last_pixel': 0})
write_simple_attrs(self.h5_results_grp, self.parms_dict)
if self.verbose and self.mpi_rank == 0:
print('created group: {} with attributes:'.format(self.h5_results_grp.name))
print(get_attributes(self.h5_results_grp))
# One of those rare instances when the result is exactly the same as the source
self.h5_i_corrected = create_empty_dataset(self.h5_main, np.float32, 'Corrected_Current', h5_group=self.h5_results_grp)
if self.verbose and self.mpi_rank == 0:
print('Created I Corrected')
# print_tree(self.h5_results_grp)
# For some reason, we cannot specify chunks or compression!
# The resistance dataset requires the creation of a new spectroscopic dimension
self.h5_resistance = write_main_dataset(self.h5_results_grp, (num_pos, self.num_x_steps), 'Resistance', 'Resistance',
'GOhms', None, Dimension('Bias', 'V', self.num_x_steps),
dtype=np.float32, # chunks=(1, self.num_x_steps), #compression='gzip',
h5_pos_inds=self.h5_main.h5_pos_inds,
h5_pos_vals=self.h5_main.h5_pos_vals)
if self.verbose and self.mpi_rank == 0:
print('Created Resistance')
# print_tree(self.h5_results_grp)
assert isinstance(self.h5_resistance, USIDataset) # only here for PyCharm
self.h5_new_spec_vals = self.h5_resistance.h5_spec_vals
# The variance is identical to the resistance dataset
self.h5_variance = create_empty_dataset(self.h5_resistance, np.float32, 'R_variance')
if self.verbose and self.mpi_rank == 0:
print('Created Variance')
# print_tree(self.h5_results_grp)
# The capacitance dataset requires new spectroscopic dimensions as well
self.h5_cap = write_main_dataset(self.h5_results_grp, (num_pos, 1), 'Capacitance', 'Capacitance', 'pF', None,
Dimension('Direction', '', [1]), h5_pos_inds=self.h5_main.h5_pos_inds,
h5_pos_vals=self.h5_main.h5_pos_vals, dtype=cap_dtype, #compression='gzip',
aux_spec_prefix='Cap_Spec_')
if self.verbose and self.mpi_rank == 0:
print('Created Capacitance')
# print_tree(self.h5_results_grp)
print('Done creating all results datasets!')
if self.mpi_size > 1:
self.mpi_comm.Barrier()
self.h5_main.file.flush()
def _create_results_datasets(self):
"""
Creates all the datasets necessary for holding all parameters + data.
"""
self.h5_results_grp = create_results_group(self.h5_main,
self.process_name)
self.params_dict.update({
'last_pixel':
0,
'algorithm':
'pycroscopy_AdaptiveBayesianInference'
})
# Write in our full_V and num_pixels as attributes to this new group
write_simple_attrs(self.h5_results_grp, self.params_dict)
assert isinstance(self.h5_results_grp, h5py.Group)
# If we ended up parsing down the data, create new spectral datasets (i.e. smaller full_V's)
# By convention, we convert the full_V back to a sine wave.
if self.parse_mod != 1:
h5_spec_inds_new, h5_spec_vals_new = write_ind_val_dsets(
self.h5_results_grp,
Dimension("Bias", "V", self.full_V.size),
is_spectral=True)
h5_spec_vals_new[()] = get_unshifted_response(
self.full_V, self.shift_index)
else:
h5_spec_inds_new = self.h5_main.h5_spec_inds
h5_spec_vals_new = self.h5_main.h5_spec_vals
# Also make some new spectroscopic datasets for R and R_sig
h5_spec_inds_R, h5_spec_vals_R = write_ind_val_dsets(
self.h5_results_grp,
Dimension("Bias", "V", 2 * self.M),
is_spectral=True,
base_name="Spectroscopic_R")
h5_spec_vals_R[()] = np.concatenate((self.x, self.x)).T
# Initialize our datasets
# Note by convention, the spectroscopic values are stored as a sine wave
# so i_recon and i_corrected are shifted at the end of bayesian_utils.process_pixel
# accordingly.
self.h5_R = write_main_dataset(self.h5_results_grp,
(self.h5_main.shape[0], 2 * self.M),
"Resistance",
"Resistance",
"GOhms",
None,
None,
dtype=np.float64,
h5_pos_inds=self.h5_main.h5_pos_inds,
h5_pos_vals=self.h5_main.h5_pos_vals,
h5_spec_inds=h5_spec_inds_R,
h5_spec_vals=h5_spec_vals_R)
assert isinstance(self.h5_R, usid.USIDataset) # Quick sanity check
self.h5_R_sig = create_empty_dataset(self.h5_R, np.float64, "R_sig")
self.h5_capacitance = write_main_dataset(
self.h5_results_grp, (self.h5_main.shape[0], 2),
"Capacitance",
"Capacitance",
"pF",
None,
Dimension("Direction", "", 2),
h5_pos_inds=self.h5_main.h5_pos_inds,
h5_pos_vals=self.h5_main.h5_pos_vals,
dtype=np.float64,
aux_spec_prefix="Cap_Spec_")
# Not sure what units this should be so tentatively storing it as amps
self.h5_i_recon = write_main_dataset(
self.h5_results_grp, (self.h5_main.shape[0], self.full_V.size),
"Reconstructed_Current",
"Current",
"nA",
None,
None,
dtype=np.float64,
h5_pos_inds=self.h5_main.h5_pos_inds,
h5_pos_vals=self.h5_main.h5_pos_vals,
h5_spec_inds=h5_spec_inds_new,
h5_spec_vals=h5_spec_vals_new)
self.h5_i_corrected = create_empty_dataset(self.h5_i_recon, np.float64,
"Corrected_Current")
'''
# Initialize our datasets
# Note, each pixel of the datasets will hold the forward and reverse sweeps concatenated together.
# R and R_sig are plotted against [x, -x], and i_recon and i_corrected are plotted against full_V.
self.h5_R = h5_results_grp.create_dataset("R", shape=(self.h5_main.shape[0], 2*self.M), dtype=np.float)
self.h5_R_sig = h5_results_grp.create_dataset("R_sig", shape=(self.h5_main.shape[0], 2*self.M), dtype=np.float)
self.h5_capacitance = h5_results_grp.create_dataset("capacitance", shape=(self.h5_main.shape[0], 2), dtype=np.float)
self.h5_i_recon = h5_results_grp.create_dataset("i_recon", shape=(self.h5_main.shape[0], self.full_V.size), dtype=np.float)
self.h5_i_corrected = h5_results_grp.create_dataset("i_corrected", shape=(self.h5_main.shape[0], self.full_V.size), dtype=np.float)
'''
if self.verbose: print("results datasets set up")
self.h5_main.file.flush()