def translate(self, h5_path, force_patch=False, **kwargs):
"""
Add the needed references and attributes to the h5 file that are not created by the
LabView data aquisition program.
Parameters
----------
h5_path : str
path to the h5 file
force_patch : bool, optional
Should the check to see if the file has already been patched be ignored.
Default False.
Returns
-------
h5_file : h5py.File
patched hdf5 file
"""
# Open the file and check if a patch is needed
h5_file = h5py.File(os.path.abspath(h5_path), 'r+')
if h5_file.attrs.get('translator') is not None and not force_patch:
print('File is already Pycroscopy ready.')
return h5_file
'''
Get the list of all Raw_Data Datasets
Loop over the list and update the needed attributes
'''
raw_list = find_dataset(h5_file, 'Raw_Data')
for h5_raw in raw_list:
if 'quantity' not in h5_raw.attrs:
h5_raw.attrs['quantity'] = 'quantity'
if 'units' not in h5_raw.attrs:
h5_raw.attrs['units'] = 'a.u.'
# Grab the channel and measurement group of the data to check some needed attributes
h5_chan = h5_raw.parent
try:
c_type = get_attr(h5_chan, 'channel_type')
except KeyError:
warn_str = "'channel_type' was not found as an attribute of {}.\n".format(
h5_chan.name)
warn_str += "If this is BEPS or BELine data from the LabView aquisition software, " + \
"please run the following piece of code. Afterwards, run this function again.\n" + \
"CODE: " \
"hdf.file['{}'].attrs['channel_type'] = 'BE'".format(h5_chan.name)
warn(warn_str)
return h5_file
except:
raise
if c_type != 'BE':
continue
h5_meas = h5_chan.parent
h5_meas.attrs['num_UDVS_steps'] = h5_meas.attrs['num_steps']
# Get the object handles for the Indices and Values datasets
h5_pos_inds = h5_chan['Position_Indices']
h5_pos_vals = h5_chan['Position_Values']
h5_spec_inds = h5_chan['Spectroscopic_Indices']
h5_spec_vals = h5_chan['Spectroscopic_Values']
# Make sure we have correct spectroscopic indices for the given values
ds_spec_inds = create_spec_inds_from_vals(h5_spec_vals[()])
if not np.allclose(ds_spec_inds, h5_spec_inds[()]):
h5_spec_inds[:, :] = ds_spec_inds[:, :]
h5_file.flush()
# Get the labels and units for the Spectroscopic datasets
h5_spec_labels = h5_spec_inds.attrs['labels']
inds_and_vals = [
h5_pos_inds, h5_pos_vals, h5_spec_inds, h5_spec_vals
]
for dset in inds_and_vals:
spec_labels = dset.attrs['labels']
try:
spec_units = dset.attrs['units']
if len(spec_units) != len(spec_labels):
raise KeyError
except KeyError:
dset['units'] = ['' for _ in spec_labels]
except:
raise
for ilabel, label in enumerate(h5_spec_labels):
label_slice = (slice(ilabel, ilabel + 1), slice(None))
if label == '':
label = 'Step'
h5_spec_inds.attrs[label] = h5_spec_inds.regionref[label_slice]
h5_spec_vals.attrs[label] = h5_spec_vals.regionref[label_slice]
# Link the references to the Indices and Values datasets to the Raw_Data
link_as_main(h5_raw, h5_pos_inds, h5_pos_vals, h5_spec_inds,
h5_spec_vals)
# Also link the Bin_Frequencies and Bin_Wfm_Type datasets
h5_freqs = h5_chan['Bin_Frequencies']
aux_dset_names = ['Bin_Frequencies']
aux_dset_refs = [h5_freqs.ref]
check_and_link_ancillary(h5_raw,
aux_dset_names,
anc_refs=aux_dset_refs)
'''
Get all SHO_Fit groups for the Raw_Data and loop over them
Get the Guess and Spectroscopic Datasets for each SHO_Fit group
'''
sho_list = find_results_groups(h5_raw, 'SHO_Fit')
for h5_sho in sho_list:
h5_sho_guess = h5_sho['Guess']
h5_sho_spec_inds = h5_sho['Spectroscopic_Indices']
h5_sho_spec_vals = h5_sho['Spectroscopic_Values']
# Make sure we have correct spectroscopic indices for the given values
ds_sho_spec_inds = create_spec_inds_from_vals(
h5_sho_spec_inds[()])
if not np.allclose(ds_sho_spec_inds, h5_sho_spec_inds[()]):
h5_sho_spec_inds[:, :] = ds_sho_spec_inds[:, :]
# Get the labels and units for the Spectroscopic datasets
h5_sho_spec_labels = get_attr(h5_sho_spec_inds, 'labels')
link_as_main(h5_sho_guess, h5_pos_inds, h5_pos_vals,
h5_sho_spec_inds, h5_sho_spec_vals)
sho_inds_and_vals = [h5_sho_spec_inds, h5_sho_spec_vals]
for dset in sho_inds_and_vals:
spec_labels = get_attr(dset, 'labels')
try:
spec_units = get_attr(dset, 'units')
if len(spec_units) != len(spec_labels):
raise KeyError
except KeyError:
spec_units = [''.encode('utf-8') for _ in spec_labels]
dset.attrs['units'] = spec_units
except:
raise
# Make region references in the
for ilabel, label in enumerate(h5_sho_spec_labels):
label_slice = (slice(ilabel, ilabel + 1), slice(None))
if label == '':
label = 'Step'.encode('utf-8')
h5_sho_spec_inds.attrs[label] = h5_sho_spec_inds.regionref[
label_slice]
h5_sho_spec_vals.attrs[label] = h5_sho_spec_vals.regionref[
label_slice]
h5_file.flush()
h5_file.attrs['translator'] = 'V3patcher'.encode('utf-8')
return h5_file
def translate(self, h5_path, force_patch=False, **kwargs):
"""
Add the needed references and attributes to the h5 file that are not created by the
LabView data aquisition program.
Parameters
----------
h5_path : str
path to the h5 file
force_patch : bool, optional
Should the check to see if the file has already been patched be ignored.
Default False.
Returns
-------
h5_file : str
path to the patched dataset
"""
# Open the file and check if a patch is needed
h5_file = h5py.File(os.path.abspath(h5_path), 'r+')
if h5_file.attrs.get('translator') is not None and not force_patch:
print('File is already Pycroscopy ready.')
h5_file.close()
return h5_path
'''
Get the list of all Raw_Data Datasets
Loop over the list and update the needed attributes
'''
raw_list = find_dataset(h5_file, 'Raw_Data')
for h5_raw in raw_list:
if 'quantity' not in h5_raw.attrs:
h5_raw.attrs['quantity'] = 'quantity'
if 'units' not in h5_raw.attrs:
h5_raw.attrs['units'] = 'a.u.'
# Grab the channel and measurement group of the data to check some needed attributes
h5_chan = h5_raw.parent
try:
c_type = get_attr(h5_chan, 'channel_type')
except KeyError:
warn_str = "'channel_type' was not found as an attribute of {}.\n".format(
h5_chan.name)
warn_str += "If this is BEPS or BELine data from the LabView aquisition software, " + \
"please run the following piece of code. Afterwards, run this function again.\n" + \
"CODE: " \
"hdf.file['{}'].attrs['channel_type'] = 'BE'".format(h5_chan.name)
warn(warn_str)
h5_file.close()
return h5_path
except:
raise
if c_type != 'BE':
continue
h5_meas = h5_chan.parent
h5_meas.attrs['num_UDVS_steps'] = h5_meas.attrs['num_steps']
# Get the object handles for the Indices and Values datasets
h5_pos_inds = h5_chan['Position_Indices']
h5_pos_vals = h5_chan['Position_Values']
h5_spec_inds = h5_chan['Spectroscopic_Indices']
h5_spec_vals = h5_chan['Spectroscopic_Values']
# Make sure we have correct spectroscopic indices for the given values
ds_spec_inds = create_spec_inds_from_vals(h5_spec_vals[()])
if not np.allclose(ds_spec_inds, h5_spec_inds[()]):
h5_spec_inds[:, :] = ds_spec_inds[:, :]
h5_file.flush()
# Get the labels and units for the Spectroscopic datasets
h5_spec_labels = h5_spec_inds.attrs['labels']
inds_and_vals = [
h5_pos_inds, h5_pos_vals, h5_spec_inds, h5_spec_vals
]
for dset in inds_and_vals:
spec_labels = dset.attrs['labels']
try:
spec_units = dset.attrs['units']
if len(spec_units) != len(spec_labels):
raise KeyError
except KeyError:
dset['units'] = ['' for _ in spec_labels]
except:
raise
""""
In early versions, too many spectroscopic dimension labels and
units were listed compared to the number of rows. Remove here:
"""
remove_non_exist_spec_dim_labs(h5_spec_inds,
h5_spec_vals,
h5_meas,
verbose=False)
"""
Add back some standard metadata to be consistent with older
BE data
"""
missing_metadata = dict()
if 'File_file_name' not in h5_meas.attrs.keys():
missing_metadata['File_file_name'] = os.path.split(
h5_raw.file.filename)[-1].replace('.h5', '')
if 'File_date_and_time' not in h5_meas.attrs.keys():
try:
date_str = get_attr(h5_raw.file, 'date_string')
time_str = get_attr(h5_raw.file, 'time_string')
full_str = date_str.strip() + ' ' + time_str.strip()
"""
convert:
date_string : 2018-12-05
time_string : 3:41:45 PM
to:
File_date_and_time: 19-Jun-2009 18:44:56
"""
try:
dt_obj = datetime.datetime.strptime(
full_str, "%Y-%m-%d %I:%M:%S %p")
missing_metadata[
'File_date_and_time'] = dt_obj.strftime(
'%d-%b-%Y %H:%M:%S')
except ValueError:
pass
except KeyError:
pass
# Now write to measurement group:
if len(missing_metadata) > 0:
write_simple_attrs(h5_meas, missing_metadata)
# Link the references to the Indices and Values datasets to the Raw_Data
link_as_main(h5_raw, h5_pos_inds, h5_pos_vals, h5_spec_inds,
h5_spec_vals)
# Also link the Bin_Frequencies and Bin_Wfm_Type datasets
h5_freqs = h5_chan['Bin_Frequencies']
aux_dset_names = ['Bin_Frequencies']
aux_dset_refs = [h5_freqs.ref]
check_and_link_ancillary(h5_raw,
aux_dset_names,
anc_refs=aux_dset_refs)
'''
Get all SHO_Fit groups for the Raw_Data and loop over them
Get the Guess and Spectroscopic Datasets for each SHO_Fit group
'''
sho_list = find_results_groups(h5_raw, 'SHO_Fit')
for h5_sho in sho_list:
h5_sho_guess = h5_sho['Guess']
h5_sho_spec_inds = h5_sho['Spectroscopic_Indices']
h5_sho_spec_vals = h5_sho['Spectroscopic_Values']
# Make sure we have correct spectroscopic indices for the given values
ds_sho_spec_inds = create_spec_inds_from_vals(
h5_sho_spec_inds[()])
if not np.allclose(ds_sho_spec_inds, h5_sho_spec_inds[()]):
h5_sho_spec_inds[:, :] = ds_sho_spec_inds[:, :]
# Get the labels and units for the Spectroscopic datasets
h5_sho_spec_labels = get_attr(h5_sho_spec_inds, 'labels')
link_as_main(h5_sho_guess, h5_pos_inds, h5_pos_vals,
h5_sho_spec_inds, h5_sho_spec_vals)
sho_inds_and_vals = [h5_sho_spec_inds, h5_sho_spec_vals]
for dset in sho_inds_and_vals:
spec_labels = get_attr(dset, 'labels')
try:
spec_units = get_attr(dset, 'units')
if len(spec_units) != len(spec_labels):
raise KeyError
except KeyError:
spec_units = [''.encode('utf-8') for _ in spec_labels]
dset.attrs['units'] = spec_units
except:
raise
h5_file.flush()
h5_file.attrs['translator'] = 'V3patcher'.encode('utf-8')
h5_file.close()
return h5_path
def _setup_h5(self, data_gen_parms):
"""
Setups up the hdf5 file structure before doing the actual generation
Parameters
----------
data_gen_parms : dict
Dictionary containing the parameters to write to the Measurement Group as attributes
Returns
-------
"""
'''
Build the group structure down to the channel group
'''
# Set up the basic group structure
root_grp = VirtualGroup('')
root_parms = generate_dummy_main_parms()
root_parms['translator'] = 'FAKEBEPS'
root_parms['data_type'] = data_gen_parms['data_type']
root_grp.attrs = root_parms
meas_grp = VirtualGroup('Measurement_')
chan_grp = VirtualGroup('Channel_')
meas_grp.attrs.update(data_gen_parms)
# Create the Position and Spectroscopic datasets for the Raw Data
ds_pos_inds, ds_pos_vals, ds_spec_inds, ds_spec_vals = self._build_ancillary_datasets(
)
raw_chunking = calc_chunks([self.n_pixels, self.n_spec_bins],
np.complex64(0).itemsize,
unit_chunks=[1, self.n_bins])
ds_raw_data = VirtualDataset(
'Raw_Data',
data=None,
maxshape=[self.n_pixels, self.n_spec_bins],
dtype=np.complex64,
compression='gzip',
chunking=raw_chunking,
parent=meas_grp)
chan_grp.add_children([
ds_pos_inds, ds_pos_vals, ds_spec_inds, ds_spec_vals, ds_raw_data
])
meas_grp.add_children([chan_grp])
root_grp.add_children([meas_grp])
hdf = HDFwriter(self.h5_path)
hdf.delete()
h5_refs = hdf.write(root_grp)
# Delete the MicroDatasets to save memory
del ds_raw_data, ds_spec_inds, ds_spec_vals, ds_pos_inds, ds_pos_vals
# Get the file and Raw_Data objects
h5_raw = get_h5_obj_refs(['Raw_Data'], h5_refs)[0]
h5_chan_grp = h5_raw.parent
# Get the Position and Spectroscopic dataset objects
h5_pos_inds = get_h5_obj_refs(['Position_Indices'], h5_refs)[0]
h5_pos_vals = get_h5_obj_refs(['Position_Values'], h5_refs)[0]
h5_spec_inds = get_h5_obj_refs(['Spectroscopic_Indices'], h5_refs)[0]
h5_spec_vals = get_h5_obj_refs(['Spectroscopic_Values'], h5_refs)[0]
# Link the Position and Spectroscopic datasets as attributes of Raw_Data
link_as_main(h5_raw, h5_pos_inds, h5_pos_vals, h5_spec_inds,
h5_spec_vals)
'''
Build the SHO Group
'''
sho_grp = VirtualGroup('Raw_Data-SHO_Fit_', parent=h5_chan_grp.name)
# Build the Spectroscopic datasets for the SHO Guess and Fit
sho_spec_starts = np.where(
h5_spec_inds[h5_spec_inds.attrs['Frequency']].squeeze() == 0)[0]
sho_spec_labs = get_attr(h5_spec_inds, 'labels')
ds_sho_spec_inds, ds_sho_spec_vals = build_reduced_spec_dsets(
h5_spec_inds,
h5_spec_vals,
keep_dim=sho_spec_labs != 'Frequency',
step_starts=sho_spec_starts)
sho_chunking = calc_chunks([self.n_pixels, self.n_sho_bins],
sho32.itemsize,
unit_chunks=[1, 1])
ds_sho_fit = VirtualDataset('Fit',
data=None,
maxshape=[self.n_pixels, self.n_sho_bins],
dtype=sho32,
compression='gzip',
chunking=sho_chunking,
parent=sho_grp)
ds_sho_guess = VirtualDataset(
'Guess',
data=None,
maxshape=[self.n_pixels, self.n_sho_bins],
dtype=sho32,
compression='gzip',
chunking=sho_chunking,
parent=sho_grp)
sho_grp.add_children(
[ds_sho_fit, ds_sho_guess, ds_sho_spec_inds, ds_sho_spec_vals])
# Write the SHO group and datasets to the file and delete the MicroDataset objects
h5_sho_refs = hdf.write(sho_grp)
del ds_sho_fit, ds_sho_guess, ds_sho_spec_inds, ds_sho_spec_vals
# Get the dataset handles for the fit and guess
h5_sho_fit = get_h5_obj_refs(['Fit'], h5_sho_refs)[0]
h5_sho_guess = get_h5_obj_refs(['Guess'], h5_sho_refs)[0]
# Get the dataset handles for the SHO Spectroscopic datasets
h5_sho_spec_inds = get_h5_obj_refs(['Spectroscopic_Indices'],
h5_sho_refs)[0]
h5_sho_spec_vals = get_h5_obj_refs(['Spectroscopic_Values'],
h5_sho_refs)[0]
# Link the Position and Spectroscopic datasets as attributes of the SHO Fit and Guess
link_as_main(h5_sho_fit, h5_pos_inds, h5_pos_vals, h5_sho_spec_inds,
h5_sho_spec_vals)
link_as_main(h5_sho_guess, h5_pos_inds, h5_pos_vals, h5_sho_spec_inds,
h5_sho_spec_vals)
'''
Build the loop group
'''
loop_grp = VirtualGroup('Fit-Loop_Fit_', parent=h5_sho_fit.parent.name)
# Build the Spectroscopic datasets for the loops
loop_spec_starts = np.where(h5_sho_spec_inds[
h5_sho_spec_inds.attrs['DC_Offset']].squeeze() == 0)[0]
loop_spec_labs = get_attr(h5_sho_spec_inds, 'labels')
ds_loop_spec_inds, ds_loop_spec_vals = build_reduced_spec_dsets(
h5_sho_spec_inds,
h5_sho_spec_vals,
keep_dim=loop_spec_labs != 'DC_Offset',
step_starts=loop_spec_starts)
# Create the loop fit and guess MicroDatasets
loop_chunking = calc_chunks([self.n_pixels, self.n_loops],
loop_fit32.itemsize,
unit_chunks=[1, 1])
ds_loop_fit = VirtualDataset('Fit',
data=None,
maxshape=[self.n_pixels, self.n_loops],
dtype=loop_fit32,
compression='gzip',
chunking=loop_chunking,
parent=loop_grp)
ds_loop_guess = VirtualDataset('Guess',
data=None,
maxshape=[self.n_pixels, self.n_loops],
dtype=loop_fit32,
compression='gzip',
chunking=loop_chunking,
parent=loop_grp)
# Add the datasets to the loop group then write it to the file
loop_grp.add_children(
[ds_loop_fit, ds_loop_guess, ds_loop_spec_inds, ds_loop_spec_vals])
h5_loop_refs = hdf.write(loop_grp)
# Delete the MicroDatasets
del ds_loop_spec_vals, ds_loop_spec_inds, ds_loop_guess, ds_loop_fit
# Get the handles to the datasets
h5_loop_fit = get_h5_obj_refs(['Fit'], h5_loop_refs)[0]
h5_loop_guess = get_h5_obj_refs(['Guess'], h5_loop_refs)[0]
h5_loop_spec_inds = get_h5_obj_refs(['Spectroscopic_Indices'],
h5_loop_refs)[0]
h5_loop_spec_vals = get_h5_obj_refs(['Spectroscopic_Values'],
h5_loop_refs)[0]
# Link the Position and Spectroscopic datasets to the Loop Guess and Fit
link_as_main(h5_loop_fit, h5_pos_inds, h5_pos_vals, h5_loop_spec_inds,
h5_loop_spec_vals)
link_as_main(h5_loop_guess, h5_pos_inds, h5_pos_vals,
h5_loop_spec_inds, h5_loop_spec_vals)
self.h5_raw = USIDataset(h5_raw)
self.h5_sho_guess = USIDataset(h5_sho_guess)
self.h5_sho_fit = USIDataset(h5_sho_fit)
self.h5_loop_guess = USIDataset(h5_loop_guess)
self.h5_loop_fit = USIDataset(h5_loop_fit)
self.h5_spec_vals = h5_spec_vals
self.h5_spec_inds = h5_spec_inds
self.h5_sho_spec_inds = h5_sho_spec_inds
self.h5_sho_spec_vals = h5_sho_spec_vals
self.h5_loop_spec_inds = h5_loop_spec_inds
self.h5_loop_spec_vals = h5_loop_spec_vals
self.h5_file = h5_raw.file
return
def translate(self, h5_path, force_patch=False, **kwargs):
"""
Add the needed references and attributes to the h5 file that are not created by the
LabView data aquisition program.
Parameters
----------
h5_path : str
path to the h5 file
force_patch : bool, optional
Should the check to see if the file has already been patched be ignored.
Default False.
Returns
-------
h5_file : h5py.File
patched hdf5 file
"""
# Open the file and check if a patch is needed
h5_file = h5py.File(os.path.abspath(h5_path), 'r+')
if h5_file.attrs.get('translator') is not None and not force_patch:
print('File is already Pycroscopy ready.')
return h5_file
'''
Get the list of all Raw_Data Datasets
Loop over the list and update the needed attributes
'''
raw_list = find_dataset(h5_file, 'Raw_Data')
for h5_raw in raw_list:
if 'quantity' not in h5_raw.attrs:
h5_raw.attrs['quantity'] = 'quantity'
if 'units' not in h5_raw.attrs:
h5_raw.attrs['units'] = 'a.u.'
# Grab the channel and measurement group of the data to check some needed attributes
h5_chan = h5_raw.parent
try:
c_type = get_attr(h5_chan, 'channel_type')
except KeyError:
warn_str = "'channel_type' was not found as an attribute of {}.\n".format(h5_chan.name)
warn_str += "If this is BEPS or BELine data from the LabView aquisition software, " + \
"please run the following piece of code. Afterwards, run this function again.\n" + \
"CODE: " \
"hdf.file['{}'].attrs['channel_type'] = 'BE'".format(h5_chan.name)
warn(warn_str)
return h5_file
except:
raise
if c_type != 'BE':
continue
h5_meas = h5_chan.parent
h5_meas.attrs['num_UDVS_steps'] = h5_meas.attrs['num_steps']
# Get the object handles for the Indices and Values datasets
h5_pos_inds = h5_chan['Position_Indices']
h5_pos_vals = h5_chan['Position_Values']
h5_spec_inds = h5_chan['Spectroscopic_Indices']
h5_spec_vals = h5_chan['Spectroscopic_Values']
# Make sure we have correct spectroscopic indices for the given values
ds_spec_inds = create_spec_inds_from_vals(h5_spec_vals[()])
if not np.allclose(ds_spec_inds, h5_spec_inds[()]):
h5_spec_inds[:, :] = ds_spec_inds[:, :]
h5_file.flush()
# Get the labels and units for the Spectroscopic datasets
h5_spec_labels = h5_spec_inds.attrs['labels']
inds_and_vals = [h5_pos_inds, h5_pos_vals, h5_spec_inds, h5_spec_vals]
for dset in inds_and_vals:
spec_labels = dset.attrs['labels']
try:
spec_units = dset.attrs['units']
if len(spec_units) != len(spec_labels):
raise KeyError
except KeyError:
dset['units'] = ['' for _ in spec_labels]
except:
raise
for ilabel, label in enumerate(h5_spec_labels):
label_slice = (slice(ilabel, ilabel + 1), slice(None))
if label == '':
label = 'Step'
h5_spec_inds.attrs[label] = h5_spec_inds.regionref[label_slice]
h5_spec_vals.attrs[label] = h5_spec_vals.regionref[label_slice]
# Link the references to the Indices and Values datasets to the Raw_Data
link_as_main(h5_raw, h5_pos_inds, h5_pos_vals, h5_spec_inds, h5_spec_vals)
# Also link the Bin_Frequencies and Bin_Wfm_Type datasets
h5_freqs = h5_chan['Bin_Frequencies']
aux_dset_names = ['Bin_Frequencies']
aux_dset_refs = [h5_freqs.ref]
check_and_link_ancillary(h5_raw, aux_dset_names, anc_refs=aux_dset_refs)
'''
Get all SHO_Fit groups for the Raw_Data and loop over them
Get the Guess and Spectroscopic Datasets for each SHO_Fit group
'''
sho_list = find_results_groups(h5_raw, 'SHO_Fit')
for h5_sho in sho_list:
h5_sho_guess = h5_sho['Guess']
h5_sho_spec_inds = h5_sho['Spectroscopic_Indices']
h5_sho_spec_vals = h5_sho['Spectroscopic_Values']
# Make sure we have correct spectroscopic indices for the given values
ds_sho_spec_inds = create_spec_inds_from_vals(h5_sho_spec_inds[()])
if not np.allclose(ds_sho_spec_inds, h5_sho_spec_inds[()]):
h5_sho_spec_inds[:, :] = ds_sho_spec_inds[:, :]
# Get the labels and units for the Spectroscopic datasets
h5_sho_spec_labels = get_attr(h5_sho_spec_inds, 'labels')
link_as_main(h5_sho_guess, h5_pos_inds, h5_pos_vals, h5_sho_spec_inds, h5_sho_spec_vals)
sho_inds_and_vals = [h5_sho_spec_inds, h5_sho_spec_vals]
for dset in sho_inds_and_vals:
spec_labels = get_attr(dset, 'labels')
try:
spec_units = get_attr(dset, 'units')
if len(spec_units) != len(spec_labels):
raise KeyError
except KeyError:
spec_units = [''.encode('utf-8') for _ in spec_labels]
dset.attrs['units'] = spec_units
except:
raise
# Make region references in the
for ilabel, label in enumerate(h5_sho_spec_labels):
label_slice = (slice(ilabel, ilabel + 1), slice(None))
if label == '':
label = 'Step'.encode('utf-8')
h5_sho_spec_inds.attrs[label] = h5_sho_spec_inds.regionref[label_slice]
h5_sho_spec_vals.attrs[label] = h5_sho_spec_vals.regionref[label_slice]
h5_file.flush()
h5_file.attrs['translator'] = 'V3patcher'.encode('utf-8')
return h5_file
def _setup_h5(self, data_gen_parms):
"""
Setups up the hdf5 file structure before doing the actual generation
Parameters
----------
data_gen_parms : dict
Dictionary containing the parameters to write to the Measurement Group as attributes
Returns
-------
"""
'''
Build the group structure down to the channel group
'''
# Set up the basic group structure
root_grp = VirtualGroup('')
root_parms = generate_dummy_main_parms()
root_parms['translator'] = 'FAKEBEPS'
root_parms['data_type'] = data_gen_parms['data_type']
root_grp.attrs = root_parms
meas_grp = VirtualGroup('Measurement_')
chan_grp = VirtualGroup('Channel_')
meas_grp.attrs.update(data_gen_parms)
# Create the Position and Spectroscopic datasets for the Raw Data
ds_pos_inds, ds_pos_vals, ds_spec_inds, ds_spec_vals = self._build_ancillary_datasets()
raw_chunking = calc_chunks([self.n_pixels,
self.n_spec_bins],
np.complex64(0).itemsize,
unit_chunks=[1, self.n_bins])
ds_raw_data = VirtualDataset('Raw_Data', data=None,
maxshape=[self.n_pixels, self.n_spec_bins],
dtype=np.complex64,
compression='gzip',
chunking=raw_chunking,
parent=meas_grp)
chan_grp.add_children([ds_pos_inds, ds_pos_vals, ds_spec_inds, ds_spec_vals,
ds_raw_data])
meas_grp.add_children([chan_grp])
root_grp.add_children([meas_grp])
hdf = HDFwriter(self.h5_path)
hdf.delete()
h5_refs = hdf.write(root_grp)
# Delete the MicroDatasets to save memory
del ds_raw_data, ds_spec_inds, ds_spec_vals, ds_pos_inds, ds_pos_vals
# Get the file and Raw_Data objects
h5_raw = get_h5_obj_refs(['Raw_Data'], h5_refs)[0]
h5_chan_grp = h5_raw.parent
# Get the Position and Spectroscopic dataset objects
h5_pos_inds = get_h5_obj_refs(['Position_Indices'], h5_refs)[0]
h5_pos_vals = get_h5_obj_refs(['Position_Values'], h5_refs)[0]
h5_spec_inds = get_h5_obj_refs(['Spectroscopic_Indices'], h5_refs)[0]
h5_spec_vals = get_h5_obj_refs(['Spectroscopic_Values'], h5_refs)[0]
# Link the Position and Spectroscopic datasets as attributes of Raw_Data
link_as_main(h5_raw, h5_pos_inds, h5_pos_vals, h5_spec_inds, h5_spec_vals)
'''
Build the SHO Group
'''
sho_grp = VirtualGroup('Raw_Data-SHO_Fit_', parent=h5_chan_grp.name)
# Build the Spectroscopic datasets for the SHO Guess and Fit
sho_spec_starts = np.where(h5_spec_inds[h5_spec_inds.attrs['Frequency']].squeeze() == 0)[0]
sho_spec_labs = get_attr(h5_spec_inds, 'labels')
ds_sho_spec_inds, ds_sho_spec_vals = build_reduced_spec_dsets(h5_spec_inds,
h5_spec_vals,
keep_dim=sho_spec_labs != 'Frequency',
step_starts=sho_spec_starts)
sho_chunking = calc_chunks([self.n_pixels,
self.n_sho_bins],
sho32.itemsize,
unit_chunks=[1, 1])
ds_sho_fit = VirtualDataset('Fit', data=None,
maxshape=[self.n_pixels, self.n_sho_bins],
dtype=sho32,
compression='gzip',
chunking=sho_chunking,
parent=sho_grp)
ds_sho_guess = VirtualDataset('Guess', data=None,
maxshape=[self.n_pixels, self.n_sho_bins],
dtype=sho32,
compression='gzip',
chunking=sho_chunking,
parent=sho_grp)
sho_grp.add_children([ds_sho_fit, ds_sho_guess, ds_sho_spec_inds, ds_sho_spec_vals])
# Write the SHO group and datasets to the file and delete the MicroDataset objects
h5_sho_refs = hdf.write(sho_grp)
del ds_sho_fit, ds_sho_guess, ds_sho_spec_inds, ds_sho_spec_vals
# Get the dataset handles for the fit and guess
h5_sho_fit = get_h5_obj_refs(['Fit'], h5_sho_refs)[0]
h5_sho_guess = get_h5_obj_refs(['Guess'], h5_sho_refs)[0]
# Get the dataset handles for the SHO Spectroscopic datasets
h5_sho_spec_inds = get_h5_obj_refs(['Spectroscopic_Indices'], h5_sho_refs)[0]
h5_sho_spec_vals = get_h5_obj_refs(['Spectroscopic_Values'], h5_sho_refs)[0]
# Link the Position and Spectroscopic datasets as attributes of the SHO Fit and Guess
link_as_main(h5_sho_fit, h5_pos_inds, h5_pos_vals, h5_sho_spec_inds, h5_sho_spec_vals)
link_as_main(h5_sho_guess, h5_pos_inds, h5_pos_vals, h5_sho_spec_inds, h5_sho_spec_vals)
'''
Build the loop group
'''
loop_grp = VirtualGroup('Fit-Loop_Fit_', parent=h5_sho_fit.parent.name)
# Build the Spectroscopic datasets for the loops
loop_spec_starts = np.where(h5_sho_spec_inds[h5_sho_spec_inds.attrs['DC_Offset']].squeeze() == 0)[0]
loop_spec_labs = get_attr(h5_sho_spec_inds, 'labels')
ds_loop_spec_inds, ds_loop_spec_vals = build_reduced_spec_dsets(h5_sho_spec_inds,
h5_sho_spec_vals,
keep_dim=loop_spec_labs != 'DC_Offset',
step_starts=loop_spec_starts)
# Create the loop fit and guess MicroDatasets
loop_chunking = calc_chunks([self.n_pixels, self.n_loops],
loop_fit32.itemsize,
unit_chunks=[1, 1])
ds_loop_fit = VirtualDataset('Fit', data=None,
maxshape=[self.n_pixels, self.n_loops],
dtype=loop_fit32,
compression='gzip',
chunking=loop_chunking,
parent=loop_grp)
ds_loop_guess = VirtualDataset('Guess', data=None,
maxshape=[self.n_pixels, self.n_loops],
dtype=loop_fit32,
compression='gzip',
chunking=loop_chunking,
parent=loop_grp)
# Add the datasets to the loop group then write it to the file
loop_grp.add_children([ds_loop_fit, ds_loop_guess, ds_loop_spec_inds, ds_loop_spec_vals])
h5_loop_refs = hdf.write(loop_grp)
# Delete the MicroDatasets
del ds_loop_spec_vals, ds_loop_spec_inds, ds_loop_guess, ds_loop_fit
# Get the handles to the datasets
h5_loop_fit = get_h5_obj_refs(['Fit'], h5_loop_refs)[0]
h5_loop_guess = get_h5_obj_refs(['Guess'], h5_loop_refs)[0]
h5_loop_spec_inds = get_h5_obj_refs(['Spectroscopic_Indices'], h5_loop_refs)[0]
h5_loop_spec_vals = get_h5_obj_refs(['Spectroscopic_Values'], h5_loop_refs)[0]
# Link the Position and Spectroscopic datasets to the Loop Guess and Fit
link_as_main(h5_loop_fit, h5_pos_inds, h5_pos_vals, h5_loop_spec_inds, h5_loop_spec_vals)
link_as_main(h5_loop_guess, h5_pos_inds, h5_pos_vals, h5_loop_spec_inds, h5_loop_spec_vals)
self.h5_raw = USIDataset(h5_raw)
self.h5_sho_guess = USIDataset(h5_sho_guess)
self.h5_sho_fit = USIDataset(h5_sho_fit)
self.h5_loop_guess = USIDataset(h5_loop_guess)
self.h5_loop_fit = USIDataset(h5_loop_fit)
self.h5_spec_vals = h5_spec_vals
self.h5_spec_inds = h5_spec_inds
self.h5_sho_spec_inds = h5_sho_spec_inds
self.h5_sho_spec_vals = h5_sho_spec_vals
self.h5_loop_spec_inds = h5_loop_spec_inds
self.h5_loop_spec_vals = h5_loop_spec_vals
self.h5_file = h5_raw.file
return