def load_pixel_averaged_from_raw(h5_file, verbose=True, loadverbose=True):
"""
Creates a new group FF_Avg where the FF_raw file is averaged together.
This is more useful as pixel-wise averages are more relevant in FF-processing
This Dataset is (n_pixels*n_rows, n_pnts_per_avg)
:param h5_file: H5 File to be examined. File typically set as h5_file = hdf.file
hdf = px.ioHDF5(h5_path), h5_path = path to disk
:type h5_file: h5py File
:param verbose: Display outputs of each function or not
:type verbose: bool, optional
:param loadverbose: Whether to print any simple "loading Line X" statements for feedback
:type loadverbose: bool, optional
:returns: The new averaged Dataset
:rtype: Dataset
"""
hdf = h5py.File(h5_file)
h5_main = usid.hdf_utils.find_dataset(hdf.file, 'FF_Raw')[0]
try:
ff_avg_group = h5_main.parent.create_group('FF_Avg')
except:
ff_avg_group = usid.hdf_utils.create_indexed_group(h5_main.parent, 'FF_Avg')
parm_dict = get_attributes(h5_main.parent)
num_rows = parm_dict['num_rows']
num_cols = parm_dict['num_cols']
pnts_per_avg = parm_dict['pnts_per_avg']
pnts_per_line = parm_dict['pnts_per_line']
pnts_per_pixel = parm_dict['pnts_per_pixel']
parm_dict['pnts_per_pixel'] = 1 # only 1 average per pixel now
parm_dict['pnts_per_line'] = num_cols # equivalent now with averaged data
n_pix = int(pnts_per_line / pnts_per_pixel)
dt = 1 / parm_dict['sampling_rate']
# Set up the position vectors for the data
pos_desc = [Dimension('X', 'm', np.linspace(0, parm_dict['FastScanSize'], num_cols)),
Dimension('Y', 'm', np.linspace(0, parm_dict['SlowScanSize'], num_rows))]
spec_desc = [Dimension('Time', 's', np.linspace(0, parm_dict['total_time'], pnts_per_avg))]
for p in parm_dict:
ff_avg_group.attrs[p] = parm_dict[p]
ff_avg_group.attrs['pnts_per_line'] = num_cols # to change number of pnts in a line
ff_avg_group.attrs['pnts_per_pixel'] = 1 # to change number of pnts in a pixel
h5_avg = usid.hdf_utils.write_main_dataset(ff_avg_group, # parent HDF5 group
(num_rows * num_cols, pnts_per_avg), # shape of Main dataset
'FF_Avg', # Name of main dataset
'Deflection', # Physical quantity contained in Main dataset
'V', # Units for the physical quantity
pos_desc, # Position dimensions
spec_desc, # Spectroscopic dimensions
dtype=np.float32, # data type / precision
compression='gzip',
main_dset_attrs=parm_dict)
# Uses get_line to extract line. Averages and returns to the Dataset FF_Avg
# We can operate on the dataset array directly, get_line is used for future_proofing if
# we want to add additional operation (such as create an Image class)
for i in range(num_rows):
if loadverbose == True:
print('#### Row:', i, '####')
_ll = get_utils.get_line(h5_main, pnts=pnts_per_line, line_num=i, array_form=False, avg=False)
_ll = _ll.pixel_wise_avg()
h5_avg[i * num_cols:(i + 1) * num_cols, :] = _ll[:, :]
if verbose == True:
usid.hdf_utils.print_tree(hdf.file, rel_paths=True)
h5_avg = usid.hdf_utils.find_dataset(hdf.file, 'FF_Avg')[0]
print('H5_avg of size:', h5_avg.shape)
hdf.flush()
return h5_avg
def process(h5_file,
ds='FF_Raw',
ref='',
clear_filter=False,
verbose=True,
liveplots=True,
**kwargs):
"""
Processes FF_Raw dataset in the HDF5 file
This then saves within the h5 file in FF_Group-processed
This uses the dataset in this priority:
\*A relative path specific by ref, e.g. '/FF_Group/FF_Avg/FF_Avg'
\*A dataset specified by ds, returning the last found, e.g. 'FF_Raw'
\*FF_Group/FF_Raw found via searching from hdf_utils folder
Typical usage:
>> import pycroscopy as px
>> h5_file = px.io.HDFwriter('path_to_h5_file.h5').file
>> from ffta import analyze_h5
>> tfp, shift, inst_freq = analyze_h5.process(h5_file, ref = '/FF_Group/FF_Avg/FF_Avg')
Parameters
----------
h5_file : h5Py file or str
Path to a specific h5 file on the disk or an hdf.file
ds : str, optional
The Dataset to search for in the file
ref : str, optional
A path to a specific dataset in the file.
e.g. h5_file['/FF_Group/FF_Avg/FF_Avg']
clear_filter : bool, optional
For data already filtered, calls Line's clear_filter function to
skip FIR/windowing steps
verbose : bool, optional,
Whether to write data to the command line
liveplots : bool
Displaying can sometimes cause the window to pop in front of other active windows
in Matplotlib. This disables it, with an obvious drawback of no feedback.
Returns
-------
tfp : ndarray
time-to-first-peak image array
shift : ndarray
frequency shift image array
inst_freq : ndarray (2D)
instantaneous frequency array, an N x p array of N=rows\*cols points
and where p = points_per_signal (e.g. 16000 for 1.6 ms @10 MHz sampling)
h5_if : USIDataset of h5_if (instantaneous frequency)
"""
# logging.basicConfig(filename='error.log', level=logging.INFO)
ftype = str(type(h5_file))
if ('str' in ftype) or ('File' in ftype) or ('Dataset' in ftype):
h5_file = px.io.HDFwriter(h5_file).file
else:
raise TypeError('Must be string path, e.g. E:\Test.h5')
# Looks for a ref first before searching for ds, h5_ds is group to process
if any(ref):
h5_ds = h5_file[ref]
parameters = get_utils.get_params(h5_ds)
elif ds != 'FF_Raw':
h5_ds = usid.hdf_utils.find_dataset(h5_file, ds)[-1]
parameters = get_utils.get_params(h5_ds)
else:
h5_ds, parameters = find_FF(h5_file)
if isinstance(h5_ds, h5py.Dataset):
h5_gp = h5_ds.parent
# Initialize file and read parameters
num_cols = parameters['num_cols']
num_rows = parameters['num_rows']
pnts_per_pixel = parameters['pnts_per_pixel']
pnts_per_avg = parameters['pnts_per_avg']
for key, value in kwargs.items():
_temp = parameters[key]
parameters[key] = value
if verbose:
print('Changing', key, 'from', _temp, 'to', value)
if verbose:
print('Recombination: ', parameters['recombination'])
print('ROI: ', parameters['roi'])
# Initialize arrays.
tfp = np.zeros([num_rows, num_cols])
shift = np.zeros([num_rows, num_cols])
inst_freq = np.zeros([num_rows * num_cols, pnts_per_avg])
# Initialize plotting.
plt.ion()
fig, a = plt.subplots(nrows=2, ncols=2, figsize=(13, 6))
tfp_ax = a[0][1]
shift_ax = a[1][1]
img_length = parameters['FastScanSize']
img_height = parameters['SlowScanSize']
kwargs = {
'origin': 'lower',
'x_vec': img_length * 1e6,
'y_vec': img_height * 1e6,
'num_ticks': 5,
'stdevs': 3
}
try:
ht = h5_file['/height/Raw_Data'][:, 0]
ht = np.reshape(ht, [num_cols, num_rows]).transpose()
ht_ax = a[0][0]
ht_image, cbar = usid.viz.plot_utils.plot_map(ht_ax,
ht * 1e9,
cmap='gray',
**kwargs)
cbar.set_label('Height (nm)', rotation=270, labelpad=16)
except:
pass
tfp_ax.set_title('tFP Image')
shift_ax.set_title('Shift Image')
tfp_image, cbar_tfp = usid.viz.plot_utils.plot_map(tfp_ax,
tfp * 1e6,
cmap='inferno',
show_cbar=False,
**kwargs)
shift_image, cbar_sh = usid.viz.plot_utils.plot_map(shift_ax,
shift,
cmap='inferno',
show_cbar=False,
**kwargs)
text = tfp_ax.text(num_cols / 2, num_rows + 3, '')
plt.show()
print('Analyzing with roi of', parameters['roi'])
# Load every file in the file list one by one.
for i in range(num_rows):
line_inst = get_utils.get_line(h5_ds, i, params=parameters)
if clear_filter:
line_inst.clear_filter_flags()
_tfp, _shf, _if = line_inst.analyze()
tfp[i, :] = _tfp.T
shift[i, :] = _shf.T
inst_freq[i * num_cols:(i + 1) * num_cols, :] = _if.T
if liveplots:
tfp_image, _ = usid.viz.plot_utils.plot_map(tfp_ax,
tfp * 1e6,
cmap='inferno',
show_cbar=False,
**kwargs)
shift_image, _ = usid.viz.plot_utils.plot_map(shift_ax,
shift,
cmap='inferno',
show_cbar=False,
**kwargs)
tfp_sc = tfp[tfp.nonzero()] * 1e6
tfp_image.set_clim(vmin=tfp_sc.min(), vmax=tfp_sc.max())
shift_sc = shift[shift.nonzero()]
shift_image.set_clim(vmin=shift_sc.min(), vmax=shift_sc.max())
tfpmean = 1e6 * tfp[i, :].mean()
tfpstd = 1e6 * tfp[i, :].std()
if verbose:
string = ("Line {0:.0f}, average tFP (us) ="
" {1:.2f} +/- {2:.2f}".format(
i + 1, tfpmean, tfpstd))
print(string)
text.remove()
text = tfp_ax.text((num_cols - len(string)) / 2, num_rows + 4,
string)
# plt.draw()
plt.pause(0.0001)
del line_inst # Delete the instance to open up memory.
tfp_image, cbar_tfp = usid.viz.plot_utils.plot_map(tfp_ax,
tfp * 1e6,
cmap='inferno',
**kwargs)
cbar_tfp.set_label('Time (us)', rotation=270, labelpad=16)
shift_image, cbar_sh = usid.viz.plot_utils.plot_map(shift_ax,
shift,
cmap='inferno',
**kwargs)
cbar_sh.set_label('Frequency Shift (Hz)', rotation=270, labelpad=16)
text = tfp_ax.text(num_cols / 2, num_rows + 3, '')
plt.show()
h5_if = save_IF(h5_ds.parent, inst_freq, parameters)
_, _, tfp_fixed = save_ht_outs(h5_if.parent, tfp, shift)
# save_CSV(h5_path, tfp, shift, tfp_fixed, append=ds)
if verbose:
print(
'Please remember to close the H5 file explicitly when you are done to retain these data',
'e.g.:', 'h5_if.file.close()',
'...and then reopen the file as needed.')
return tfp, shift, inst_freq, h5_if
def hdf_commands(h5_path, ds='FF_Raw'):
"""
Creates a bunch of typical workspace HDF5 variables for scripting use
Parameters
----------
h5_path : str
String path to H5PY file
ds : str, optional
The dataset to search for and set as h5_main.
This prints the valid commands to the workspace. Then just highlight and
copy-paste to execute
h5_path : str
Path to hdf5 file on disk
"""
commands = ['***Copy-paste all commands below this line, then hit ENTER***',
'import h5py']
if not isinstance(h5_path, str):
raise TypeError('Pass a file path (string), not an H5 file')
try:
hdf = h5py.File(h5_path, 'r+')
commands.append("hdf = h5py.File(h5_path, 'r+')")
except:
pass
try:
h5_file = hdf.file
commands.append("h5_file = hdf.file")
except:
pass
try:
h5_main = usid.hdf_utils.find_dataset(hdf.file, ds)[0]
commands.append("h5_main = usid.hdf_utils.find_dataset(hdf.file, '" + ds + "')[0]")
except:
pass
try:
h5_if = usid.hdf_utils.find_dataset(hdf.file, 'inst_freq')[-1]
commands.append("h5_if = usid.hdf_utils.find_dataset(hdf.file, 'inst_freq')[-1]")
except:
pass
try:
h5_if = usid.hdf_utils.find_dataset(hdf.file, 'Inst_Freq')[-1]
commands.append("h5_if = usid.hdf_utils.find_dataset(hdf.file, 'Inst_Freq')[-1]")
except:
pass
try:
h5_tfp = usid.hdf_utils.find_dataset(hdf.file, 'tfp')[-1]
commands.append("h5_tfp= usid.hdf_utils.find_dataset(hdf.file, 'tfp')[-1]")
except:
pass
try:
h5_shift = usid.hdf_utils.find_dataset(hdf.file, 'shift')[-1]
commands.append("h5_shift= usid.hdf_utils.find_dataset(hdf.file, 'shift')[-1]")
except:
pass
try:
h5_avg = usid.hdf_utils.find_dataset(hdf.file, 'FF_Avg')[-1]
commands.append("h5_avg = usid.hdf_utils.find_dataset(hdf.file, 'FF_Avg')[-1]")
except:
pass
try:
h5_filt = usid.hdf_utils.find_dataset(hdf.file, 'Filtered_Data')[-1]
commands.append("h5_filt = usid.hdf_utils.find_dataset(hdf.file, 'Filtered_Data')[-1]")
except:
pass
try:
h5_rb = usid.hdf_utils.find_dataset(hdf.file, 'Rebuilt_Data')[-1]
commands.append("h5_rb = usid.hdf_utils.find_dataset(hdf.file, 'Rebuilt_Data')[-1]")
except:
pass
try:
parameters = usid.hdf_utils.get_attributes(h5_avg)
commands.append("parameters = usid.hdf_utils.get_attributes(h5_avg)")
except:
pass
try:
h5_ll = get_utils.get_line(h5_if, line_num=0)
commands.append("h5_ll = ffta.load.get_utils.get_line(h5_if, line_num=0)")
except:
pass
try:
h5_px = get_utils.get_pixel(h5_if, rc=[0, 0])
commands.append("h5_px = ffta.load.get_utils.get_pixel(h5_if, rc=[0,0])")
except:
pass
try:
h5_svd = usid.hdf_utils.find_dataset(hdf.file, 'U')[-1]
commands.append("h5_svd = usid.hdf_utils.find_dataset(hdf.file, 'U')[-1].parent")
except:
pass
try:
h5_cpd = usid.hdf_utils.find_dataset(hdf.file, 'cpd')[-1]
commands.append("h5_cpd = usid.hdf_utils.find_dataset(hdf.file, 'cpd')[-1]")
except:
pass
try:
h5_ytime = usid.hdf_utils.find_dataset(hdf.file, 'y_time')[-1]
commands.append("h5_ytime = usid.hdf_utils.find_dataset(hdf.file, 'y_time')[-1]")
except:
pass
try:
h5_Y = usid.hdf_utils.find_dataset(hdf.file, 'Y')[-1]
commands.append("h5_Y = usid.hdf_utils.find_dataset(hdf.file, 'Y')[-1]")
except:
pass
for i in commands:
print(i)
return
