def svd_filter(h5_main, clean_components=None): """ Filters data given the array clean_components :param h5_main: Dataset to be filtered and reconstructed. This must be the same as where SVD was performed :type h5_main: h5py :param clean_components: Clean_components has 2 components will filter from start to finish Clean_components has 3+ components will use those individual components :type clean_components: :returns: :rtype: """ if not (isinstance(h5_main, usid.USIDataset)): h5_main = usid.USIDataset(h5_main) h5_rb = rebuild_svd(h5_main, components=clean_components) parameters = get_utils.get_params(h5_main) for key in parameters: if key not in h5_rb.attrs: h5_rb.attrs[key] = parameters[key] return h5_rb
def find_FF(h5_path): """ :param h5_path: Location of the relevant datasets to be saved/plotted. e.g. h5_rb.name :type h5_path: str :returns: tuple (h5_gp, parameters) WHERE [type] h5_gp is... [type] parameters is... """ parameters = get_utils.get_params(h5_path) h5_gp = hdf_utils._which_h5_group(h5_path) return h5_gp, parameters
def find_FF(h5_path): parameters = get_utils.get_params(h5_path) h5_gp = hdf_utils._which_h5_group(h5_path) return h5_gp, parameters
def plot_tfps(h5_file, h5_path='/', append='', savefig=True, stdevs=2): """ Plots the relevant tfp, inst_freq, and shift values as separate image files Parameters ---------- h5_file : h5Py File h5_path : str, optional Location of the relevant datasets to be saved/plotted. e.g. h5_rb.name append : str, optional A string to include in the saved figure filename savefig : bool, optional Whether or not to save the image stdevs : int, optional Number of standard deviations to display """ h5_file = px.io.HDFwriter(h5_file).file try: h5_if = usid.hdf_utils.find_dataset(h5_file[h5_path], 'Inst_Freq')[0] except: h5_if = usid.hdf_utils.find_dataset(h5_file[h5_path], 'inst_freq')[0] parm_dict = get_utils.get_params(h5_if) # parm_dict = usid.hdf_utils.get_attributes(h5_file[h5_path]) if 'Dataset' in str(type(h5_file[h5_path])): h5_path = h5_file[h5_path].parent.name tfp = usid.hdf_utils.find_dataset(h5_file[h5_path], 'tfp')[0][()] shift = usid.hdf_utils.find_dataset(h5_file[h5_path], 'shift')[0][()] if tfp.shape[1] == 1: # Forgot to reconvert and/or needs to be converted [ysz, xsz] = h5_if.pos_dim_sizes tfp = np.reshape(tfp, [ysz, xsz]) shift = np.reshape(shift, [ysz, xsz]) try: tfp_fixed = usid.hdf_utils.find_dataset(h5_file[h5_path], 'tfp_fixed')[0][()] except: tfp_fixed, _ = badpixels.fix_array(tfp, threshold=2) tfp_fixed = np.array(tfp_fixed) xs = parm_dict['FastScanSize'] ys = parm_dict['SlowScanSize'] asp = ys / xs if asp != 1: asp = asp * 2 fig, a = plt.subplots(nrows=3, figsize=(8, 9)) [vmint, vmaxt] = np.mean(tfp) - 2 * np.std(tfp), np.mean(tfp) - 2 * np.std(tfp) [vmins, vmaxs ] = np.mean(shift) - 2 * np.std(shift), np.mean(shift) - 2 * np.std(shift) _, cbar_t = usid.viz.plot_utils.plot_map(a[0], tfp_fixed * 1e6, x_vec=xs * 1e6, y_vec=ys * 1e6, aspect=asp, cmap='inferno', stdevs=stdevs) _, cbar_r = usid.viz.plot_utils.plot_map(a[1], 1 / (1e3 * tfp_fixed), x_vec=xs * 1e6, y_vec=ys * 1e6, aspect=asp, cmap='inferno', stdevs=stdevs) _, cbar_s = usid.viz.plot_utils.plot_map(a[2], shift, x_vec=xs * 1e6, y_vec=ys * 1e6, aspect=asp, cmap='inferno', stdevs=stdevs) cbar_t.set_label('tfp (us)', rotation=270, labelpad=16) a[0].set_title('tfp', fontsize=12) cbar_r.set_label('Rate (kHz)', rotation=270, labelpad=16) a[1].set_title('1/tfp', fontsize=12) cbar_s.set_label('shift (Hz)', rotation=270, labelpad=16) a[2].set_title('shift', fontsize=12) fig.tight_layout() if savefig: path = h5_file.file.filename.replace('\\', '/') path = '/'.join(path.split('/')[:-1]) + '/' os.chdir(path) fig.savefig('tfp_shift_' + append + '_.tif', format='tiff') return
def process(h5_file, ds='FF_Raw', ref='', clear_filter=False, verbose=True, liveplots=True, **kwargs): """ Processes FF_Raw dataset in the HDF5 file This then saves within the h5 file in FF_Group-processed This uses the dataset in this priority: \*A relative path specific by ref, e.g. '/FF_Group/FF_Avg/FF_Avg' \*A dataset specified by ds, returning the last found, e.g. 'FF_Raw' \*FF_Group/FF_Raw found via searching from hdf_utils folder Typical usage: >> import pycroscopy as px >> h5_file = px.io.HDFwriter('path_to_h5_file.h5').file >> from ffta import analyze_h5 >> tfp, shift, inst_freq = analyze_h5.process(h5_file, ref = '/FF_Group/FF_Avg/FF_Avg') Parameters ---------- h5_file : h5Py file or str Path to a specific h5 file on the disk or an hdf.file ds : str, optional The Dataset to search for in the file ref : str, optional A path to a specific dataset in the file. e.g. h5_file['/FF_Group/FF_Avg/FF_Avg'] clear_filter : bool, optional For data already filtered, calls Line's clear_filter function to skip FIR/windowing steps verbose : bool, optional, Whether to write data to the command line liveplots : bool Displaying can sometimes cause the window to pop in front of other active windows in Matplotlib. This disables it, with an obvious drawback of no feedback. Returns ------- tfp : ndarray time-to-first-peak image array shift : ndarray frequency shift image array inst_freq : ndarray (2D) instantaneous frequency array, an N x p array of N=rows\*cols points and where p = points_per_signal (e.g. 16000 for 1.6 ms @10 MHz sampling) h5_if : USIDataset of h5_if (instantaneous frequency) """ # logging.basicConfig(filename='error.log', level=logging.INFO) ftype = str(type(h5_file)) if ('str' in ftype) or ('File' in ftype) or ('Dataset' in ftype): h5_file = px.io.HDFwriter(h5_file).file else: raise TypeError('Must be string path, e.g. E:\Test.h5') # Looks for a ref first before searching for ds, h5_ds is group to process if any(ref): h5_ds = h5_file[ref] parameters = get_utils.get_params(h5_ds) elif ds != 'FF_Raw': h5_ds = usid.hdf_utils.find_dataset(h5_file, ds)[-1] parameters = get_utils.get_params(h5_ds) else: h5_ds, parameters = find_FF(h5_file) if isinstance(h5_ds, h5py.Dataset): h5_gp = h5_ds.parent # Initialize file and read parameters num_cols = parameters['num_cols'] num_rows = parameters['num_rows'] pnts_per_pixel = parameters['pnts_per_pixel'] pnts_per_avg = parameters['pnts_per_avg'] for key, value in kwargs.items(): _temp = parameters[key] parameters[key] = value if verbose: print('Changing', key, 'from', _temp, 'to', value) if verbose: print('Recombination: ', parameters['recombination']) print('ROI: ', parameters['roi']) # Initialize arrays. tfp = np.zeros([num_rows, num_cols]) shift = np.zeros([num_rows, num_cols]) inst_freq = np.zeros([num_rows * num_cols, pnts_per_avg]) # Initialize plotting. plt.ion() fig, a = plt.subplots(nrows=2, ncols=2, figsize=(13, 6)) tfp_ax = a[0][1] shift_ax = a[1][1] img_length = parameters['FastScanSize'] img_height = parameters['SlowScanSize'] kwargs = { 'origin': 'lower', 'x_vec': img_length * 1e6, 'y_vec': img_height * 1e6, 'num_ticks': 5, 'stdevs': 3 } try: ht = h5_file['/height/Raw_Data'][:, 0] ht = np.reshape(ht, [num_cols, num_rows]).transpose() ht_ax = a[0][0] ht_image, cbar = usid.viz.plot_utils.plot_map(ht_ax, ht * 1e9, cmap='gray', **kwargs) cbar.set_label('Height (nm)', rotation=270, labelpad=16) except: pass tfp_ax.set_title('tFP Image') shift_ax.set_title('Shift Image') tfp_image, cbar_tfp = usid.viz.plot_utils.plot_map(tfp_ax, tfp * 1e6, cmap='inferno', show_cbar=False, **kwargs) shift_image, cbar_sh = usid.viz.plot_utils.plot_map(shift_ax, shift, cmap='inferno', show_cbar=False, **kwargs) text = tfp_ax.text(num_cols / 2, num_rows + 3, '') plt.show() print('Analyzing with roi of', parameters['roi']) # Load every file in the file list one by one. for i in range(num_rows): line_inst = get_utils.get_line(h5_ds, i, params=parameters) if clear_filter: line_inst.clear_filter_flags() _tfp, _shf, _if = line_inst.analyze() tfp[i, :] = _tfp.T shift[i, :] = _shf.T inst_freq[i * num_cols:(i + 1) * num_cols, :] = _if.T if liveplots: tfp_image, _ = usid.viz.plot_utils.plot_map(tfp_ax, tfp * 1e6, cmap='inferno', show_cbar=False, **kwargs) shift_image, _ = usid.viz.plot_utils.plot_map(shift_ax, shift, cmap='inferno', show_cbar=False, **kwargs) tfp_sc = tfp[tfp.nonzero()] * 1e6 tfp_image.set_clim(vmin=tfp_sc.min(), vmax=tfp_sc.max()) shift_sc = shift[shift.nonzero()] shift_image.set_clim(vmin=shift_sc.min(), vmax=shift_sc.max()) tfpmean = 1e6 * tfp[i, :].mean() tfpstd = 1e6 * tfp[i, :].std() if verbose: string = ("Line {0:.0f}, average tFP (us) =" " {1:.2f} +/- {2:.2f}".format( i + 1, tfpmean, tfpstd)) print(string) text.remove() text = tfp_ax.text((num_cols - len(string)) / 2, num_rows + 4, string) # plt.draw() plt.pause(0.0001) del line_inst # Delete the instance to open up memory. tfp_image, cbar_tfp = usid.viz.plot_utils.plot_map(tfp_ax, tfp * 1e6, cmap='inferno', **kwargs) cbar_tfp.set_label('Time (us)', rotation=270, labelpad=16) shift_image, cbar_sh = usid.viz.plot_utils.plot_map(shift_ax, shift, cmap='inferno', **kwargs) cbar_sh.set_label('Frequency Shift (Hz)', rotation=270, labelpad=16) text = tfp_ax.text(num_cols / 2, num_rows + 3, '') plt.show() h5_if = save_IF(h5_ds.parent, inst_freq, parameters) _, _, tfp_fixed = save_ht_outs(h5_if.parent, tfp, shift) # save_CSV(h5_path, tfp, shift, tfp_fixed, append=ds) if verbose: print( 'Please remember to close the H5 file explicitly when you are done to retain these data', 'e.g.:', 'h5_if.file.close()', '...and then reopen the file as needed.') return tfp, shift, inst_freq, h5_if
def __init__(self, h5_main, data_avg, mask, results=None, isCPD=False, parms_dict=None): """ :param h5_main: File type to be processed. :type h5_main : h5Py dataset or str :param data_avg: The file to use as the "averaged" data upon which to apply the mask and do calculations e.g. 'tfp' searches within the h5_main parent folder for 'tfp' :type data_avg: str :param mask: mask file to be loaded, is expected to be of 1 (transparent) and 0 (opaque) loaded e.g. mask = mask_utils.loadmask('E:/ORNL/20191221_BAPI/nice_masks/Mask_BAPI20_0008.txt') :type mask: ndarray :param results: If you want to pass some previously-calculated results.. :type results: clustering results :param isCPD: toggle between GMode and FFtrEFM data, this is just for plotting :type isCPD: bool, optional, :param parms_dict: Parameters file used for the image. Must contain at minimum: num_rows, num_cols, sampling_rate, FastScanSize, SlowScanSize, total_time :type parms_dict: dict """ self.h5_main = h5_main if isinstance(data_avg, str): self.data_avg = usid.hdf_utils.find_dataset( h5_main.parent, data_avg)[0].value elif isinstance(data_avg, np.ndarray): self.data_avg = usid.hdf_utils.find_dataset(h5_main.parent, 'tfp')[0].value else: raise ValueError('Wrong format for data_avg') if results: self.results = results self.isCPD = isCPD # Set up datasets data self.data = self.h5_main[()] self.parms_dict = parms_dict if parms_dict == None: print('get params') self.parms_dict = get_utils.get_params(h5_main) self._params() # Create mask for grain boundaries if not mask.any(): mask = np.ones([self.num_rows, self.num_cols]) self.mask = mask self.mask_nan, self.mask_on_1D, self.mask_off_1D = mask_utils.load_masks( self.mask) self._idx_1D = np.copy(self.mask_off_1D) return
def add_standard_sets(h5_path, group, fast_x=32e-6, slow_y=8e-6, parm_dict={}, ds='FF_Raw', verbose=False): """ Adds Position_Indices and Position_Value datasets to a folder within the h5_file Uses the values of fast_x and fast_y to determine the values Parameters ---------- h5_path : h5 File or str Points to a path to process group : str or H5PY group Location to process data to, either as str or H5PY parms_dict : dict, optional Parameters to be passed. By default this should be at the command line for FFtrEFM data ds : str, optional Dataset name to search for within this group and set as h5_main verbose : bool, optional Whether to write to the command line """ hdf = px.io.HDFwriter(h5_path) if not any(parm_dict): parm_dict = get_utils.get_params(h5_path) if 'FastScanSize' in parm_dict: fast_x = parm_dict['FastScanSize'] if 'SlowScanSize' in parm_dict: slow_y = parm_dict['SlowScanSize'] try: num_rows = parm_dict['num_rows'] num_cols = parm_dict['num_cols'] pnts_per_avg = parm_dict['pnts_per_avg'] dt = 1 / parm_dict['sampling_rate'] except: # some defaults warnings.warn('Improper parameters specified.') num_rows = 64 num_cols = 128 pnts_per_avg = 1 dt = 1 try: grp = px.io.VirtualGroup(group) except: grp = px.io.VirtualGroup(group.name) pos_desc = [ Dimension('X', 'm', np.linspace(0, parm_dict['FastScanSize'], num_cols)), Dimension('Y', 'm', np.linspace(0, parm_dict['SlowScanSize'], num_rows)) ] ds_pos_ind, ds_pos_val = build_ind_val_dsets(pos_desc, is_spectral=False, verbose=verbose) spec_desc = [Dimension('Time', 's', np.linspace(0, pnts_per_avg, dt))] ds_spec_inds, ds_spec_vals = build_ind_val_matrices(spec_desc, is_spectral=True, verbose=verbose) aux_ds_names = [ 'Position_Indices', 'Position_Values', 'Spectroscopic_Indices', 'Spectroscopic_Values' ] grp.add_children([ds_pos_ind, ds_pos_val, ds_spec_inds, ds_spec_vals]) h5_refs = hdf.write(grp, print_log=verbose) h5_main = hdf.file[grp.name] if any(ds): h5_main = usid.hdf_utils.find_dataset(hdf.file[grp.name], ds)[0] try: usid.hdf_utils.link_h5_objects_as_attrs( h5_main, usid.hdf_utils.get_h5_obj_refs(aux_ds_names, h5_refs)) except: usid.hdf_utils.link_h5_objects_as_attrs( h5_main, usid.hdf_utils.get_h5_obj_refs(aux_ds_names, h5_refs)) hdf.flush() return h5_main
def test_filter(hdf_file, freq_filts, parameters={}, pixelnum=[0, 0], noise_tolerance=5e-7, show_plots=True, check_filter=True): """ Applies FFT Filter to the file at a specific line and displays the result :param hdf_file: hdf_file to work on, e.g. hdf.file['/FF-raw'] if that's a Dataset if ndarray, uses passed or default parameters Use ndarray.flatten() to ensure correct dimensions :type hdf_file: h5Py file or Nx1 NumPy array (preferred is NumPy array) :param freq_filts: Contains the filters to apply to the test signal :type freq_filts: list of FrequencyFilter class objects :param parameters: Contains parameters in FF-raw file for constructing filters. Automatic if a Dataset/File Must contain num_pts and samp_rate to be functional :type parameters: dict, optional :param pixelnum: For extracting a specific pixel to do FFT Filtering on :type pixelnum: int, optional :param noise_tolerance: Amount of noise below which signal is set to 0 :type noise_tolerance: float 0 to 1 :param show_plots: Turns on FFT plots from Pycroscopy :type show_plots : bool, optional :returns: tuple (filt_line, freq_filts, fig_filt, axes_filt) WHERE numpy.ndarray filt_line is filtered signal of hdf_file list freq_filts is the filter parameters to be passed to SignalFilter matplotlib controls fig_filt, axes_filt - Only functional if show_plots is on """ reshape = False ftype = str(type(hdf_file)) if ('h5py' in ftype) or ('Dataset' in ftype): # hdf file parameters = get_utils.get_params(hdf_file) hdf_file = get_utils.get_pixel(hdf_file, [pixelnum[0], pixelnum[1]], array_form=True, transpose=False) hdf_file = hdf_file.flatten() if len(hdf_file.shape) == 2: reshape = True hdf_file = hdf_file.flatten() sh = hdf_file.shape # Test filter on a single line: filt_line, fig_filt, axes_filt = px.processing.gmode_utils.test_filter( hdf_file, frequency_filters=freq_filts, noise_threshold=noise_tolerance, show_plots=show_plots) # If need to reshape if reshape: filt_line = np.reshape(filt_line, sh) # Test filter out in Pixel if check_filter: plt.figure() plt.plot(hdf_file, 'b') plt.plot(filt_line, 'k') h5_px_filt = pixel.Pixel(filt_line, parameters) h5_px_filt.clear_filter_flags() h5_px_filt.analyze() h5_px_filt.plot(newplot=True) h5_px_raw = pixel.Pixel(hdf_file, parameters) h5_px_raw.analyze() h5_px_raw.plot(newplot=True) # h5_px_raw_unfilt = pixel.Pixel(hdf_file, parameters) # h5_px_raw_unfilt.clear_filter_flags() # h5_px_raw_unfilt.analyze() # h5_px_raw_unfilt.plot(newplot=False,c1='y', c2='c') return filt_line, freq_filts, fig_filt, axes_filt
def plot_tfps(h5_file, h5_path='/', append='', savefig=True, stdevs=2, scale=None): """ Plots the relevant tfp, inst_freq, and shift values as separate image files :param h5_file: :type h5_file: h5Py File :param h5_path: Location of the relevant datasets to be saved/plotted. e.g. h5_rb.name :type h5_path: str, optional :param append: A string to include in the saved figure filename :type append: str, optional :param savefig: Whether or not to save the image :type savefig: bool, optional :param stdevs: Number of standard deviations to display :type stdevs: int, optional :param scale: Scale bar size, in microns :type scale: float, optional :returns: tuple (fig, ax) WHERE [type] fig is... [type] ax is... """ try: h5_if = usid.hdf_utils.find_dataset(h5_file[h5_path], 'Inst_Freq')[0] except: h5_if = usid.hdf_utils.find_dataset(h5_file[h5_path], 'inst_freq')[0] parm_dict = get_utils.get_params(h5_if) # parm_dict = usid.hdf_utils.get_attributes(h5_file[h5_path]) if 'Dataset' in str(type(h5_file[h5_path])): h5_path = h5_file[h5_path].parent.name tfp = usid.hdf_utils.find_dataset(h5_file[h5_path], 'tfp')[0][()] shift = usid.hdf_utils.find_dataset(h5_file[h5_path], 'shift')[0][()] if tfp.shape[1] == 1: # Forgot to reconvert and/or needs to be converted [ysz, xsz] = h5_if.pos_dim_sizes tfp = np.reshape(tfp, [ysz, xsz]) shift = np.reshape(shift, [ysz, xsz]) try: tfp_fixed = usid.hdf_utils.find_dataset(h5_file[h5_path], 'tfp_fixed')[0][()] except: tfp_fixed, _ = badpixels.fix_array(tfp, threshold=2) tfp_fixed = np.array(tfp_fixed) xs = parm_dict['FastScanSize'] ys = parm_dict['SlowScanSize'] asp = ys / xs if asp != 1: asp = asp * 2 fig, ax = plt.subplots(nrows=3, figsize=(8, 9), facecolor='white') [vmint, vmaxt] = np.mean(tfp) - 2 * np.std(tfp), np.mean(tfp) - 2 * np.std(tfp) [vmins, vmaxs ] = np.mean(shift) - 2 * np.std(shift), np.mean(shift) - 2 * np.std(shift) _, cbar_t = usid.viz.plot_utils.plot_map(ax[0], tfp_fixed * 1e6, x_vec=xs * 1e6, y_vec=ys * 1e6, aspect=asp, cmap='inferno', stdevs=stdevs) _, cbar_r = usid.viz.plot_utils.plot_map(ax[1], 1 / (1e3 * tfp_fixed), x_vec=xs * 1e6, y_vec=ys * 1e6, aspect=asp, cmap='inferno', stdevs=stdevs) _, cbar_s = usid.viz.plot_utils.plot_map(ax[2], shift, x_vec=xs * 1e6, y_vec=ys * 1e6, aspect=asp, cmap='inferno', stdevs=stdevs) cbar_t.set_label('tfp (us)', rotation=270, labelpad=16) ax[0].set_title('tfp', fontsize=12) if scale: sz = np.floor(xs * 1e6 / scale) ratio = int(tfp_fixed.shape[1] / sz) if scale > 1: sz = str(scale) + ' $\mu m$' else: sz = str(int(scale * 1000)) + ' nm' fontprops = fm.FontProperties(size=14, weight='bold') bar1 = AnchoredSizeBar(ax[0].transData, ratio, sz, frameon=False, loc='lower right', size_vertical=2, pad=0.2, color='white', fontproperties=fontprops) ax[0].add_artist(bar1) cbar_r.set_label('Rate (kHz)', rotation=270, labelpad=16) ax[1].set_title('1/tfp', fontsize=12) if scale: bar1 = AnchoredSizeBar(ax[0].transData, ratio, sz, frameon=False, loc='lower right', size_vertical=2, pad=0.2, color='white', fontproperties=fontprops) ax[1].add_artist(bar1) cbar_s.set_label('shift (Hz)', rotation=270, labelpad=16) ax[2].set_title('shift', fontsize=12) if scale: bar1 = AnchoredSizeBar(ax[0].transData, ratio, sz, frameon=False, loc='lower right', size_vertical=2, pad=0.2, color='white', fontproperties=fontprops) ax[2].add_artist(bar1) fig.tight_layout() if savefig: path = h5_file.file.filename.replace('\\', '/') path = '/'.join(path.split('/')[:-1]) + '/' os.chdir(path) fig.savefig('tfp_shift_' + append + '_.tif', format='tiff') return fig, ax