def analyze_pixel(ibw_file, param_file): ''' Analyzes a single pixel Parameters ---------- ibw_file : str path to \*.ibw file param_file : str path to parameters.cfg file Returns ------- pixel : Pixel The pixel object read and analyzed ''' signal_array = signal(ibw_file) n_pixels, params = configuration(param_file) pixel = Pixel(signal_array, params=params) pixel.analyze() pixel.plot() plt.xlabel('Time Step') plt.ylabel('Freq Shift (Hz)') print('tFP is', pixel.tfp, 's') return pixel.tfp
def process_line(args): """Wrapper function for line class, used in parallel processing.""" signal_file, params, n_pixels = args signal_array = load.signal(signal_file) line_inst = line.Line(signal_array, params, n_pixels) tfp, shift, _ = line_inst.analyze() return tfp, shift
def createHDF5_file(signal, parm_dict, h5_path='', ds_name='FF_Raw'): """ Generates the HDF5 file given path to a specific file and a parameters dictionary Parameters ---------- h5_path : string Path to desired h5 file. signal : str, ndarray Path to the data file to be converted or a workspace array parm_dict : dict Scan parameters Returns ------- h5_path: str The filename path to the H5 file create """ sg = signal if 'str' in str(type(signal)): sg = load.signal(signal) if not any(h5_path): # if not passed, auto-generate name fname = signal.replace('/', '\\') h5_path = fname[:-4] + '.h5' else: fname = h5_path hdf = px.ioHDF5(h5_path) usid.hdf_utils.print_tree(hdf.file) ff_group = px.MicroDataGroup('FF_Group', parent='/') root_group = px.MicroDataGroup('/') # fname = fname.split('\\')[-1][:-4] sg = px.MicroDataset(ds_name, data=sg, dtype=np.float32, parent=ff_group) if 'pnts_per_pixel' not in parm_dict.keys(): parm_dict['pnts_per_avg'] = signal.shape[1] parm_dict['pnts_per_pixel'] = 1 parm_dict['pnts_per_line'] = parm_dict['num_cols'] ff_group.addChildren([sg]) ff_group.attrs = parm_dict # Get reference for writing the data h5_refs = hdf.writeData(ff_group, print_log=True) hdf.flush()
def main(argv=None): """Main function of the executable file.""" logging.basicConfig(filename='error.log', level=logging.INFO) # Get the CPU count to display in help. cpu_count = multiprocessing.cpu_count() if argv is None: argv = sys.argv[1:] # Parse arguments from the command line, and print out help. parser = ap.ArgumentParser(description='Analysis software for FF-trEFM') parser.add_argument('path', nargs='?', default=os.getcwd(), help='path to directory') parser.add_argument('-p', help='parallel computing option should be' 'followed by the number of CPUs.', type=int, choices=range(2, cpu_count + 1)) parser.add_argument('-v', action='version', version='FFtr-EFM 2.0 Release Candidate') args = parser.parse_args(argv) # Scan the path for .ibw and .cfg files. path = args.path filelist = os.listdir(path) data_files = [os.path.join(path, name) for name in filelist if name[-3:] == 'ibw'] config_file = [os.path.join(path, name) for name in filelist if name[-3:] == 'cfg'][0] # Load parameters from .cfg file. n_pixels, parameters = load.configuration(config_file) print('Recombination: ', parameters['recombination']) if 'phase_fitting' in parameters: print('Phase fitting: ', parameters['phase_fitting']) print( 'ROI: ', parameters['roi']) if not args.p: # Initialize arrays. tfp = np.zeros((len(data_files), n_pixels)) shift = np.zeros((len(data_files), n_pixels)) # Initialize plotting. plt.ion() fig = plt.figure(figsize=(12, 6), tight_layout=True) grid = gs.GridSpec(1, 2) tfp_ax = plt.subplot(grid[0, 0]) shift_ax = plt.subplot(grid[0, 1]) plt.setp(tfp_ax.get_xticklabels(), visible=False) plt.setp(tfp_ax.get_yticklabels(), visible=False) plt.setp(shift_ax.get_xticklabels(), visible=False) plt.setp(shift_ax.get_yticklabels(), visible=False) tfp_ax.set_title('tFP Image') shift_ax.set_title('Shift Image') kwargs = {'origin': 'lower', 'aspect': 'equal'} tfp_image = tfp_ax.imshow(tfp * 1e6, cmap='afmhot', **kwargs) shift_image = shift_ax.imshow(shift, cmap='cubehelix', **kwargs) text = plt.figtext(0.4, 0.1, '') plt.show() # Load every file in the file list one by one. for i, data_file in enumerate(data_files): signal_array = load.signal(data_file) line_inst = line.Line(signal_array, parameters, n_pixels) tfp[i, :], shift[i, :], _ = line_inst.analyze() # line_inst = line.Line(signal_array, parameters, n_pixels,fitphase=True) # tfpphase[i, :], _, _ = line_inst.analyze() tfp_image = tfp_ax.imshow(tfp * 1e6, cmap='inferno', **kwargs) shift_image = shift_ax.imshow(shift, cmap='cubehelix', **kwargs) tfp_sc = tfp[tfp.nonzero()] * 1e6 tfp_image.set_clim(vmin=tfp_sc.min(), vmax=tfp_sc.max()) shift_sc = shift[shift.nonzero()] shift_image.set_clim(vmin=shift_sc.min(), vmax=shift_sc.max()) tfpmean = 1e6 * tfp[i, :].mean() tfpstd = 1e6 * tfp[i, :].std() string = ("Line {0:.0f}, average tFP (us) =" " {1:.2f} +/- {2:.2f}".format(i + 1, tfpmean, tfpstd)) text.remove() text = plt.figtext(0.35, 0.1, string) plt.draw() plt.pause(0.0001) del line_inst # Delete the instance to open up memory. elif args.p: print('Starting parallel processing, using {0:1d} \ CPUs.'.format(args.p)) start_time = time.time() # Keep when it's started. # Create a pool of workers. pool = multiprocessing.Pool(processes=args.p) # Create the iterable and map onto the function. n_files = len(data_files) iterable = zip(data_files, [parameters] * n_files, [n_pixels] * n_files) result = pool.map(process_line, iterable) # Do not forget to close spawned processes. pool.close() pool.join() # Unzip the result. tfp_list, shift_list = zip(*result) # Initialize arrays. tfp = np.zeros((n_files, n_pixels)) shift = np.zeros((n_files, n_pixels)) # Convert list of arrays to 2D array. for i in range(n_files): tfp[i, :] = tfp_list[i] shift[i, :] = shift_list[i] elapsed_time = time.time() - start_time print ('It took {0:.1f} seconds.'.format(elapsed_time)) # Filter bad pixels tfp_fixed, _ = badpixels.fix_array(tfp, threshold=2) tfp_fixed = np.array(tfp_fixed) # Save csv files. os.chdir(path) np.savetxt('tfp.csv', np.fliplr(tfp).T, delimiter=',') np.savetxt('shift.csv', np.fliplr(shift).T, delimiter=',') np.savetxt('tfp_fixed.csv', np.fliplr(tfp_fixed).T, delimiter=',') return
def load_FF(data_files, parm_dict, h5_path, verbose=False, loadverbose=True, average=True, mirror=True): """ Generates the HDF5 file given path to data_files and parameters dictionary Creates a Datagroup FFtrEFM_Group with a single dataset in chunks :param data_files: List of the \*.ibw files to be invidually scanned. This is generated by load_folder above :type data_files: list :param parm_dict: Scan parameters to be saved as attributes. This is generated by load_folder above, or you can pass this explicitly. :type parm_dict: dict :param h5_path: :type h5_path : string :param verbose: Display outputs of each function or not :type verbose: bool, optional :param loadverbose: Whether to print any simple "loading Line X" statements for feedback :type loadverbose: bool, optional :param average: Whether to average each pixel before saving to H5. This saves both time and space :type average: bool, optional :param mirror: Mirrors the data when saving. This parameter is to match the FFtrEFM data with the associate topography as FFtrEFM is acquired during a retrace while topo is saved during a forward trace :type mirror: bool, optional :returns: The filename path to the H5 file created :rtype: str """ # Prepare data for writing to HDF num_rows = parm_dict['num_rows'] num_cols = parm_dict['num_cols'] pnts_per_avg = parm_dict['pnts_per_avg'] name = 'FF_Raw' if average: parm_dict['pnts_per_pixel'] = 1 parm_dict['pnts_per_line'] = num_cols name = 'FF_Avg' pnts_per_pixel = parm_dict['pnts_per_pixel'] pnts_per_line = parm_dict['pnts_per_line'] dt = 1 / parm_dict['sampling_rate'] def_vec = np.arange(0, parm_dict['total_time'], dt) if def_vec.shape[0] != parm_dict['pnts_per_avg']: def_vec = def_vec[:-1] # warnings.warn('Time-per-point calculation error') # To do: Fix the labels/attributes on the relevant data sets try: hdf = h5py.File(h5_path, 'r+') except: print('Creating HDF5 file...') hdf = h5py.File(h5_path, 'w') try: ff_group = hdf.create_group('FF_Group') except: print('Group already exists, creating new one') ff_group = usid.hdf_utils.create_indexed_group(hdf['/'], 'FF_Group') # Set up the position vectors for the data pos_desc = [Dimension('X', 'm', np.linspace(0, parm_dict['FastScanSize'], num_cols * pnts_per_pixel)), Dimension('Y', 'm', np.linspace(0, parm_dict['SlowScanSize'], num_rows))] spec_desc = [Dimension('Time', 's', np.linspace(0, parm_dict['total_time'], pnts_per_avg))] for p in parm_dict: ff_group.attrs[p] = parm_dict[p] ff_group.attrs['pnts_per_line'] = num_cols h5_ff = usid.hdf_utils.write_main_dataset(ff_group, # parent HDF5 group (num_rows * num_cols * pnts_per_pixel, pnts_per_avg), # shape of Main dataset name, # Name of main dataset 'Deflection', # Physical quantity contained in Main dataset 'V', # Units for the physical quantity pos_desc, # Position dimensions spec_desc, # Spectroscopic dimensions dtype=np.float32, # data type / precision compression='gzip', main_dset_attrs=parm_dict) pnts_per_line = parm_dict['pnts_per_line'] # Cycles through the remaining files. This takes a while (~few minutes) for k, num in zip(data_files, np.arange(0, len(data_files))): if loadverbose: fname = k.replace('/', '\\') print('####', fname.split('\\')[-1], '####') fname = str(num).rjust(4, '0') line_file = load.signal(k) if average: _ll = line.Line(line_file, parm_dict, n_pixels=num_cols, pycroscopy=False) _ll = _ll.pixel_wise_avg().T else: _ll = line_file.transpose() f = hdf.file[h5_ff.name] if mirror: f[pnts_per_line * num:pnts_per_line * (num + 1), :] = np.flipud(_ll[:, :]) else: f[pnts_per_line * num:pnts_per_line * (num + 1), :] = _ll[:, :] if verbose == True: usid.hdf_utils.print_tree(hdf.file, rel_paths=True) hdf.flush() return h5_ff
def load_folder(folder_path='', xy_scansize=[0, 0], file_name='FF_H5', textload=False, verbose=False): """ Sets up loading the HDF5 files. Parses the data file list and creates the .H5 file path :param folder_path: Path to folder you want to process :type folder_path: string :param xy_scansize: Width by Height in meters (e.g. [8e-6, 4e-6]), if not in parameters file :type xy_scansize: 2-float array :param file_name: Desired file name, otherwise is auto-generated :type file_name: str :param textload: If you have a folder of .txt instead of .ibw (older files, some synthetic data) :type textload: bool, optional :param verbose: Whether to output the datasets being processed :type verbose: bool, optional :returns: tuple (h5_path, data_files, parm_dict) WHERE str h5_path is the filename path to the H5 file created List data_files is the list of \*.ibw files in the folder to be processed dict parm_dict is the dictionary of relevant scan parameters """ if any(xy_scansize) and len(xy_scansize) != 2: raise Exception('XY Scan Size must be either empty (in .cfg) or length-2') if not any(folder_path): folder_path = sidpy.io.interface_utils.openfile_dialog(caption='Select Config File in FF-trEFM folder', file_types='Config Files (*.cfg)') folder_path = '/'.join(folder_path.split('/')[:-1]) print(folder_path, 'folder path') filelist = sorted(os.listdir(folder_path)) if textload == False: data_files = [os.path.join(folder_path, name) for name in filelist if (name[-3:] == 'ibw' and 'FFtrEFM' in name)] else: data_files = [os.path.join(folder_path, name) for name in filelist if name[-3:] == 'txt'] if not data_files: raise OSError('No data files found! Are these text files?') config_file = [os.path.join(folder_path, name) for name in filelist if name[-3:] == 'cfg'][0] n_pixels, parm_dict = load.configuration(config_file) parm_dict['num_rows'] = len(data_files) parm_dict['num_cols'] = n_pixels # Add dimensions if not in the config file if 'FastScanSize' not in parm_dict.keys(): if not any(xy_scansize): raise Exception('Need XY Scan Size! Save "Width" and "Height" in Config or pass xy_scansize') [width, height] = xy_scansize if width > 1e-3: # if entering as microns width = width * 1e-6 height = height * 1e-6 parm_dict['FastScanSize'] = width parm_dict['SlowScanSize'] = height # sometimes use width/height in config files if 'width' in parm_dict.keys(): parm_dict['FastScanSize'] = width parm_dict['SlowScanSize'] = height # Check ratio is correct ratio = np.round(parm_dict['FastScanSize'] * 1e6, 4) / np.round(parm_dict['SlowScanSize'] * 1e6, 4) if n_pixels / len(data_files) != ratio: print(ratio) print(parm_dict['FastScanSize'], parm_dict['SlowScanSize'], n_pixels / len(data_files), len(data_files)) raise Exception('X-Y Dimensions do not match filelist. Add manually to config file. Check n-pixels.') # add associated dimension info # # e.g. if a 16000 point signal with 2000 averages and 10 pixels # (10MHz sampling of a 1.6 ms long signal=16000, 200 averages per pixel) # parm_dict['pnts_per_pixel'] = 200 (# signals at each pixel) # ['pnts_per_avg'] = 16000 (# pnts per signal, called an "average") # ['pnts_per_line'] = 2000 (# signals in each line) if 'pnts_per_pixel' not in parm_dict.keys(): print('Loading first signal') # Uses first data set to determine parameters line_file = load.signal(data_files[0]) parm_dict['pnts_per_avg'] = int(line_file.shape[0]) try: # for 1 average per pixel, this will fail parm_dict['pnts_per_pixel'] = int(line_file.shape[1] / parm_dict['num_cols']) parm_dict['pnts_per_line'] = int(line_file.shape[1]) except: parm_dict['pnts_per_pixel'] = 1 parm_dict['pnts_per_line'] = 1 folder_path = folder_path.replace('/', '\\') if os.path.exists(file_name) == False: h5_path = os.path.join(folder_path, file_name) + '.h5' else: h5_path = file_name return h5_path, data_files, parm_dict
def load_pixel_averaged_FF(data_files, parm_dict, h5_path, verbose=False, loadverbose=True, mirror=False): """ Creates a new group FF_Avg where the raw FF files are averaged together This function does not process the Raw data and is more useful when the resulting Raw data matrix is very large (causing memory errors) This is more useful as pixel-wise averages are more relevant in FF-processing This Dataset is (n_pixels*n_rows, n_pnts_per_avg) Parameters ---------- h5_file : h5py File H5 File to be examined. File typically set as h5_file = hdf.file hdf = px.ioHDF5(h5_path), h5_path = path to disk verbose : bool, optional Display outputs of each function or not loadverbose : Boolean (optional) Whether to print any simple "loading Line X" statements for feedback Returns ------- h5_avg : Dataset The new averaged Dataset """ hdf = px.io.HDFwriter(h5_path) try: ff_avg_group = hdf.file.create_group('FF_Group') except: ff_avg_group = usid.hdf_utils.create_indexed_group(hdf.file['/'], 'FF_Group') try: ff_avg_group = hdf.file[ff_avg_group.name].create_group('FF_Avg') except: ff_avg_group = usid.hdf_utils.create_indexed_group(ff_avg_group, 'FF_Avg') num_rows = parm_dict['num_rows'] num_cols = parm_dict['num_cols'] pnts_per_avg = parm_dict['pnts_per_avg'] pnts_per_line = parm_dict['pnts_per_line'] pnts_per_pixel = parm_dict['pnts_per_pixel'] parm_dict['pnts_per_pixel'] = 1 # only 1 average per pixel now parm_dict['pnts_per_line'] = num_cols # equivalent now with averaged data n_pix = int(pnts_per_line / pnts_per_pixel) dt = 1 / parm_dict['sampling_rate'] # Set up the position vectors for the data pos_desc = [Dimension('X', 'm', np.linspace(0, parm_dict['FastScanSize'], num_cols)), Dimension('Y', 'm', np.linspace(0, parm_dict['SlowScanSize'], num_rows))] ds_pos_ind, ds_pos_val = build_ind_val_dsets(pos_desc, is_spectral=False) spec_desc = [Dimension('Time', 's', np.linspace(0, parm_dict['total_time'], pnts_per_avg))] ds_spec_inds, ds_spec_vals = build_ind_val_dsets(spec_desc, is_spectral=True) for p in parm_dict: ff_avg_group.attrs[p] = parm_dict[p] ff_avg_group.attrs['pnts_per_line'] = num_cols # to change number of pnts in a line ff_avg_group.attrs['pnts_per_pixel'] = 1 # to change number of pnts in a pixel h5_avg = usid.hdf_utils.write_main_dataset(ff_avg_group, # parent HDF5 group (num_rows * num_cols, pnts_per_avg), # shape of Main dataset 'FF_Avg', # Name of main dataset 'Deflection', # Physical quantity contained in Main dataset 'V', # Units for the physical quantity pos_desc, # Position dimensions spec_desc, # Spectroscopic dimensions dtype=np.float32, # data type / precision compression='gzip', main_dset_attrs=parm_dict) # Generates a line from each data file, averages, then saves the data for k, n in zip(data_files, np.arange(0, len(data_files))): if loadverbose: fname = k.replace('/', '\\') print('####', fname.split('\\')[-1], '####') fname = str(n).rjust(4, '0') line_file = load.signal(k) _ll = line.Line(line_file, parm_dict, n_pixels=n_pix, pycroscopy=False) _ll = _ll.pixel_wise_avg().T if mirror: h5_avg[n * num_cols:(n + 1) * num_cols, :] = np.flipud(_ll[:, :]) else: h5_avg[n * num_cols:(n + 1) * num_cols, :] = _ll[:, :] if verbose == True: usid.hdf_utils.print_tree(hdf.file, rel_paths=True) h5_avg = usid.hdf_utils.find_dataset(hdf.file, 'FF_Avg')[0] print('H5_avg of size:', h5_avg.shape) hdf.flush() return h5_avg