def test_wrong_type(self): file_path = 'link_h5_objects_as_attrs.h5' data_utils.delete_existing_file(file_path) with h5py.File(file_path, mode='w') as h5_f: h5_main = h5_f.create_dataset('main', data=np.arange(5)) with self.assertRaises(TypeError): hdf_utils.link_h5_objects_as_attrs(h5_main, np.arange(4)) with self.assertRaises(TypeError): hdf_utils.link_h5_objects_as_attrs(np.arange(4), h5_main) os.remove(file_path)
def test_legal(self): file_path = 'link_h5_objects_as_attrs.h5' data_utils.delete_existing_file(file_path) with h5py.File(file_path, mode='w') as h5_f: h5_main = h5_f.create_dataset('main', data=np.arange(5)) h5_anc = h5_f.create_dataset('Ancillary', data=np.arange(3)) h5_group = h5_f.create_group('Results') hdf_utils.link_h5_objects_as_attrs(h5_f, [h5_anc, h5_main, h5_group]) for exp, name in zip([h5_main, h5_anc, h5_group], ['main', 'Ancillary', 'Results']): self.assertEqual(exp, h5_f[h5_f.attrs[name]]) # Single object hdf_utils.link_h5_objects_as_attrs(h5_main, h5_anc) self.assertEqual(h5_f[h5_main.attrs['Ancillary']], h5_anc) # Linking to a group: hdf_utils.link_h5_objects_as_attrs(h5_group, [h5_anc, h5_main]) for exp, name in zip([h5_main, h5_anc], ['main', 'Ancillary']): self.assertEqual(exp, h5_group[h5_group.attrs[name]]) os.remove(file_path)
def translate(self, parm_path): """ Basic method that translates .mat data files to a single .h5 file Parameters ------------ parm_path : string / unicode Absolute file path of the parameters .mat file. Returns ---------- h5_path : string / unicode Absolute path of the translated h5 file """ self.parm_path = path.abspath(parm_path) (folder_path, file_name) = path.split(parm_path) (file_name, base_name) = path.split(folder_path) h5_path = path.join(folder_path, base_name + '.h5') # Read parameters parm_dict = readGmodeParms(parm_path) # Add the w^2 specific parameters to this list parm_data = loadmat(parm_path, squeeze_me=True, struct_as_record=True) freq_sweep_parms = parm_data['freqSweepParms'] parm_dict['freq_sweep_delay'] = np.float( freq_sweep_parms['delay'].item()) gen_sig = parm_data['genSig'] parm_dict['wfm_fix_d_fast'] = np.int32(gen_sig['restrictT'].item()) freq_array = np.float32(parm_data['freqArray']) # prepare and write spectroscopic values samp_rate = parm_dict['IO_down_samp_rate_[Hz]'] num_bins = int(parm_dict['wfm_n_cycles'] * parm_dict['wfm_p_slow'] * samp_rate) w_vec = np.arange(-0.5 * samp_rate, 0.5 * samp_rate, np.float32(samp_rate / num_bins)) # There is most likely a more elegant solution to this but I don't have the time... Maybe np.meshgrid spec_val_mat = np.zeros((len(freq_array) * num_bins, 2), dtype=VALUES_DTYPE) spec_val_mat[:, 0] = np.tile(w_vec, len(freq_array)) spec_val_mat[:, 1] = np.repeat(freq_array, num_bins) spec_ind_mat = np.zeros((2, len(freq_array) * num_bins), dtype=np.int32) spec_ind_mat[0, :] = np.tile(np.arange(num_bins), len(freq_array)) spec_ind_mat[1, :] = np.repeat(np.arange(len(freq_array)), num_bins) num_rows = parm_dict['grid_num_rows'] num_cols = parm_dict['grid_num_cols'] parm_dict['data_type'] = 'GmodeW2' num_pix = num_rows * num_cols global_parms = dict() global_parms['grid_size_x'] = parm_dict['grid_num_cols'] global_parms['grid_size_y'] = parm_dict['grid_num_rows'] # assuming that the experiment was completed: global_parms['current_position_x'] = parm_dict['grid_num_cols'] - 1 global_parms['current_position_y'] = parm_dict['grid_num_rows'] - 1 global_parms['data_type'] = parm_dict[ 'data_type'] # self.__class__.__name__ global_parms['translator'] = 'W2' # Now start creating datasets and populating: if path.exists(h5_path): remove(h5_path) h5_f = h5py.File(h5_path, 'w') write_simple_attrs(h5_f, global_parms) meas_grp = create_indexed_group(h5_f, 'Measurement') chan_grp = create_indexed_group(meas_grp, 'Channel') write_simple_attrs(chan_grp, parm_dict) pos_dims = [ Dimension('X', 'nm', num_rows), Dimension('Y', 'nm', num_cols) ] spec_dims = [ Dimension('Response Bin', 'a.u.', num_bins), Dimension('Excitation Frequency ', 'Hz', len(freq_array)) ] # Minimize file size to the extent possible. # DAQs are rated at 16 bit so float16 should be most appropriate. # For some reason, compression is more effective on time series data h5_main = write_main_dataset(chan_grp, (num_pix, num_bins), 'Raw_Data', 'Deflection', 'V', pos_dims, spec_dims, chunks=(1, num_bins), dtype=np.float32) h5_ex_freqs = chan_grp.create_dataset('Excitation_Frequencies', freq_array) h5_bin_freq = chan_grp.create_dataset('Bin_Frequencies', w_vec) # Now doing link_h5_objects_as_attrs: link_h5_objects_as_attrs(h5_main, [h5_ex_freqs, h5_bin_freq]) # Now read the raw data files: pos_ind = 0 for row_ind in range(1, num_rows + 1): for col_ind in range(1, num_cols + 1): file_path = path.join( folder_path, 'fSweep_r' + str(row_ind) + '_c' + str(col_ind) + '.mat') print('Working on row {} col {}'.format(row_ind, col_ind)) if path.exists(file_path): # Load data file pix_data = loadmat(file_path, squeeze_me=True) pix_mat = pix_data['AI_mat'] # Take the inverse FFT on 2nd dimension pix_mat = np.fft.ifft(np.fft.ifftshift(pix_mat, axes=1), axis=1) # Verified with Matlab - no conjugate required here. pix_vec = pix_mat.transpose().reshape(pix_mat.size) h5_main[pos_ind, :] = np.float32(pix_vec) h5_f.flush() # flush from memory! else: print('File not found for: row {} col {}'.format( row_ind, col_ind)) pos_ind += 1 if (100.0 * pos_ind / num_pix) % 10 == 0: print('completed translating {} %'.format( int(100 * pos_ind / num_pix))) h5_f.close() return h5_path
def translate(self, parm_path): """ Basic method that translates .mat data files to a single .h5 file Parameters ------------ parm_path : string / unicode Absolute file path of the parameters .mat file. Returns ---------- h5_path : string / unicode Absolute path of the translated h5 file """ parm_path = path.abspath(parm_path) (folder_path, file_name) = path.split(parm_path) (file_name, base_name) = path.split(folder_path) h5_path = path.join(folder_path, base_name + '.h5') # Read parameters print('reading parameter files') parm_dict, excit_wfm, spec_ind_mat = self.__readparms(parm_path) parm_dict['data_type'] = 'SPORC' num_rows = parm_dict['grid_num_rows'] num_cols = parm_dict['grid_num_cols'] num_pix = num_rows * num_cols # new data format spec_ind_mat = np.transpose(VALUES_DTYPE(spec_ind_mat)) # Now start creating datasets and populating: pos_desc = [Dimension('Y', 'm', np.arange(num_rows)), Dimension('X', 'm', np.arange(num_cols))] ds_pos_ind, ds_pos_val = build_ind_val_dsets(pos_desc, is_spectral=False) spec_ind_labels = ['x index', 'y index', 'loop index', 'repetition index', 'slope index'] spec_ind_dict = dict() for col_ind, col_name in enumerate(spec_ind_labels): spec_ind_dict[col_name] = (slice(col_ind, col_ind + 1), slice(None)) ds_spec_inds = VirtualDataset('Spectroscopic_Indices', INDICES_DTYPE(spec_ind_mat)) ds_spec_inds.attrs['labels'] = spec_ind_dict ds_spec_vals = VirtualDataset('Spectroscopic_Values', spec_ind_mat) ds_spec_vals.attrs['labels'] = spec_ind_dict ds_spec_vals.attrs['units'] = ['V', 'V', '', '', ''] ds_excit_wfm = VirtualDataset('Excitation_Waveform', np.float32(excit_wfm)) ds_raw_data = VirtualDataset('Raw_Data', data=[], maxshape=(num_pix, len(excit_wfm)), dtype=np.float16, chunking=(1, len(excit_wfm)), compression='gzip') # technically should change the date, etc. chan_grp = VirtualGroup('Channel_000') chan_grp.attrs = parm_dict chan_grp.add_children([ds_pos_ind, ds_pos_val, ds_spec_inds, ds_spec_vals, ds_excit_wfm, ds_raw_data]) global_parms = generate_dummy_main_parms() global_parms['grid_size_x'] = parm_dict['grid_num_cols'] global_parms['grid_size_y'] = parm_dict['grid_num_rows'] # assuming that the experiment was completed: global_parms['current_position_x'] = parm_dict['grid_num_cols'] - 1 global_parms['current_position_y'] = parm_dict['grid_num_rows'] - 1 global_parms['data_type'] = parm_dict['data_type'] global_parms['translator'] = 'SPORC' meas_grp = VirtualGroup('Measurement_000') meas_grp.add_children([chan_grp]) spm_data = VirtualGroup('') spm_data.attrs = global_parms spm_data.add_children([meas_grp]) if path.exists(h5_path): remove(h5_path) # Write everything except for the main data. hdf = HDFwriter(h5_path) h5_refs = hdf.write(spm_data) h5_main = get_h5_obj_refs(['Raw_Data'], h5_refs)[0] # Now doing link_h5_objects_as_attrs: aux_ds_names = ['Excitation_Waveform', 'Position_Indices', 'Position_Values', 'Spectroscopic_Indices', 'Spectroscopic_Values'] link_h5_objects_as_attrs(h5_main, get_h5_obj_refs(aux_ds_names, h5_refs)) print('reading raw data now...') # Now read the raw data files: pos_ind = 0 for row_ind in range(1, num_rows + 1): for col_ind in range(1, num_cols + 1): file_path = path.join(folder_path, 'result_r' + str(row_ind) + '_c' + str(col_ind) + '.mat') # print('Working on row {} col {}'.format(row_ind,col_ind)) if path.exists(file_path): # Load data file pix_data = loadmat(file_path, squeeze_me=True) # Take the inverse FFT on 1st dimension pix_vec = np.fft.ifft(np.fft.ifftshift(pix_data['data'])) # Verified with Matlab - no conjugate required here. h5_main[pos_ind, :] = np.float16(np.real(pix_vec)) hdf.flush() # flush from memory! else: print('File for row {} col {} not found'.format(row_ind, col_ind)) pos_ind += 1 if (100.0 * pos_ind / num_pix) % 10 == 0: print('Finished reading {} % of data'.format(int(100 * pos_ind / num_pix))) hdf.close() return h5_path
def translate(self, parm_path): """ Basic method that translates .mat data files to a single .h5 file Parameters ------------ parm_path : string / unicode Absolute file path of the parameters .mat file. Returns ---------- h5_path : string / unicode Absolute path of the translated h5 file """ parm_path = path.abspath(parm_path) (folder_path, file_name) = path.split(parm_path) (file_name, base_name) = path.split(folder_path) h5_path = path.join(folder_path, base_name + '.h5') # Read parameters print('reading parameter files') parm_dict, excit_wfm, spec_ind_mat = self.__readparms(parm_path) parm_dict['data_type'] = 'SPORC' num_rows = parm_dict['grid_num_rows'] num_cols = parm_dict['grid_num_cols'] num_pix = num_rows * num_cols # new data format spec_ind_mat = np.transpose(VALUES_DTYPE(spec_ind_mat)) # Now start creating datasets and populating: pos_desc = [ Dimension('Y', 'm', np.arange(num_rows)), Dimension('X', 'm', np.arange(num_cols)) ] ds_pos_ind, ds_pos_val = build_ind_val_dsets(pos_desc, is_spectral=False) spec_ind_labels = [ 'x index', 'y index', 'loop index', 'repetition index', 'slope index' ] spec_ind_dict = dict() for col_ind, col_name in enumerate(spec_ind_labels): spec_ind_dict[col_name] = (slice(col_ind, col_ind + 1), slice(None)) ds_spec_inds = VirtualDataset('Spectroscopic_Indices', INDICES_DTYPE(spec_ind_mat)) ds_spec_inds.attrs['labels'] = spec_ind_dict ds_spec_vals = VirtualDataset('Spectroscopic_Values', spec_ind_mat) ds_spec_vals.attrs['labels'] = spec_ind_dict ds_spec_vals.attrs['units'] = ['V', 'V', '', '', ''] ds_excit_wfm = VirtualDataset('Excitation_Waveform', np.float32(excit_wfm)) ds_raw_data = VirtualDataset('Raw_Data', data=[], maxshape=(num_pix, len(excit_wfm)), dtype=np.float16, chunking=(1, len(excit_wfm)), compression='gzip') # technically should change the date, etc. chan_grp = VirtualGroup('Channel_000') chan_grp.attrs = parm_dict chan_grp.add_children([ ds_pos_ind, ds_pos_val, ds_spec_inds, ds_spec_vals, ds_excit_wfm, ds_raw_data ]) global_parms = generate_dummy_main_parms() global_parms['grid_size_x'] = parm_dict['grid_num_cols'] global_parms['grid_size_y'] = parm_dict['grid_num_rows'] # assuming that the experiment was completed: global_parms['current_position_x'] = parm_dict['grid_num_cols'] - 1 global_parms['current_position_y'] = parm_dict['grid_num_rows'] - 1 global_parms['data_type'] = parm_dict['data_type'] global_parms['translator'] = 'SPORC' meas_grp = VirtualGroup('Measurement_000') meas_grp.add_children([chan_grp]) spm_data = VirtualGroup('') spm_data.attrs = global_parms spm_data.add_children([meas_grp]) if path.exists(h5_path): remove(h5_path) # Write everything except for the main data. hdf = HDFwriter(h5_path) h5_refs = hdf.write(spm_data) h5_main = get_h5_obj_refs(['Raw_Data'], h5_refs)[0] # Now doing link_h5_objects_as_attrs: aux_ds_names = [ 'Excitation_Waveform', 'Position_Indices', 'Position_Values', 'Spectroscopic_Indices', 'Spectroscopic_Values' ] link_h5_objects_as_attrs(h5_main, get_h5_obj_refs(aux_ds_names, h5_refs)) print('reading raw data now...') # Now read the raw data files: pos_ind = 0 for row_ind in range(1, num_rows + 1): for col_ind in range(1, num_cols + 1): file_path = path.join( folder_path, 'result_r' + str(row_ind) + '_c' + str(col_ind) + '.mat') # print('Working on row {} col {}'.format(row_ind,col_ind)) if path.exists(file_path): # Load data file pix_data = loadmat(file_path, squeeze_me=True) # Take the inverse FFT on 1st dimension pix_vec = np.fft.ifft(np.fft.ifftshift(pix_data['data'])) # Verified with Matlab - no conjugate required here. h5_main[pos_ind, :] = np.float16(np.real(pix_vec)) hdf.flush() # flush from memory! else: print('File for row {} col {} not found'.format( row_ind, col_ind)) pos_ind += 1 if (100.0 * pos_ind / num_pix) % 10 == 0: print('Finished reading {} % of data'.format( int(100 * pos_ind / num_pix))) hdf.close() return h5_path
def __close_meas_group(self, h5_refs, show_plots, save_plots, do_histogram): """ Performs following operations : * Updates the number of pixels attribute in the measurement group * Writes Noise floor axis labels as region references * Writes position values and indices along with region references * Links all ancilliary datasets to the main data set * Writes the spatiall averaged plot data Parameters ---------- h5_refs : list of HDF references References to the written datasets show_plots : Boolean Whether or not to show plots save_plots : Boolean Whether or not to save the generated plots do_histogram : Boolean Whether or not to generate and save 2D histograms of the raw data Returns ------- None """ # Update the number of pixels in the attributes meas_grp = self.ds_main.parent meas_grp.attrs['num_pix'] = self.ds_pixel_index # Write position specific datasets now that the dataset is complete pos_slice_dict = dict() for spat_ind, spat_dim in enumerate(self.pos_labels): pos_slice_dict[spat_dim] = (slice(None), slice(spat_ind, spat_ind + 1)) ds_pos_ind = VirtualDataset('Position_Indices', self.pos_mat[self.ds_pixel_start_indx:self.ds_pixel_start_indx + self.ds_pixel_index, :], dtype=INDICES_DTYPE) ds_pos_ind.attrs['labels'] = pos_slice_dict ds_pos_ind.attrs['units'] = self.pos_units self.pos_vals_list = np.array(self.pos_vals_list) # Ensuring that the X and Y values vary from 0 to N instead of -0.5 N to + 0.5 N for col_ind in range(2): min_val = np.min(self.pos_vals_list[:, col_ind]) self.pos_vals_list[:, col_ind] -= min_val self.pos_vals_list[:, col_ind] *= 1E+6 # convert to microns if np.max(self.pos_vals_list[:, 2]) > 1E-3: # Setpoint spectroscopy if 'Z' in self.pos_labels: dim_ind = self.pos_labels.index('Z') # TODO: Find a way to correct the labels # self.pos_labels[dim_ind] = 'Setpoint' self.pos_units[dim_ind] = 'defl V' else: # Z spectroscopy self.pos_vals_list[:, 2] *= 1E+6 # convert to microns pos_val_mat = VALUES_DTYPE(self.pos_mat[self.ds_pixel_start_indx:self.ds_pixel_start_indx + self.ds_pixel_index, :]) for col_ind, targ_dim_name in enumerate(['X', 'Y', 'Z']): if targ_dim_name in self.pos_labels: dim_ind = self.pos_labels.index(targ_dim_name) # Replace indices with the x, y, z values from the pixels pos_val_mat[:, dim_ind] = self.pos_vals_list[:, col_ind] ds_pos_val = VirtualDataset('Position_Values', pos_val_mat) ds_pos_val.attrs['labels'] = pos_slice_dict ds_pos_val.attrs['units'] = self.pos_units meas_grp = VirtualGroup(meas_grp.name, '/') meas_grp.add_children([ds_pos_ind, ds_pos_val]) h5_refs += self.hdf.write(meas_grp) # Do all the reference linking: aux_ds_names = ['Excitation_Waveform', 'Position_Indices', 'Position_Values', 'UDVS_Indices', 'Spectroscopic_Indices', 'Bin_Step', 'Bin_Indices', 'Bin_Wfm_Type', 'Bin_Frequencies', 'Bin_FFT', 'UDVS', 'UDVS_Labels', 'Noise_Floor', 'Spectroscopic_Values'] link_h5_objects_as_attrs(self.ds_main, get_h5_obj_refs(aux_ds_names, h5_refs)) # While we have all the references and mean data, write the plot groups as well: generatePlotGroups(USIDataset(self.ds_main), self.mean_resp, self.folder_path, self.basename, self.max_resp, self.min_resp, max_mem_mb=self.max_ram, spec_label=self.spec_label, show_plots=show_plots, save_plots=save_plots, do_histogram=do_histogram, debug=self.debug) # Now that everything about this dataset is complete: self.dset_index += 1
def translate(self, parm_path): """ The main function that translates the provided file into a .h5 file Parameters ------------ parm_path : string / unicode Absolute file path of the parameters .mat file. Returns ---------- h5_path : string / unicode Absolute path of the translated h5 file """ parm_path = path.abspath(parm_path) parm_dict, excit_wfm = self._read_parms(parm_path) self._parse_file_path(parm_path) num_dat_files = len(self.file_list) f = open(self.file_list[0], 'rb') spectrogram_size, count_vals = self._parse_spectrogram_size(f) print("spectrogram size:", spectrogram_size) num_pixels = parm_dict['grid_num_rows'] * parm_dict['grid_num_cols'] print('Number of pixels: ', num_pixels) print('Count Values: ', count_vals) if (num_pixels + 1) != count_vals: print( "Data size does not match number of pixels expected. Cannot continue" ) # Now start creating datasets and populating: ds_spec_inds, ds_spec_vals = build_ind_val_dsets(Dimension( 'Bias', 'V', excit_wfm), is_spectral=True, verbose=False) ds_spec_vals.data = np.atleast_2d( excit_wfm) # The data generated above varies linearly. Override. pos_desc = [ Dimension('X', 'a.u.', np.arange(parm_dict['grid_num_cols'])), Dimension('Y', 'a.u.', np.arange(parm_dict['grid_num_rows'])) ] ds_pos_ind, ds_pos_val = build_ind_val_dsets(pos_desc, is_spectral=False, verbose=False) ds_raw_data = VirtualDataset('Raw_Data', data=[], maxshape=(ds_pos_ind.shape[0], spectrogram_size - 5), dtype=np.complex64, chunking=(1, spectrogram_size - 5), compression='gzip') ds_raw_data.attrs['quantity'] = ['Complex'] aux_ds_names = [ 'Position_Indices', 'Position_Values', 'Spectroscopic_Indices', 'Spectroscopic_Values' ] num_ai_chans = np.int(num_dat_files / 2) # Division by 2 due to real/imaginary # technically should change the date, etc. spm_data = VirtualGroup('') global_parms = generate_dummy_main_parms() global_parms['data_type'] = 'trKPFM' global_parms['translator'] = 'trKPFM' spm_data.attrs = global_parms meas_grp = VirtualGroup('Measurement_000') meas_grp.attrs = parm_dict spm_data.add_children([meas_grp]) hdf = HDFwriter(self.h5_path) # spm_data.showTree() hdf.write(spm_data, print_log=False) self.raw_datasets = list() for chan_index in range(num_ai_chans): chan_grp = VirtualGroup( '{:s}{:03d}'.format('Channel_', chan_index), '/Measurement_000/') if chan_index == 0: chan_grp.attrs = {'Harmonic': 1} else: chan_grp.attrs = {'Harmonic': 2} chan_grp.add_children([ ds_pos_ind, ds_pos_val, ds_spec_inds, ds_spec_vals, ds_raw_data ]) h5_refs = hdf.write(chan_grp, print_log=False) h5_raw = get_h5_obj_refs(['Raw_Data'], h5_refs)[0] link_h5_objects_as_attrs(h5_raw, get_h5_obj_refs(aux_ds_names, h5_refs)) self.raw_datasets.append(h5_raw) self.raw_datasets.append(h5_raw) # Now that the N channels have been made, populate them with the actual data.... self._read_data(parm_dict, parm_path, spectrogram_size) hdf.close() return self.h5_path
def translate(self, parm_path): """ Basic method that translates .mat data files to a single .h5 file Parameters ------------ parm_path : string / unicode Absolute file path of the parameters .mat file. Returns ---------- h5_path : string / unicode Absolute path of the translated h5 file """ self.parm_path = path.abspath(parm_path) (folder_path, file_name) = path.split(parm_path) (file_name, base_name) = path.split(folder_path) h5_path = path.join(folder_path, base_name + '.h5') # Read parameters parm_dict = readGmodeParms(parm_path) # Add the w^2 specific parameters to this list parm_data = loadmat(parm_path, squeeze_me=True, struct_as_record=True) #freq_sweep_parms = parm_data['freqSweepParms'] #parm_dict['freq_sweep_delay'] = np.float(freq_sweep_parms['delay'].item()) gen_sig = parm_data['genSig'] #parm_dict['wfm_fix_d_fast'] = np.int32(gen_sig['restrictT'].item()) #freq_array = np.float32(parm_data['freqArray']) # prepare and write spectroscopic values samp_rate = parm_dict['IO_down_samp_rate_[Hz]'] num_bins = int(parm_dict['wfm_n_cycles'] * parm_dict['wfm_p_slow'] * samp_rate) w_vec = np.arange(-0.5 * samp_rate, 0.5 * samp_rate, np.float32(samp_rate / num_bins)) # There is most likely a more elegant solution to this but I don't have the time... Maybe np.meshgrid spec_val_mat = np.zeros((len(freq_array) * num_bins, 2), dtype=VALUES_DTYPE) spec_val_mat[:, 0] = np.tile(w_vec, len(freq_array)) spec_val_mat[:, 1] = np.repeat(freq_array, num_bins) spec_ind_mat = np.zeros((2, len(freq_array) * num_bins), dtype=np.int32) spec_ind_mat[0, :] = np.tile(np.arange(num_bins), len(freq_array)) spec_ind_mat[1, :] = np.repeat(np.arange(len(freq_array)), num_bins) num_rows = parm_dict['grid_num_rows'] num_cols = parm_dict['grid_num_cols'] parm_dict['data_type'] = 'GVS' num_pix = num_rows * num_cols global_parms = generate_dummy_main_parms() global_parms['grid_size_x'] = parm_dict['grid_num_cols'] global_parms['grid_size_y'] = parm_dict['grid_num_rows'] # assuming that the experiment was completed: global_parms['current_position_x'] = parm_dict['grid_num_cols'] - 1 global_parms['current_position_y'] = parm_dict['grid_num_rows'] - 1 global_parms['data_type'] = parm_dict['data_type'] # self.__class__.__name__ global_parms['translator'] = 'GVS' # Now start creating datasets and populating: if path.exists(h5_path): remove(h5_path) h5_f = h5py.File(h5_path, 'w') write_simple_attrs(h5_f, global_parms) meas_grp = create_indexed_group(h5_f, 'Measurement') chan_grp = create_indexed_group(meas_grp, 'Channel') write_simple_attrs(chan_grp, parm_dict) pos_dims = [Dimension('X', 'nm', num_rows), Dimension('Y', 'nm', num_cols)] spec_dims = [Dimension('Response Bin', 'a.u.', num_bins), Dimension('Excitation Frequency ', 'Hz', len(freq_array))] # Minimize file size to the extent possible. # DAQs are rated at 16 bit so float16 should be most appropriate. # For some reason, compression is more effective on time series data h5_main = write_main_dataset(chan_grp, (num_pix, num_bins), 'Raw_Data', 'Deflection', 'V', pos_dims, spec_dims, chunks=(1, num_bins), dtype=np.float32) h5_ex_freqs = chan_grp.create_dataset('Excitation_Frequencies', freq_array) h5_bin_freq = chan_grp.create_dataset('Bin_Frequencies', w_vec) # Now doing link_h5_objects_as_attrs: link_h5_objects_as_attrs(h5_main, [h5_ex_freqs, h5_bin_freq]) # Now read the raw data files: pos_ind = 0 for row_ind in range(1, num_rows + 1): for col_ind in range(1, num_cols + 1): file_path = path.join(folder_path, 'fSweep_r' + str(row_ind) + '_c' + str(col_ind) + '.mat') print('Working on row {} col {}'.format(row_ind, col_ind)) if path.exists(file_path): # Load data file pix_data = loadmat(file_path, squeeze_me=True) pix_mat = pix_data['AI_mat'] # Take the inverse FFT on 2nd dimension pix_mat = np.fft.ifft(np.fft.ifftshift(pix_mat, axes=1), axis=1) # Verified with Matlab - no conjugate required here. pix_vec = pix_mat.transpose().reshape(pix_mat.size) h5_main[pos_ind, :] = np.float32(pix_vec) h5_f.flush() # flush from memory! else: print('File not found for: row {} col {}'.format(row_ind, col_ind)) pos_ind += 1 if (100.0 * pos_ind / num_pix) % 10 == 0: print('completed translating {} %'.format(int(100 * pos_ind / num_pix))) h5_f.close() return h5_path