def test_nd_h5_illegal(self): with h5py.File(file_path, mode='r') as h5_f: with self.assertRaises(TypeError): _ = dtype_utils.stack_real_to_compound(h5_f['compound'], struc_dtype) with self.assertRaises(TypeError): _ = dtype_utils.stack_real_to_compound(h5_f['complex'], struc_dtype)
def test_illegal(self): num_elems = (3, 5) r_vals = np.random.random(size=num_elems) with self.assertRaises(TypeError): _ = dtype_utils.stack_real_to_compound(r_vals, np.float32) with self.assertRaises(ValueError): _ = dtype_utils.stack_real_to_compound(r_vals, struc_dtype) with self.assertRaises(ValueError): _ = dtype_utils.stack_real_to_compound([1, 'a', { 'a': 1 }, False], struc_dtype)
def test_1d(self): num_elems = 5 structured_array = np.zeros(shape=num_elems, dtype=struc_dtype) structured_array['r'] = r_vals = np.random.random(size=num_elems) structured_array['g'] = g_vals = np.random.randint(0, high=1024, size=num_elems) structured_array['b'] = b_vals = np.random.random(size=num_elems) real_val = np.concatenate((r_vals, g_vals, b_vals)) actual = dtype_utils.stack_real_to_compound(real_val, struc_dtype) self.assertTrue(compare_structured_arrays(actual, structured_array))
def base_nd_h5_legal(self, lazy): with h5py.File(file_path, mode='r') as h5_f: h5_real = h5_f['real2'] structured_array = np.zeros( shape=list(h5_real.shape)[:-1] + [h5_real.shape[-1] // len(struc_dtype.names)], dtype=struc_dtype) for name_ind, name in enumerate(struc_dtype.names): i_start = name_ind * structured_array.shape[-1] i_end = (name_ind + 1) * structured_array.shape[-1] structured_array[name] = h5_real[..., i_start:i_end] actual = dtype_utils.stack_real_to_compound(h5_real, struc_dtype, lazy=lazy) if lazy: self.assertIsInstance(actual, da.core.Array) actual = actual.compute() self.assertTrue(compare_structured_arrays(actual, structured_array))
def base_nd(self, in_lazy, out_lazy): num_elems = (2, 3, 5, 7) structured_array = np.zeros(shape=num_elems, dtype=struc_dtype) structured_array['r'] = r_vals = np.random.random(size=num_elems) structured_array['g'] = g_vals = np.random.randint(0, high=1024, size=num_elems) structured_array['b'] = b_vals = np.random.random(size=num_elems) real_val = np.concatenate((r_vals, g_vals, b_vals), axis=len(num_elems) - 1) if in_lazy: real_val = da.from_array(real_val, chunks=real_val.shape) actual = dtype_utils.stack_real_to_compound(real_val, struc_dtype, lazy=out_lazy) if out_lazy: self.assertIsInstance(actual, da.core.Array) actual = actual.compute() self.assertTrue(compare_structured_arrays(actual, structured_array))
def _calc_sho(self, coef_OF_mat, coef_IF_mat, amp_noise=0.1, phase_noise=0.1, q_noise=0.2, resp_noise=0.01): """ Build the SHO dataset from the coefficient matrices Parameters ---------- coef_OF_mat : numpy.ndarray Out-of-field coefficients coef_IF_mat : numpy.ndarray In-field coefficients amp_noise : float Noise factor for amplitude parameter phase_noise : float Noise factor for phase parameter q_noise : float Noise factor for Q-value parameter resp_noise : float Noide factor for w0 parameter Returns ------- None """ print(list(self.h5_sho_spec_vals.attrs)) vdc_vec = self.h5_sho_spec_vals[self.h5_sho_fit.spec_dim_labels.index( 'DC_Offset')].squeeze() sho_field = self.h5_sho_spec_vals[ self.h5_sho_fit.spec_dim_labels.index('Field')].squeeze() sho_of_inds = sho_field == 0 sho_if_inds = sho_field == 1 # determine how many pixels can be read at once mem_per_pix = vdc_vec.size * np.float32(0).itemsize #free_mem = self.max_ram - vdc_vec.size * vdc_vec.dtype.itemsize * 6 free_mem = 1024 batch_size = int(free_mem / mem_per_pix) batches = gen_batches(self.n_pixels, batch_size) one_cycle_length = vdc_vec[sho_of_inds].shape[-1] for pix_batch in batches: roll_len = one_cycle_length // 4 vdc_OF_rolled = np.roll(vdc_vec[sho_of_inds], -1 * roll_len) vdc_IF_rolled = np.roll(vdc_vec[sho_if_inds], -1 * roll_len) R_OF = np.array([ np.roll(loop_fit_function(vdc_OF_rolled, coef), roll_len) for coef in coef_OF_mat[pix_batch] ]) R_OF = R_OF - R_OF.mean() R_IF = np.array([ np.roll(loop_fit_function(vdc_IF_rolled, coef), roll_len) for coef in coef_IF_mat[pix_batch] ]) R_IF = R_IF - R_IF.mean() R_mat = np.hstack([R_IF[:, np.newaxis, :], R_OF[:, np.newaxis, :]]) R_mat = np.rollaxis(R_mat, 1, R_mat.ndim).reshape(R_mat.shape[0], -1) del R_OF, R_IF amp = np.abs(R_mat) resp = coef_OF_mat[pix_batch, 9, None] * np.ones_like(R_mat) q_val = coef_OF_mat[pix_batch, 10, None] * np.ones_like(R_mat) * 10 phase = np.sign(R_mat) * np.pi / 2 + np.pi / 2 self.h5_sho_fit[pix_batch, :] = stack_real_to_compound( np.hstack([amp, resp, q_val, phase, np.ones_like(R_mat)]), sho32) self.h5_sho_guess[pix_batch, :] = stack_real_to_compound( np.hstack([ amp * get_noise_vec(self.n_sho_bins, amp_noise), resp * get_noise_vec(self.n_sho_bins, resp_noise), q_val * get_noise_vec(self.n_sho_bins, q_noise), phase * get_noise_vec(self.n_sho_bins, phase_noise), np.ones_like(R_mat) ]), sho32) self.h5_file.flush() return
def translate(self, h5_path, n_steps=32, n_bins=37, start_freq=300E+3, end_freq=350E+3, data_type='BEPSData', mode='DC modulation mode', field_mode='in and out-of-field', n_cycles=1, FORC_cycles=1, FORC_repeats=1, loop_a=3, loop_b=4, cycle_frac='full', image_folder=beps_image_folder, bin_factor=None, bin_func=np.mean, image_type='.tif', simple_coefs=False): """ Parameters ---------- h5_path : str Desired path to write the new HDF5 file n_steps : uint, optional Number of voltage steps Default - 32 n_bins : uint, optional Number of frequency bins Default - 37 start_freq : float, optional Starting frequency in Hz Default - 300E+3 end_freq : float, optional Final freqency in Hz Default - 350E+3 data_type : str, optional Type of data to generate Options - 'BEPSData', 'BELineData' Default - 'BEPSData' mode : str, optional Modulation mode to use when generating the data. Options - 'DC modulation mode', 'AC modulation mode' Default - 'DC modulation mode' field_mode : str, optional Field mode Options - 'in-field', 'out-of-field', 'in and out-of-field' Default - 'in and out-of-field' n_cycles : uint, optional Number of cycles Default - 1 FORC_cycles : uint, optional Number of FORC cycles Default - 1 FORC_repeats : uint, optional Number of FORC repeats Default - 1 loop_a : float, optional Loop coefficient a Default - 1 loop_b : float, optional Loop coefficient b cycle_frac : str Cycle fraction parameter. Default - 'full' image_folder : str Path to the images that will be used to generate the loop coefficients. There must be 11 images named '1.tif', '2.tif', ..., '11.tif' Default - pycroscopy.io.translators.df_utils.beps_gen_utils.beps_image_folder bin_factor : array_like of uint, optional Downsampling factor for each dimension. Default is None. bin_func : callable, optional Function which will be called to calculate the return value of each block. Function must implement an axis parameter, i.e. numpy.mean. Ignored if bin_factor is None. Default is numpy.mean. image_type : str File extension of images to be read. Default '.tif' simple_coefs : bool Should a simpler coefficient generation be used. Ensures loops, but all loops are identical. Default False Returns ------- """ # Setup shared parameters self.n_steps = n_steps self.n_bins = n_bins self.start_freq = start_freq self.end_freq = end_freq self.n_cycles = n_cycles self.forc_cycles = FORC_cycles self.forc_repeats = FORC_repeats self.loop_a = loop_a self.loop_b = loop_b self.data_type = data_type self.mode = mode self.field_mode = field_mode self.cycle_fraction = cycle_frac self.bin_factor = bin_factor self.bin_func = bin_func if field_mode == 'in and out-of-field': self.n_fields = 2 else: self.n_fields = 1 self.n_loops = FORC_cycles * FORC_repeats * n_cycles * self.n_fields self.n_sho_bins = n_steps * self.n_loops self.n_spec_bins = n_bins * self.n_sho_bins self.h5_path = h5_path self.image_ext = image_type self.simple_coefs = simple_coefs ''' Check if a bin_factor is given. Set up binning objects if it is. ''' if bin_factor is not None: self.rebin = True if isinstance(bin_factor, int): self.bin_factor = (bin_factor, bin_factor) elif len(bin_factor) == 2: self.bin_factor = tuple(bin_factor) else: raise ValueError( 'Input parameter `bin_factor` must be a length 2 array_like or an integer.\n' + '{} was given.'.format(bin_factor)) self.binning_func = block_reduce self.bin_func = bin_func print('Image folder is {}'.format(image_folder)) images = self._read_data(image_folder) self.images = images data_gen_parms = { 'N_x': self.N_x, 'N_y': self.N_y, 'n_steps;:': n_steps, 'n_bins': n_bins, 'start_freq': start_freq, 'end_freq': end_freq, 'n_cycles': n_cycles, 'forc_cycles': FORC_cycles, 'forc_repeats': FORC_repeats, 'loop_a': loop_a, 'loop_b': loop_b, 'data_type': data_type, 'VS_mode': mode, 'field_mode': field_mode, 'num_udvs_steps': self.n_spec_bins, 'VS_cycle_fraction': cycle_frac } # Build the hdf5 file and get the datasets to write the data to self._setup_h5(data_gen_parms) # Calculate the loop parameters coef_mat = self.calc_loop_coef_mat(images) # In-and-out of field coefficients if field_mode != 'in-field': coef_OF_mat = np.copy(coef_mat) if field_mode != 'out-of-field': coef_IF_mat = np.copy(coef_mat) coef_IF_mat[:, 4] -= 0.05 self.coef_mat = coef_mat # Calculate the SHO fit and guess from the loop coefficients self._calc_sho(coef_OF_mat, coef_IF_mat) # Save the loop guess and fit to file coef_OF_mat = np.hstack( (coef_OF_mat[:, :9], np.ones([coef_OF_mat.shape[0], 1]))) coef_IF_mat = np.hstack( (coef_IF_mat[:, :9], np.ones([coef_IF_mat.shape[0], 1]))) coef_mat = np.hstack( [coef_IF_mat[:, np.newaxis, :], coef_OF_mat[:, np.newaxis, :]]) coef_mat = np.rollaxis(coef_mat, 1, coef_mat.ndim).reshape([coef_mat.shape[0], -1]) self.h5_loop_fit[:] = np.tile( stack_real_to_compound(coef_mat, loop_fit32), [1, int(self.n_loops / self.n_fields)]) self.h5_loop_guess[:] = np.tile( stack_real_to_compound( coef_mat * get_noise_vec(coef_mat.shape, 0.1), loop_fit32), [1, int(self.n_loops / self.n_fields)]) self.h5_file.flush() self._calc_raw() self.h5_file.flush() return self.h5_path
def fit_atom_positions_dset(h5_grp, fitting_parms=None, num_cores=None): """ A temporary substitute for a full-fledged process class. Computes the guess and fit coefficients for the provided atom guess positions and writes these results to the given h5 group Parameters ---------- h5_grp : h5py.Group reference Group containing the atom guess positions, cropped clean image and some necessary parameters fitting_parms : dictionary Parameters used for atom position fitting num_cores : unsigned int (Optional. Default = available logical cores - 2) Number of cores to compute with Returns ------- h5_grp : h5py.Group reference Same group as the parameter but now with the 'Guess' and 'Fit' datasets """ cropped_clean_image = h5_grp['Cropped_Clean_Image'][()] h5_guess = h5_grp['Guess_Positions'] all_atom_guesses = np.transpose(np.vstack( (h5_guess['x'], h5_guess['y']))) # leave out the atom type for now win_size = h5_grp.attrs['motif_win_size'] psf_width = h5_grp.attrs['psf_width'] num_atoms = all_atom_guesses.shape[0] # number of atoms # build distance matrix pos_vec = all_atom_guesses[:, 0] + 1j * all_atom_guesses[:, 1] pos_mat1 = np.tile(np.transpose(np.atleast_2d(pos_vec)), [1, num_atoms]) pos_mat2 = np.transpose(pos_mat1) d_mat = np.abs(pos_mat2 - pos_mat1) # matrix of distances between all atoms # sort the distance matrix and keep only the atoms within the nearest neighbor limit neighbor_dist_order = np.argsort(d_mat) if fitting_parms is None: num_nearest_neighbors = 6 # to consider when fitting fitting_parms = { 'fit_region_size': win_size * 0.80, # region to consider when fitting 'gauss_width_guess': psf_width * 2, 'num_nearest_neighbors': num_nearest_neighbors, 'min_amplitude': 0, # min amplitude limit for gauss fit 'max_amplitude': 2, # max amplitude limit for gauss fit 'position_range': win_size / 2, # range that the fitted position can go from initial guess position[pixels] 'max_function_evals': 100, 'min_gauss_width_ratio': 0.5, # min width of gauss fit ratio, 'max_gauss_width_ratio': 2, # max width of gauss fit ratio 'fitting_tolerance': 1E-4 } num_nearest_neighbors = fitting_parms['num_nearest_neighbors'] # neighbor dist order has the (indices of the) neighbors for each atom sorted by distance closest_neighbors_mat = neighbor_dist_order[:, 1:num_nearest_neighbors + 1] parm_dict = { 'atom_pos_guess': all_atom_guesses, 'nearest_neighbors': closest_neighbors_mat, 'cropped_cleaned_image': cropped_clean_image } # do the parallel fitting fitting_results = fit_atom_positions_parallel(parm_dict, fitting_parms, num_cores=num_cores) # Make datasets to write back to file: guess_parms = np.zeros(shape=(num_atoms, num_nearest_neighbors + 1), dtype=atom_coeff_dtype) fit_parms = np.zeros(shape=guess_parms.shape, dtype=guess_parms.dtype) for atom_ind, single_atom_results in enumerate(fitting_results): guess_coeff, fit_coeff = single_atom_results num_neighbors_used = guess_coeff.shape[0] guess_parms[atom_ind, :num_neighbors_used] = np.squeeze( stack_real_to_compound(guess_coeff, guess_parms.dtype)) fit_parms[atom_ind, :num_neighbors_used] = np.squeeze( stack_real_to_compound(fit_coeff, guess_parms.dtype)) ds_atom_guesses = VirtualDataset('Guess', data=guess_parms) ds_atom_fits = VirtualDataset('Fit', data=fit_parms) dgrp_atom_finding = VirtualGroup(h5_grp.name.split('/')[-1], parent=h5_grp.parent.name) dgrp_atom_finding.attrs = fitting_parms dgrp_atom_finding.add_children([ds_atom_guesses, ds_atom_fits]) hdf = HDFwriter(h5_grp.file) h5_atom_refs = hdf.write(dgrp_atom_finding) return h5_grp
def _write_results_chunk(self): """ Writes data chunks back to the h5 file """ if self.verbose: print( 'Rank {} - Started accumulating results for this chunk'.format( self.mpi_rank)) num_pixels = len(self.forward_results) cap_mat = np.zeros((num_pixels, 2), dtype=np.float32) r_inf_mat = np.zeros((num_pixels, self.num_x_steps), dtype=np.float32) r_var_mat = np.zeros((num_pixels, self.num_x_steps), dtype=np.float32) i_cor_sin_mat = np.zeros((num_pixels, self.single_ao.size), dtype=np.float32) for pix_ind, i_meas, forw_results, rev_results in zip( range(num_pixels), self.data, self.forward_results, self.reverse_results): full_results = dict() for item in ['cValue']: full_results[item] = np.hstack( (forw_results[item], rev_results[item])) # print(item, full_results[item].shape) # Capacitance is always doubled - halve it now (locally): # full_results['cValue'] *= 0.5 cap_val = np.mean(full_results['cValue']) * 0.5 # Compensating the resistance.. """ omega = 2 * np.pi * self.ex_freq i_cap = cap_val * omega * self.rolled_bias """ i_cap = cap_val * self.dvdt i_extra = self.r_extra * 2 * cap_val * self.single_ao i_corr_sine = i_meas - i_cap - i_extra # Equivalent to flipping the X: rev_results['x'] *= -1 # Stacking the results - no flipping required for reverse: for item in ['x', 'mR', 'vR']: full_results[item] = np.hstack( (forw_results[item], rev_results[item])) i_cor_sin_mat[pix_ind] = i_corr_sine cap_mat[pix_ind] = full_results[ 'cValue'] * 1000 # convert from nF to pF r_inf_mat[pix_ind] = full_results['mR'] r_var_mat[pix_ind] = full_results['vR'] # Now write to h5 files: if self.verbose: print( 'Rank {} - Finished accumulating results. Writing results of chunk to h5' .format(self.mpi_rank)) if self.__first_batch: self.h5_new_spec_vals[0, :] = full_results[ 'x'] # Technically this needs to only be done once self.__first_batch = False # Get access to the private variable: pos_in_batch = self._get_pixels_in_current_batch() self.h5_cap[pos_in_batch, :] = np.atleast_2d( stack_real_to_compound(cap_mat, cap_dtype)).T self.h5_variance[pos_in_batch, :] = r_var_mat self.h5_resistance[pos_in_batch, :] = r_inf_mat self.h5_i_corrected[pos_in_batch, :] = i_cor_sin_mat