def test_real_to_compound_illegal(self):
num_elems = (3, 5)
r_vals = np.random.random(size=num_elems)
with self.assertRaises(TypeError):
_ = dtype_utils.stack_real_to_compound(r_vals, np.float32)
with self.assertRaises(ValueError):
_ = dtype_utils.stack_real_to_compound(r_vals, struc_dtype)
def test_nd_h5_illegal(self):
with h5py.File(file_path, mode='r') as h5_f:
with self.assertRaises(TypeError):
_ = dtype_utils.stack_real_to_compound(h5_f['compound'], struc_dtype)
with self.assertRaises(TypeError):
_ = dtype_utils.stack_real_to_compound(h5_f['complex'], struc_dtype)
def _reformat_results(self, results, strategy='BE_LOOP'):
"""
Reformat loop fit results to target compound dataset
Parameters
----------
results : list
list of loop fit / guess results objects
strategy : string / unicode (optional)
Name of the computational strategy
verbose : Boolean (optional)
Whether or not to print debugging statements
Returns
-------
temp : 1D compound array
An array of the loop parameters in the target compound datatype
"""
if self._verbose:
print('Strategy to use: {}'.format(strategy))
# Create an empty array to store the guess parameters
if self._verbose:
print('Raw results and compound Loop vector of shape {}'.format(len(results)))
if strategy in ['BE_LOOP']:
temp = np.array([np.hstack([result.x, result.fun]) for result in results])
temp = stack_real_to_compound(temp, loop_fit32)
return temp
def test_1d_list(self):
num_elems = 5
structured_array = np.zeros(shape=num_elems, dtype=struc_dtype)
structured_array['r'] = r_vals = np.random.random(size=num_elems)
structured_array['g'] = g_vals = np.random.randint(0, high=1024, size=num_elems)
structured_array['b'] = b_vals = np.random.random(size=num_elems)
real_val = np.concatenate((r_vals, g_vals, b_vals))
actual = dtype_utils.stack_real_to_compound(list(real_val), struc_dtype)
self.assertTrue(compare_structured_arrays(actual, structured_array))
def base_nd_h5_legal(self, lazy):
with h5py.File(file_path, mode='r') as h5_f:
h5_real = h5_f['real2']
structured_array = np.zeros(shape=list(h5_real.shape)[:-1] + [h5_real.shape[-1] // len(struc_dtype.names)],
dtype=struc_dtype)
for name_ind, name in enumerate(struc_dtype.names):
i_start = name_ind * structured_array.shape[-1]
i_end = (name_ind + 1) * structured_array.shape[-1]
structured_array[name] = h5_real[..., i_start:i_end]
actual = dtype_utils.stack_real_to_compound(h5_real, struc_dtype, lazy=lazy)
if lazy:
self.assertIsInstance(actual, da.core.Array)
actual = actual.compute()
self.assertTrue(compare_structured_arrays(actual, structured_array))
def base_nd(self, in_lazy, out_lazy):
num_elems = (2, 3, 5, 7)
structured_array = np.zeros(shape=num_elems, dtype=struc_dtype)
structured_array['r'] = r_vals = np.random.random(size=num_elems)
structured_array['g'] = g_vals = np.random.randint(0, high=1024, size=num_elems)
structured_array['b'] = b_vals = np.random.random(size=num_elems)
real_val = np.concatenate((r_vals, g_vals, b_vals), axis=len(num_elems) - 1)
if in_lazy:
real_val = da.from_array(real_val, chunks=real_val.shape)
actual = dtype_utils.stack_real_to_compound(real_val, struc_dtype, lazy=out_lazy)
if out_lazy:
self.assertIsInstance(actual, da.core.Array)
actual = actual.compute()
self.assertTrue(compare_structured_arrays(actual, structured_array))
def _write_results_chunk(self):
"""
Writes data chunks back to the h5 file
"""
if self.verbose:
print(
'Rank {} - Started accumulating results for this chunk'.format(
self.mpi_rank))
num_pixels = len(self.forward_results)
cap_mat = np.zeros((num_pixels, 2), dtype=np.float32)
r_inf_mat = np.zeros((num_pixels, self.num_x_steps), dtype=np.float32)
r_var_mat = np.zeros((num_pixels, self.num_x_steps), dtype=np.float32)
i_cor_sin_mat = np.zeros((num_pixels, self.single_ao.size),
dtype=np.float32)
for pix_ind, i_meas, forw_results, rev_results in zip(
range(num_pixels), self.data, self.forward_results,
self.reverse_results):
full_results = dict()
for item in ['cValue']:
full_results[item] = np.hstack(
(forw_results[item], rev_results[item]))
# print(item, full_results[item].shape)
# Capacitance is always doubled - halve it now (locally):
# full_results['cValue'] *= 0.5
cap_val = np.mean(full_results['cValue']) * 0.5
# Compensating the resistance..
"""
omega = 2 * np.pi * self.ex_freq
i_cap = cap_val * omega * self.rolled_bias
"""
i_cap = cap_val * self.dvdt
i_extra = self.r_extra * 2 * cap_val * self.single_ao
i_corr_sine = i_meas - i_cap - i_extra
# Equivalent to flipping the X:
rev_results['x'] *= -1
# Stacking the results - no flipping required for reverse:
for item in ['x', 'mR', 'vR']:
full_results[item] = np.hstack(
(forw_results[item], rev_results[item]))
i_cor_sin_mat[pix_ind] = i_corr_sine
cap_mat[pix_ind] = full_results[
'cValue'] * 1000 # convert from nF to pF
r_inf_mat[pix_ind] = full_results['mR']
r_var_mat[pix_ind] = full_results['vR']
# Now write to h5 files:
if self.verbose:
print(
'Rank {} - Finished accumulating results. Writing results of chunk to h5'
.format(self.mpi_rank))
if self.__first_batch:
self.h5_new_spec_vals[0, :] = full_results[
'x'] # Technically this needs to only be done once
self.__first_batch = False
# Get access to the private variable:
pos_in_batch = self._get_pixels_in_current_batch()
self.h5_cap[pos_in_batch, :] = np.atleast_2d(
stack_real_to_compound(cap_mat, cap_dtype)).T
self.h5_variance[pos_in_batch, :] = r_var_mat
self.h5_resistance[pos_in_batch, :] = r_inf_mat
self.h5_i_corrected[pos_in_batch, :] = i_cor_sin_mat
def _guess_loops(vdc_vec, projected_loops_2d):
"""
Provides loop parameter guesses for a given set of loops
Parameters
----------
vdc_vec : 1D numpy float numpy array
DC voltage offsets for the loops
projected_loops_2d : 2D numpy float array
Projected loops arranged as [instance or position x dc voltage steps]
Returns
-------
guess_parms : 1D compound numpy array
Loop parameter guesses for the provided projected loops
"""
def _loop_fit_tree(tree, guess_mat, fit_results, vdc_shifted, shift_ind):
"""
Recursive function that fits a tree object describing the cluster results
Parameters
----------
tree : ClusterTree object
Tree describing the clustering results
guess_mat : 1D numpy float array
Loop parameters that serve as guesses for the loops in the tree
fit_results : 1D numpy float array
Loop parameters that serve as fits for the loops in the tree
vdc_shifted : 1D numpy float array
DC voltages shifted be 1/4 cycle
shift_ind : unsigned int
Number of units to shift loops by
Returns
-------
guess_mat : 1D numpy float array
Loop parameters that serve as guesses for the loops in the tree
fit_results : 1D numpy float array
Loop parameters that serve as fits for the loops in the tree
"""
# print('Now fitting cluster #{}'.format(tree.name))
# I already have a guess. Now fit myself
curr_fit_results = fit_loop(vdc_shifted, np.roll(tree.value, shift_ind), guess_mat[tree.name])
# keep all the fit results
fit_results[tree.name] = curr_fit_results
for child in tree.children:
# Use my fit as a guess for the lower layers:
guess_mat[child.name] = curr_fit_results[0].x
# Fit this child:
guess_mat, fit_mat = _loop_fit_tree(child, guess_mat, fit_results, vdc_shifted, shift_ind)
return guess_mat, fit_results
num_clusters = max(2, int(projected_loops_2d.shape[0] ** 0.5)) # change this to 0.6 if necessary
estimators = KMeans(num_clusters)
results = estimators.fit(projected_loops_2d)
centroids = results.cluster_centers_
labels = results.labels_
# Get the distance between cluster means
distance_mat = pdist(centroids)
# get hierarchical pairings of clusters
linkage_pairing = linkage(distance_mat, 'weighted')
# Normalize the pairwise distance with the maximum distance
linkage_pairing[:, 2] = linkage_pairing[:, 2] / max(linkage_pairing[:, 2])
# Now use the tree class:
cluster_tree = ClusterTree(linkage_pairing[:, :2], labels,
distances=linkage_pairing[:, 2],
centroids=centroids)
num_nodes = len(cluster_tree.nodes)
# prepare the guess and fit matrices
loop_guess_mat = np.zeros(shape=(num_nodes, 9), dtype=np.float32)
# loop_fit_mat = np.zeros(shape=loop_guess_mat.shape, dtype=loop_guess_mat.dtype)
loop_fit_results = list(np.arange(num_nodes, dtype=np.uint16)) # temporary placeholder
shift_ind, vdc_shifted = BELoopFitter.shift_vdc(vdc_vec)
# guess the top (or last) node
loop_guess_mat[-1] = generate_guess(vdc_vec, cluster_tree.tree.value)
# Now guess the rest of the tree
loop_guess_mat, loop_fit_results = _loop_fit_tree(cluster_tree.tree, loop_guess_mat, loop_fit_results,
vdc_shifted, shift_ind)
# Prepare guesses for each pixel using the fit of the cluster it belongs to:
guess_parms = np.zeros(shape=projected_loops_2d.shape[0], dtype=loop_fit32)
for clust_id in range(num_clusters):
pix_inds = np.where(labels == clust_id)[0]
temp = np.atleast_2d(loop_fit_results[clust_id][0].x)
# convert to the appropriate dtype as well:
r2 = 1 - np.sum(np.abs(loop_fit_results[clust_id][0].fun ** 2))
guess_parms[pix_inds] = stack_real_to_compound(np.hstack([temp, np.atleast_2d(r2)]), loop_fit32)
return guess_parms
def _write_results_chunk(self):
"""
Writes data chunks back to the h5 file
"""
#if self.verbose:
# print('Rank {} - Started accumulating results for this chunk'.format(self.mpi_rank))
num_pixels = len(self.forward_results)
cap_mat = np.zeros((num_pixels, 2), dtype=np.float32)
r_inf_mat = np.zeros((num_pixels, self.num_x_steps), dtype=np.float32)
r_var_mat = np.zeros((num_pixels, self.num_x_steps), dtype=np.float32)
i_cor_sin_mat = np.zeros((num_pixels, self.single_ao.size),
dtype=np.float32)
for pix_ind, i_meas, forw_results, rev_results in zip(
range(num_pixels), self.data, self.forward_results,
self.reverse_results):
full_results = dict()
for item in ['cValue']:
full_results[item] = np.hstack(
(forw_results[item], rev_results[item]))
# print(item, full_results[item].shape)
# Capacitance is always doubled - halve it now (locally):
# full_results['cValue'] *= 0.5
cap_val = np.mean(full_results['cValue']) * 0.5
# Compensating the resistance..
"""
omega = 2 * np.pi * self.ex_freq
i_cap = cap_val * omega * self.rolled_bias
"""
i_cap = cap_val * self.dvdt
i_extra = self.r_extra * 2 * cap_val * self.single_ao
i_corr_sine = i_meas - i_cap - i_extra
# Equivalent to flipping the X:
rev_results['x'] *= -1
# Stacking the results - no flipping required for reverse:
for item in ['x', 'mR', 'vR']:
full_results[item] = np.hstack(
(forw_results[item], rev_results[item]))
i_cor_sin_mat[pix_ind] = i_corr_sine
cap_mat[pix_ind] = full_results[
'cValue'] * 1000 # convert from nF to pF
r_inf_mat[pix_ind] = full_results['mR']
r_var_mat[pix_ind] = full_results['vR']
# Now write to h5 files:
if self.verbose:
print(
'Rank {} - Finished accumulating results. Writing results of chunk to h5'
.format(self.mpi_rank))
if self._start_pos == 0:
self.h5_new_spec_vals[0, :] = full_results[
'x'] # Technically this needs to only be done once
pos_slice = slice(self._start_pos, self._end_pos)
self.h5_cap[pos_slice] = np.atleast_2d(
stack_real_to_compound(cap_mat, cap_dtype)).T
self.h5_variance[pos_slice] = r_var_mat
self.h5_resistance[pos_slice] = r_inf_mat
self.h5_i_corrected[pos_slice] = i_cor_sin_mat
# Leaving in this provision that will allow restarting of processes
self.h5_results_grp.attrs['last_pixel'] = self._end_pos
# Disabling flush because h5py-parallel doesn't like it
# self.h5_main.file.flush()
print('Rank {} - Finished processing up to pixel {} of {}'
'.'.format(self.mpi_rank, self._end_pos, self._rank_end_pos))
# Now update the start position
self._start_pos = self._end_pos
def fit_atom_positions_dset(h5_grp, fitting_parms=None, num_cores=None):
"""
A temporary substitute for a full-fledged process class.
Computes the guess and fit coefficients for the provided atom guess positions and writes these results to the
given h5 group
Parameters
----------
h5_grp : h5py.Group reference
Group containing the atom guess positions, cropped clean image and some necessary parameters
fitting_parms : dictionary
Parameters used for atom position fitting
num_cores : unsigned int (Optional. Default = available logical cores - 2)
Number of cores to compute with
Returns
-------
h5_grp : h5py.Group reference
Same group as the parameter but now with the 'Guess' and 'Fit' datasets
"""
cropped_clean_image = h5_grp['Cropped_Clean_Image'][()]
h5_guess = h5_grp['Guess_Positions']
all_atom_guesses = np.transpose(np.vstack((h5_guess['x'], h5_guess['y']))) # leave out the atom type for now
win_size = h5_grp.attrs['motif_win_size']
psf_width = h5_grp.attrs['psf_width']
num_atoms = all_atom_guesses.shape[0] # number of atoms
# build distance matrix
pos_vec = all_atom_guesses[:, 0] + 1j * all_atom_guesses[:, 1]
pos_mat1 = np.tile(np.transpose(np.atleast_2d(pos_vec)), [1, num_atoms])
pos_mat2 = np.transpose(pos_mat1)
d_mat = np.abs(pos_mat2 - pos_mat1) # matrix of distances between all atoms
# sort the distance matrix and keep only the atoms within the nearest neighbor limit
neighbor_dist_order = np.argsort(d_mat)
if fitting_parms is None:
num_nearest_neighbors = 6 # to consider when fitting
fitting_parms = {'fit_region_size': win_size * 0.80, # region to consider when fitting
'gauss_width_guess': psf_width * 2,
'num_nearest_neighbors': num_nearest_neighbors,
'min_amplitude': 0, # min amplitude limit for gauss fit
'max_amplitude': 2, # max amplitude limit for gauss fit
'position_range': win_size / 2,
# range that the fitted position can go from initial guess position[pixels]
'max_function_evals': 100,
'min_gauss_width_ratio': 0.5, # min width of gauss fit ratio,
'max_gauss_width_ratio': 2, # max width of gauss fit ratio
'fitting_tolerance': 1E-4}
num_nearest_neighbors = fitting_parms['num_nearest_neighbors']
# neighbor dist order has the (indices of the) neighbors for each atom sorted by distance
closest_neighbors_mat = neighbor_dist_order[:, 1:num_nearest_neighbors + 1]
parm_dict = {'atom_pos_guess': all_atom_guesses,
'nearest_neighbors': closest_neighbors_mat,
'cropped_cleaned_image': cropped_clean_image}
# do the parallel fitting
fitting_results = fit_atom_positions_parallel(parm_dict, fitting_parms, num_cores=num_cores)
# Make datasets to write back to file:
guess_parms = np.zeros(shape=(num_atoms, num_nearest_neighbors + 1), dtype=atom_coeff_dtype)
fit_parms = np.zeros(shape=guess_parms.shape, dtype=guess_parms.dtype)
for atom_ind, single_atom_results in enumerate(fitting_results):
guess_coeff, fit_coeff = single_atom_results
num_neighbors_used = guess_coeff.shape[0]
guess_parms[atom_ind, :num_neighbors_used] = np.squeeze(stack_real_to_compound(guess_coeff, guess_parms.dtype))
fit_parms[atom_ind, :num_neighbors_used] = np.squeeze(stack_real_to_compound(fit_coeff, guess_parms.dtype))
ds_atom_guesses = VirtualDataset('Guess', data=guess_parms)
ds_atom_fits = VirtualDataset('Fit', data=fit_parms)
dgrp_atom_finding = VirtualGroup(h5_grp.name.split('/')[-1], parent=h5_grp.parent.name)
dgrp_atom_finding.attrs = fitting_parms
dgrp_atom_finding.add_children([ds_atom_guesses, ds_atom_fits])
hdf = HDFwriter(h5_grp.file)
h5_atom_refs = hdf.write(dgrp_atom_finding)
return h5_grp
def _calc_sho(self,
coef_OF_mat,
coef_IF_mat,
amp_noise=0.1,
phase_noise=0.1,
q_noise=0.2,
resp_noise=0.01):
"""
Build the SHO dataset from the coefficient matrices
Parameters
----------
coef_OF_mat : numpy.ndarray
Out-of-field coefficients
coef_IF_mat : numpy.ndarray
In-field coefficients
amp_noise : float
Noise factor for amplitude parameter
phase_noise : float
Noise factor for phase parameter
q_noise : float
Noise factor for Q-value parameter
resp_noise : float
Noide factor for w0 parameter
Returns
-------
None
"""
# TODO: Fix sho parameter generation
vdc_vec = self.h5_sho_spec_vals[
self.h5_sho_spec_vals.attrs['DC_Offset']].squeeze()
sho_field = self.h5_sho_spec_vals[
self.h5_sho_spec_vals.attrs['Field']].squeeze()
sho_of_inds = sho_field == 0
sho_if_inds = sho_field == 1
# determine how many pixels can be read at once
mem_per_pix = vdc_vec.size * np.float32(0).itemsize
free_mem = self.max_ram - vdc_vec.size * vdc_vec.dtype.itemsize * 6
batch_size = int(free_mem / mem_per_pix)
batches = gen_batches(self.n_pixels, batch_size)
for pix_batch in batches:
R_OF = np.array([
loop_fit_function(vdc_vec[sho_of_inds], coef)
for coef in coef_OF_mat[pix_batch]
])
R_IF = np.array([
loop_fit_function(vdc_vec[sho_if_inds], coef)
for coef in coef_IF_mat[pix_batch]
])
R_mat = np.hstack([R_IF[:, np.newaxis, :], R_OF[:, np.newaxis, :]])
R_mat = np.rollaxis(R_mat, 1,
R_mat.ndim).reshape(R_mat.shape[0], -1)
del R_OF, R_IF
amp = np.abs(R_mat)
resp = coef_OF_mat[pix_batch, 9, None] * np.ones_like(R_mat)
q_val = coef_OF_mat[pix_batch, 10, None] * np.ones_like(R_mat) * 10
phase = np.sign(R_mat) * np.pi / 2
self.h5_sho_fit[pix_batch, :] = stack_real_to_compound(
np.hstack([amp, resp, q_val, phase,
np.ones_like(R_mat)]), sho32)
self.h5_sho_guess[pix_batch, :] = stack_real_to_compound(
np.hstack([
amp * get_noise_vec(self.n_sho_bins, amp_noise),
resp * get_noise_vec(self.n_sho_bins, resp_noise),
q_val * get_noise_vec(self.n_sho_bins, q_noise),
phase * get_noise_vec(self.n_sho_bins, phase_noise),
np.ones_like(R_mat)
]), sho32)
self.h5_file.flush()
return
def translate(self,
h5_path,
n_steps=32,
n_bins=37,
start_freq=300E+3,
end_freq=350E+3,
data_type='BEPSData',
mode='DC modulation mode',
field_mode='in and out-of-field',
n_cycles=1,
FORC_cycles=1,
FORC_repeats=1,
loop_a=1,
loop_b=4,
cycle_frac='full',
image_folder=beps_image_folder,
bin_factor=None,
bin_func=np.mean,
image_type='.tif',
simple_coefs=False):
"""
Parameters
----------
h5_path : str
Desired path to write the new HDF5 file
n_steps : uint, optional
Number of voltage steps
Default - 32
n_bins : uint, optional
Number of frequency bins
Default - 37
start_freq : float, optional
Starting frequency in Hz
Default - 300E+3
end_freq : float, optional
Final freqency in Hz
Default - 350E+3
data_type : str, optional
Type of data to generate
Options - 'BEPSData', 'BELineData'
Default - 'BEPSData'
mode : str, optional
Modulation mode to use when generating the data.
Options - 'DC modulation mode', 'AC modulation mode'
Default - 'DC modulation mode'
field_mode : str, optional
Field mode
Options - 'in-field', 'out-of-field', 'in and out-of-field'
Default - 'in and out-of-field'
n_cycles : uint, optional
Number of cycles
Default - 1
FORC_cycles : uint, optional
Number of FORC cycles
Default - 1
FORC_repeats : uint, optional
Number of FORC repeats
Default - 1
loop_a : float, optional
Loop coefficient a
Default - 1
loop_b : float, optional
Loop coefficient b
cycle_frac : str
Cycle fraction parameter.
Default - 'full'
image_folder : str
Path to the images that will be used to generate the loop coefficients. There must be 11 images named
'1.tif', '2.tif', ..., '11.tif'
Default - pycroscopy.io.translators.df_utils.beps_gen_utils.beps_image_folder
bin_factor : array_like of uint, optional
Downsampling factor for each dimension. Default is None.
bin_func : callable, optional
Function which will be called to calculate the return value
of each block. Function must implement an axis parameter,
i.e. numpy.mean. Ignored if bin_factor is None. Default is
numpy.mean.
image_type : str
File extension of images to be read. Default '.tif'
simple_coefs : bool
Should a simpler coefficient generation be used. Ensures loops, but all loops are identical.
Default False
Returns
-------
"""
# Setup shared parameters
self.n_steps = n_steps
self.n_bins = n_bins
self.start_freq = start_freq
self.end_freq = end_freq
self.n_cycles = n_cycles
self.forc_cycles = FORC_cycles
self.forc_repeats = FORC_repeats
self.loop_a = loop_a
self.loop_b = loop_b
self.data_type = data_type
self.mode = mode
self.field_mode = field_mode
self.cycle_fraction = cycle_frac
self.bin_factor = bin_factor
self.bin_func = bin_func
if field_mode == 'in and out-of-field':
self.n_fields = 2
else:
self.n_fields = 1
self.n_loops = FORC_cycles * FORC_repeats * n_cycles * self.n_fields
self.n_sho_bins = n_steps * self.n_loops
self.n_spec_bins = n_bins * self.n_sho_bins
self.h5_path = h5_path
self.image_ext = image_type
self.simple_coefs = simple_coefs
'''
Check if a bin_factor is given. Set up binning objects if it is.
'''
if bin_factor is not None:
self.rebin = True
if isinstance(bin_factor, int):
self.bin_factor = (bin_factor, bin_factor)
elif len(bin_factor) == 2:
self.bin_factor = tuple(bin_factor)
else:
raise ValueError(
'Input parameter `bin_factor` must be a length 2 array_like or an integer.\n'
+ '{} was given.'.format(bin_factor))
self.binning_func = block_reduce
self.bin_func = bin_func
images = self._read_data(image_folder)
data_gen_parms = {
'N_x': self.N_x,
'N_y': self.N_y,
'n_steps;:': n_steps,
'n_bins': n_bins,
'start_freq': start_freq,
'end_freq': end_freq,
'n_cycles': n_cycles,
'forc_cycles': FORC_cycles,
'forc_repeats': FORC_repeats,
'loop_a': loop_a,
'loop_b': loop_b,
'data_type': data_type,
'VS_mode': mode,
'field_mode': field_mode,
'num_udvs_steps': self.n_spec_bins,
'VS_cycle_fraction': cycle_frac
}
# Build the hdf5 file and get the datasets to write the data to
self._setup_h5(data_gen_parms)
# Calculate the loop parameters
coef_mat = self.calc_loop_coef_mat(images)
# In-and-out of field coefficients
if field_mode != 'in-field':
coef_OF_mat = np.copy(coef_mat)
if field_mode != 'out-of-field':
coef_IF_mat = np.copy(coef_mat)
coef_IF_mat[:, 4] -= 0.05
# Calculate the SHO fit and guess from the loop coefficients
self._calc_sho(coef_OF_mat, coef_IF_mat)
# Save the loop guess and fit to file
coef_OF_mat = np.hstack(
(coef_OF_mat[:, :9], np.ones([coef_OF_mat.shape[0], 1])))
coef_IF_mat = np.hstack(
(coef_IF_mat[:, :9], np.ones([coef_IF_mat.shape[0], 1])))
coef_mat = np.hstack(
[coef_IF_mat[:, np.newaxis, :], coef_OF_mat[:, np.newaxis, :]])
coef_mat = np.rollaxis(coef_mat, 1,
coef_mat.ndim).reshape([coef_mat.shape[0], -1])
self.h5_loop_fit[:] = np.tile(
stack_real_to_compound(coef_mat, loop_fit32),
[1, int(self.n_loops / self.n_fields)])
self.h5_loop_guess[:] = np.tile(
stack_real_to_compound(
coef_mat * get_noise_vec(coef_mat.shape, 0.1), loop_fit32),
[1, int(self.n_loops / self.n_fields)])
self.h5_file.flush()
self._calc_raw()
self.h5_file.flush()
# Test in-field
# positions = np.random.randint(0, coef_mat.shape[0], 100)
# for pos in positions:
# is_close = np.all(np.isclose(self.h5_loop_fit[pos, 0].tolist(), coef_IF_mat[pos]))
# if is_close:
# print('Close for {}'.format(pos))
# else:
# print('Not close for {}'.format(pos))
# for h5_coef, coef in zip(self.h5_loop_fit[pos, 0].tolist(), coef_IF_mat[pos]):
# print('h5: {}, \t mat: {}'.format(h5_coef, coef))
# Test out-of-field
# positions = np.random.randint(0, coef_mat.shape[0], 100)
# for pos in positions:
# is_close = np.all(np.isclose(self.h5_loop_fit[pos, 1].tolist(), coef_OF_mat[pos]))
# if is_close:
# print('Close for {}'.format(pos))
# else:
# print('Not close for {}'.format(pos))
# for h5_coef, coef in zip(self.h5_loop_fit[pos, 1].tolist(), coef_OF_mat[pos]):
# print('h5: {}, \t mat: {}'.format(h5_coef, coef))
self.h5_file.close()
return self.h5_path
def fit_atom_positions_dset(h5_grp, fitting_parms=None, num_cores=None):
"""
A temporary substitute for a full-fledged process class.
Computes the guess and fit coefficients for the provided atom guess positions and writes these results to the
given h5 group
Parameters
----------
h5_grp : h5py.Group reference
Group containing the atom guess positions, cropped clean image and some necessary parameters
fitting_parms : dictionary
Parameters used for atom position fitting
num_cores : unsigned int (Optional. Default = available logical cores - 2)
Number of cores to compute with
Returns
-------
h5_grp : h5py.Group reference
Same group as the parameter but now with the 'Guess' and 'Fit' datasets
"""
cropped_clean_image = h5_grp['Cropped_Clean_Image'][()]
h5_guess = h5_grp['Guess_Positions']
all_atom_guesses = np.transpose(np.vstack(
(h5_guess['x'], h5_guess['y']))) # leave out the atom type for now
win_size = h5_grp.attrs['motif_win_size']
psf_width = h5_grp.attrs['psf_width']
num_atoms = all_atom_guesses.shape[0] # number of atoms
# build distance matrix
pos_vec = all_atom_guesses[:, 0] + 1j * all_atom_guesses[:, 1]
pos_mat1 = np.tile(np.transpose(np.atleast_2d(pos_vec)), [1, num_atoms])
pos_mat2 = np.transpose(pos_mat1)
d_mat = np.abs(pos_mat2 -
pos_mat1) # matrix of distances between all atoms
# sort the distance matrix and keep only the atoms within the nearest neighbor limit
neighbor_dist_order = np.argsort(d_mat)
if fitting_parms is None:
num_nearest_neighbors = 6 # to consider when fitting
fitting_parms = {
'fit_region_size':
win_size * 0.80, # region to consider when fitting
'gauss_width_guess': psf_width * 2,
'num_nearest_neighbors': num_nearest_neighbors,
'min_amplitude': 0, # min amplitude limit for gauss fit
'max_amplitude': 2, # max amplitude limit for gauss fit
'position_range': win_size / 2,
# range that the fitted position can go from initial guess position[pixels]
'max_function_evals': 100,
'min_gauss_width_ratio': 0.5, # min width of gauss fit ratio,
'max_gauss_width_ratio': 2, # max width of gauss fit ratio
'fitting_tolerance': 1E-4
}
num_nearest_neighbors = fitting_parms['num_nearest_neighbors']
# neighbor dist order has the (indices of the) neighbors for each atom sorted by distance
closest_neighbors_mat = neighbor_dist_order[:, 1:num_nearest_neighbors + 1]
parm_dict = {
'atom_pos_guess': all_atom_guesses,
'nearest_neighbors': closest_neighbors_mat,
'cropped_cleaned_image': cropped_clean_image
}
# do the parallel fitting
fitting_results = fit_atom_positions_parallel(parm_dict,
fitting_parms,
num_cores=num_cores)
# Make datasets to write back to file:
guess_parms = np.zeros(shape=(num_atoms, num_nearest_neighbors + 1),
dtype=atom_coeff_dtype)
fit_parms = np.zeros(shape=guess_parms.shape, dtype=guess_parms.dtype)
for atom_ind, single_atom_results in enumerate(fitting_results):
guess_coeff, fit_coeff = single_atom_results
num_neighbors_used = guess_coeff.shape[0]
guess_parms[atom_ind, :num_neighbors_used] = np.squeeze(
stack_real_to_compound(guess_coeff, guess_parms.dtype))
fit_parms[atom_ind, :num_neighbors_used] = np.squeeze(
stack_real_to_compound(fit_coeff, guess_parms.dtype))
ds_atom_guesses = VirtualDataset('Guess', data=guess_parms)
ds_atom_fits = VirtualDataset('Fit', data=fit_parms)
dgrp_atom_finding = VirtualGroup(h5_grp.name.split('/')[-1],
parent=h5_grp.parent.name)
dgrp_atom_finding.attrs = fitting_parms
dgrp_atom_finding.add_children([ds_atom_guesses, ds_atom_fits])
hdf = HDFwriter(h5_grp.file)
h5_atom_refs = hdf.write(dgrp_atom_finding)
return h5_grp
def _calc_sho(self, coef_OF_mat, coef_IF_mat, amp_noise=0.1, phase_noise=0.1, q_noise=0.2, resp_noise=0.01):
"""
Build the SHO dataset from the coefficient matrices
Parameters
----------
coef_OF_mat : numpy.ndarray
Out-of-field coefficients
coef_IF_mat : numpy.ndarray
In-field coefficients
amp_noise : float
Noise factor for amplitude parameter
phase_noise : float
Noise factor for phase parameter
q_noise : float
Noise factor for Q-value parameter
resp_noise : float
Noide factor for w0 parameter
Returns
-------
None
"""
# TODO: Fix sho parameter generation
vdc_vec = self.h5_sho_spec_vals[self.h5_sho_spec_vals.attrs['DC_Offset']].squeeze()
sho_field = self.h5_sho_spec_vals[self.h5_sho_spec_vals.attrs['Field']].squeeze()
sho_of_inds = sho_field == 0
sho_if_inds = sho_field == 1
# determine how many pixels can be read at once
mem_per_pix = vdc_vec.size * np.float32(0).itemsize
free_mem = self.max_ram - vdc_vec.size * vdc_vec.dtype.itemsize * 6
batch_size = int(free_mem / mem_per_pix)
batches = gen_batches(self.n_pixels, batch_size)
for pix_batch in batches:
R_OF = np.array([loop_fit_function(vdc_vec[sho_of_inds], coef) for coef in coef_OF_mat[pix_batch]])
R_IF = np.array([loop_fit_function(vdc_vec[sho_if_inds], coef) for coef in coef_IF_mat[pix_batch]])
R_mat = np.hstack([R_IF[:, np.newaxis, :], R_OF[:, np.newaxis, :]])
R_mat = np.rollaxis(R_mat, 1, R_mat.ndim).reshape(R_mat.shape[0], -1)
del R_OF, R_IF
amp = np.abs(R_mat)
resp = coef_OF_mat[pix_batch, 9, None] * np.ones_like(R_mat)
q_val = coef_OF_mat[pix_batch, 10, None] * np.ones_like(R_mat) * 10
phase = np.sign(R_mat) * np.pi / 2
self.h5_sho_fit[pix_batch, :] = stack_real_to_compound(np.hstack([amp,
resp,
q_val,
phase,
np.ones_like(R_mat)]),
sho32)
self.h5_sho_guess[pix_batch, :] = stack_real_to_compound(np.hstack([amp * get_noise_vec(self.n_sho_bins,
amp_noise),
resp * get_noise_vec(self.n_sho_bins,
resp_noise),
q_val * get_noise_vec(self.n_sho_bins,
q_noise),
phase * get_noise_vec(self.n_sho_bins,
phase_noise),
np.ones_like(R_mat)]),
sho32)
self.h5_file.flush()
return
def translate(self, h5_path, n_steps=32, n_bins=37, start_freq=300E+3, end_freq=350E+3,
data_type='BEPSData', mode='DC modulation mode', field_mode='in and out-of-field',
n_cycles=1, FORC_cycles=1, FORC_repeats=1, loop_a=1, loop_b=4,
cycle_frac='full', image_folder=beps_image_folder, bin_factor=None,
bin_func=np.mean, image_type='.tif', simple_coefs=False):
"""
Parameters
----------
h5_path : str
Desired path to write the new HDF5 file
n_steps : uint, optional
Number of voltage steps
Default - 32
n_bins : uint, optional
Number of frequency bins
Default - 37
start_freq : float, optional
Starting frequency in Hz
Default - 300E+3
end_freq : float, optional
Final freqency in Hz
Default - 350E+3
data_type : str, optional
Type of data to generate
Options - 'BEPSData', 'BELineData'
Default - 'BEPSData'
mode : str, optional
Modulation mode to use when generating the data.
Options - 'DC modulation mode', 'AC modulation mode'
Default - 'DC modulation mode'
field_mode : str, optional
Field mode
Options - 'in-field', 'out-of-field', 'in and out-of-field'
Default - 'in and out-of-field'
n_cycles : uint, optional
Number of cycles
Default - 1
FORC_cycles : uint, optional
Number of FORC cycles
Default - 1
FORC_repeats : uint, optional
Number of FORC repeats
Default - 1
loop_a : float, optional
Loop coefficient a
Default - 1
loop_b : float, optional
Loop coefficient b
cycle_frac : str
Cycle fraction parameter.
Default - 'full'
image_folder : str
Path to the images that will be used to generate the loop coefficients. There must be 11 images named
'1.tif', '2.tif', ..., '11.tif'
Default - pycroscopy.io.translators.df_utils.beps_gen_utils.beps_image_folder
bin_factor : array_like of uint, optional
Downsampling factor for each dimension. Default is None.
bin_func : callable, optional
Function which will be called to calculate the return value
of each block. Function must implement an axis parameter,
i.e. numpy.mean. Ignored if bin_factor is None. Default is
numpy.mean.
image_type : str
File extension of images to be read. Default '.tif'
simple_coefs : bool
Should a simpler coefficient generation be used. Ensures loops, but all loops are identical.
Default False
Returns
-------
"""
# Setup shared parameters
self.n_steps = n_steps
self.n_bins = n_bins
self.start_freq = start_freq
self.end_freq = end_freq
self.n_cycles = n_cycles
self.forc_cycles = FORC_cycles
self.forc_repeats = FORC_repeats
self.loop_a = loop_a
self.loop_b = loop_b
self.data_type = data_type
self.mode = mode
self.field_mode = field_mode
self.cycle_fraction = cycle_frac
self.bin_factor = bin_factor
self.bin_func = bin_func
if field_mode == 'in and out-of-field':
self.n_fields = 2
else:
self.n_fields = 1
self.n_loops = FORC_cycles * FORC_repeats * n_cycles * self.n_fields
self.n_sho_bins = n_steps * self.n_loops
self.n_spec_bins = n_bins * self.n_sho_bins
self.h5_path = h5_path
self.image_ext = image_type
self.simple_coefs = simple_coefs
'''
Check if a bin_factor is given. Set up binning objects if it is.
'''
if bin_factor is not None:
self.rebin = True
if isinstance(bin_factor, int):
self.bin_factor = (bin_factor, bin_factor)
elif len(bin_factor) == 2:
self.bin_factor = tuple(bin_factor)
else:
raise ValueError('Input parameter `bin_factor` must be a length 2 array_like or an integer.\n' +
'{} was given.'.format(bin_factor))
self.binning_func = block_reduce
self.bin_func = bin_func
images = self._read_data(image_folder)
data_gen_parms = {'N_x': self.N_x, 'N_y': self.N_y, 'n_steps;:': n_steps,
'n_bins': n_bins, 'start_freq': start_freq,
'end_freq': end_freq, 'n_cycles': n_cycles,
'forc_cycles': FORC_cycles, 'forc_repeats': FORC_repeats,
'loop_a': loop_a, 'loop_b': loop_b, 'data_type': data_type,
'VS_mode': mode, 'field_mode': field_mode, 'num_udvs_steps': self.n_spec_bins,
'VS_cycle_fraction': cycle_frac}
# Build the hdf5 file and get the datasets to write the data to
self._setup_h5(data_gen_parms)
# Calculate the loop parameters
coef_mat = self.calc_loop_coef_mat(images)
# In-and-out of field coefficients
if field_mode != 'in-field':
coef_OF_mat = np.copy(coef_mat)
if field_mode != 'out-of-field':
coef_IF_mat = np.copy(coef_mat)
coef_IF_mat[:, 4] -= 0.05
# Calculate the SHO fit and guess from the loop coefficients
self._calc_sho(coef_OF_mat, coef_IF_mat)
# Save the loop guess and fit to file
coef_OF_mat = np.hstack((coef_OF_mat[:, :9], np.ones([coef_OF_mat.shape[0], 1])))
coef_IF_mat = np.hstack((coef_IF_mat[:, :9], np.ones([coef_IF_mat.shape[0], 1])))
coef_mat = np.hstack([coef_IF_mat[:, np.newaxis, :], coef_OF_mat[:, np.newaxis, :]])
coef_mat = np.rollaxis(coef_mat, 1, coef_mat.ndim).reshape([coef_mat.shape[0], -1])
self.h5_loop_fit[:] = np.tile(stack_real_to_compound(coef_mat, loop_fit32),
[1, int(self.n_loops / self.n_fields)])
self.h5_loop_guess[:] = np.tile(stack_real_to_compound(coef_mat * get_noise_vec(coef_mat.shape, 0.1),
loop_fit32),
[1, int(self.n_loops / self.n_fields)])
self.h5_file.flush()
self._calc_raw()
self.h5_file.flush()
# Test in-field
# positions = np.random.randint(0, coef_mat.shape[0], 100)
# for pos in positions:
# is_close = np.all(np.isclose(self.h5_loop_fit[pos, 0].tolist(), coef_IF_mat[pos]))
# if is_close:
# print('Close for {}'.format(pos))
# else:
# print('Not close for {}'.format(pos))
# for h5_coef, coef in zip(self.h5_loop_fit[pos, 0].tolist(), coef_IF_mat[pos]):
# print('h5: {}, \t mat: {}'.format(h5_coef, coef))
# Test out-of-field
# positions = np.random.randint(0, coef_mat.shape[0], 100)
# for pos in positions:
# is_close = np.all(np.isclose(self.h5_loop_fit[pos, 1].tolist(), coef_OF_mat[pos]))
# if is_close:
# print('Close for {}'.format(pos))
# else:
# print('Not close for {}'.format(pos))
# for h5_coef, coef in zip(self.h5_loop_fit[pos, 1].tolist(), coef_OF_mat[pos]):
# print('h5: {}, \t mat: {}'.format(h5_coef, coef))
self.h5_file.close()
return self.h5_path
def _write_results_chunk(self):
"""
Writes data chunks back to the h5 file
"""
if self.verbose:
print('Rank {} - Started accumulating results for this chunk'.format(self.mpi_rank))
num_pixels = len(self.forward_results)
cap_mat = np.zeros((num_pixels, 2), dtype=np.float32)
r_inf_mat = np.zeros((num_pixels, self.num_x_steps), dtype=np.float32)
r_var_mat = np.zeros((num_pixels, self.num_x_steps), dtype=np.float32)
i_cor_sin_mat = np.zeros((num_pixels, self.single_ao.size), dtype=np.float32)
for pix_ind, i_meas, forw_results, rev_results in zip(range(num_pixels), self.data,
self.forward_results, self.reverse_results):
full_results = dict()
for item in ['cValue']:
full_results[item] = np.hstack((forw_results[item], rev_results[item]))
# print(item, full_results[item].shape)
# Capacitance is always doubled - halve it now (locally):
# full_results['cValue'] *= 0.5
cap_val = np.mean(full_results['cValue']) * 0.5
# Compensating the resistance..
"""
omega = 2 * np.pi * self.ex_freq
i_cap = cap_val * omega * self.rolled_bias
"""
i_cap = cap_val * self.dvdt
i_extra = self.r_extra * 2 * cap_val * self.single_ao
i_corr_sine = i_meas - i_cap - i_extra
# Equivalent to flipping the X:
rev_results['x'] *= -1
# Stacking the results - no flipping required for reverse:
for item in ['x', 'mR', 'vR']:
full_results[item] = np.hstack((forw_results[item], rev_results[item]))
i_cor_sin_mat[pix_ind] = i_corr_sine
cap_mat[pix_ind] = full_results['cValue'] * 1000 # convert from nF to pF
r_inf_mat[pix_ind] = full_results['mR']
r_var_mat[pix_ind] = full_results['vR']
# Now write to h5 files:
if self.verbose:
print('Rank {} - Finished accumulating results. Writing results of chunk to h5'.format(self.mpi_rank))
if self.__first_batch:
self.h5_new_spec_vals[0, :] = full_results['x'] # Technically this needs to only be done once
self.__first_batch = False
# Get access to the private variable:
pos_in_batch = self._get_pixels_in_current_batch()
self.h5_cap[pos_in_batch, :] = np.atleast_2d(stack_real_to_compound(cap_mat, cap_dtype)).T
self.h5_variance[pos_in_batch, :] = r_var_mat
self.h5_resistance[pos_in_batch, :] = r_inf_mat
self.h5_i_corrected[pos_in_batch, :] = i_cor_sin_mat
