def test_nd_h5_illegal(self):
with h5py.File(file_path, mode='r') as h5_f:
with self.assertRaises(TypeError):
_ = dtype_utils.stack_real_to_compound(h5_f['compound'],
struc_dtype)
with self.assertRaises(TypeError):
_ = dtype_utils.stack_real_to_compound(h5_f['complex'],
struc_dtype)
def test_illegal(self):
num_elems = (3, 5)
r_vals = np.random.random(size=num_elems)
with self.assertRaises(TypeError):
_ = dtype_utils.stack_real_to_compound(r_vals, np.float32)
with self.assertRaises(ValueError):
_ = dtype_utils.stack_real_to_compound(r_vals, struc_dtype)
with self.assertRaises(ValueError):
_ = dtype_utils.stack_real_to_compound([1, 'a', {
'a': 1
}, False], struc_dtype)
def test_1d(self):
num_elems = 5
structured_array = np.zeros(shape=num_elems, dtype=struc_dtype)
structured_array['r'] = r_vals = np.random.random(size=num_elems)
structured_array['g'] = g_vals = np.random.randint(0,
high=1024,
size=num_elems)
structured_array['b'] = b_vals = np.random.random(size=num_elems)
real_val = np.concatenate((r_vals, g_vals, b_vals))
actual = dtype_utils.stack_real_to_compound(real_val, struc_dtype)
self.assertTrue(compare_structured_arrays(actual, structured_array))
def base_nd_h5_legal(self, lazy):
with h5py.File(file_path, mode='r') as h5_f:
h5_real = h5_f['real2']
structured_array = np.zeros(
shape=list(h5_real.shape)[:-1] +
[h5_real.shape[-1] // len(struc_dtype.names)],
dtype=struc_dtype)
for name_ind, name in enumerate(struc_dtype.names):
i_start = name_ind * structured_array.shape[-1]
i_end = (name_ind + 1) * structured_array.shape[-1]
structured_array[name] = h5_real[..., i_start:i_end]
actual = dtype_utils.stack_real_to_compound(h5_real,
struc_dtype,
lazy=lazy)
if lazy:
self.assertIsInstance(actual, da.core.Array)
actual = actual.compute()
self.assertTrue(compare_structured_arrays(actual,
structured_array))
def base_nd(self, in_lazy, out_lazy):
num_elems = (2, 3, 5, 7)
structured_array = np.zeros(shape=num_elems, dtype=struc_dtype)
structured_array['r'] = r_vals = np.random.random(size=num_elems)
structured_array['g'] = g_vals = np.random.randint(0,
high=1024,
size=num_elems)
structured_array['b'] = b_vals = np.random.random(size=num_elems)
real_val = np.concatenate((r_vals, g_vals, b_vals),
axis=len(num_elems) - 1)
if in_lazy:
real_val = da.from_array(real_val, chunks=real_val.shape)
actual = dtype_utils.stack_real_to_compound(real_val,
struc_dtype,
lazy=out_lazy)
if out_lazy:
self.assertIsInstance(actual, da.core.Array)
actual = actual.compute()
self.assertTrue(compare_structured_arrays(actual, structured_array))
def _calc_sho(self,
coef_OF_mat,
coef_IF_mat,
amp_noise=0.1,
phase_noise=0.1,
q_noise=0.2,
resp_noise=0.01):
"""
Build the SHO dataset from the coefficient matrices
Parameters
----------
coef_OF_mat : numpy.ndarray
Out-of-field coefficients
coef_IF_mat : numpy.ndarray
In-field coefficients
amp_noise : float
Noise factor for amplitude parameter
phase_noise : float
Noise factor for phase parameter
q_noise : float
Noise factor for Q-value parameter
resp_noise : float
Noide factor for w0 parameter
Returns
-------
None
"""
print(list(self.h5_sho_spec_vals.attrs))
vdc_vec = self.h5_sho_spec_vals[self.h5_sho_fit.spec_dim_labels.index(
'DC_Offset')].squeeze()
sho_field = self.h5_sho_spec_vals[
self.h5_sho_fit.spec_dim_labels.index('Field')].squeeze()
sho_of_inds = sho_field == 0
sho_if_inds = sho_field == 1
# determine how many pixels can be read at once
mem_per_pix = vdc_vec.size * np.float32(0).itemsize
#free_mem = self.max_ram - vdc_vec.size * vdc_vec.dtype.itemsize * 6
free_mem = 1024
batch_size = int(free_mem / mem_per_pix)
batches = gen_batches(self.n_pixels, batch_size)
one_cycle_length = vdc_vec[sho_of_inds].shape[-1]
for pix_batch in batches:
roll_len = one_cycle_length // 4
vdc_OF_rolled = np.roll(vdc_vec[sho_of_inds], -1 * roll_len)
vdc_IF_rolled = np.roll(vdc_vec[sho_if_inds], -1 * roll_len)
R_OF = np.array([
np.roll(loop_fit_function(vdc_OF_rolled, coef), roll_len)
for coef in coef_OF_mat[pix_batch]
])
R_OF = R_OF - R_OF.mean()
R_IF = np.array([
np.roll(loop_fit_function(vdc_IF_rolled, coef), roll_len)
for coef in coef_IF_mat[pix_batch]
])
R_IF = R_IF - R_IF.mean()
R_mat = np.hstack([R_IF[:, np.newaxis, :], R_OF[:, np.newaxis, :]])
R_mat = np.rollaxis(R_mat, 1,
R_mat.ndim).reshape(R_mat.shape[0], -1)
del R_OF, R_IF
amp = np.abs(R_mat)
resp = coef_OF_mat[pix_batch, 9, None] * np.ones_like(R_mat)
q_val = coef_OF_mat[pix_batch, 10, None] * np.ones_like(R_mat) * 10
phase = np.sign(R_mat) * np.pi / 2 + np.pi / 2
self.h5_sho_fit[pix_batch, :] = stack_real_to_compound(
np.hstack([amp, resp, q_val, phase,
np.ones_like(R_mat)]), sho32)
self.h5_sho_guess[pix_batch, :] = stack_real_to_compound(
np.hstack([
amp * get_noise_vec(self.n_sho_bins, amp_noise),
resp * get_noise_vec(self.n_sho_bins, resp_noise),
q_val * get_noise_vec(self.n_sho_bins, q_noise),
phase * get_noise_vec(self.n_sho_bins, phase_noise),
np.ones_like(R_mat)
]), sho32)
self.h5_file.flush()
return
def translate(self,
h5_path,
n_steps=32,
n_bins=37,
start_freq=300E+3,
end_freq=350E+3,
data_type='BEPSData',
mode='DC modulation mode',
field_mode='in and out-of-field',
n_cycles=1,
FORC_cycles=1,
FORC_repeats=1,
loop_a=3,
loop_b=4,
cycle_frac='full',
image_folder=beps_image_folder,
bin_factor=None,
bin_func=np.mean,
image_type='.tif',
simple_coefs=False):
"""
Parameters
----------
h5_path : str
Desired path to write the new HDF5 file
n_steps : uint, optional
Number of voltage steps
Default - 32
n_bins : uint, optional
Number of frequency bins
Default - 37
start_freq : float, optional
Starting frequency in Hz
Default - 300E+3
end_freq : float, optional
Final freqency in Hz
Default - 350E+3
data_type : str, optional
Type of data to generate
Options - 'BEPSData', 'BELineData'
Default - 'BEPSData'
mode : str, optional
Modulation mode to use when generating the data.
Options - 'DC modulation mode', 'AC modulation mode'
Default - 'DC modulation mode'
field_mode : str, optional
Field mode
Options - 'in-field', 'out-of-field', 'in and out-of-field'
Default - 'in and out-of-field'
n_cycles : uint, optional
Number of cycles
Default - 1
FORC_cycles : uint, optional
Number of FORC cycles
Default - 1
FORC_repeats : uint, optional
Number of FORC repeats
Default - 1
loop_a : float, optional
Loop coefficient a
Default - 1
loop_b : float, optional
Loop coefficient b
cycle_frac : str
Cycle fraction parameter.
Default - 'full'
image_folder : str
Path to the images that will be used to generate the loop coefficients. There must be 11 images named
'1.tif', '2.tif', ..., '11.tif'
Default - pycroscopy.io.translators.df_utils.beps_gen_utils.beps_image_folder
bin_factor : array_like of uint, optional
Downsampling factor for each dimension. Default is None.
bin_func : callable, optional
Function which will be called to calculate the return value
of each block. Function must implement an axis parameter,
i.e. numpy.mean. Ignored if bin_factor is None. Default is
numpy.mean.
image_type : str
File extension of images to be read. Default '.tif'
simple_coefs : bool
Should a simpler coefficient generation be used. Ensures loops, but all loops are identical.
Default False
Returns
-------
"""
# Setup shared parameters
self.n_steps = n_steps
self.n_bins = n_bins
self.start_freq = start_freq
self.end_freq = end_freq
self.n_cycles = n_cycles
self.forc_cycles = FORC_cycles
self.forc_repeats = FORC_repeats
self.loop_a = loop_a
self.loop_b = loop_b
self.data_type = data_type
self.mode = mode
self.field_mode = field_mode
self.cycle_fraction = cycle_frac
self.bin_factor = bin_factor
self.bin_func = bin_func
if field_mode == 'in and out-of-field':
self.n_fields = 2
else:
self.n_fields = 1
self.n_loops = FORC_cycles * FORC_repeats * n_cycles * self.n_fields
self.n_sho_bins = n_steps * self.n_loops
self.n_spec_bins = n_bins * self.n_sho_bins
self.h5_path = h5_path
self.image_ext = image_type
self.simple_coefs = simple_coefs
'''
Check if a bin_factor is given. Set up binning objects if it is.
'''
if bin_factor is not None:
self.rebin = True
if isinstance(bin_factor, int):
self.bin_factor = (bin_factor, bin_factor)
elif len(bin_factor) == 2:
self.bin_factor = tuple(bin_factor)
else:
raise ValueError(
'Input parameter `bin_factor` must be a length 2 array_like or an integer.\n'
+ '{} was given.'.format(bin_factor))
self.binning_func = block_reduce
self.bin_func = bin_func
print('Image folder is {}'.format(image_folder))
images = self._read_data(image_folder)
self.images = images
data_gen_parms = {
'N_x': self.N_x,
'N_y': self.N_y,
'n_steps;:': n_steps,
'n_bins': n_bins,
'start_freq': start_freq,
'end_freq': end_freq,
'n_cycles': n_cycles,
'forc_cycles': FORC_cycles,
'forc_repeats': FORC_repeats,
'loop_a': loop_a,
'loop_b': loop_b,
'data_type': data_type,
'VS_mode': mode,
'field_mode': field_mode,
'num_udvs_steps': self.n_spec_bins,
'VS_cycle_fraction': cycle_frac
}
# Build the hdf5 file and get the datasets to write the data to
self._setup_h5(data_gen_parms)
# Calculate the loop parameters
coef_mat = self.calc_loop_coef_mat(images)
# In-and-out of field coefficients
if field_mode != 'in-field':
coef_OF_mat = np.copy(coef_mat)
if field_mode != 'out-of-field':
coef_IF_mat = np.copy(coef_mat)
coef_IF_mat[:, 4] -= 0.05
self.coef_mat = coef_mat
# Calculate the SHO fit and guess from the loop coefficients
self._calc_sho(coef_OF_mat, coef_IF_mat)
# Save the loop guess and fit to file
coef_OF_mat = np.hstack(
(coef_OF_mat[:, :9], np.ones([coef_OF_mat.shape[0], 1])))
coef_IF_mat = np.hstack(
(coef_IF_mat[:, :9], np.ones([coef_IF_mat.shape[0], 1])))
coef_mat = np.hstack(
[coef_IF_mat[:, np.newaxis, :], coef_OF_mat[:, np.newaxis, :]])
coef_mat = np.rollaxis(coef_mat, 1,
coef_mat.ndim).reshape([coef_mat.shape[0], -1])
self.h5_loop_fit[:] = np.tile(
stack_real_to_compound(coef_mat, loop_fit32),
[1, int(self.n_loops / self.n_fields)])
self.h5_loop_guess[:] = np.tile(
stack_real_to_compound(
coef_mat * get_noise_vec(coef_mat.shape, 0.1), loop_fit32),
[1, int(self.n_loops / self.n_fields)])
self.h5_file.flush()
self._calc_raw()
self.h5_file.flush()
return self.h5_path
def fit_atom_positions_dset(h5_grp, fitting_parms=None, num_cores=None):
"""
A temporary substitute for a full-fledged process class.
Computes the guess and fit coefficients for the provided atom guess positions and writes these results to the
given h5 group
Parameters
----------
h5_grp : h5py.Group reference
Group containing the atom guess positions, cropped clean image and some necessary parameters
fitting_parms : dictionary
Parameters used for atom position fitting
num_cores : unsigned int (Optional. Default = available logical cores - 2)
Number of cores to compute with
Returns
-------
h5_grp : h5py.Group reference
Same group as the parameter but now with the 'Guess' and 'Fit' datasets
"""
cropped_clean_image = h5_grp['Cropped_Clean_Image'][()]
h5_guess = h5_grp['Guess_Positions']
all_atom_guesses = np.transpose(np.vstack(
(h5_guess['x'], h5_guess['y']))) # leave out the atom type for now
win_size = h5_grp.attrs['motif_win_size']
psf_width = h5_grp.attrs['psf_width']
num_atoms = all_atom_guesses.shape[0] # number of atoms
# build distance matrix
pos_vec = all_atom_guesses[:, 0] + 1j * all_atom_guesses[:, 1]
pos_mat1 = np.tile(np.transpose(np.atleast_2d(pos_vec)), [1, num_atoms])
pos_mat2 = np.transpose(pos_mat1)
d_mat = np.abs(pos_mat2 -
pos_mat1) # matrix of distances between all atoms
# sort the distance matrix and keep only the atoms within the nearest neighbor limit
neighbor_dist_order = np.argsort(d_mat)
if fitting_parms is None:
num_nearest_neighbors = 6 # to consider when fitting
fitting_parms = {
'fit_region_size':
win_size * 0.80, # region to consider when fitting
'gauss_width_guess': psf_width * 2,
'num_nearest_neighbors': num_nearest_neighbors,
'min_amplitude': 0, # min amplitude limit for gauss fit
'max_amplitude': 2, # max amplitude limit for gauss fit
'position_range': win_size / 2,
# range that the fitted position can go from initial guess position[pixels]
'max_function_evals': 100,
'min_gauss_width_ratio': 0.5, # min width of gauss fit ratio,
'max_gauss_width_ratio': 2, # max width of gauss fit ratio
'fitting_tolerance': 1E-4
}
num_nearest_neighbors = fitting_parms['num_nearest_neighbors']
# neighbor dist order has the (indices of the) neighbors for each atom sorted by distance
closest_neighbors_mat = neighbor_dist_order[:, 1:num_nearest_neighbors + 1]
parm_dict = {
'atom_pos_guess': all_atom_guesses,
'nearest_neighbors': closest_neighbors_mat,
'cropped_cleaned_image': cropped_clean_image
}
# do the parallel fitting
fitting_results = fit_atom_positions_parallel(parm_dict,
fitting_parms,
num_cores=num_cores)
# Make datasets to write back to file:
guess_parms = np.zeros(shape=(num_atoms, num_nearest_neighbors + 1),
dtype=atom_coeff_dtype)
fit_parms = np.zeros(shape=guess_parms.shape, dtype=guess_parms.dtype)
for atom_ind, single_atom_results in enumerate(fitting_results):
guess_coeff, fit_coeff = single_atom_results
num_neighbors_used = guess_coeff.shape[0]
guess_parms[atom_ind, :num_neighbors_used] = np.squeeze(
stack_real_to_compound(guess_coeff, guess_parms.dtype))
fit_parms[atom_ind, :num_neighbors_used] = np.squeeze(
stack_real_to_compound(fit_coeff, guess_parms.dtype))
ds_atom_guesses = VirtualDataset('Guess', data=guess_parms)
ds_atom_fits = VirtualDataset('Fit', data=fit_parms)
dgrp_atom_finding = VirtualGroup(h5_grp.name.split('/')[-1],
parent=h5_grp.parent.name)
dgrp_atom_finding.attrs = fitting_parms
dgrp_atom_finding.add_children([ds_atom_guesses, ds_atom_fits])
hdf = HDFwriter(h5_grp.file)
h5_atom_refs = hdf.write(dgrp_atom_finding)
return h5_grp
def _write_results_chunk(self):
"""
Writes data chunks back to the h5 file
"""
if self.verbose:
print(
'Rank {} - Started accumulating results for this chunk'.format(
self.mpi_rank))
num_pixels = len(self.forward_results)
cap_mat = np.zeros((num_pixels, 2), dtype=np.float32)
r_inf_mat = np.zeros((num_pixels, self.num_x_steps), dtype=np.float32)
r_var_mat = np.zeros((num_pixels, self.num_x_steps), dtype=np.float32)
i_cor_sin_mat = np.zeros((num_pixels, self.single_ao.size),
dtype=np.float32)
for pix_ind, i_meas, forw_results, rev_results in zip(
range(num_pixels), self.data, self.forward_results,
self.reverse_results):
full_results = dict()
for item in ['cValue']:
full_results[item] = np.hstack(
(forw_results[item], rev_results[item]))
# print(item, full_results[item].shape)
# Capacitance is always doubled - halve it now (locally):
# full_results['cValue'] *= 0.5
cap_val = np.mean(full_results['cValue']) * 0.5
# Compensating the resistance..
"""
omega = 2 * np.pi * self.ex_freq
i_cap = cap_val * omega * self.rolled_bias
"""
i_cap = cap_val * self.dvdt
i_extra = self.r_extra * 2 * cap_val * self.single_ao
i_corr_sine = i_meas - i_cap - i_extra
# Equivalent to flipping the X:
rev_results['x'] *= -1
# Stacking the results - no flipping required for reverse:
for item in ['x', 'mR', 'vR']:
full_results[item] = np.hstack(
(forw_results[item], rev_results[item]))
i_cor_sin_mat[pix_ind] = i_corr_sine
cap_mat[pix_ind] = full_results[
'cValue'] * 1000 # convert from nF to pF
r_inf_mat[pix_ind] = full_results['mR']
r_var_mat[pix_ind] = full_results['vR']
# Now write to h5 files:
if self.verbose:
print(
'Rank {} - Finished accumulating results. Writing results of chunk to h5'
.format(self.mpi_rank))
if self.__first_batch:
self.h5_new_spec_vals[0, :] = full_results[
'x'] # Technically this needs to only be done once
self.__first_batch = False
# Get access to the private variable:
pos_in_batch = self._get_pixels_in_current_batch()
self.h5_cap[pos_in_batch, :] = np.atleast_2d(
stack_real_to_compound(cap_mat, cap_dtype)).T
self.h5_variance[pos_in_batch, :] = r_var_mat
self.h5_resistance[pos_in_batch, :] = r_inf_mat
self.h5_i_corrected[pos_in_batch, :] = i_cor_sin_mat