def get_add_electrical(samps, scaler_ch):
for samp in samps:
# calculate the scale factor (in amperes) according to electrical settings of scan
# this for loop takes advantage of the order of
# 'c_stanford', 'beam_conv', and 'c_lockin' lists in the sample dicts
xbic_scale_factors = [
(stan * 1E-9) / (V2F * lock) for stan, V2F, lock in zip(
samp['c_stanford'], samp['beam_conv'], samp['c_lockin'])
]
# store scaler containing electrical in list
raw_xbic = [h5['/MAPS/scalers'][scaler_ch] for h5 in samp['XBIC_h5s']]
# convert scaler channel into amps
xbic_scaled = [
ds_ic * scale for ds_ic, scale in zip(raw_xbic, xbic_scale_factors)
]
# import XBIC_maps to sample dict
build_dict(samp, 'XBIC_maps', xbic_scaled)
xbiv_scale_factors = [
1 / (V2F * lock)
for V2F, lock in zip(samp['beam_conv'], samp['c_lockin'])
]
raw_xbiv = [h5['/MAPS/scalers'][scaler_ch]
for h5 in samp['XBIV_h5s']] #grab xbic channels
xbiv_scaled = [
ds_ic * scale for ds_ic, scale in zip(raw_xbiv, xbiv_scale_factors)
]
build_dict(samp, 'XBIV_maps', xbiv_scaled)
return
def join_corrected_beamtimes(samples, keys, new_key):
for sample in samples:
all_corr = [sample[key] for key in keys]
flatten = lambda l: [item for sublist in l for item in sublist]
all_corr = flatten(all_corr)
samp_dict_grow.build_dict(sample, new_key, all_corr)
return
def get_layer_iios(samples, elements, beam_settings, layer):
for sample in samples:
STACK = sample['STACK']
layer_idx = list(STACK.keys()).index(layer)
if layer_idx != 0:
# retrieve info of upstream layers
layers_before = {
k: v
for idx, (k, v) in enumerate(STACK.items()) if idx < layer_idx
}
# percent incoming beam transmitted to layer
upstream_attn_in = get_upstream_iioIN(layers_before, beam_settings)
# percent outgoing XRF transmitted by upstream layers
upstream_attn_out = get_upstream_iioOUT(layers_before, elements,
beam_settings)
cumulative_upstream_attenuation = upstream_attn_in * upstream_attn_out
# percent outgoing XRF transmitted
ele_avg_iios = get_avg_internal_attn(
STACK[layer], layer, elements, beam_settings,
cumulative_upstream_attenuation)
#print(ele_avg_iios)
#iio_arrays.append(ele_avg_iios)
samp_dict_grow.build_dict(sample,
beam_settings['beamtime'] + '_iios',
ele_avg_iios)
else:
print('you have chosen the first layer of the stack.')
print(
'this program is not currently configured to calculate element iios for the first layer.'
)
print(
'please either enter another layer, or look into modifying this function'
)
return
def remove_outliers(samples, dict_data, bad_idx, sigma, new_dict_data):
for sample in samples:
no_outliers = []
for scan_arr in sample[dict_data]:
bad_arr = scan_arr[:, bad_idx]
good_indices = get_limits(bad_arr, sigma)
no_outlier = scan_arr[good_indices]
no_outliers.append(no_outlier)
samp_dict_grow.build_dict(sample, new_dict_data, no_outliers)
return
def stat_arrs(samples, dict_data, new_dict_data):
for sample in samples:
stat_scans = []
for maps in sample[dict_data]:
no_nan_maps = maps[:, :, :-2]
z3Dto2D_rows = np.shape(no_nan_maps)[1] * np.shape(no_nan_maps)[2]
z3Dto2D_cols = np.shape(no_nan_maps)[0]
stat_arrs = no_nan_maps.T.reshape(z3Dto2D_rows, z3Dto2D_cols)
stat_scans.append(stat_arrs)
samp_dict_grow.build_dict(sample, new_dict_data, stat_scans)
return
def unmasked_mapcorr(samp, scans, data_key):
correlations_of_each_scan = []
for scan in scans:
data = samp[data_key][scan]
map_corrcoeffs = np.corrcoef(data.T)
correlations_of_each_scan.append(map_corrcoeffs)
corrs_of_scans_regavg_matrices = np.array(correlations_of_each_scan)
scan_avg = np.mean(corrs_of_scans_regavg_matrices, axis=0)
scan_stdev = np.std(corrs_of_scans_regavg_matrices, axis=0)
samp_dict_grow.build_dict(samp, 'nomask_avg', scan_avg)
samp_dict_grow.build_dict(samp, 'nomask_std', scan_stdev)
return
def get_add_h5s(samps, path):
for samp in samps:
# make list containing full path+file for each scan
# make list containing h5 files
# add h5s to sample dictionary
c_filenames = [(path + '/2idd_0' + scan + '.h5') for scan in str_list(samp['XBIC_scans'])]
xbic_h5s = [h5py.File(file, 'r') for file in c_filenames]
build_dict(samp, 'XBIC_h5s', xbic_h5s)
v_filenames = [(path + '/2idd_0' + scan + '.h5') for scan in str_list(samp['XBIV_scans'])]
xbiv_h5s = [h5py.File(file, 'r') for file in v_filenames]
build_dict(samp, 'XBIV_h5s', xbiv_h5s)
return
def stand_arrs(samples, dict_data, new_dict_data):
scaler = skpp.StandardScaler()
for sample in samples:
stand_scans = []
for stat_arrs in sample[dict_data]:
stand_arrs_sep = [
scaler.fit_transform(column.reshape(-1, 1))
for column in stat_arrs.T
] # is this necessary...? can i apply standardization on whole matrix...
stand_arrs_comb = np.concatenate(stand_arrs_sep, axis=1)
stand_scans.append(stand_arrs_comb)
samp_dict_grow.build_dict(sample, new_dict_data, stand_scans)
return
def import_maps(samps, switch, scaler_ch, elements, flux_norm, fit_norm):
for samp in samps:
## electrical deal
# find electrical scaler channel; store cts/s
if switch == 'XBIC':
raw_maps = [
h5['/MAPS/scalers'][scaler_ch] for h5 in samp['XBIC_h5s']
]
# calc scale factor for each scan (nanoamps to amps for stanford)
scale_factors = [
(stan * 1E-9) / (V2F * lock) for stan, V2F, lock in zip(
samp['c_stanford'], samp['beam_conv'], samp['c_lockin'])
]
# scale cts/s to amperes
elect_maps = [
ds_ic * scale for ds_ic, scale in zip(raw_maps, scale_factors)
]
elif switch == 'XBIV':
raw_maps = [
h5['/MAPS/scalers'][scaler_ch] for h5 in samp['XBIV_h5s']
]
scale_factors = [
1 / (V2F * lock)
for V2F, lock in zip(samp['beam_conv'], samp['v_lockin'])
]
elect_maps = [
ds_ic * scale for ds_ic, scale in zip(raw_maps, scale_factors)
]
## element deal
# get element indices for each scan inside the scan h5
ele_indices = [
get_ele_idxs(elements, file['/MAPS/channel_names'])
for file in samp['XBIC_h5s']
]
# get normalization keys inside h5
flxnorm_idx, nav_keys = get_normalization_keys(flux_norm, fit_norm)
# initialize map storage
scan_maps = []
# normalize each element map (cts/s to ug/cm2)
for scan_h5, scan_elect, scan_eles in zip(samp['XBIC_h5s'], elect_maps,
ele_indices):
norm_ele_maps = [(scan_h5[nav_keys[0]][ele_idx, :, :] /
scan_h5['/MAPS/scalers'][flxnorm_idx, :, :] /
scan_h5[nav_keys[1]][flxnorm_idx, 0, ele_idx])
for ele_idx in scan_eles]
norm_ele_maps = np.array(norm_ele_maps)
all_scan_maps = np.insert(norm_ele_maps, 0, scan_elect, axis=0)
scan_maps.append(all_scan_maps)
samp_dict_grow.build_dict(samp, switch + '_maps', scan_maps)
return
def extract_norm_ele_maps(sample_dicts, fluxnorm, fitnorm):
flxnorm_idx, nav_keys = get_normalization_keys(fluxnorm, fitnorm)
for samp in sample_dicts:
c_ele_maps = []
for h5, ch_inds in zip(samp['XBIC_h5s'], samp['XBIC_eles_i']):
"""Calculate quantified element matrix in ug/cm2"""
maps_of_eles_in_scan = [(h5[nav_keys[0]][elementindex, :, :] / h5['/MAPS/scalers'][flxnorm_idx, :, :] /
h5[nav_keys[1]][flxnorm_idx, 0, elementindex]) for elementindex in ch_inds]
c_ele_maps.append(maps_of_eles_in_scan)
samp_dict_grow.build_dict(samp, 'elXBIC', c_ele_maps)
v_ele_maps = []
for h5, ch_inds in zip(samp['XBIV_h5s'], samp['XBIV_eles_i']):
maps_of_eles_in_scan = [(h5[nav_keys[0]][elementindex, :, :] / h5['/MAPS/scalers'][flxnorm_idx, :, :] /
h5[nav_keys[1]][flxnorm_idx, 0, elementindex]) for elementindex in ch_inds]
v_ele_maps.append(maps_of_eles_in_scan)
samp_dict_grow.build_dict(samp, 'elXBIV', v_ele_maps)
return
def apply_iios(samples, electrical_key, sample_scan_idxs, iio_arr_key,
ele_map_idxs, ele_iio_idxs, new_dict_key):
for sample, scan_idxs in zip(samples, sample_scan_idxs):
correct_scans = []
iio_arr = sample[iio_arr_key]
for scan_idx in scan_idxs:
scan_raw_maps = sample[electrical_key + '_maps'][scan_idx]
correct_maps = scan_raw_maps.copy() # create copy to overwrite
for ele_idx, iio_idx in zip(ele_map_idxs, ele_iio_idxs):
map_to_correct = scan_raw_maps[ele_idx, :, :] # extract map
correct_map = map_to_correct / iio_arr[iio_idx] # correct map
correct_maps[ele_idx, :, :] = correct_map # store map
correct_scans.append(correct_maps)
samp_dict_grow.build_dict(
sample,
new_dict_key, #'{e_key}{date_key}corr'.format(e_key=electrical_key, date_key=iio_arr_key[0:8])
correct_scans)
return
def kmeans_trials(samps, data_key, mask_chan, clust_num, iter_num, new_data_key):
for samp in samps:
scans_models = []
for i, scan_arr in enumerate(samp[data_key]):
model_data = scan_arr[:,mask_chan]
model_data = model_data.reshape(-1,1)
model = KMeans(init='k-means++', n_clusters=clust_num, n_init=10)
models_labels = []
count = 0
while count < iter_num:
model_fit = model.fit(model_data)
model_labels = model_fit.labels_
models_labels.append(model_labels)
count = count+1
models_labels = np.array(models_labels)
scans_models.append(models_labels)
samp_dict_grow.build_dict(samp, new_data_key, scans_models)
return
def make_mol_maps(samples, elements, dict_data, new_dict_data):
elements = [element[0:2] for element in elements]
mol_masses = [
xl.AtomicWeight(xl.SymbolToAtomicNumber(element))
for element in elements
]
#ug/cm2 * (g/ug) * (mol/g) == mol/cm2
conv_factors = [(1 / 1E6) / (1 / mol_mass) for mol_mass in mol_masses]
for sample in samples:
mol_scans = []
for scan_raw_maps in sample[dict_data]:
mol_maps = scan_raw_maps.copy()
for ele_idx, factor in enumerate(conv_factors):
map_idx = ele_idx + 1
ele_map = scan_raw_maps[map_idx, :, :]
mol_map = ele_map * factor
mol_maps[map_idx, :, :] = mol_map
mol_scans.append(mol_maps)
samp_dict_grow.build_dict(sample, new_dict_data, mol_scans)
return
def reduce_arrs_actual(samps, bad_XRF, loaded_XRF, sigma_control,
original_data, new_data):
for samp in samps:
c_reduced_arrs = []
for full_data_array in samp[original_data[0]]:
bad_channel_column_index = get_channel_column_index(
bad_XRF, loaded_XRF) # find bad column
bad_channel_column_index = bad_channel_column_index + 1 # add 1; xbic in position 0
bad_chan_column = full_data_array[:,
bad_channel_column_index] # isolate bad XRF column
lwr_lim = np.mean(bad_chan_column) - sigma_control * np.std(
bad_chan_column) # determine bounds
upr_lim = np.mean(
bad_chan_column) + sigma_control * np.std(bad_chan_column)
good_indices = np.where(
np.logical_and(bad_chan_column >= lwr_lim, bad_chan_column <=
upr_lim)) # get indices within these bounds
reduced_data_array = full_data_array[
good_indices] # take good indices from whole array
c_reduced_arrs.append(reduced_data_array) # store data
samp_dict_grow.build_dict(samp, new_data[0], c_reduced_arrs)
v_reduced_arrs = []
for full_data_array in samp[original_data[1]]:
bad_channel_column_index = get_channel_column_index(
bad_XRF, loaded_XRF) # find bad column
bad_channel_column_index = bad_channel_column_index + 1 # add 1; xbic in position 0
bad_chan_column = full_data_array[:,
bad_channel_column_index] # isolate bad XRF column
lwr_lim = np.mean(bad_chan_column) - sigma_control * np.std(
bad_chan_column) # determine bounds
upr_lim = np.mean(
bad_chan_column) + sigma_control * np.std(bad_chan_column)
good_indices = np.where(
np.logical_and(bad_chan_column >= lwr_lim, bad_chan_column <=
upr_lim)) # get indices within these bounds
reduced_data_array = full_data_array[
good_indices] # take good indices from whole array
v_reduced_arrs.append(reduced_data_array) # store data
samp_dict_grow.build_dict(samp, new_data[1], v_reduced_arrs)
return
def standardize_channels(samps, dict_data, new_keys):
scaler = skpp.StandardScaler()
for samp in samps:
c_standardized_stats = []
for scan_arrays in samp[dict_data[0]]:
c_stand_arrs = [
scaler.fit_transform(column.reshape(-1, 1))
for column in scan_arrays.T
]
combine_stand_arrs_of_scan = np.concatenate(c_stand_arrs, axis=1)
c_standardized_stats.append(combine_stand_arrs_of_scan)
samp_dict_grow.build_dict(samp, new_keys[0], c_standardized_stats)
v_standardized_stats = []
for scan_arrays in samp[dict_data[1]]:
c_stand_arrs = [
scaler.fit_transform(column.reshape(-1, 1))
for column in scan_arrays.T
]
combine_stand_arrs_of_scan = np.concatenate(c_stand_arrs, axis=1)
v_standardized_stats.append(combine_stand_arrs_of_scan)
samp_dict_grow.build_dict(samp, new_keys[1], v_standardized_stats)
return
def make_stat_arrays(samps, dict_data, new_keys):
for samp in samps:
# array XBIC scans first
c_stats_of_scans = []
for scan_i, eles in enumerate(
samp[dict_data[0]]): # samp['elXBIC_corr'][scan][element]
stat_XBIC_map = samp['XBIC_maps'][
scan_i][:, :-2] # get XBIC and chop off nans
held_maps = [ele[:, :-2]
for ele in eles] # get element maps and chop off nans
held_maps.insert(
0, stat_XBIC_map) # combine maps into list with XBIC @ index 0
stats_of_scan = [m.reshape(-1, 1)
for m in held_maps] # array each map
combine_stats_of_scan = np.concatenate(
stats_of_scan, axis=1) # combine the arrays
c_stats_of_scans.append(
combine_stats_of_scan) # add these arrays to sample dictionary
samp_dict_grow.build_dict(samp, new_keys[0],
c_stats_of_scans) # add comb arrs to dict
v_stats_of_scans = []
for scan_i, eles in enumerate(
samp[dict_data[1]]): # samp['elXBIC_corr'][scan][element]
stat_XBIC_map = samp['XBIV_maps'][
scan_i][:, :-2] # get XBIV and chop off nans
held_maps = [ele[:, :-2]
for ele in eles] # get element maps and chop off nans
held_maps.insert(
0, stat_XBIC_map) # combine maps into list with XBIC @ index 0
stats_of_scan = [m.reshape(-1, 1)
for m in held_maps] # array each map
combine_stats_of_scan = np.concatenate(
stats_of_scan, axis=1) # combine the arrays
v_stats_of_scans.append(
combine_stats_of_scan) # add these arrays to sample dictionary
samp_dict_grow.build_dict(samp, new_keys[1],
v_stats_of_scans) # add comb arrs to dict
return
def correlation_stats(samp, scans, data_key, trials_key,
number_of_clusters, focus_cluster_row, focus_channel_col,
new_data_keys):
# rethink storage structure...
sCorrs = []; sPvals = [] ### include 'kcorrs = []' and 'kpvals = []' if you want to store the individual trials (2)
for scan in scans:
real_data = samp[data_key][scan]
kmeans_trials = samp[trials_key][scan]
# returns list of properties from ktrials
trials_list = correlations_of_kmeans_trials(real_data,
kmeans_trials,
number_of_clusters,
focus_cluster_row,
focus_channel_col)
### change these indices if you included individual ktrials (kcorrs, and kpvals) (3)
corrs_stats = trials_list[0:2]; corrs_stats=np.array(corrs_stats) #the average and std of correlations
pvals_stats = trials_list[2:4]; pvals_stats=np.array(pvals_stats) #the average and std of pvalues
sCorrs.append(corrs_stats); sPvals.append(pvals_stats)
sCorrs = np.array(sCorrs); sPvals = np.array(sPvals)
CORRS = np.mean(sCorrs[:,0,:,:], axis=0); PVALS = np.mean(sPvals[:,0,:,:], axis=0) # global average between scans
CORRS_std=np.std(sCorrs[:,0,:,:], axis=0); PVALS_std=np.std(sPvals[:,0,:,:], axis=0) # std dev of global average between scans
samp_dict_grow.build_dict(samp, new_data_keys[0], [CORRS, CORRS_std])
samp_dict_grow.build_dict(samp, new_data_keys[1], [PVALS, PVALS_std])
return