def get_add_electrical(samps, scaler_ch): for samp in samps: # calculate the scale factor (in amperes) according to electrical settings of scan # this for loop takes advantage of the order of # 'c_stanford', 'beam_conv', and 'c_lockin' lists in the sample dicts xbic_scale_factors = [ (stan * 1E-9) / (V2F * lock) for stan, V2F, lock in zip( samp['c_stanford'], samp['beam_conv'], samp['c_lockin']) ] # store scaler containing electrical in list raw_xbic = [h5['/MAPS/scalers'][scaler_ch] for h5 in samp['XBIC_h5s']] # convert scaler channel into amps xbic_scaled = [ ds_ic * scale for ds_ic, scale in zip(raw_xbic, xbic_scale_factors) ] # import XBIC_maps to sample dict build_dict(samp, 'XBIC_maps', xbic_scaled) xbiv_scale_factors = [ 1 / (V2F * lock) for V2F, lock in zip(samp['beam_conv'], samp['c_lockin']) ] raw_xbiv = [h5['/MAPS/scalers'][scaler_ch] for h5 in samp['XBIV_h5s']] #grab xbic channels xbiv_scaled = [ ds_ic * scale for ds_ic, scale in zip(raw_xbiv, xbiv_scale_factors) ] build_dict(samp, 'XBIV_maps', xbiv_scaled) return
def join_corrected_beamtimes(samples, keys, new_key): for sample in samples: all_corr = [sample[key] for key in keys] flatten = lambda l: [item for sublist in l for item in sublist] all_corr = flatten(all_corr) samp_dict_grow.build_dict(sample, new_key, all_corr) return
def get_layer_iios(samples, elements, beam_settings, layer): for sample in samples: STACK = sample['STACK'] layer_idx = list(STACK.keys()).index(layer) if layer_idx != 0: # retrieve info of upstream layers layers_before = { k: v for idx, (k, v) in enumerate(STACK.items()) if idx < layer_idx } # percent incoming beam transmitted to layer upstream_attn_in = get_upstream_iioIN(layers_before, beam_settings) # percent outgoing XRF transmitted by upstream layers upstream_attn_out = get_upstream_iioOUT(layers_before, elements, beam_settings) cumulative_upstream_attenuation = upstream_attn_in * upstream_attn_out # percent outgoing XRF transmitted ele_avg_iios = get_avg_internal_attn( STACK[layer], layer, elements, beam_settings, cumulative_upstream_attenuation) #print(ele_avg_iios) #iio_arrays.append(ele_avg_iios) samp_dict_grow.build_dict(sample, beam_settings['beamtime'] + '_iios', ele_avg_iios) else: print('you have chosen the first layer of the stack.') print( 'this program is not currently configured to calculate element iios for the first layer.' ) print( 'please either enter another layer, or look into modifying this function' ) return
def remove_outliers(samples, dict_data, bad_idx, sigma, new_dict_data): for sample in samples: no_outliers = [] for scan_arr in sample[dict_data]: bad_arr = scan_arr[:, bad_idx] good_indices = get_limits(bad_arr, sigma) no_outlier = scan_arr[good_indices] no_outliers.append(no_outlier) samp_dict_grow.build_dict(sample, new_dict_data, no_outliers) return
def stat_arrs(samples, dict_data, new_dict_data): for sample in samples: stat_scans = [] for maps in sample[dict_data]: no_nan_maps = maps[:, :, :-2] z3Dto2D_rows = np.shape(no_nan_maps)[1] * np.shape(no_nan_maps)[2] z3Dto2D_cols = np.shape(no_nan_maps)[0] stat_arrs = no_nan_maps.T.reshape(z3Dto2D_rows, z3Dto2D_cols) stat_scans.append(stat_arrs) samp_dict_grow.build_dict(sample, new_dict_data, stat_scans) return
def unmasked_mapcorr(samp, scans, data_key): correlations_of_each_scan = [] for scan in scans: data = samp[data_key][scan] map_corrcoeffs = np.corrcoef(data.T) correlations_of_each_scan.append(map_corrcoeffs) corrs_of_scans_regavg_matrices = np.array(correlations_of_each_scan) scan_avg = np.mean(corrs_of_scans_regavg_matrices, axis=0) scan_stdev = np.std(corrs_of_scans_regavg_matrices, axis=0) samp_dict_grow.build_dict(samp, 'nomask_avg', scan_avg) samp_dict_grow.build_dict(samp, 'nomask_std', scan_stdev) return
def get_add_h5s(samps, path): for samp in samps: # make list containing full path+file for each scan # make list containing h5 files # add h5s to sample dictionary c_filenames = [(path + '/2idd_0' + scan + '.h5') for scan in str_list(samp['XBIC_scans'])] xbic_h5s = [h5py.File(file, 'r') for file in c_filenames] build_dict(samp, 'XBIC_h5s', xbic_h5s) v_filenames = [(path + '/2idd_0' + scan + '.h5') for scan in str_list(samp['XBIV_scans'])] xbiv_h5s = [h5py.File(file, 'r') for file in v_filenames] build_dict(samp, 'XBIV_h5s', xbiv_h5s) return
def stand_arrs(samples, dict_data, new_dict_data): scaler = skpp.StandardScaler() for sample in samples: stand_scans = [] for stat_arrs in sample[dict_data]: stand_arrs_sep = [ scaler.fit_transform(column.reshape(-1, 1)) for column in stat_arrs.T ] # is this necessary...? can i apply standardization on whole matrix... stand_arrs_comb = np.concatenate(stand_arrs_sep, axis=1) stand_scans.append(stand_arrs_comb) samp_dict_grow.build_dict(sample, new_dict_data, stand_scans) return
def import_maps(samps, switch, scaler_ch, elements, flux_norm, fit_norm): for samp in samps: ## electrical deal # find electrical scaler channel; store cts/s if switch == 'XBIC': raw_maps = [ h5['/MAPS/scalers'][scaler_ch] for h5 in samp['XBIC_h5s'] ] # calc scale factor for each scan (nanoamps to amps for stanford) scale_factors = [ (stan * 1E-9) / (V2F * lock) for stan, V2F, lock in zip( samp['c_stanford'], samp['beam_conv'], samp['c_lockin']) ] # scale cts/s to amperes elect_maps = [ ds_ic * scale for ds_ic, scale in zip(raw_maps, scale_factors) ] elif switch == 'XBIV': raw_maps = [ h5['/MAPS/scalers'][scaler_ch] for h5 in samp['XBIV_h5s'] ] scale_factors = [ 1 / (V2F * lock) for V2F, lock in zip(samp['beam_conv'], samp['v_lockin']) ] elect_maps = [ ds_ic * scale for ds_ic, scale in zip(raw_maps, scale_factors) ] ## element deal # get element indices for each scan inside the scan h5 ele_indices = [ get_ele_idxs(elements, file['/MAPS/channel_names']) for file in samp['XBIC_h5s'] ] # get normalization keys inside h5 flxnorm_idx, nav_keys = get_normalization_keys(flux_norm, fit_norm) # initialize map storage scan_maps = [] # normalize each element map (cts/s to ug/cm2) for scan_h5, scan_elect, scan_eles in zip(samp['XBIC_h5s'], elect_maps, ele_indices): norm_ele_maps = [(scan_h5[nav_keys[0]][ele_idx, :, :] / scan_h5['/MAPS/scalers'][flxnorm_idx, :, :] / scan_h5[nav_keys[1]][flxnorm_idx, 0, ele_idx]) for ele_idx in scan_eles] norm_ele_maps = np.array(norm_ele_maps) all_scan_maps = np.insert(norm_ele_maps, 0, scan_elect, axis=0) scan_maps.append(all_scan_maps) samp_dict_grow.build_dict(samp, switch + '_maps', scan_maps) return
def extract_norm_ele_maps(sample_dicts, fluxnorm, fitnorm): flxnorm_idx, nav_keys = get_normalization_keys(fluxnorm, fitnorm) for samp in sample_dicts: c_ele_maps = [] for h5, ch_inds in zip(samp['XBIC_h5s'], samp['XBIC_eles_i']): """Calculate quantified element matrix in ug/cm2""" maps_of_eles_in_scan = [(h5[nav_keys[0]][elementindex, :, :] / h5['/MAPS/scalers'][flxnorm_idx, :, :] / h5[nav_keys[1]][flxnorm_idx, 0, elementindex]) for elementindex in ch_inds] c_ele_maps.append(maps_of_eles_in_scan) samp_dict_grow.build_dict(samp, 'elXBIC', c_ele_maps) v_ele_maps = [] for h5, ch_inds in zip(samp['XBIV_h5s'], samp['XBIV_eles_i']): maps_of_eles_in_scan = [(h5[nav_keys[0]][elementindex, :, :] / h5['/MAPS/scalers'][flxnorm_idx, :, :] / h5[nav_keys[1]][flxnorm_idx, 0, elementindex]) for elementindex in ch_inds] v_ele_maps.append(maps_of_eles_in_scan) samp_dict_grow.build_dict(samp, 'elXBIV', v_ele_maps) return
def apply_iios(samples, electrical_key, sample_scan_idxs, iio_arr_key, ele_map_idxs, ele_iio_idxs, new_dict_key): for sample, scan_idxs in zip(samples, sample_scan_idxs): correct_scans = [] iio_arr = sample[iio_arr_key] for scan_idx in scan_idxs: scan_raw_maps = sample[electrical_key + '_maps'][scan_idx] correct_maps = scan_raw_maps.copy() # create copy to overwrite for ele_idx, iio_idx in zip(ele_map_idxs, ele_iio_idxs): map_to_correct = scan_raw_maps[ele_idx, :, :] # extract map correct_map = map_to_correct / iio_arr[iio_idx] # correct map correct_maps[ele_idx, :, :] = correct_map # store map correct_scans.append(correct_maps) samp_dict_grow.build_dict( sample, new_dict_key, #'{e_key}{date_key}corr'.format(e_key=electrical_key, date_key=iio_arr_key[0:8]) correct_scans) return
def kmeans_trials(samps, data_key, mask_chan, clust_num, iter_num, new_data_key): for samp in samps: scans_models = [] for i, scan_arr in enumerate(samp[data_key]): model_data = scan_arr[:,mask_chan] model_data = model_data.reshape(-1,1) model = KMeans(init='k-means++', n_clusters=clust_num, n_init=10) models_labels = [] count = 0 while count < iter_num: model_fit = model.fit(model_data) model_labels = model_fit.labels_ models_labels.append(model_labels) count = count+1 models_labels = np.array(models_labels) scans_models.append(models_labels) samp_dict_grow.build_dict(samp, new_data_key, scans_models) return
def make_mol_maps(samples, elements, dict_data, new_dict_data): elements = [element[0:2] for element in elements] mol_masses = [ xl.AtomicWeight(xl.SymbolToAtomicNumber(element)) for element in elements ] #ug/cm2 * (g/ug) * (mol/g) == mol/cm2 conv_factors = [(1 / 1E6) / (1 / mol_mass) for mol_mass in mol_masses] for sample in samples: mol_scans = [] for scan_raw_maps in sample[dict_data]: mol_maps = scan_raw_maps.copy() for ele_idx, factor in enumerate(conv_factors): map_idx = ele_idx + 1 ele_map = scan_raw_maps[map_idx, :, :] mol_map = ele_map * factor mol_maps[map_idx, :, :] = mol_map mol_scans.append(mol_maps) samp_dict_grow.build_dict(sample, new_dict_data, mol_scans) return
def reduce_arrs_actual(samps, bad_XRF, loaded_XRF, sigma_control, original_data, new_data): for samp in samps: c_reduced_arrs = [] for full_data_array in samp[original_data[0]]: bad_channel_column_index = get_channel_column_index( bad_XRF, loaded_XRF) # find bad column bad_channel_column_index = bad_channel_column_index + 1 # add 1; xbic in position 0 bad_chan_column = full_data_array[:, bad_channel_column_index] # isolate bad XRF column lwr_lim = np.mean(bad_chan_column) - sigma_control * np.std( bad_chan_column) # determine bounds upr_lim = np.mean( bad_chan_column) + sigma_control * np.std(bad_chan_column) good_indices = np.where( np.logical_and(bad_chan_column >= lwr_lim, bad_chan_column <= upr_lim)) # get indices within these bounds reduced_data_array = full_data_array[ good_indices] # take good indices from whole array c_reduced_arrs.append(reduced_data_array) # store data samp_dict_grow.build_dict(samp, new_data[0], c_reduced_arrs) v_reduced_arrs = [] for full_data_array in samp[original_data[1]]: bad_channel_column_index = get_channel_column_index( bad_XRF, loaded_XRF) # find bad column bad_channel_column_index = bad_channel_column_index + 1 # add 1; xbic in position 0 bad_chan_column = full_data_array[:, bad_channel_column_index] # isolate bad XRF column lwr_lim = np.mean(bad_chan_column) - sigma_control * np.std( bad_chan_column) # determine bounds upr_lim = np.mean( bad_chan_column) + sigma_control * np.std(bad_chan_column) good_indices = np.where( np.logical_and(bad_chan_column >= lwr_lim, bad_chan_column <= upr_lim)) # get indices within these bounds reduced_data_array = full_data_array[ good_indices] # take good indices from whole array v_reduced_arrs.append(reduced_data_array) # store data samp_dict_grow.build_dict(samp, new_data[1], v_reduced_arrs) return
def standardize_channels(samps, dict_data, new_keys): scaler = skpp.StandardScaler() for samp in samps: c_standardized_stats = [] for scan_arrays in samp[dict_data[0]]: c_stand_arrs = [ scaler.fit_transform(column.reshape(-1, 1)) for column in scan_arrays.T ] combine_stand_arrs_of_scan = np.concatenate(c_stand_arrs, axis=1) c_standardized_stats.append(combine_stand_arrs_of_scan) samp_dict_grow.build_dict(samp, new_keys[0], c_standardized_stats) v_standardized_stats = [] for scan_arrays in samp[dict_data[1]]: c_stand_arrs = [ scaler.fit_transform(column.reshape(-1, 1)) for column in scan_arrays.T ] combine_stand_arrs_of_scan = np.concatenate(c_stand_arrs, axis=1) v_standardized_stats.append(combine_stand_arrs_of_scan) samp_dict_grow.build_dict(samp, new_keys[1], v_standardized_stats) return
def make_stat_arrays(samps, dict_data, new_keys): for samp in samps: # array XBIC scans first c_stats_of_scans = [] for scan_i, eles in enumerate( samp[dict_data[0]]): # samp['elXBIC_corr'][scan][element] stat_XBIC_map = samp['XBIC_maps'][ scan_i][:, :-2] # get XBIC and chop off nans held_maps = [ele[:, :-2] for ele in eles] # get element maps and chop off nans held_maps.insert( 0, stat_XBIC_map) # combine maps into list with XBIC @ index 0 stats_of_scan = [m.reshape(-1, 1) for m in held_maps] # array each map combine_stats_of_scan = np.concatenate( stats_of_scan, axis=1) # combine the arrays c_stats_of_scans.append( combine_stats_of_scan) # add these arrays to sample dictionary samp_dict_grow.build_dict(samp, new_keys[0], c_stats_of_scans) # add comb arrs to dict v_stats_of_scans = [] for scan_i, eles in enumerate( samp[dict_data[1]]): # samp['elXBIC_corr'][scan][element] stat_XBIC_map = samp['XBIV_maps'][ scan_i][:, :-2] # get XBIV and chop off nans held_maps = [ele[:, :-2] for ele in eles] # get element maps and chop off nans held_maps.insert( 0, stat_XBIC_map) # combine maps into list with XBIC @ index 0 stats_of_scan = [m.reshape(-1, 1) for m in held_maps] # array each map combine_stats_of_scan = np.concatenate( stats_of_scan, axis=1) # combine the arrays v_stats_of_scans.append( combine_stats_of_scan) # add these arrays to sample dictionary samp_dict_grow.build_dict(samp, new_keys[1], v_stats_of_scans) # add comb arrs to dict return
def correlation_stats(samp, scans, data_key, trials_key, number_of_clusters, focus_cluster_row, focus_channel_col, new_data_keys): # rethink storage structure... sCorrs = []; sPvals = [] ### include 'kcorrs = []' and 'kpvals = []' if you want to store the individual trials (2) for scan in scans: real_data = samp[data_key][scan] kmeans_trials = samp[trials_key][scan] # returns list of properties from ktrials trials_list = correlations_of_kmeans_trials(real_data, kmeans_trials, number_of_clusters, focus_cluster_row, focus_channel_col) ### change these indices if you included individual ktrials (kcorrs, and kpvals) (3) corrs_stats = trials_list[0:2]; corrs_stats=np.array(corrs_stats) #the average and std of correlations pvals_stats = trials_list[2:4]; pvals_stats=np.array(pvals_stats) #the average and std of pvalues sCorrs.append(corrs_stats); sPvals.append(pvals_stats) sCorrs = np.array(sCorrs); sPvals = np.array(sPvals) CORRS = np.mean(sCorrs[:,0,:,:], axis=0); PVALS = np.mean(sPvals[:,0,:,:], axis=0) # global average between scans CORRS_std=np.std(sCorrs[:,0,:,:], axis=0); PVALS_std=np.std(sPvals[:,0,:,:], axis=0) # std dev of global average between scans samp_dict_grow.build_dict(samp, new_data_keys[0], [CORRS, CORRS_std]) samp_dict_grow.build_dict(samp, new_data_keys[1], [PVALS, PVALS_std]) return