def householder_reflection(col): n_height = len(col) m_matrix = np.eye(n_height, 1) I = np.eye(n_height) v = np.array([x + y for x, y in zip(col, norm(col) * m_matrix)]) u = v / norm(v) return I - 2 * u * u.T
def calc_mom_TT(step, prevstep): moment = helpers.norm( helpers.list_mult_num([ step["x"] - prevstep["x"], step["y"] - prevstep["y"], step["z"] - prevstep["z"] ], 0.5)) return moment
def run_step(self, prev, params): data = prev df = pd.read_excel(data, sheet_name='10.2') df.columns = [ norm(x.strip().lower().replace(' ', '_').replace('-', '_').replace( '%', 'perc')) for x in df.columns ] print(df.columns) df.columns = [ 'year', 'country', 'investment_type', 'value', 'count', 'value_c' ] df = df.loc[~df['year'].astype(str).str.contains('Total')].copy() df = df.loc[df['value_c'] != 'C'].copy() df.drop(columns=['value'], inplace=True) dim_country = get_dimensions()[1] df['country'].replace(COUNTRY_REPLACE, inplace=True) df['country'] = df['country'].replace( dict(zip(dim_country['country_name_es'], dim_country['iso3']))) df[['year', 'count', 'value_c']] = df[['year', 'count', 'value_c']].astype(float) df['investment_type'].replace(INVESTMENT_TYPE, inplace=True) return df
def NRM_dir_stat(sc): """ Function to calculate the NRM direction statistics, DANG, free floating direction and NRMdev. input: Mdec_free,Minc_free, x, y, z, Yint output: DANG, NRMdev, dir_vec_free """ # input: directional_statistics/ mean_dir_stat[Mdec_free,Minc_free, x, y, z] # arai_statistics/ intercept_stats[Yint] # output: directional_statistics/ NRM_dir_stat[DANG, NRMdev, dir_vec_free] x = sc["directional_statistics"]["mean_dir_stat"]["x"] y = sc["directional_statistics"]["mean_dir_stat"]["y"] z = sc["directional_statistics"]["mean_dir_stat"]["z"] Mdec_free = sc["directional_statistics"]["mean_dir_stat"]["Mdec_free"] Minc_free = sc["directional_statistics"]["mean_dir_stat"]["Minc_free"] Yint = sc["arai_statistics"]["intercept_stats"]["Yint"] mean_x = sum(x) / len(x) mean_y = sum(y) / len(y) mean_z = sum(z) / len(z) center_of_mass = [mean_x, mean_y, mean_z] # calculate DANG & NRMdev dir_vec_free = helpers.dir2cart(Mdec_free, Minc_free, 1) DANG = helpers.get_angle_diff(dir_vec_free, center_of_mass) NRMdev = ((math.sin(math.radians(DANG)) * helpers.norm(center_of_mass)) / abs(Yint)) * 100 sc["directional_statistics"]["NRM_dir_stat"]["dir_vec_free"] = dir_vec_free sc["directional_statistics"]["NRM_dir_stat"]["DANG"] = DANG sc["directional_statistics"]["NRM_dir_stat"]["NRMdev"] = NRMdev return sc
def run_step(self, prev, params): data = prev df = pd.read_excel(data, sheet_name='3') df.columns = [ norm(x.strip().lower().replace(' ', '_').replace('-', '_').replace( '%', 'perc')) for x in df.columns ] df = df.loc[~df['entidad_federativa'].str.contains('Total')].copy() # get country, end_id dimensions dim_geo, dim_country = get_dimensions() df['entidad_federativa'].replace(dict( zip(dim_geo['ent_name'], dim_geo['ent_id'])), inplace=True) df['pais'].replace(COUNTRY_REPLACE, inplace=True) df['pais'] = df['pais'].replace( dict(zip(dim_country['country_name_es'], dim_country['iso3']))).replace(missing_replacements) df.columns = ['ent_id', 'year', 'country', 'value', 'count', 'value_c'] df.drop(columns=['value'], inplace=True) df = df.loc[df['value_c'].astype(str).str.lower() != 'c'].copy() df['value_c'] = df['value_c'].astype(float) return df
def run_step(self, prev, params): data = prev df = pd.read_excel(data, sheet_name='9.5') df.columns = [ norm(x.strip().lower().replace(' ', '_').replace('-', '_').replace( '%', 'perc')) for x in df.columns ] print(df.columns) df.columns = [ 'year', 'quarter', 'value_between_companies', 'value_new_investments', 'value_re_investments', 'count_between_companies', 'count_new_investments', 'count_re_investments', 'value_between_companies_c', 'value_new_investments_c', 'value_re_investments_c' ] df = df.loc[~df['year'].astype(str).str.contains('Total')].copy() df['quarter_id'] = df['year'].astype(str) + df['quarter'].astype( int).astype(str) df.drop(columns=[ 'year', 'quarter', 'value_between_companies', 'value_new_investments', 'value_re_investments' ], inplace=True) for col in df.columns: df[col] = df[col].astype(float) base = ['quarter_id'] df_final = pd.DataFrame() for option in [ 'between_companies', 'new_investments', 're_investments' ]: temp = df[base + ['count_{}'.format(option), 'value_{}_c'.format(option) ]].copy() temp.columns = ['quarter_id', 'count', 'value_c'] temp.dropna(subset=['value_c'], inplace=True) temp['investment_type'] = option df_final = df_final.append(temp) df = df_final.copy() df['investment_type'].replace(INVESTMENT_TYPE, inplace=True) return df
def H(self, idx=None, heliocentric=False): if idx is None: T = sum(map(lambda body: body.kinetic_energy(), self.particles)) else: T = self.particles[idx].kinetic_energy() U = 0 # self-potential energy for i in range(len(self.particles)): if idx is not None and idx != i: continue # skip if not specific particle for j in range(i + 1, len(self.particles)): if heliocentric and self.particles[j].name.lower() != "sun": continue denom = norm(self.particles[i].qx - self.particles[j].qx, self.particles[i].qy - self.particles[j].qy) U -= G * self.particles[i].m * self.particles[j].m / denom return T + U
def VDS_stats(sc): """ Vector difference sum (VDS) calculations. These use the entire NRM vector. calculated here: the VDS, FRAC and GAP-MAX. FRAC and GAP-MAX proposed by (Shaar and Tauxe, 2013). FRAC is the NRM fraction used for the best-fit line. GAP-MAX is the maximum gap between two points. input: The entire NRM vector and the NRM vector of the selection output: VDS, FRAC, GAP_MAX """ # input: preprocessed/ basics[NRM_vec_all, NRM_vec_select] # output: arai_statistics/ VDS_stats[VDS, FRAC, GAP_MAX] NRM_vec_select = sc["preprocessed"]["basics"]["NRM_vec_select"] NRM_vec_all = sc["preprocessed"]["basics"]["NRM_vec_all"] # calculate difference vector, use a helper function def calc_diff(MagVec): diff = [] for i in range(len(MagVec) - 1): diff.append(helpers.difference(MagVec[i], MagVec[i + 1])) return diff diff_total_NRM = calc_diff( NRM_vec_all ) # this is only the second part of the equation, need to add NRMmax # calculate norm of the last NRM step, the NRM_max NRM_max = helpers.norm(NRM_vec_all[len(NRM_vec_all) - 1]) diff_total_NRM.append(NRM_max) # append NRM max tot the VDS vector VDS = sum(diff_total_NRM) # take the sum of all elements # Calculation of FRAC # calculate the difference vector for the correct selection # NRM_vec_select is the vector only containing the NRM of the selection of interest diff_select_NRM = calc_diff(NRM_vec_select) FRAC = 0 FRAC = sum(diff_select_NRM) / VDS # Calculation of GAP-MAX GAP_MAX = 0 GAP_MAX = max(diff_select_NRM) / sum(diff_select_NRM) sc["arai_statistics"]["VDS_stats"]["VDS"] = VDS sc["arai_statistics"]["VDS_stats"]["FRAC"] = FRAC sc["arai_statistics"]["VDS_stats"]["GAP_MAX"] = GAP_MAX return sc
def global_process_batch(log_probs, original_tensors, batch_size, is_trt=True): '''Accumulates prediction evaluations for batches across an epoch is_trt determines which global dictionary will be used. To get WER at any point, use global_process_epoch. For one-off WER evaluations, use get_results() ''' # State-based approach for full WER comparison across a dataset. greedy_decoder = GreedyCTCDecoder() predicts = norm(greedy_decoder(log_probs=log_probs)) values_dict = dict( predictions=[predicts], transcript=[original_tensors[2][0:batch_size, ...]], transcript_length=[original_tensors[3][0:batch_size, ...]], ) dict_to_process = _global_trt_dict if is_trt else _global_pyt_dict process_evaluation_batch(values_dict, dict_to_process, labels=get_vocab())
def Additivity_check_stat(sc): """ Function to calculate additivity statistics. The number of checks (n_add) and the maximum absolute additivity check difference normalized by the total TRM (obtained from the intersection of the best-fit line and the x-axis on an Arai plot; Leonhardt et al., 2004a). input: additivity check info (add_check_steps, add_check_step, add_check_vec), info for the pTRM (x_ptrm_all, x_temp_all), the intercept with the x-asix (Xint) output: n_add, d_AC """ # input: preprocessed/ msrmnts[add_check_steps] # checks[add_check_step, add_check_vec] # basics[x_ptrm_all, x_temp_all] # arai_statistics/ intercept_stats[Xint] # output: check_statistics/ Additivity_check[n_add, d_AC] # Determine the Additivity check for all checks with Ti < Tmax, that is, all checks that are in the selection, use the ptrm of all steps add_check_steps = sc["preprocessed"]["msrmnts"]["add_check_steps"] add_check_step = sc["preprocessed"]["checks"]["add_check_step"] add_check_vec = sc["preprocessed"]["checks"]["add_check_vec"] x_ptrm_all = sc["preprocessed"]["basics"]["x_ptrm_all"] x_temp_all = sc["preprocessed"]["basics"]["x_temp_all"] Xint = sc["arai_statistics"]["intercept_stats"]["Xint"] n_add = [] d_AC = [] if len(add_check_steps) != 0: # then do calculations AC = [] for i in range(len(add_check_vec)): for j in range(len(x_ptrm_all)): if add_check_step[i][0] == x_temp_all[ j]: # step for the add check and the ptrm should be the same AC.append(helpers.norm(add_check_vec[i]) - x_ptrm_all[j]) n_add = len(AC) AC_abs = [] for l in range(len(AC)): AC_abs.append(abs(AC[l])) d_AC = 100 * max(AC_abs) / abs(Xint) sc["check_statistics"]["Additivity_check"]["n_add"] = n_add sc["check_statistics"]["Additivity_check"]["d_AC"] = d_AC return sc
def run_step(self, prev, params): data = prev df = pd.read_excel(data, sheet_name='9.2') df.columns = [ norm(x.strip().lower().replace(' ', '_').replace('-', '_').replace( '%', 'perc')) for x in df.columns ] print(df.columns) df.columns = ['year', 'value', 'count', 'value_c'] df['year'] = df['year'].astype(str) df = df.loc[~df['year'].str.contains('Total')].copy() df.drop(columns=['value'], inplace=True) for col in ['count', 'year']: df[col] = df[col].astype(int) return df
def neural_net_gui(game, board, model, start, fast): feat = copy.copy(game) i = 0 while (1): out = model.predict(feat.reshape((1, 9, 9, 1))) out = out.squeeze() pred = np.argmax(out, axis=1).reshape((9, 9)) + 1 prob = np.around(np.max(out, axis=1).reshape((9, 9)), 2) feat = denorm(feat).reshape((9, 9)) mask = (feat == 0) if (mask.sum() == 0): break prob_new = prob * mask ## involve constraints here - further processing on prob ind = np.argmax(prob_new) x, y = (ind // 9), (ind % 9) val = pred[x][y] feat[x][y] = val feat = norm(feat) i += 1 ''' if i == 1: print(i) print('pred:', pred) print('prob:', prob_new) print() ''' cnn_gui(val, x, y, board, round(time.time() - start)) if not fast: pygame.time.delay(100)
def get_results(log_probs, original_tensors, batch_size): ''' Returns WER and predictions for the outputs of the acoustic model Used for one-off batches. Epoch-wide evaluation should use global_process_batch and global_process_epoch ''' # Used to get WER and predictions for one-off batches greedy_decoder = GreedyCTCDecoder() predicts = norm(greedy_decoder(log_probs=log_probs)) values_dict = dict( predictions=[predicts], transcript=[original_tensors[2][0:batch_size, ...]], transcript_length=[original_tensors[3][0:batch_size, ...]], ) temp_dict = { 'predictions': [], 'transcripts': [], } process_evaluation_batch(values_dict, temp_dict, labels=get_vocab()) predictions = temp_dict['predictions'] wer, _ = process_evaluation_epoch(temp_dict) return wer, predictions
def basics_pTh(sc): """ Function to calculate more basics from the ARM_dem_steps and ARM_acq_steps_all for the pseudo-Thellier method [NAA protocol only, NRMdemag, ARM-acquisition, ARMdemag]. This addes on the previous function that calculates the NRM demag basics. input: the ARM_dem_steps and ARM_acq_steps_all vectors with the "step", "x", "y", and "z" values output: basic vectors and lists for the ARM demag """ # input: preprocessed/ msrmnts[ARM_dem_steps, ARM_acq_steps_all] # output: preprocessed/ basics_pTh[arm_step, y_arm_d, ARMd_vec,yBar_arm ] ARM_dem_steps = sc["preprocessed"]["msrmnts"][ "ARM_dem_steps"] # pseudo Thellier NAA format ARM_acq_steps_all = sc["preprocessed"]["msrmnts"]["ARM_acq_steps_all"] if len(ARM_dem_steps) != 0: # pseudo-Thellier NAA x0 = ARM_acq_steps_all[0]["x"] y0 = ARM_acq_steps_all[0]["y"] z0 = ARM_acq_steps_all[0]["z"] y_arm_d = [] ARMd_vec = [] arm_step = [] for dstep in ARM_dem_steps: ARMd_vec.append( [dstep["x"] - x0, dstep["y"] - y0, dstep["z"] - z0]) y_arm_d.append( helpers.norm( [dstep["x"] - x0, dstep["y"] - y0, dstep["z"] - z0])) arm_step.append(dstep["step"]) # calculate average yBar_arm = sum(y_arm_d) / len(y_arm_d) sc["preprocessed"]["basics_pTh"]["arm_step"] = arm_step sc["preprocessed"]["basics_pTh"]["y_arm_d"] = y_arm_d sc["preprocessed"]["basics_pTh"]["ARMd_vec"] = ARMd_vec sc["preprocessed"]["basics_pTh"]["yBar_arm"] = yBar_arm return sc
def params_noCorr_corr(c_str, alpha, m0, m1, m2, m3, m4): Q_DB = [] Q_DSC = [] mu_ds = [] H_max = [] H_est = [] Eps_alt = [] Eps_alt_abs = [] Err_alt = [] Err_ds = [] Err_total = [] Err_alt_abs = [] num_specimens = len(m0) for i in range(num_specimens): name = m0[i]["specimen"] # specimen name is the same for m0-m4 H_lab = m1[i][ "lab_field"] # the x-axis coordinate is the AF field of the m1-m4 steps # do the calculations m_m0 = m0[i]["total_m"] m_m1 = m1[i]["total_m"] m_m2 = m2[i]["total_m"] m_m3 = m3[i]["total_m"] m_m4 = m4[i]["total_m"] if (m_m2 == None) or (m_m3 == None) or (m_m4 == None): Q_DB.append([name, H_lab, (m_m1 - m_m0) / m_m0]) Q_DSC.append([name, None, None]) Eps_alt.append([name, None]) else: # calculate also the Q_DSC ratio and all the parameters Q_DB.append([name, H_lab, (m_m1 - m_m0) / m_m0 ]) # no corrected version, only "normal version" # first check for the corrected version of un-corrected version for Q_DSC & parameter calculations if (c_str == "_corr"): # calculate corrections m0M = [m0[i]["x"], m0[i]["y"], m0[i]["z"]] m1M = [m1[i]["x"], m1[i]["y"], m1[i]["z"]] m2M = [m2[i]["x"], m2[i]["y"], m2[i]["z"]] m3M = [m3[i]["x"], m3[i]["y"], m3[i]["z"]] m4M = [m4[i]["x"], m4[i]["y"], m4[i]["z"]] NRMrem = helpers.list_mult_num( helpers.list_plus_list(m1M, m2M), 0.5) m1pTRM = helpers.list_min_list(m1M, NRMrem) m2pTRM = helpers.list_min_list(m2M, NRMrem) m3pTRM = helpers.list_min_list(m3M, NRMrem) m4pTRM = helpers.list_min_list(m4M, NRMrem) m_m0 = m0[i]["total_m"] # m_m0_corr m_m1 = helpers.norm(NRMrem) + helpers.norm( m1pTRM) # m_m1_corr m_m2 = helpers.norm(NRMrem) - helpers.norm( m2pTRM) # exception to the rule m_m3 = helpers.norm(NRMrem) + helpers.norm(m3pTRM) m_m4 = helpers.norm(NRMrem) + helpers.norm(m4pTRM) Q_DSC.append([ name, H_lab, 2 * ((1 + alpha) * m_m1 - m_m0 - alpha * m_m3) / (2 * m_m0 - m_m1 - m_m2) ]) Eps = (m_m4 - m_m1) / m_m1 Eps_alt.append([name, Eps]) sc["MSP_Q_calc" + c_str]["Q_DB"] = Q_DB sc["MSP_Q_calc" + c_str]["Q_DSC"] = Q_DSC sc["MSP_Q_calc" + c_str]["Eps_alt"] = Eps_alt
def dpal_ptrm_check_stat(sc): """ Function to calculate dpal statistic. A measure of cumulative alteration determined by the difference of the alteration corrected intensity estimate (Valet et al., 1996) and the uncorrected estimate, normalized by the uncorrected estimate (Leonhardt et al., 2004a). input: slope of the best-fit line, ptrm_checks, ptrm, y_nrm, yBar output: d_pal """ # input: preprocessed/ checks[ptrm_check] # preprocessed/ basics[ptrm, y_nrm, yBar] # arai_statistics/ PI_est[b_slope] # output: check_statistics/ dpal_ptrm_check_stat[d_pal] ptrm_check = sc["preprocessed"]["checks"]["ptrm_check"] ptrm = sc["preprocessed"]["basics"]["ptrm"] y_nrm = sc["preprocessed"]["basics"]["y_nrm"] yBar = sc["preprocessed"]["basics"]["yBar"] b_slope = sc["arai_statistics"]["PI_est"]["b_slope"] d_pal = [] dT = [] # first check is you have ptrm_checks in the data, if not do nothing if len(ptrm_check) != 0: # first determine d_pTRM, which is the vector difference between ptrm - ptrmcheck for each step, if no ptrm check performed this is 0, 0, 0 s = 0 sprev = 0 for T in ptrm: # look trough the calculated ptrm gained specimen data for C in ptrm_check: if (T["step"] == C["step"]): dT.append( [T["x"] - C["x"], T["y"] - C["y"], T["z"] - C["z"]]) s += 1 if s <= sprev: dT.append([0, 0, 0]) sprev = s # make the cumulative sum vector C C = [] Ci = [0, 0, 0] for i in range(len(ptrm)): Ci = helpers.list_plus_list(Ci, dT[i]) C.append(Ci) ptrm_list = [] # add C to TRM for T in ptrm: ptrm_list.append([T["x"], T["y"], T["z"]]) TRM_star = [] for j in range(len(ptrm_list)): TRM_star.append(helpers.list_plus_list(ptrm_list[j], C[j])) x_ptrm_star = [] for i in range(len(TRM_star)): x_ptrm_star.append(helpers.norm(TRM_star[i])) # calculate the "new slope" # copy form sc_arai_statiscics n = len(x_ptrm_star) xBar_star = sum(x_ptrm_star) / len(x_ptrm_star) # Part (1) of b_slope equation sum_xy = 0 for i in range(0, len(x_ptrm_star)): sum_xy += (x_ptrm_star[i] - xBar_star) * (y_nrm[i] - yBar) if sum_xy < 0: sign = -1 elif sum_xy > 0: sign = 1 else: sign = 0 # part (2) of b_slope equation sumx en sumy sumx = 0 sumy = 0 for i in range(0, len(x_ptrm_star)): sumx += (x_ptrm_star[i] - xBar_star)**2 sumy += (y_nrm[i] - yBar)**2 # part(1) * part(2) gives b_slope b_slope_star = sign * math.sqrt(sumy / sumx) ## stop copy d_pal = abs((b_slope - b_slope_star) / b_slope) * 100 sc["check_statistics"]["dpal_ptrm_check_stat"]["d_pal"] = d_pal return sc
def Tail_check_dtstar_stat(sc): """ Function to calculate the extent of a pTRM tail after correction for angular dependence (Leonhardt et al., 2004a; 2004b). input: y_tail_check, y_temp_tail_check, y_nrm, y_nrm_all, x_temp_all, y_temp_all, NRM_vec_select, NRM_vec_all, xy_temp, field_dir_vec, Yint, Xint, b_slope output: dt_star """ # input: preprocessed/ checks[y_tail_check, y_temp_tail_check] # basics[y_nrm, y_nrm_all, x_temp_all, y_temp_all, NRM_vec_select, NRM_vec_all, xy_temp] # field_basics[field_dir_vec] # arai_statistics/ intercept_stats[Yint, Xint] # PI_est[b_slope] # output: check_statistics/ Tail_check_stat[dt_star] tail_check_vec = sc["preprocessed"]["checks"]["tail_check_vec"] y_temp_tail_check = sc["preprocessed"]["checks"]["y_temp_tail_check"] y_nrm = sc["preprocessed"]["basics"]["y_nrm"] y_nrm_all = sc["preprocessed"]["basics"]["y_nrm_all"] x_temp_all = sc["preprocessed"]["basics"]["x_temp_all"] y_temp_all = sc["preprocessed"]["basics"]["y_temp_all"] xy_temp = sc["preprocessed"]["basics"]["xy_temp"] NRM_vec_select = sc["preprocessed"]["basics"]["NRM_vec_select"] NRM_vec_all = sc["preprocessed"]["basics"]["NRM_vec_all"] field_dir_vec = sc["preprocessed"]["field_basics"]["field_dir_vec"] b_slope = sc["arai_statistics"]["PI_est"]["b_slope"] Yint = sc["arai_statistics"]["intercept_stats"]["Yint"] Xint = sc["arai_statistics"]["intercept_stats"]["Xint"] dt_star = [] t_star = [] if len(y_temp_tail_check ) != 0: # then you have at least one tailcheck, do calculations # make an index matrix for the steps that step tail = step nrm_all ind = [] for i in range(len(y_temp_tail_check)): # number of tail checks for j in range(len(y_temp_all)): # loop over all NRM steps if y_temp_tail_check[i] == y_temp_all[ j]: # if step of tail check equals step op NRM, write index to "ind" this is used below ind.append(j) # do calculations per tail check step for i in range(len(y_temp_tail_check)): MDx = tail_check_vec[i][0] MDy = tail_check_vec[i][1] MDz = tail_check_vec[i][2] Nx = NRM_vec_all[ind[i]][ 0] # take the NRM vector with same step as tail_check Ny = NRM_vec_all[ind[i]][1] Nz = NRM_vec_all[ind[i]][2] a = helpers.list_div_num(NRM_vec_all[ind[i]], y_nrm_all[ ind[i]]) # divide the NRM list by the corresponding moment theta_dt = helpers.get_angle_diff( a, field_dir_vec ) # get angle between nomalized NRM vector and field direction # calculate dH and dZ and inc_diff # according to orientation of Blab if abs(field_dir_vec[0] / helpers.norm(field_dir_vec)) == 1: # then allong x dH = math.sqrt(Ny**2 + Nz**2) - math.sqrt(MDy**2 + MDz**2) dZ = Nx - MDx Dec_F, Inc_F, R_F = helpers.cart2dir(field_dir_vec[0], field_dir_vec[1], field_dir_vec[2]) Dec_N, Inc_N, R_N = helpers.cart2dir(Nx, Ny, Nz) inc_diff = Inc_F - Inc_N elif abs(field_dir_vec[1] / helpers.norm(field_dir_vec)) == 1: # then allong y dH = math.sqrt(Nx**2 + Nz**2) - math.sqrt(MDx**2 + MDz**2) dZ = Ny - MDy Dec_F, Inc_F, R_F = helpers.cart2dir(field_dir_vec[0], field_dir_vec[1], field_dir_vec[2]) Dec_N, Inc_N, R_N = helpers.cart2dir(Nx, Ny, Nz) inc_diff = Inc_F - Inc_N elif abs(field_dir_vec[2] / helpers.norm(field_dir_vec)) == 1: # then allong z dH = math.sqrt(Nx**2 + Ny**2) - math.sqrt(MDx**2 + MDy**2) dZ = Nz - MDz Dec_F, Inc_F, R_F = helpers.cart2dir(field_dir_vec[0], field_dir_vec[1], field_dir_vec[2]) Dec_N, Inc_N, R_N = helpers.cart2dir(Nx, Ny, Nz) inc_diff = Inc_F - Inc_N else: # labfiels is not along any of the principal axis, do not calulate dt* dH = 9999 if dH != 9999: # if it is, the labfield is not allong pinciple axix and no calculations are there B = dH / math.tan(theta_dt) Lim_upp = 2968 # rad *1000 -> 2.968 rad = 170 degrees Lim_low = 175 # rad *1000 -> 0.175 rad = 10 degrees # check if data is within limits # floor rounds to the nearest small integer if (math.floor(theta_dt * 1000)) < Lim_upp and (math.floor( theta_dt * 1000)) > Lim_low: if inc_diff > 0: t_star.append( (-dZ + B) * abs(b_slope) * 100 / abs(Yint)) else: t_star.append( (dZ - B) * abs(b_slope) * 100 / abs(Yint)) if (math.floor(theta_dt * 1000)) < Lim_low: t_star.append(0) if (math.floor(theta_dt * 1000)) > Lim_upp: t_star.append(-dZ * 100 / (abs(Yint) + abs(Xint))) if max(t_star) > 0: dt_star = max(t_star) else: dt_star = 0 sc["check_statistics"]["Tail_check_stat"]["dt_star"] = dt_star return sc
def boostrap(Q_str, c_str, site, selection, alpha, NumCycles, Confidence): # split in measurements m0 m1 m2 m3 m4 with multiple specimens per list m0 = list(filter(lambda m: m['type'] == 0, selection)) m1 = list(filter(lambda m: m['type'] == 1, selection)) m2 = list(filter(lambda m: m['type'] == 2, selection)) m3 = list(filter(lambda m: m['type'] == 3, selection)) m4 = list(filter(lambda m: m['type'] == 4, selection)) m1_all = list(filter(lambda m: m['type'] == 1, site)) # get the steps for the labfield array, this is done by looking at all the data from one site and find the min and maximum used labfields. fields = [] num_specimens = len(m1_all) # all the data and not only the selection for j in range(num_specimens): fields.append(m1_all[j]["lab_field"]) # append all used labfields # find min and max labfield used and determine the step minField = min(fields) maxField = max(fields) numsteps = 11 # Moster et al., 2015 shows that 11 lab steps give the best results step = (minField + maxField) / (numsteps - 1. ) # This is (Hmin+Hmax)/10 # append the step to a list of labfields -> Hlist Hlist = [] for i in range(numsteps): Hlist.append(i * step) # set minimum standard deviation for Hlab stdevH_min = 10 N2 = [] stdevHl = [] aa = [] bb = [] intercept = [] H0 = [] H1 = [] H2 = [] H3 = [] H4 = [] H5 = [] H6 = [] H7 = [] H8 = [] H9 = [] H10 = [] m = 0 killCounter = 0 while m < (NumCycles) and killCounter < (NumCycles * 5): Hlab_DB = [] Hlab_DSC = [] Q_DB_error = [] Q_DSC_error = [] num_specimens = len(m0) for j in range(num_specimens): # get N times a random specimen # get the index of a random specimen i = int(helpers.rand_num() * num_specimens) # random number between 0 & N # get moment per random specimen m_m0 = m0[i]["total_m"] m_m1 = m1[i]["total_m"] # get corresponding error for that specimen, for Q_DB only m0 & m1 e_m0 = m0[i]["error"] e_m1 = m1[i]["error"] # calculate new m0_err and m1_err to calculate new Q_DB_error frac_m0 = helpers.rand_num() * (0.02 * e_m0) + 1 - 0.01 * e_m0 m0_err = frac_m0 * m_m0 frac_m1 = helpers.rand_num() * (0.02 * e_m1) + 1 - 0.01 * e_m1 m1_err = frac_m1 * m_m1 Q_DB_error.append((m1_err - m0_err) / m0_err) Hlab_DB.append(m1[i]["lab_field"]) if Q_str == "DSC": if m2[i]["total_m"] != None: m_m2 = m2[i]["total_m"] m_m3 = m3[i]["total_m"] m_m4 = m4[i]["total_m"] e_m2 = m2[i]["error"] e_m3 = m3[i]["error"] e_m4 = m4[i]["error"] # and check for the corrected version, if so replace the moments if (c_str == "_corr"): m0M = [m0[i]["x"], m0[i]["y"], m0[i]["z"]] m1M = [m1[i]["x"], m1[i]["y"], m1[i]["z"]] m2M = [m2[i]["x"], m2[i]["y"], m2[i]["z"]] m3M = [m3[i]["x"], m3[i]["y"], m3[i]["z"]] m4M = [m4[i]["x"], m4[i]["y"], m4[i]["z"]] NRMrem = helpers.list_mult_num( helpers.list_plus_list(m1M, m2M), 0.5) m1pTRM = helpers.list_min_list(m1M, NRMrem) m2pTRM = helpers.list_min_list(m2M, NRMrem) m3pTRM = helpers.list_min_list(m3M, NRMrem) m4pTRM = helpers.list_min_list(m4M, NRMrem) m_m0 = m0[i]["total_m"] # m_m0_corr m_m1 = helpers.norm(NRMrem) + helpers.norm( m1pTRM) # m_m1_corr m_m2 = helpers.norm(NRMrem) - helpers.norm( m2pTRM) # exception to the rule m_m3 = helpers.norm(NRMrem) + helpers.norm(m3pTRM) m_m4 = helpers.norm(NRMrem) + helpers.norm(m4pTRM) frac_m0 = helpers.rand_num() * (0.02 * e_m0) + 1 - 0.01 * e_m0 m0_err = frac_m0 * m_m0 frac_m1 = helpers.rand_num() * (0.02 * e_m1) + 1 - 0.01 * e_m1 m1_err = frac_m1 * m_m1 frac_m2 = helpers.rand_num() * (0.02 * e_m2) + 1 - 0.01 * e_m2 m2_err = frac_m2 * m_m2 frac_m3 = helpers.rand_num() * (0.02 * e_m3) + 1 - 0.01 * e_m3 m3_err = frac_m3 * m_m3 Q_DSC_error.append( 2 * ((1 + alpha) * m1_err - m0_err - alpha * m3_err) / (2 * m0_err - m1_err - m2_err)) Hlab_DSC.append(m2[i]["lab_field"]) if (Q_str == "DB"): Q_error = Q_DB_error Hlab = Hlab_DB elif (Q_str == "DSC"): Q_error = Q_DSC_error Hlab = Hlab_DSC N = len(Q_error) if N > 1: avgH = sum(Hlab) / N # calculate standard deviation on Hlab, and determine x and y stdevH1 = [] x = [] y = [] for k in range(N): stdevH1.append((Hlab[k] - avgH)**2) x.append(Hlab[k]) y.append(Q_error[k]) stdevH = math.sqrt(sum(stdevH1) / (N - 1)) # calculate Sx, Sy, Sxx, Syy, Sxy Sx = sum(x) Sy = sum(y) Sxy = helpers.dot_product(x, y) Sxx = helpers.dot_product(x, x) # calculate linear fit is not all at the same Hlab if stdevH > stdevH_min: b = (N * Sxy - Sx * Sy) / (N * Sxx - Sx**2) a = Sy / N - b * Sx / N PI = -1 * a / b N2.append(N) stdevHl.append(stdevH) aa.append(a) bb.append(b) intercept.append(PI) H0.append(a + b * Hlist[0]) H1.append(a + b * Hlist[1]) H2.append(a + b * Hlist[2]) H3.append(a + b * Hlist[3]) H4.append(a + b * Hlist[4]) H5.append(a + b * Hlist[5]) H6.append(a + b * Hlist[6]) H7.append(a + b * Hlist[7]) H8.append(a + b * Hlist[8]) H9.append(a + b * Hlist[9]) H10.append(a + b * Hlist[10]) # end of the big while loop, add one to m (this should be within the if statement) m += 1 killCounter += 1 # sort columns and apply cut-off cutOffValue = 0.01 * (100 - Confidence) / 2 cutOff = int(NumCycles * cutOffValue) H0.sort() H1.sort() H2.sort() H3.sort() H4.sort() H5.sort() H6.sort() H7.sort() H8.sort() H9.sort() H10.sort() Q_Hlist = [H0, H1, H2, H3, H4, H5, H6, H7, H8, H9, H10] # determine the average of the bootstrap over the 11 labfields # take the average of each of the labfield specified in Q_Hlist Boot_int_min = [] Boot_int_max = [] Boot_avg = [] if len(Q_Hlist[0]) != 0: h = 0 for el in Q_Hlist: Boot_avg.append([Hlist[h], sum(el) / len(el)]) h += 1 F = cutOff # the minimum value F first L = m - cutOff - 1 # the maximum value L last ( -1 because python counts from 0) y_min = [] y_max = [] for w in range(len(Q_Hlist)): y_min.append(Q_Hlist[w][F]) y_max.append(Q_Hlist[w][L]) for w in range(len(Hlist)): Boot_int_min.append([Hlist[w], y_min[w]]) Boot_int_max.append([Hlist[w], y_max[w]]) # determine the x axis intercept for lower bound ind_min = 999 for i in range(len(y_min) - 1): if (y_min[i] < 0) & (y_min[i + 1] > 0): ind_min = i if ind_min == 999: ictLow = None else: slope_min = (y_min[ind_min + 1] - y_min[ind_min]) / ( Hlist[ind_min + 1] - Hlist[ind_min]) ictLow = -1 * (y_min[ind_min] - Hlist[ind_min] * slope_min) / slope_min # determine the x axis intercept for upper bound ind_max = 999 for j in range(len(y_max) - 1): if (y_max[j] < 0) & (y_max[j + 1] > 0): ind_max = j if ind_max == 999: ictHigh = None else: slope_max = (y_max[ind_max + 1] - y_max[ind_max]) / ( Hlist[ind_max + 1] - Hlist[ind_max]) ictHigh = -1 * (y_max[ind_max] - Hlist[ind_max] * slope_max) / slope_max # write corresponding PI min and max values, these are the intercepts of the bootstrap intervals PI_min = ictHigh PI_max = ictLow else: PI_min = None PI_max = None return [PI_min, PI_max, Boot_int_min, Boot_int_max, Boot_avg]
def run_step(self, prev, params): data = prev df = pd.read_excel(data, sheet_name='2.4') df.columns = [ norm(x.strip().lower().replace(' ', '_').replace('-', '_').replace( '%', 'perc')) for x in df.columns ] print(df.columns) df = df.loc[~df['entidad_federativa'].str.contains('Total')].copy() # get end_id dimension dim_geo = get_dimensions()[0] df['entidad_federativa'].replace(dict( zip(dim_geo['ent_name'], dim_geo['ent_id'])), inplace=True) df.columns = [ 'ent_id', 'year', 'quarter_id', 'value_between_companies', 'value_new_investments', 'value_re_investments', 'count_between_companies', 'count_new_investments', 'count_re_investments', 'value_between_companies_c', 'value_new_investments_c', 'value_re_investments_c' ] df['quarter_id'] = df['year'].astype(int).astype( str) + df['quarter_id'].astype(int).astype(str) df['quarter_id'] = df['quarter_id'].astype(int) df.drop(columns=[ 'value_between_companies', 'value_new_investments', 'value_re_investments' ], inplace=True) base = ['ent_id', 'year', 'quarter_id'] df_final = pd.DataFrame() for option in [ 'between_companies', 'new_investments', 're_investments' ]: temp = df[base + ['count_{}'.format(option), 'value_{}_c'.format(option) ]].copy() temp.columns = ['ent_id', 'year', 'quarter_id', 'count', 'value_c'] temp.dropna(subset=['value_c'], inplace=True) temp['investment_type'] = option df_final = df_final.append(temp) df = df_final.copy() df['investment_type'].replace(INVESTMENT_TYPE, inplace=True) temp = pd.DataFrame() for ent in list(df['ent_id'].unique()): temp = temp.append( validate_category(df.loc[(df['ent_id'] == ent)], 'investment_type', 'value_c', 'c')) df = temp.copy() temp = pd.DataFrame() df = df.loc[df['value_c'].astype(str).str.lower() != 'c'].copy() df['value_c'] = df['value_c'].astype(float) return df
def specimen_fail_pass(Q_str, c_str, site, Boot_min, Boot_max): # split in measurements m0 m1 m2 m3 m4 with multiple specimens per list m0 = list(filter(lambda m: m['type'] == 0, site)) m1 = list(filter(lambda m: m['type'] == 1, site)) m2 = list(filter(lambda m: m['type'] == 2, site)) m3 = list(filter(lambda m: m['type'] == 3, site)) m4 = list(filter(lambda m: m['type'] == 4, site)) # determine if specimen is above / below bootstrap interval and FAIL or PASS specimen num_specimens = len(m0) # calculate original Q_DB & Q_DSC (also corrected) for ALL specimens Hlab = [] Q_DB = [] Q_DSC = [] B_specimen_pass_fail = [] for i in range(num_specimens): # get labfield and name specimen Hlab = m1[i]["lab_field"] name = m0[i]["specimen"] # specimen name is the same for m0-m4 m_m0 = m0[i]["total_m"] m_m1 = m1[i]["total_m"] m_m2 = m2[i]["total_m"] m_m3 = m3[i]["total_m"] m_m4 = m4[i]["total_m"] if (m_m2 == None) or (m_m3 == None) or (m_m4 == None): Q_DB.append([name, Hlab, (m_m1 - m_m0) / m_m0]) Q_DSC.append([name, None, None]) else: # MSP-DSC calculate Q_DB & Q_DSC (check for corr) Q_DB.append([name, Hlab, (m_m1 - m_m0) / m_m0]) # no Corr if (c_str == "_corr"): m0M = [m0[i]["x"], m0[i]["y"], m0[i]["z"]] m1M = [m1[i]["x"], m1[i]["y"], m1[i]["z"]] m2M = [m2[i]["x"], m2[i]["y"], m2[i]["z"]] m3M = [m3[i]["x"], m3[i]["y"], m3[i]["z"]] m4M = [m4[i]["x"], m4[i]["y"], m4[i]["z"]] NRMrem = helpers.list_mult_num( helpers.list_plus_list(m1M, m2M), 0.5) m1pTRM = helpers.list_min_list(m1M, NRMrem) m2pTRM = helpers.list_min_list(m2M, NRMrem) m3pTRM = helpers.list_min_list(m3M, NRMrem) m4pTRM = helpers.list_min_list(m4M, NRMrem) m_m0 = m0[i]["total_m"] # m_m0_corr m_m1 = helpers.norm(NRMrem) + helpers.norm( m1pTRM) # m_m1_corr m_m2 = helpers.norm(NRMrem) - helpers.norm( m2pTRM) # exception to the rule m_m3 = helpers.norm(NRMrem) + helpers.norm(m3pTRM) m_m4 = helpers.norm(NRMrem) + helpers.norm(m4pTRM) Q_DSC.append([ name, Hlab, 2 * ((1 + alpha) * m_m1 - m_m0 - alpha * m_m3) / (2 * m_m0 - m_m1 - m_m2) ]) # loop over all specimens and check closest labfields from the Boot_min and Boot_max for i in range(num_specimens): Hsam = Q_DB[i][1] if (Q_str == "DB"): # get the y for specimen for Q_DB ysam = Q_DB[i][2] elif (Q_str == "DSC"): # get the y for specimen for Q_DSC ysam = Q_DSC[i][2] name = m0[i]["specimen"] # specimen name is the same for m0-m4 if ysam == None: B_specimen_pass_fail.append([name, "None"]) else: ind_min = 999 for j in range(len(Boot_min) - 1): if (Boot_min[j][0] <= Hsam) and (Boot_min[j + 1][0] > Hsam): ind = j a_min = (Boot_min[ind + 1][1] - Boot_min[ind][1]) / ( Boot_min[ind + 1][0] - Boot_min[ind][0]) ycalc_min = (Hsam - Boot_min[ind][0]) * a_min + Boot_min[ind][1] a_max = (Boot_max[ind + 1][1] - Boot_max[ind][1]) / ( Boot_max[ind + 1][0] - Boot_max[ind][0]) ycalc_max = (Hsam - Boot_max[ind][0]) * a_max + Boot_max[ind][1] if (ysam > ycalc_max) or (ysam < ycalc_min): B_specimen_pass_fail.append([name, "fail"]) else: B_specimen_pass_fail.append([name, "pass"]) return [B_specimen_pass_fail]
def calc_nrm_ptrm(sc): """ Function to calculate/split the basic nrm and ptrm vectors and measurements for the different methods and protocols. input: preprocessed measurement data, for all protocols and methods from the suitcase output: basic vectors and list, like the NRM remaining and the ptrm vectors """ # input: preprocessed/ msrmnts[zerofield_steps, zerofield_steps_all, infield_steps, infield_steps_all, infield_antiparallel_steps, infield_antiparallel_steps_all, ARM_acq_steps, ARM_acq_steps_all] # output: preprocessed/ basics[nrm0, ptrm, NRM_rem, NRM_vec_all, ptrm_vec_all, x_ptrm_all, x_temp_all, y_nrm_all, y_temp_all, pstep_all] zerofield_steps = sc["preprocessed"]["msrmnts"]["zerofield_steps"] zerofield_steps_all = sc["preprocessed"]["msrmnts"]["zerofield_steps_all"] infield_steps = sc["preprocessed"]["msrmnts"]["infield_steps"] infield_steps_all = sc["preprocessed"]["msrmnts"]["infield_steps_all"] infield_antiparallel_steps = sc["preprocessed"]["msrmnts"][ "infield_antiparallel_steps"] # Thellier - Thellier infield_antiparallel_steps_all = sc["preprocessed"]["msrmnts"][ "infield_antiparallel_steps_all"] # Thellier - Thellier ARM_acq_steps = sc["preprocessed"]["msrmnts"][ "ARM_acq_steps"] # pseudo Thellier NAA format ARM_acq_steps_all = sc["preprocessed"]["msrmnts"][ "ARM_acq_steps_all"] # pseudo Thellier NAA format # get NRM0, the first zerofield step, this is true for both modified Thellier and classical TT and pseudo Thellier (GF & NAA) nrm0 = zerofield_steps_all[0]["total_m"] NRMrem_list = [] ptrm_list = [] NRM_vec_all = [] ptrm_vec_all = [] pstep_all = [] x_ptrm_all = [] x_temp_all = [] y_nrm_all = [] y_temp_all = [] # "classical thellier Thelier-Thellier (TT)" if len(infield_antiparallel_steps) != 0: # first the NRM, which is always a zerofield step, also for the Thellier - Thellier protocol, # but not always in the selection so check the length to see if it is in the selection if len(zerofield_steps) != 0: # then in selection NRMrem_list.append([ zerofield_steps[0]["step"], zerofield_steps[0]["x"], zerofield_steps[0]["y"], zerofield_steps[0]["z"] ]) ptrm_list.append([zerofield_steps[0]["step"], 0, 0, 0]) # this is new 06-02 if len(zerofield_steps_all ) != 0: # this is probably always the case # this is new 06-02 NRM_vec_all.append([ zerofield_steps_all[0]["x"], zerofield_steps_all[0]["y"], zerofield_steps_all[0]["z"] ]) ptrm_vec_all.append([0, 0, 0]) x_ptrm_all.append(0) y_nrm_all.append( helpers.norm([ zerofield_steps_all[0]["x"], zerofield_steps_all[0]["y"], zerofield_steps_all[0]["z"] ])) x_temp_all.append(zerofield_steps_all[0]["step"]) y_temp_all.append(zerofield_steps_all[0]["step"]) # determine the NRM remaining & ptrm gained for the selection for i_step in infield_steps: for ai_step in infield_antiparallel_steps: if i_step["step"] == ai_step["step"]: NRMrem_list.append([ i_step["step"], (i_step["x"] + ai_step["x"]) / 2., (i_step["y"] + ai_step["y"]) / 2., (i_step["z"] + ai_step["z"]) / 2. ]) ptrm_list.append([ i_step["step"], (i_step["x"] - ai_step["x"]) / 2., (i_step["y"] - ai_step["y"]) / 2., (i_step["z"] - ai_step["z"]) / 2. ]) # determine the NRM remaining & ptrm gained for ALL steps for i_step in infield_steps_all: for ai_step in infield_antiparallel_steps_all: if i_step["step"] == ai_step["step"]: tempor_ptrm = [(i_step["x"] - ai_step["x"]) / 2., (i_step["y"] - ai_step["y"]) / 2., (i_step["z"] - ai_step["z"]) / 2.] tempor_nrm = [(i_step["x"] + ai_step["x"]) / 2., (i_step["y"] + ai_step["y"]) / 2., (i_step["z"] + ai_step["z"]) / 2.] NRM_vec_all.append(tempor_nrm) ptrm_vec_all.append(tempor_ptrm) x_ptrm_all.append(helpers.norm(tempor_ptrm)) x_temp_all.append(i_step["step"]) y_nrm_all.append(helpers.norm(tempor_nrm)) y_temp_all.append(i_step["step"]) # "modified thellier" if len(infield_antiparallel_steps) == 0: # pseudo Thellier GF or Thermal Thellier or Microwave Thellier if len(ARM_acq_steps) == 0: # if there are more zerofiels steps that infield steps, add steps to the ptrm_list with zeros, assumption is that extra zerofield steps are always before first ptrm step # for i in range(len(zerofield_steps) - len(infield_steps)): if (len(zerofield_steps) - len(infield_steps)) > 0: ptrm_list.append( [zerofield_steps_all[0]["step"], 0, 0, 0]) # this takes care off the first y-axis point # make ptrm_list for i_step in infield_steps: for z_step in zerofield_steps: if i_step["step"] == z_step["step"]: ptrm_list.append([ i_step["step"], (i_step["x"] - z_step["x"]), (i_step["y"] - z_step["y"]), (i_step["z"] - z_step["z"]) ]) # pseudo-Thellier NAA if len(ARM_acq_steps) != 0: # get the first ARM step, this is your NRM remaining and the ptrm should be corrected for this x0 = ARM_acq_steps_all[0]["x"] y0 = ARM_acq_steps_all[0]["y"] z0 = ARM_acq_steps_all[0]["z"] for a_step in ARM_acq_steps: ptrm_list.append([ a_step["step"], a_step["x"] - x0, a_step["y"] - y0, a_step["z"] - z0 ]) # make NRMrem_list for z_step in zerofield_steps: NRMrem_list.append( [z_step["step"], z_step["x"], z_step["y"], z_step["z"]]) for all_step in zerofield_steps_all: NRM_vec_all.append([all_step["x"], all_step["y"], all_step["z"]]) y_nrm_all.append( helpers.norm([all_step["x"], all_step["y"], all_step["z"]])) y_temp_all.append(all_step["step"]) for all_infield in infield_steps_all: # pTh-NAA is not getting through this if statement if all_infield["step"] == all_step["step"]: ptrm_vec_all.append([ all_infield["x"] - all_step["x"], all_infield["y"] - all_step["y"], all_infield["z"] - all_step["z"] ]) pstep_all.append(all_infield["step"]) x_temp_all.append(all_infield["step"]) x_ptrm_all.append( helpers.norm([ all_infield["x"] - all_step["x"], all_infield["y"] - all_step["y"], all_infield["z"] - all_step["z"] ])) for a_step in ARM_acq_steps_all: # pTh -NAA format ptrm vec ptrm_vec_all.append( [a_step["x"] - x0, a_step["y"] - y0, a_step["z"] - z0]) x_ptrm_all.append( helpers.norm( [a_step["x"] - x0, a_step["y"] - y0, a_step["z"] - z0])) x_temp_all.append(a_step["step"]) # rewrite format ptrm = [] for msrmt in ptrm_list: ptrm.append({ "step": msrmt[0], "x": msrmt[1], "y": msrmt[2], "z": msrmt[3] }) NRM_rem = [] for msrmt in NRMrem_list: NRM_rem.append({ "step": msrmt[0], "x": msrmt[1], "y": msrmt[2], "z": msrmt[3] }) sc["preprocessed"]["basics"]["nrm0"] = nrm0 sc["preprocessed"]["basics"]["ptrm"] = ptrm sc["preprocessed"]["basics"]["NRM_rem"] = NRM_rem sc["preprocessed"]["basics"]["NRM_vec_all"] = NRM_vec_all sc["preprocessed"]["basics"]["ptrm_vec_all"] = ptrm_vec_all sc["preprocessed"]["basics"]["x_ptrm_all"] = x_ptrm_all sc["preprocessed"]["basics"]["x_temp_all"] = x_temp_all sc["preprocessed"]["basics"]["y_nrm_all"] = y_nrm_all sc["preprocessed"]["basics"]["y_temp_all"] = y_temp_all sc["preprocessed"]["basics"]["pstep_all"] = pstep_all return sc
def anisotropy_calc(sc): """ Function that calculates the paleointensity correction factor when anisotropy data is available. It uses the previously calculated s-tensor. First the direction of the NRM for the selected part of the Arai plot is determined to give Mhat_ChRM. Which is used to get the anisotropy correction c. The anisotropy correction in multiplied with Banc to get Banc_aniso_corr. input: s-tensor, the direction of the applied labfield, direction of the data selection, the original Banc output: the anisotropy correction anis_c, the anisotropy corrected paleointensity estimate Banc_aniso_corr """ # input: preprocessed/ field_basics [field_dir_vec] # s_tensor [s1_list, s2_list, s3_list, s4_list, s5_list, s6_list, scheck_list] # arai_statistics/ PI_Banc_est [B_anc] # directional_statistics/ mean_dir_stat [Mdec_free, Minc_free] # output: anisotropy_statistics/ Anisotropy_Correction [anis_c, Banc_aniso_corr] Mdec_free = sc["directional_statistics"]["mean_dir_stat"]["Mdec_free"] Minc_free = sc["directional_statistics"]["mean_dir_stat"]["Minc_free"] field_dir_vec = sc["preprocessed"]["field_basics"]["field_dir_vec"] B_anc = sc["arai_statistics"]["PI_Banc_est"]["B_anc"] Xp_list = sc["preprocessed"]["aniso_trm"]["x+"] s_tensor = sc["anisotropy_statistics"]["Aniso_tensor"]["s_tensor"] if Xp_list != None: # then there is anisotropy data, do calculations, if no anisotropy data present do nothing anis_c = [] # the direction of the NRM, should be determined from the selected Arai plot # this is the Mdec_free & Minc_free, the unit vecotr of this gives -> Mhat_ChRM # get mean unit vector Mhat_ChRM = helpers.dir2cart(Mdec_free, Minc_free, 1) Blab_orient = field_dir_vec # it was: helpers.dir2cart(Mdec_free, Minc_free, 1), that is wrong!! (13 feb 2020) s1 = s_tensor[0] s2 = s_tensor[1] s3 = s_tensor[2] s4 = s_tensor[3] s5 = s_tensor[4] s6 = s_tensor[5] A1 = [s1, s4, s6] A2 = [s4, s2, s5] A3 = [s6, s5, s3] A = [A1, A2, A3] # make A and Mhat_ChRM into a numpy arrays (1) A = numpy.array(A) Mhat_ChRM = numpy.array(Mhat_ChRM) # do numpy calculation (2) Hanc = numpy.linalg.solve(A, Mhat_ChRM) # back to python lists (3) A = A.tolist() Mhat_ChRM = Mhat_ChRM.tolist() Hanc = Hanc.tolist() # unit vector in the direction of the ancient field Hanc_hat = helpers.list_div_num(Hanc, helpers.norm(Hanc)) Manc = [] Mlab = [] for i in range(len(A)): Manc.append(helpers.dot_product(A[i], Hanc_hat)) Mlab.append(helpers.dot_product(A[i], field_dir_vec)) aniso_c = helpers.norm(Mlab) / helpers.norm(Manc) Banc_aniso_corr = aniso_c * B_anc sc["anisotropy_statistics"]["Anisotropy_Correction"][ "aniso_c"] = aniso_c sc["anisotropy_statistics"]["Anisotropy_Correction"][ "Banc_aniso_corr"] = Banc_aniso_corr return (sc)
def kinetic_energy(self): return norm(self.px, self.py)**2 / self.m / 2
def params_noCorr_corr(c_str, alpha, m0, m1, m2, m3, m4): Q_DB = [] Q_DSC = [] mu_ds = [] H_max = [] H_est = [] Eps_alt = [] Eps_alt_abs = [] Err_alt = [] Err_ds = [] Err_total = [] Err_alt_abs = [] num_specimens = len(m0) for i in range(num_specimens): name = m0[i]["specimen"] # specimen name is the same for m0-m4 H_lab = m1[i][ "lab_field"] # the x-axis coordinate is the AF field of the m1-m4 steps # do the calculations m_m0 = m0[i]["total_m"] m_m1 = m1[i]["total_m"] m_m2 = m2[i]["total_m"] m_m3 = m3[i]["total_m"] m_m4 = m4[i]["total_m"] if (m_m2 == None) or (m_m3 == None) or (m_m4 == None): Q_DB.append([name, H_lab, (m_m1 - m_m0) / m_m0]) Q_DSC.append([name, None, None]) mu_ds.append([name, None]) H_max.append([name, None]) H_est.append([name, None]) Eps_alt.append([name, None]) Eps_alt_abs.append([name, None]) Err_alt.append([name, None]) Err_ds.append([name, None]) Err_total.append([name, None]) else: # calculate also the Q_DSC ratio and all the parameters Q_DB.append([name, H_lab, (m_m1 - m_m0) / m_m0 ]) # no corrected version, only "normal version" # check for the corrected version of un-corrected version for Q_DSC & prameter calculations if (c_str == "_corr"): m0M = [m0[i]["x"], m0[i]["y"], m0[i]["z"]] m1M = [m1[i]["x"], m1[i]["y"], m1[i]["z"]] m2M = [m2[i]["x"], m2[i]["y"], m2[i]["z"]] m3M = [m3[i]["x"], m3[i]["y"], m3[i]["z"]] m4M = [m4[i]["x"], m4[i]["y"], m4[i]["z"]] NRMrem = helpers.list_mult_num( helpers.list_plus_list(m1M, m2M), 0.5) m1pTRM = helpers.list_min_list(m1M, NRMrem) m2pTRM = helpers.list_min_list(m2M, NRMrem) m3pTRM = helpers.list_min_list(m3M, NRMrem) m4pTRM = helpers.list_min_list(m4M, NRMrem) m_m0 = m0[i]["total_m"] # m_m0_corr m_m1 = helpers.norm(NRMrem) + helpers.norm( m1pTRM) # m_m1_corr m_m2 = helpers.norm(NRMrem) - helpers.norm( m2pTRM) # exception to the rule m_m3 = helpers.norm(NRMrem) + helpers.norm(m3pTRM) m_m4 = helpers.norm(NRMrem) + helpers.norm(m4pTRM) Q_DSC.append([ name, H_lab, 2 * ((1 + alpha) * m_m1 - m_m0 - alpha * m_m3) / (2 * m_m0 - m_m1 - m_m2) ]) mu_ds.append( [name, (m_m1 - m_m3) / (m_m3 - 0.5 * (m_m1 + m_m2))]) H_max.append( [name, (2 * m_m0 - m_m1 - m_m2) / (m_m1 - m_m2) * H_lab]) H_est.append([ name, (2 * m_m0 - m_m1 - m_m2) / ((1 + 2 * alpha) * m_m1 - 2 * alpha * m_m3 - m_m2) * H_lab ]) Eps = (m_m4 - m_m1) / m_m1 Eps_alt.append([name, Eps]) Eps_alt_abs.append([name, abs((m_m1 - m_m4) / m_m1)]) # calculate the error estimates # nummerator & denominator num = 2 * ((1 + alpha) * m_m1 - m_m0 - alpha * m_m3) den = (2 * m_m0) - m_m1 - m_m2 # partial derivatives of Q_DSC d_num_m1 = 2 * (1 + alpha) d_num_m2 = 0 d_num_m3 = -2 * alpha d_den_m1 = -1 d_den_m2 = -1 d_den_m3 = 0 # terms Term_1 = m_m1 * ((den * d_num_m1) - (num * d_den_m1)) / (den)**2 Term_2 = m_m2 * ((den * d_num_m2) - (num * d_den_m2)) / (den)**2 Term_3 = m_m3 * ((den * d_num_m3) - (num * d_den_m3)) / (den)**2 dQ_DSC_alt = Eps**2 * (Term_1**2 + Term_2**2 + Term_3**2) dQ_DSC_ds = (((m_m3 - m_m1) / den)**2) / 3 Err_alt.append([name, dQ_DSC_alt]) Err_ds.append([name, dQ_DSC_ds]) Err_total.append([name, dQ_DSC_alt + dQ_DSC_ds]) return [ Q_DB, Q_DSC, mu_ds, H_max, H_est, Eps_alt, Eps_alt_abs, Err_alt, Err_ds, Err_total ]
def train(tileSize=64, numPuzzles=23, num_permutations=10, batch_size=16): # On server with PET and PCT in image_dir = "/hepgpu3-data1/dmcsween/Data128/ResampleData/PlanningCT" print("Load Data") image_data, __image, __label = load.data_reader(image_dir, image_dir, image_dir) image_array, image_affine = image_data.get_data() moving_array, moving_affine = __image.get_data() dvf_array, dvf_affine = __label.get_data() """ list_avail_keys = help.get_moveable_keys(image_array) hamming_set = pd.read_csv( "hamming_set_PCT.txt", sep=",", header=None) """ avail_keys = pd.read_csv("avail_keys_both.txt", sep=",", header=None) print("Len keys:", len(avail_keys)) list_avail_keys = [(avail_keys.loc[i, 0], avail_keys.loc[i, 1], avail_keys.loc[i, 2]) for i in range(len(avail_keys))] print(list_avail_keys) # Get hamming set print("Load hamming Set") hamming_set = pd.read_csv("hamming_set.txt", sep=",", header=None) #hamming_set = hamming_set.loc[:9] print("Ham Len", len(hamming_set)) print(hamming_set) fixed_array, moving_array, dvf_array = helper.shuffle_inplace( image_array, moving_array, dvf_array) # Ignore moving and dvf validation_dataset, validation_moving, validation_dvf, train_dataset, train_moving, train_dvf = helper.split_data( fixed_array, moving_array, dvf_array, split_ratio=0.15) normalised_train = helper.norm(train_dataset) normalised_val = helper.norm(validation_dataset) # Output all data from a training session into a dated folder outputPath = "./logs" # hamming_list = [0, 1, 2, 3, 4] # img_idx = [0, 1, 2, 3, 4] # callbacks checkpoint = ModelCheckpoint(outputPath + '/best_model.h5', monitor='val_acc', verbose=1, save_best_only=True, period=1) reduce_lr_plateau = ReduceLROnPlateau(monitor='val_acc', patience=10, verbose=1) # early_stop = EarlyStopping(monitor='val_acc', patience=5, verbose=1) tensorboard = TrainValTensorBoard(write_graph=False) callbacks = [checkpoint, reduce_lr_plateau, tensorboard] # BUILD Model model = createSharedAlexnet3D_onemodel() # for layer in model.layers: # print(layer.name, layer.output_shape) opt = optimizers.SGD(lr=0.01) plot_model(model, to_file='model.png') print(model.summary()) model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) model.fit_generator(generator=gen.generator(normalised_train, list_avail_keys, hamming_set, batch_size=batch_size, N=num_permutations), epochs=1000, verbose=1, steps_per_epoch=normalised_train.shape[0] // batch_size, validation_data=gen.generator(normalised_val, list_avail_keys, hamming_set, batch_size=batch_size, N=num_permutations), validation_steps=normalised_val.shape[0] // batch_size, callbacks=callbacks, shuffle=False) model.save('model_best.h5')