def householder_reflection(col):
n_height = len(col)
m_matrix = np.eye(n_height, 1)
I = np.eye(n_height)
v = np.array([x + y for x, y in zip(col, norm(col) * m_matrix)])
u = v / norm(v)
return I - 2 * u * u.T
def calc_mom_TT(step, prevstep):
moment = helpers.norm(
helpers.list_mult_num([
step["x"] - prevstep["x"], step["y"] - prevstep["y"],
step["z"] - prevstep["z"]
], 0.5))
return moment
def run_step(self, prev, params):
data = prev
df = pd.read_excel(data, sheet_name='10.2')
df.columns = [
norm(x.strip().lower().replace(' ', '_').replace('-', '_').replace(
'%', 'perc')) for x in df.columns
]
print(df.columns)
df.columns = [
'year', 'country', 'investment_type', 'value', 'count', 'value_c'
]
df = df.loc[~df['year'].astype(str).str.contains('Total')].copy()
df = df.loc[df['value_c'] != 'C'].copy()
df.drop(columns=['value'], inplace=True)
dim_country = get_dimensions()[1]
df['country'].replace(COUNTRY_REPLACE, inplace=True)
df['country'] = df['country'].replace(
dict(zip(dim_country['country_name_es'], dim_country['iso3'])))
df[['year', 'count', 'value_c']] = df[['year', 'count',
'value_c']].astype(float)
df['investment_type'].replace(INVESTMENT_TYPE, inplace=True)
return df
def NRM_dir_stat(sc):
"""
Function to calculate the NRM direction statistics, DANG, free floating direction and NRMdev.
input: Mdec_free,Minc_free, x, y, z, Yint
output: DANG, NRMdev, dir_vec_free
"""
# input: directional_statistics/ mean_dir_stat[Mdec_free,Minc_free, x, y, z]
# arai_statistics/ intercept_stats[Yint]
# output: directional_statistics/ NRM_dir_stat[DANG, NRMdev, dir_vec_free]
x = sc["directional_statistics"]["mean_dir_stat"]["x"]
y = sc["directional_statistics"]["mean_dir_stat"]["y"]
z = sc["directional_statistics"]["mean_dir_stat"]["z"]
Mdec_free = sc["directional_statistics"]["mean_dir_stat"]["Mdec_free"]
Minc_free = sc["directional_statistics"]["mean_dir_stat"]["Minc_free"]
Yint = sc["arai_statistics"]["intercept_stats"]["Yint"]
mean_x = sum(x) / len(x)
mean_y = sum(y) / len(y)
mean_z = sum(z) / len(z)
center_of_mass = [mean_x, mean_y, mean_z]
# calculate DANG & NRMdev
dir_vec_free = helpers.dir2cart(Mdec_free, Minc_free, 1)
DANG = helpers.get_angle_diff(dir_vec_free, center_of_mass)
NRMdev = ((math.sin(math.radians(DANG)) * helpers.norm(center_of_mass)) / abs(Yint)) * 100
sc["directional_statistics"]["NRM_dir_stat"]["dir_vec_free"] = dir_vec_free
sc["directional_statistics"]["NRM_dir_stat"]["DANG"] = DANG
sc["directional_statistics"]["NRM_dir_stat"]["NRMdev"] = NRMdev
return sc
def run_step(self, prev, params):
data = prev
df = pd.read_excel(data, sheet_name='3')
df.columns = [
norm(x.strip().lower().replace(' ', '_').replace('-', '_').replace(
'%', 'perc')) for x in df.columns
]
df = df.loc[~df['entidad_federativa'].str.contains('Total')].copy()
# get country, end_id dimensions
dim_geo, dim_country = get_dimensions()
df['entidad_federativa'].replace(dict(
zip(dim_geo['ent_name'], dim_geo['ent_id'])),
inplace=True)
df['pais'].replace(COUNTRY_REPLACE, inplace=True)
df['pais'] = df['pais'].replace(
dict(zip(dim_country['country_name_es'],
dim_country['iso3']))).replace(missing_replacements)
df.columns = ['ent_id', 'year', 'country', 'value', 'count', 'value_c']
df.drop(columns=['value'], inplace=True)
df = df.loc[df['value_c'].astype(str).str.lower() != 'c'].copy()
df['value_c'] = df['value_c'].astype(float)
return df
def run_step(self, prev, params):
data = prev
df = pd.read_excel(data, sheet_name='9.5')
df.columns = [
norm(x.strip().lower().replace(' ', '_').replace('-', '_').replace(
'%', 'perc')) for x in df.columns
]
print(df.columns)
df.columns = [
'year', 'quarter', 'value_between_companies',
'value_new_investments', 'value_re_investments',
'count_between_companies', 'count_new_investments',
'count_re_investments', 'value_between_companies_c',
'value_new_investments_c', 'value_re_investments_c'
]
df = df.loc[~df['year'].astype(str).str.contains('Total')].copy()
df['quarter_id'] = df['year'].astype(str) + df['quarter'].astype(
int).astype(str)
df.drop(columns=[
'year', 'quarter', 'value_between_companies',
'value_new_investments', 'value_re_investments'
],
inplace=True)
for col in df.columns:
df[col] = df[col].astype(float)
base = ['quarter_id']
df_final = pd.DataFrame()
for option in [
'between_companies', 'new_investments', 're_investments'
]:
temp = df[base +
['count_{}'.format(option), 'value_{}_c'.format(option)
]].copy()
temp.columns = ['quarter_id', 'count', 'value_c']
temp.dropna(subset=['value_c'], inplace=True)
temp['investment_type'] = option
df_final = df_final.append(temp)
df = df_final.copy()
df['investment_type'].replace(INVESTMENT_TYPE, inplace=True)
return df
def H(self, idx=None, heliocentric=False):
if idx is None:
T = sum(map(lambda body: body.kinetic_energy(), self.particles))
else:
T = self.particles[idx].kinetic_energy()
U = 0 # self-potential energy
for i in range(len(self.particles)):
if idx is not None and idx != i:
continue # skip if not specific particle
for j in range(i + 1, len(self.particles)):
if heliocentric and self.particles[j].name.lower() != "sun":
continue
denom = norm(self.particles[i].qx - self.particles[j].qx,
self.particles[i].qy - self.particles[j].qy)
U -= G * self.particles[i].m * self.particles[j].m / denom
return T + U
def VDS_stats(sc):
"""
Vector difference sum (VDS) calculations. These use the entire NRM vector. calculated here: the VDS, FRAC and GAP-MAX. FRAC and GAP-MAX proposed by (Shaar and Tauxe, 2013). FRAC is the NRM fraction used for the best-fit line. GAP-MAX is the maximum gap between two points.
input: The entire NRM vector and the NRM vector of the selection
output: VDS, FRAC, GAP_MAX
"""
# input: preprocessed/ basics[NRM_vec_all, NRM_vec_select]
# output: arai_statistics/ VDS_stats[VDS, FRAC, GAP_MAX]
NRM_vec_select = sc["preprocessed"]["basics"]["NRM_vec_select"]
NRM_vec_all = sc["preprocessed"]["basics"]["NRM_vec_all"]
# calculate difference vector, use a helper function
def calc_diff(MagVec):
diff = []
for i in range(len(MagVec) - 1):
diff.append(helpers.difference(MagVec[i], MagVec[i + 1]))
return diff
diff_total_NRM = calc_diff(
NRM_vec_all
) # this is only the second part of the equation, need to add NRMmax
# calculate norm of the last NRM step, the NRM_max
NRM_max = helpers.norm(NRM_vec_all[len(NRM_vec_all) - 1])
diff_total_NRM.append(NRM_max) # append NRM max tot the VDS vector
VDS = sum(diff_total_NRM) # take the sum of all elements
# Calculation of FRAC
# calculate the difference vector for the correct selection
# NRM_vec_select is the vector only containing the NRM of the selection of interest
diff_select_NRM = calc_diff(NRM_vec_select)
FRAC = 0
FRAC = sum(diff_select_NRM) / VDS
# Calculation of GAP-MAX
GAP_MAX = 0
GAP_MAX = max(diff_select_NRM) / sum(diff_select_NRM)
sc["arai_statistics"]["VDS_stats"]["VDS"] = VDS
sc["arai_statistics"]["VDS_stats"]["FRAC"] = FRAC
sc["arai_statistics"]["VDS_stats"]["GAP_MAX"] = GAP_MAX
return sc
def global_process_batch(log_probs, original_tensors, batch_size, is_trt=True):
'''Accumulates prediction evaluations for batches across an epoch
is_trt determines which global dictionary will be used.
To get WER at any point, use global_process_epoch.
For one-off WER evaluations, use get_results()
'''
# State-based approach for full WER comparison across a dataset.
greedy_decoder = GreedyCTCDecoder()
predicts = norm(greedy_decoder(log_probs=log_probs))
values_dict = dict(
predictions=[predicts],
transcript=[original_tensors[2][0:batch_size, ...]],
transcript_length=[original_tensors[3][0:batch_size, ...]],
)
dict_to_process = _global_trt_dict if is_trt else _global_pyt_dict
process_evaluation_batch(values_dict, dict_to_process, labels=get_vocab())
def Additivity_check_stat(sc):
"""
Function to calculate additivity statistics. The number of checks (n_add) and the maximum absolute additivity check difference normalized by the total TRM (obtained from the intersection of the best-fit line and the x-axis on an Arai plot; Leonhardt et al., 2004a).
input: additivity check info (add_check_steps, add_check_step, add_check_vec), info for the pTRM (x_ptrm_all, x_temp_all), the intercept with the x-asix (Xint)
output: n_add, d_AC
"""
# input: preprocessed/ msrmnts[add_check_steps]
# checks[add_check_step, add_check_vec]
# basics[x_ptrm_all, x_temp_all]
# arai_statistics/ intercept_stats[Xint]
# output: check_statistics/ Additivity_check[n_add, d_AC]
# Determine the Additivity check for all checks with Ti < Tmax, that is, all checks that are in the selection, use the ptrm of all steps
add_check_steps = sc["preprocessed"]["msrmnts"]["add_check_steps"]
add_check_step = sc["preprocessed"]["checks"]["add_check_step"]
add_check_vec = sc["preprocessed"]["checks"]["add_check_vec"]
x_ptrm_all = sc["preprocessed"]["basics"]["x_ptrm_all"]
x_temp_all = sc["preprocessed"]["basics"]["x_temp_all"]
Xint = sc["arai_statistics"]["intercept_stats"]["Xint"]
n_add = []
d_AC = []
if len(add_check_steps) != 0: # then do calculations
AC = []
for i in range(len(add_check_vec)):
for j in range(len(x_ptrm_all)):
if add_check_step[i][0] == x_temp_all[
j]: # step for the add check and the ptrm should be the same
AC.append(helpers.norm(add_check_vec[i]) - x_ptrm_all[j])
n_add = len(AC)
AC_abs = []
for l in range(len(AC)):
AC_abs.append(abs(AC[l]))
d_AC = 100 * max(AC_abs) / abs(Xint)
sc["check_statistics"]["Additivity_check"]["n_add"] = n_add
sc["check_statistics"]["Additivity_check"]["d_AC"] = d_AC
return sc
def run_step(self, prev, params):
data = prev
df = pd.read_excel(data, sheet_name='9.2')
df.columns = [
norm(x.strip().lower().replace(' ', '_').replace('-', '_').replace(
'%', 'perc')) for x in df.columns
]
print(df.columns)
df.columns = ['year', 'value', 'count', 'value_c']
df['year'] = df['year'].astype(str)
df = df.loc[~df['year'].str.contains('Total')].copy()
df.drop(columns=['value'], inplace=True)
for col in ['count', 'year']:
df[col] = df[col].astype(int)
return df
def neural_net_gui(game, board, model, start, fast):
feat = copy.copy(game)
i = 0
while (1):
out = model.predict(feat.reshape((1, 9, 9, 1)))
out = out.squeeze()
pred = np.argmax(out, axis=1).reshape((9, 9)) + 1
prob = np.around(np.max(out, axis=1).reshape((9, 9)), 2)
feat = denorm(feat).reshape((9, 9))
mask = (feat == 0)
if (mask.sum() == 0):
break
prob_new = prob * mask
## involve constraints here - further processing on prob
ind = np.argmax(prob_new)
x, y = (ind // 9), (ind % 9)
val = pred[x][y]
feat[x][y] = val
feat = norm(feat)
i += 1
'''
if i == 1:
print(i)
print('pred:', pred)
print('prob:', prob_new)
print()
'''
cnn_gui(val, x, y, board, round(time.time() - start))
if not fast:
pygame.time.delay(100)
def get_results(log_probs, original_tensors, batch_size):
''' Returns WER and predictions for the outputs of the acoustic model
Used for one-off batches. Epoch-wide evaluation should use
global_process_batch and global_process_epoch
'''
# Used to get WER and predictions for one-off batches
greedy_decoder = GreedyCTCDecoder()
predicts = norm(greedy_decoder(log_probs=log_probs))
values_dict = dict(
predictions=[predicts],
transcript=[original_tensors[2][0:batch_size, ...]],
transcript_length=[original_tensors[3][0:batch_size, ...]],
)
temp_dict = {
'predictions': [],
'transcripts': [],
}
process_evaluation_batch(values_dict, temp_dict, labels=get_vocab())
predictions = temp_dict['predictions']
wer, _ = process_evaluation_epoch(temp_dict)
return wer, predictions
def basics_pTh(sc):
"""
Function to calculate more basics from the ARM_dem_steps and ARM_acq_steps_all for the pseudo-Thellier method [NAA protocol only, NRMdemag, ARM-acquisition, ARMdemag]. This addes on the previous function that calculates the NRM demag basics.
input: the ARM_dem_steps and ARM_acq_steps_all vectors with the "step", "x", "y", and "z" values
output: basic vectors and lists for the ARM demag
"""
# input: preprocessed/ msrmnts[ARM_dem_steps, ARM_acq_steps_all]
# output: preprocessed/ basics_pTh[arm_step, y_arm_d, ARMd_vec,yBar_arm ]
ARM_dem_steps = sc["preprocessed"]["msrmnts"][
"ARM_dem_steps"] # pseudo Thellier NAA format
ARM_acq_steps_all = sc["preprocessed"]["msrmnts"]["ARM_acq_steps_all"]
if len(ARM_dem_steps) != 0: # pseudo-Thellier NAA
x0 = ARM_acq_steps_all[0]["x"]
y0 = ARM_acq_steps_all[0]["y"]
z0 = ARM_acq_steps_all[0]["z"]
y_arm_d = []
ARMd_vec = []
arm_step = []
for dstep in ARM_dem_steps:
ARMd_vec.append(
[dstep["x"] - x0, dstep["y"] - y0, dstep["z"] - z0])
y_arm_d.append(
helpers.norm(
[dstep["x"] - x0, dstep["y"] - y0, dstep["z"] - z0]))
arm_step.append(dstep["step"])
# calculate average
yBar_arm = sum(y_arm_d) / len(y_arm_d)
sc["preprocessed"]["basics_pTh"]["arm_step"] = arm_step
sc["preprocessed"]["basics_pTh"]["y_arm_d"] = y_arm_d
sc["preprocessed"]["basics_pTh"]["ARMd_vec"] = ARMd_vec
sc["preprocessed"]["basics_pTh"]["yBar_arm"] = yBar_arm
return sc
def params_noCorr_corr(c_str, alpha, m0, m1, m2, m3, m4):
Q_DB = []
Q_DSC = []
mu_ds = []
H_max = []
H_est = []
Eps_alt = []
Eps_alt_abs = []
Err_alt = []
Err_ds = []
Err_total = []
Err_alt_abs = []
num_specimens = len(m0)
for i in range(num_specimens):
name = m0[i]["specimen"] # specimen name is the same for m0-m4
H_lab = m1[i][
"lab_field"] # the x-axis coordinate is the AF field of the m1-m4 steps
# do the calculations
m_m0 = m0[i]["total_m"]
m_m1 = m1[i]["total_m"]
m_m2 = m2[i]["total_m"]
m_m3 = m3[i]["total_m"]
m_m4 = m4[i]["total_m"]
if (m_m2 == None) or (m_m3 == None) or (m_m4 == None):
Q_DB.append([name, H_lab, (m_m1 - m_m0) / m_m0])
Q_DSC.append([name, None, None])
Eps_alt.append([name, None])
else: # calculate also the Q_DSC ratio and all the parameters
Q_DB.append([name, H_lab, (m_m1 - m_m0) / m_m0
]) # no corrected version, only "normal version"
# first check for the corrected version of un-corrected version for Q_DSC & parameter calculations
if (c_str == "_corr"):
# calculate corrections
m0M = [m0[i]["x"], m0[i]["y"], m0[i]["z"]]
m1M = [m1[i]["x"], m1[i]["y"], m1[i]["z"]]
m2M = [m2[i]["x"], m2[i]["y"], m2[i]["z"]]
m3M = [m3[i]["x"], m3[i]["y"], m3[i]["z"]]
m4M = [m4[i]["x"], m4[i]["y"], m4[i]["z"]]
NRMrem = helpers.list_mult_num(
helpers.list_plus_list(m1M, m2M), 0.5)
m1pTRM = helpers.list_min_list(m1M, NRMrem)
m2pTRM = helpers.list_min_list(m2M, NRMrem)
m3pTRM = helpers.list_min_list(m3M, NRMrem)
m4pTRM = helpers.list_min_list(m4M, NRMrem)
m_m0 = m0[i]["total_m"] # m_m0_corr
m_m1 = helpers.norm(NRMrem) + helpers.norm(
m1pTRM) # m_m1_corr
m_m2 = helpers.norm(NRMrem) - helpers.norm(
m2pTRM) # exception to the rule
m_m3 = helpers.norm(NRMrem) + helpers.norm(m3pTRM)
m_m4 = helpers.norm(NRMrem) + helpers.norm(m4pTRM)
Q_DSC.append([
name, H_lab,
2 * ((1 + alpha) * m_m1 - m_m0 - alpha * m_m3) /
(2 * m_m0 - m_m1 - m_m2)
])
Eps = (m_m4 - m_m1) / m_m1
Eps_alt.append([name, Eps])
sc["MSP_Q_calc" + c_str]["Q_DB"] = Q_DB
sc["MSP_Q_calc" + c_str]["Q_DSC"] = Q_DSC
sc["MSP_Q_calc" + c_str]["Eps_alt"] = Eps_alt
def dpal_ptrm_check_stat(sc):
"""
Function to calculate dpal statistic. A measure of cumulative alteration determined by the difference of the alteration corrected intensity estimate (Valet et al., 1996) and the uncorrected estimate, normalized by the uncorrected estimate (Leonhardt et al., 2004a).
input: slope of the best-fit line, ptrm_checks, ptrm, y_nrm, yBar
output: d_pal
"""
# input: preprocessed/ checks[ptrm_check]
# preprocessed/ basics[ptrm, y_nrm, yBar]
# arai_statistics/ PI_est[b_slope]
# output: check_statistics/ dpal_ptrm_check_stat[d_pal]
ptrm_check = sc["preprocessed"]["checks"]["ptrm_check"]
ptrm = sc["preprocessed"]["basics"]["ptrm"]
y_nrm = sc["preprocessed"]["basics"]["y_nrm"]
yBar = sc["preprocessed"]["basics"]["yBar"]
b_slope = sc["arai_statistics"]["PI_est"]["b_slope"]
d_pal = []
dT = []
# first check is you have ptrm_checks in the data, if not do nothing
if len(ptrm_check) != 0:
# first determine d_pTRM, which is the vector difference between ptrm - ptrmcheck for each step, if no ptrm check performed this is 0, 0, 0
s = 0
sprev = 0
for T in ptrm: # look trough the calculated ptrm gained specimen data
for C in ptrm_check:
if (T["step"] == C["step"]):
dT.append(
[T["x"] - C["x"], T["y"] - C["y"], T["z"] - C["z"]])
s += 1
if s <= sprev:
dT.append([0, 0, 0])
sprev = s
# make the cumulative sum vector C
C = []
Ci = [0, 0, 0]
for i in range(len(ptrm)):
Ci = helpers.list_plus_list(Ci, dT[i])
C.append(Ci)
ptrm_list = []
# add C to TRM
for T in ptrm:
ptrm_list.append([T["x"], T["y"], T["z"]])
TRM_star = []
for j in range(len(ptrm_list)):
TRM_star.append(helpers.list_plus_list(ptrm_list[j], C[j]))
x_ptrm_star = []
for i in range(len(TRM_star)):
x_ptrm_star.append(helpers.norm(TRM_star[i]))
# calculate the "new slope"
# copy form sc_arai_statiscics
n = len(x_ptrm_star)
xBar_star = sum(x_ptrm_star) / len(x_ptrm_star)
# Part (1) of b_slope equation
sum_xy = 0
for i in range(0, len(x_ptrm_star)):
sum_xy += (x_ptrm_star[i] - xBar_star) * (y_nrm[i] - yBar)
if sum_xy < 0:
sign = -1
elif sum_xy > 0:
sign = 1
else:
sign = 0
# part (2) of b_slope equation sumx en sumy
sumx = 0
sumy = 0
for i in range(0, len(x_ptrm_star)):
sumx += (x_ptrm_star[i] - xBar_star)**2
sumy += (y_nrm[i] - yBar)**2
# part(1) * part(2) gives b_slope
b_slope_star = sign * math.sqrt(sumy / sumx)
## stop copy
d_pal = abs((b_slope - b_slope_star) / b_slope) * 100
sc["check_statistics"]["dpal_ptrm_check_stat"]["d_pal"] = d_pal
return sc
def Tail_check_dtstar_stat(sc):
"""
Function to calculate the extent of a pTRM tail after correction for angular dependence (Leonhardt et al., 2004a; 2004b).
input: y_tail_check, y_temp_tail_check, y_nrm, y_nrm_all, x_temp_all, y_temp_all, NRM_vec_select, NRM_vec_all, xy_temp, field_dir_vec, Yint, Xint, b_slope
output: dt_star
"""
# input: preprocessed/ checks[y_tail_check, y_temp_tail_check]
# basics[y_nrm, y_nrm_all, x_temp_all, y_temp_all, NRM_vec_select, NRM_vec_all, xy_temp]
# field_basics[field_dir_vec]
# arai_statistics/ intercept_stats[Yint, Xint]
# PI_est[b_slope]
# output: check_statistics/ Tail_check_stat[dt_star]
tail_check_vec = sc["preprocessed"]["checks"]["tail_check_vec"]
y_temp_tail_check = sc["preprocessed"]["checks"]["y_temp_tail_check"]
y_nrm = sc["preprocessed"]["basics"]["y_nrm"]
y_nrm_all = sc["preprocessed"]["basics"]["y_nrm_all"]
x_temp_all = sc["preprocessed"]["basics"]["x_temp_all"]
y_temp_all = sc["preprocessed"]["basics"]["y_temp_all"]
xy_temp = sc["preprocessed"]["basics"]["xy_temp"]
NRM_vec_select = sc["preprocessed"]["basics"]["NRM_vec_select"]
NRM_vec_all = sc["preprocessed"]["basics"]["NRM_vec_all"]
field_dir_vec = sc["preprocessed"]["field_basics"]["field_dir_vec"]
b_slope = sc["arai_statistics"]["PI_est"]["b_slope"]
Yint = sc["arai_statistics"]["intercept_stats"]["Yint"]
Xint = sc["arai_statistics"]["intercept_stats"]["Xint"]
dt_star = []
t_star = []
if len(y_temp_tail_check
) != 0: # then you have at least one tailcheck, do calculations
# make an index matrix for the steps that step tail = step nrm_all
ind = []
for i in range(len(y_temp_tail_check)): # number of tail checks
for j in range(len(y_temp_all)): # loop over all NRM steps
if y_temp_tail_check[i] == y_temp_all[
j]: # if step of tail check equals step op NRM, write index to "ind" this is used below
ind.append(j)
# do calculations per tail check step
for i in range(len(y_temp_tail_check)):
MDx = tail_check_vec[i][0]
MDy = tail_check_vec[i][1]
MDz = tail_check_vec[i][2]
Nx = NRM_vec_all[ind[i]][
0] # take the NRM vector with same step as tail_check
Ny = NRM_vec_all[ind[i]][1]
Nz = NRM_vec_all[ind[i]][2]
a = helpers.list_div_num(NRM_vec_all[ind[i]], y_nrm_all[
ind[i]]) # divide the NRM list by the corresponding moment
theta_dt = helpers.get_angle_diff(
a, field_dir_vec
) # get angle between nomalized NRM vector and field direction
# calculate dH and dZ and inc_diff
# according to orientation of Blab
if abs(field_dir_vec[0] /
helpers.norm(field_dir_vec)) == 1: # then allong x
dH = math.sqrt(Ny**2 + Nz**2) - math.sqrt(MDy**2 + MDz**2)
dZ = Nx - MDx
Dec_F, Inc_F, R_F = helpers.cart2dir(field_dir_vec[0],
field_dir_vec[1],
field_dir_vec[2])
Dec_N, Inc_N, R_N = helpers.cart2dir(Nx, Ny, Nz)
inc_diff = Inc_F - Inc_N
elif abs(field_dir_vec[1] /
helpers.norm(field_dir_vec)) == 1: # then allong y
dH = math.sqrt(Nx**2 + Nz**2) - math.sqrt(MDx**2 + MDz**2)
dZ = Ny - MDy
Dec_F, Inc_F, R_F = helpers.cart2dir(field_dir_vec[0],
field_dir_vec[1],
field_dir_vec[2])
Dec_N, Inc_N, R_N = helpers.cart2dir(Nx, Ny, Nz)
inc_diff = Inc_F - Inc_N
elif abs(field_dir_vec[2] /
helpers.norm(field_dir_vec)) == 1: # then allong z
dH = math.sqrt(Nx**2 + Ny**2) - math.sqrt(MDx**2 + MDy**2)
dZ = Nz - MDz
Dec_F, Inc_F, R_F = helpers.cart2dir(field_dir_vec[0],
field_dir_vec[1],
field_dir_vec[2])
Dec_N, Inc_N, R_N = helpers.cart2dir(Nx, Ny, Nz)
inc_diff = Inc_F - Inc_N
else: # labfiels is not along any of the principal axis, do not calulate dt*
dH = 9999
if dH != 9999: # if it is, the labfield is not allong pinciple axix and no calculations are there
B = dH / math.tan(theta_dt)
Lim_upp = 2968 # rad *1000 -> 2.968 rad = 170 degrees
Lim_low = 175 # rad *1000 -> 0.175 rad = 10 degrees
# check if data is within limits
# floor rounds to the nearest small integer
if (math.floor(theta_dt * 1000)) < Lim_upp and (math.floor(
theta_dt * 1000)) > Lim_low:
if inc_diff > 0:
t_star.append(
(-dZ + B) * abs(b_slope) * 100 / abs(Yint))
else:
t_star.append(
(dZ - B) * abs(b_slope) * 100 / abs(Yint))
if (math.floor(theta_dt * 1000)) < Lim_low:
t_star.append(0)
if (math.floor(theta_dt * 1000)) > Lim_upp:
t_star.append(-dZ * 100 / (abs(Yint) + abs(Xint)))
if max(t_star) > 0:
dt_star = max(t_star)
else:
dt_star = 0
sc["check_statistics"]["Tail_check_stat"]["dt_star"] = dt_star
return sc
def boostrap(Q_str, c_str, site, selection, alpha, NumCycles, Confidence):
# split in measurements m0 m1 m2 m3 m4 with multiple specimens per list
m0 = list(filter(lambda m: m['type'] == 0, selection))
m1 = list(filter(lambda m: m['type'] == 1, selection))
m2 = list(filter(lambda m: m['type'] == 2, selection))
m3 = list(filter(lambda m: m['type'] == 3, selection))
m4 = list(filter(lambda m: m['type'] == 4, selection))
m1_all = list(filter(lambda m: m['type'] == 1, site))
# get the steps for the labfield array, this is done by looking at all the data from one site and find the min and maximum used labfields.
fields = []
num_specimens = len(m1_all) # all the data and not only the selection
for j in range(num_specimens):
fields.append(m1_all[j]["lab_field"]) # append all used labfields
# find min and max labfield used and determine the step
minField = min(fields)
maxField = max(fields)
numsteps = 11 # Moster et al., 2015 shows that 11 lab steps give the best results
step = (minField + maxField) / (numsteps - 1.
) # This is (Hmin+Hmax)/10
# append the step to a list of labfields -> Hlist
Hlist = []
for i in range(numsteps):
Hlist.append(i * step)
# set minimum standard deviation for Hlab
stdevH_min = 10
N2 = []
stdevHl = []
aa = []
bb = []
intercept = []
H0 = []
H1 = []
H2 = []
H3 = []
H4 = []
H5 = []
H6 = []
H7 = []
H8 = []
H9 = []
H10 = []
m = 0
killCounter = 0
while m < (NumCycles) and killCounter < (NumCycles * 5):
Hlab_DB = []
Hlab_DSC = []
Q_DB_error = []
Q_DSC_error = []
num_specimens = len(m0)
for j in range(num_specimens): # get N times a random specimen
# get the index of a random specimen
i = int(helpers.rand_num() *
num_specimens) # random number between 0 & N
# get moment per random specimen
m_m0 = m0[i]["total_m"]
m_m1 = m1[i]["total_m"]
# get corresponding error for that specimen, for Q_DB only m0 & m1
e_m0 = m0[i]["error"]
e_m1 = m1[i]["error"]
# calculate new m0_err and m1_err to calculate new Q_DB_error
frac_m0 = helpers.rand_num() * (0.02 * e_m0) + 1 - 0.01 * e_m0
m0_err = frac_m0 * m_m0
frac_m1 = helpers.rand_num() * (0.02 * e_m1) + 1 - 0.01 * e_m1
m1_err = frac_m1 * m_m1
Q_DB_error.append((m1_err - m0_err) / m0_err)
Hlab_DB.append(m1[i]["lab_field"])
if Q_str == "DSC":
if m2[i]["total_m"] != None:
m_m2 = m2[i]["total_m"]
m_m3 = m3[i]["total_m"]
m_m4 = m4[i]["total_m"]
e_m2 = m2[i]["error"]
e_m3 = m3[i]["error"]
e_m4 = m4[i]["error"]
# and check for the corrected version, if so replace the moments
if (c_str == "_corr"):
m0M = [m0[i]["x"], m0[i]["y"], m0[i]["z"]]
m1M = [m1[i]["x"], m1[i]["y"], m1[i]["z"]]
m2M = [m2[i]["x"], m2[i]["y"], m2[i]["z"]]
m3M = [m3[i]["x"], m3[i]["y"], m3[i]["z"]]
m4M = [m4[i]["x"], m4[i]["y"], m4[i]["z"]]
NRMrem = helpers.list_mult_num(
helpers.list_plus_list(m1M, m2M), 0.5)
m1pTRM = helpers.list_min_list(m1M, NRMrem)
m2pTRM = helpers.list_min_list(m2M, NRMrem)
m3pTRM = helpers.list_min_list(m3M, NRMrem)
m4pTRM = helpers.list_min_list(m4M, NRMrem)
m_m0 = m0[i]["total_m"] # m_m0_corr
m_m1 = helpers.norm(NRMrem) + helpers.norm(
m1pTRM) # m_m1_corr
m_m2 = helpers.norm(NRMrem) - helpers.norm(
m2pTRM) # exception to the rule
m_m3 = helpers.norm(NRMrem) + helpers.norm(m3pTRM)
m_m4 = helpers.norm(NRMrem) + helpers.norm(m4pTRM)
frac_m0 = helpers.rand_num() * (0.02 *
e_m0) + 1 - 0.01 * e_m0
m0_err = frac_m0 * m_m0
frac_m1 = helpers.rand_num() * (0.02 *
e_m1) + 1 - 0.01 * e_m1
m1_err = frac_m1 * m_m1
frac_m2 = helpers.rand_num() * (0.02 *
e_m2) + 1 - 0.01 * e_m2
m2_err = frac_m2 * m_m2
frac_m3 = helpers.rand_num() * (0.02 *
e_m3) + 1 - 0.01 * e_m3
m3_err = frac_m3 * m_m3
Q_DSC_error.append(
2 *
((1 + alpha) * m1_err - m0_err - alpha * m3_err) /
(2 * m0_err - m1_err - m2_err))
Hlab_DSC.append(m2[i]["lab_field"])
if (Q_str == "DB"):
Q_error = Q_DB_error
Hlab = Hlab_DB
elif (Q_str == "DSC"):
Q_error = Q_DSC_error
Hlab = Hlab_DSC
N = len(Q_error)
if N > 1:
avgH = sum(Hlab) / N
# calculate standard deviation on Hlab, and determine x and y
stdevH1 = []
x = []
y = []
for k in range(N):
stdevH1.append((Hlab[k] - avgH)**2)
x.append(Hlab[k])
y.append(Q_error[k])
stdevH = math.sqrt(sum(stdevH1) / (N - 1))
# calculate Sx, Sy, Sxx, Syy, Sxy
Sx = sum(x)
Sy = sum(y)
Sxy = helpers.dot_product(x, y)
Sxx = helpers.dot_product(x, x)
# calculate linear fit is not all at the same Hlab
if stdevH > stdevH_min:
b = (N * Sxy - Sx * Sy) / (N * Sxx - Sx**2)
a = Sy / N - b * Sx / N
PI = -1 * a / b
N2.append(N)
stdevHl.append(stdevH)
aa.append(a)
bb.append(b)
intercept.append(PI)
H0.append(a + b * Hlist[0])
H1.append(a + b * Hlist[1])
H2.append(a + b * Hlist[2])
H3.append(a + b * Hlist[3])
H4.append(a + b * Hlist[4])
H5.append(a + b * Hlist[5])
H6.append(a + b * Hlist[6])
H7.append(a + b * Hlist[7])
H8.append(a + b * Hlist[8])
H9.append(a + b * Hlist[9])
H10.append(a + b * Hlist[10])
# end of the big while loop, add one to m (this should be within the if statement)
m += 1
killCounter += 1
# sort columns and apply cut-off
cutOffValue = 0.01 * (100 - Confidence) / 2
cutOff = int(NumCycles * cutOffValue)
H0.sort()
H1.sort()
H2.sort()
H3.sort()
H4.sort()
H5.sort()
H6.sort()
H7.sort()
H8.sort()
H9.sort()
H10.sort()
Q_Hlist = [H0, H1, H2, H3, H4, H5, H6, H7, H8, H9, H10]
# determine the average of the bootstrap over the 11 labfields
# take the average of each of the labfield specified in Q_Hlist
Boot_int_min = []
Boot_int_max = []
Boot_avg = []
if len(Q_Hlist[0]) != 0:
h = 0
for el in Q_Hlist:
Boot_avg.append([Hlist[h], sum(el) / len(el)])
h += 1
F = cutOff # the minimum value F first
L = m - cutOff - 1 # the maximum value L last ( -1 because python counts from 0)
y_min = []
y_max = []
for w in range(len(Q_Hlist)):
y_min.append(Q_Hlist[w][F])
y_max.append(Q_Hlist[w][L])
for w in range(len(Hlist)):
Boot_int_min.append([Hlist[w], y_min[w]])
Boot_int_max.append([Hlist[w], y_max[w]])
# determine the x axis intercept for lower bound
ind_min = 999
for i in range(len(y_min) - 1):
if (y_min[i] < 0) & (y_min[i + 1] > 0):
ind_min = i
if ind_min == 999:
ictLow = None
else:
slope_min = (y_min[ind_min + 1] - y_min[ind_min]) / (
Hlist[ind_min + 1] - Hlist[ind_min])
ictLow = -1 * (y_min[ind_min] -
Hlist[ind_min] * slope_min) / slope_min
# determine the x axis intercept for upper bound
ind_max = 999
for j in range(len(y_max) - 1):
if (y_max[j] < 0) & (y_max[j + 1] > 0):
ind_max = j
if ind_max == 999:
ictHigh = None
else:
slope_max = (y_max[ind_max + 1] - y_max[ind_max]) / (
Hlist[ind_max + 1] - Hlist[ind_max])
ictHigh = -1 * (y_max[ind_max] -
Hlist[ind_max] * slope_max) / slope_max
# write corresponding PI min and max values, these are the intercepts of the bootstrap intervals
PI_min = ictHigh
PI_max = ictLow
else:
PI_min = None
PI_max = None
return [PI_min, PI_max, Boot_int_min, Boot_int_max, Boot_avg]
def run_step(self, prev, params):
data = prev
df = pd.read_excel(data, sheet_name='2.4')
df.columns = [
norm(x.strip().lower().replace(' ', '_').replace('-', '_').replace(
'%', 'perc')) for x in df.columns
]
print(df.columns)
df = df.loc[~df['entidad_federativa'].str.contains('Total')].copy()
# get end_id dimension
dim_geo = get_dimensions()[0]
df['entidad_federativa'].replace(dict(
zip(dim_geo['ent_name'], dim_geo['ent_id'])),
inplace=True)
df.columns = [
'ent_id', 'year', 'quarter_id', 'value_between_companies',
'value_new_investments', 'value_re_investments',
'count_between_companies', 'count_new_investments',
'count_re_investments', 'value_between_companies_c',
'value_new_investments_c', 'value_re_investments_c'
]
df['quarter_id'] = df['year'].astype(int).astype(
str) + df['quarter_id'].astype(int).astype(str)
df['quarter_id'] = df['quarter_id'].astype(int)
df.drop(columns=[
'value_between_companies', 'value_new_investments',
'value_re_investments'
],
inplace=True)
base = ['ent_id', 'year', 'quarter_id']
df_final = pd.DataFrame()
for option in [
'between_companies', 'new_investments', 're_investments'
]:
temp = df[base +
['count_{}'.format(option), 'value_{}_c'.format(option)
]].copy()
temp.columns = ['ent_id', 'year', 'quarter_id', 'count', 'value_c']
temp.dropna(subset=['value_c'], inplace=True)
temp['investment_type'] = option
df_final = df_final.append(temp)
df = df_final.copy()
df['investment_type'].replace(INVESTMENT_TYPE, inplace=True)
temp = pd.DataFrame()
for ent in list(df['ent_id'].unique()):
temp = temp.append(
validate_category(df.loc[(df['ent_id'] == ent)],
'investment_type', 'value_c', 'c'))
df = temp.copy()
temp = pd.DataFrame()
df = df.loc[df['value_c'].astype(str).str.lower() != 'c'].copy()
df['value_c'] = df['value_c'].astype(float)
return df
def specimen_fail_pass(Q_str, c_str, site, Boot_min, Boot_max):
# split in measurements m0 m1 m2 m3 m4 with multiple specimens per list
m0 = list(filter(lambda m: m['type'] == 0, site))
m1 = list(filter(lambda m: m['type'] == 1, site))
m2 = list(filter(lambda m: m['type'] == 2, site))
m3 = list(filter(lambda m: m['type'] == 3, site))
m4 = list(filter(lambda m: m['type'] == 4, site))
# determine if specimen is above / below bootstrap interval and FAIL or PASS specimen
num_specimens = len(m0)
# calculate original Q_DB & Q_DSC (also corrected) for ALL specimens
Hlab = []
Q_DB = []
Q_DSC = []
B_specimen_pass_fail = []
for i in range(num_specimens):
# get labfield and name specimen
Hlab = m1[i]["lab_field"]
name = m0[i]["specimen"] # specimen name is the same for m0-m4
m_m0 = m0[i]["total_m"]
m_m1 = m1[i]["total_m"]
m_m2 = m2[i]["total_m"]
m_m3 = m3[i]["total_m"]
m_m4 = m4[i]["total_m"]
if (m_m2 == None) or (m_m3 == None) or (m_m4 == None):
Q_DB.append([name, Hlab, (m_m1 - m_m0) / m_m0])
Q_DSC.append([name, None, None])
else:
# MSP-DSC calculate Q_DB & Q_DSC (check for corr)
Q_DB.append([name, Hlab, (m_m1 - m_m0) / m_m0]) # no Corr
if (c_str == "_corr"):
m0M = [m0[i]["x"], m0[i]["y"], m0[i]["z"]]
m1M = [m1[i]["x"], m1[i]["y"], m1[i]["z"]]
m2M = [m2[i]["x"], m2[i]["y"], m2[i]["z"]]
m3M = [m3[i]["x"], m3[i]["y"], m3[i]["z"]]
m4M = [m4[i]["x"], m4[i]["y"], m4[i]["z"]]
NRMrem = helpers.list_mult_num(
helpers.list_plus_list(m1M, m2M), 0.5)
m1pTRM = helpers.list_min_list(m1M, NRMrem)
m2pTRM = helpers.list_min_list(m2M, NRMrem)
m3pTRM = helpers.list_min_list(m3M, NRMrem)
m4pTRM = helpers.list_min_list(m4M, NRMrem)
m_m0 = m0[i]["total_m"] # m_m0_corr
m_m1 = helpers.norm(NRMrem) + helpers.norm(
m1pTRM) # m_m1_corr
m_m2 = helpers.norm(NRMrem) - helpers.norm(
m2pTRM) # exception to the rule
m_m3 = helpers.norm(NRMrem) + helpers.norm(m3pTRM)
m_m4 = helpers.norm(NRMrem) + helpers.norm(m4pTRM)
Q_DSC.append([
name, Hlab,
2 * ((1 + alpha) * m_m1 - m_m0 - alpha * m_m3) /
(2 * m_m0 - m_m1 - m_m2)
])
# loop over all specimens and check closest labfields from the Boot_min and Boot_max
for i in range(num_specimens):
Hsam = Q_DB[i][1]
if (Q_str == "DB"): # get the y for specimen for Q_DB
ysam = Q_DB[i][2]
elif (Q_str == "DSC"): # get the y for specimen for Q_DSC
ysam = Q_DSC[i][2]
name = m0[i]["specimen"] # specimen name is the same for m0-m4
if ysam == None:
B_specimen_pass_fail.append([name, "None"])
else:
ind_min = 999
for j in range(len(Boot_min) - 1):
if (Boot_min[j][0] <= Hsam) and (Boot_min[j + 1][0] >
Hsam):
ind = j
a_min = (Boot_min[ind + 1][1] - Boot_min[ind][1]) / (
Boot_min[ind + 1][0] - Boot_min[ind][0])
ycalc_min = (Hsam -
Boot_min[ind][0]) * a_min + Boot_min[ind][1]
a_max = (Boot_max[ind + 1][1] - Boot_max[ind][1]) / (
Boot_max[ind + 1][0] - Boot_max[ind][0])
ycalc_max = (Hsam -
Boot_max[ind][0]) * a_max + Boot_max[ind][1]
if (ysam > ycalc_max) or (ysam < ycalc_min):
B_specimen_pass_fail.append([name, "fail"])
else:
B_specimen_pass_fail.append([name, "pass"])
return [B_specimen_pass_fail]
def calc_nrm_ptrm(sc):
"""
Function to calculate/split the basic nrm and ptrm vectors and measurements for the different methods and protocols.
input: preprocessed measurement data, for all protocols and methods from the suitcase
output: basic vectors and list, like the NRM remaining and the ptrm vectors
"""
# input: preprocessed/ msrmnts[zerofield_steps, zerofield_steps_all, infield_steps, infield_steps_all, infield_antiparallel_steps, infield_antiparallel_steps_all, ARM_acq_steps, ARM_acq_steps_all]
# output: preprocessed/ basics[nrm0, ptrm, NRM_rem, NRM_vec_all, ptrm_vec_all, x_ptrm_all, x_temp_all, y_nrm_all, y_temp_all, pstep_all]
zerofield_steps = sc["preprocessed"]["msrmnts"]["zerofield_steps"]
zerofield_steps_all = sc["preprocessed"]["msrmnts"]["zerofield_steps_all"]
infield_steps = sc["preprocessed"]["msrmnts"]["infield_steps"]
infield_steps_all = sc["preprocessed"]["msrmnts"]["infield_steps_all"]
infield_antiparallel_steps = sc["preprocessed"]["msrmnts"][
"infield_antiparallel_steps"] # Thellier - Thellier
infield_antiparallel_steps_all = sc["preprocessed"]["msrmnts"][
"infield_antiparallel_steps_all"] # Thellier - Thellier
ARM_acq_steps = sc["preprocessed"]["msrmnts"][
"ARM_acq_steps"] # pseudo Thellier NAA format
ARM_acq_steps_all = sc["preprocessed"]["msrmnts"][
"ARM_acq_steps_all"] # pseudo Thellier NAA format
# get NRM0, the first zerofield step, this is true for both modified Thellier and classical TT and pseudo Thellier (GF & NAA)
nrm0 = zerofield_steps_all[0]["total_m"]
NRMrem_list = []
ptrm_list = []
NRM_vec_all = []
ptrm_vec_all = []
pstep_all = []
x_ptrm_all = []
x_temp_all = []
y_nrm_all = []
y_temp_all = []
# "classical thellier Thelier-Thellier (TT)"
if len(infield_antiparallel_steps) != 0:
# first the NRM, which is always a zerofield step, also for the Thellier - Thellier protocol,
# but not always in the selection so check the length to see if it is in the selection
if len(zerofield_steps) != 0: # then in selection
NRMrem_list.append([
zerofield_steps[0]["step"], zerofield_steps[0]["x"],
zerofield_steps[0]["y"], zerofield_steps[0]["z"]
])
ptrm_list.append([zerofield_steps[0]["step"], 0, 0,
0]) # this is new 06-02
if len(zerofield_steps_all
) != 0: # this is probably always the case # this is new 06-02
NRM_vec_all.append([
zerofield_steps_all[0]["x"], zerofield_steps_all[0]["y"],
zerofield_steps_all[0]["z"]
])
ptrm_vec_all.append([0, 0, 0])
x_ptrm_all.append(0)
y_nrm_all.append(
helpers.norm([
zerofield_steps_all[0]["x"], zerofield_steps_all[0]["y"],
zerofield_steps_all[0]["z"]
]))
x_temp_all.append(zerofield_steps_all[0]["step"])
y_temp_all.append(zerofield_steps_all[0]["step"])
# determine the NRM remaining & ptrm gained for the selection
for i_step in infield_steps:
for ai_step in infield_antiparallel_steps:
if i_step["step"] == ai_step["step"]:
NRMrem_list.append([
i_step["step"], (i_step["x"] + ai_step["x"]) / 2.,
(i_step["y"] + ai_step["y"]) / 2.,
(i_step["z"] + ai_step["z"]) / 2.
])
ptrm_list.append([
i_step["step"], (i_step["x"] - ai_step["x"]) / 2.,
(i_step["y"] - ai_step["y"]) / 2.,
(i_step["z"] - ai_step["z"]) / 2.
])
# determine the NRM remaining & ptrm gained for ALL steps
for i_step in infield_steps_all:
for ai_step in infield_antiparallel_steps_all:
if i_step["step"] == ai_step["step"]:
tempor_ptrm = [(i_step["x"] - ai_step["x"]) / 2.,
(i_step["y"] - ai_step["y"]) / 2.,
(i_step["z"] - ai_step["z"]) / 2.]
tempor_nrm = [(i_step["x"] + ai_step["x"]) / 2.,
(i_step["y"] + ai_step["y"]) / 2.,
(i_step["z"] + ai_step["z"]) / 2.]
NRM_vec_all.append(tempor_nrm)
ptrm_vec_all.append(tempor_ptrm)
x_ptrm_all.append(helpers.norm(tempor_ptrm))
x_temp_all.append(i_step["step"])
y_nrm_all.append(helpers.norm(tempor_nrm))
y_temp_all.append(i_step["step"])
# "modified thellier"
if len(infield_antiparallel_steps) == 0:
# pseudo Thellier GF or Thermal Thellier or Microwave Thellier
if len(ARM_acq_steps) == 0:
# if there are more zerofiels steps that infield steps, add steps to the ptrm_list with zeros, assumption is that extra zerofield steps are always before first ptrm step
# for i in range(len(zerofield_steps) - len(infield_steps)):
if (len(zerofield_steps) - len(infield_steps)) > 0:
ptrm_list.append(
[zerofield_steps_all[0]["step"], 0, 0,
0]) # this takes care off the first y-axis point
# make ptrm_list
for i_step in infield_steps:
for z_step in zerofield_steps:
if i_step["step"] == z_step["step"]:
ptrm_list.append([
i_step["step"], (i_step["x"] - z_step["x"]),
(i_step["y"] - z_step["y"]),
(i_step["z"] - z_step["z"])
])
# pseudo-Thellier NAA
if len(ARM_acq_steps) != 0:
# get the first ARM step, this is your NRM remaining and the ptrm should be corrected for this
x0 = ARM_acq_steps_all[0]["x"]
y0 = ARM_acq_steps_all[0]["y"]
z0 = ARM_acq_steps_all[0]["z"]
for a_step in ARM_acq_steps:
ptrm_list.append([
a_step["step"], a_step["x"] - x0, a_step["y"] - y0,
a_step["z"] - z0
])
# make NRMrem_list
for z_step in zerofield_steps:
NRMrem_list.append(
[z_step["step"], z_step["x"], z_step["y"], z_step["z"]])
for all_step in zerofield_steps_all:
NRM_vec_all.append([all_step["x"], all_step["y"], all_step["z"]])
y_nrm_all.append(
helpers.norm([all_step["x"], all_step["y"], all_step["z"]]))
y_temp_all.append(all_step["step"])
for all_infield in infield_steps_all: # pTh-NAA is not getting through this if statement
if all_infield["step"] == all_step["step"]:
ptrm_vec_all.append([
all_infield["x"] - all_step["x"],
all_infield["y"] - all_step["y"],
all_infield["z"] - all_step["z"]
])
pstep_all.append(all_infield["step"])
x_temp_all.append(all_infield["step"])
x_ptrm_all.append(
helpers.norm([
all_infield["x"] - all_step["x"],
all_infield["y"] - all_step["y"],
all_infield["z"] - all_step["z"]
]))
for a_step in ARM_acq_steps_all: # pTh -NAA format ptrm vec
ptrm_vec_all.append(
[a_step["x"] - x0, a_step["y"] - y0, a_step["z"] - z0])
x_ptrm_all.append(
helpers.norm(
[a_step["x"] - x0, a_step["y"] - y0, a_step["z"] - z0]))
x_temp_all.append(a_step["step"])
# rewrite format
ptrm = []
for msrmt in ptrm_list:
ptrm.append({
"step": msrmt[0],
"x": msrmt[1],
"y": msrmt[2],
"z": msrmt[3]
})
NRM_rem = []
for msrmt in NRMrem_list:
NRM_rem.append({
"step": msrmt[0],
"x": msrmt[1],
"y": msrmt[2],
"z": msrmt[3]
})
sc["preprocessed"]["basics"]["nrm0"] = nrm0
sc["preprocessed"]["basics"]["ptrm"] = ptrm
sc["preprocessed"]["basics"]["NRM_rem"] = NRM_rem
sc["preprocessed"]["basics"]["NRM_vec_all"] = NRM_vec_all
sc["preprocessed"]["basics"]["ptrm_vec_all"] = ptrm_vec_all
sc["preprocessed"]["basics"]["x_ptrm_all"] = x_ptrm_all
sc["preprocessed"]["basics"]["x_temp_all"] = x_temp_all
sc["preprocessed"]["basics"]["y_nrm_all"] = y_nrm_all
sc["preprocessed"]["basics"]["y_temp_all"] = y_temp_all
sc["preprocessed"]["basics"]["pstep_all"] = pstep_all
return sc
def anisotropy_calc(sc):
"""
Function that calculates the paleointensity correction factor when anisotropy data is available. It uses the previously calculated s-tensor. First the direction of the NRM for the selected part of the Arai plot is determined to give Mhat_ChRM. Which is used to get the anisotropy correction c. The anisotropy correction in multiplied with Banc to get Banc_aniso_corr.
input: s-tensor, the direction of the applied labfield, direction of the data selection, the original Banc
output: the anisotropy correction anis_c, the anisotropy corrected paleointensity estimate Banc_aniso_corr
"""
# input: preprocessed/ field_basics [field_dir_vec]
# s_tensor [s1_list, s2_list, s3_list, s4_list, s5_list, s6_list, scheck_list]
# arai_statistics/ PI_Banc_est [B_anc]
# directional_statistics/ mean_dir_stat [Mdec_free, Minc_free]
# output: anisotropy_statistics/ Anisotropy_Correction [anis_c, Banc_aniso_corr]
Mdec_free = sc["directional_statistics"]["mean_dir_stat"]["Mdec_free"]
Minc_free = sc["directional_statistics"]["mean_dir_stat"]["Minc_free"]
field_dir_vec = sc["preprocessed"]["field_basics"]["field_dir_vec"]
B_anc = sc["arai_statistics"]["PI_Banc_est"]["B_anc"]
Xp_list = sc["preprocessed"]["aniso_trm"]["x+"]
s_tensor = sc["anisotropy_statistics"]["Aniso_tensor"]["s_tensor"]
if Xp_list != None: # then there is anisotropy data, do calculations, if no anisotropy data present do nothing
anis_c = []
# the direction of the NRM, should be determined from the selected Arai plot
# this is the Mdec_free & Minc_free, the unit vecotr of this gives -> Mhat_ChRM
# get mean unit vector
Mhat_ChRM = helpers.dir2cart(Mdec_free, Minc_free, 1)
Blab_orient = field_dir_vec # it was: helpers.dir2cart(Mdec_free, Minc_free, 1), that is wrong!! (13 feb 2020)
s1 = s_tensor[0]
s2 = s_tensor[1]
s3 = s_tensor[2]
s4 = s_tensor[3]
s5 = s_tensor[4]
s6 = s_tensor[5]
A1 = [s1, s4, s6]
A2 = [s4, s2, s5]
A3 = [s6, s5, s3]
A = [A1, A2, A3]
# make A and Mhat_ChRM into a numpy arrays (1)
A = numpy.array(A)
Mhat_ChRM = numpy.array(Mhat_ChRM)
# do numpy calculation (2)
Hanc = numpy.linalg.solve(A, Mhat_ChRM)
# back to python lists (3)
A = A.tolist()
Mhat_ChRM = Mhat_ChRM.tolist()
Hanc = Hanc.tolist()
# unit vector in the direction of the ancient field
Hanc_hat = helpers.list_div_num(Hanc, helpers.norm(Hanc))
Manc = []
Mlab = []
for i in range(len(A)):
Manc.append(helpers.dot_product(A[i], Hanc_hat))
Mlab.append(helpers.dot_product(A[i], field_dir_vec))
aniso_c = helpers.norm(Mlab) / helpers.norm(Manc)
Banc_aniso_corr = aniso_c * B_anc
sc["anisotropy_statistics"]["Anisotropy_Correction"][
"aniso_c"] = aniso_c
sc["anisotropy_statistics"]["Anisotropy_Correction"][
"Banc_aniso_corr"] = Banc_aniso_corr
return (sc)
def kinetic_energy(self):
return norm(self.px, self.py)**2 / self.m / 2
def params_noCorr_corr(c_str, alpha, m0, m1, m2, m3, m4):
Q_DB = []
Q_DSC = []
mu_ds = []
H_max = []
H_est = []
Eps_alt = []
Eps_alt_abs = []
Err_alt = []
Err_ds = []
Err_total = []
Err_alt_abs = []
num_specimens = len(m0)
for i in range(num_specimens):
name = m0[i]["specimen"] # specimen name is the same for m0-m4
H_lab = m1[i][
"lab_field"] # the x-axis coordinate is the AF field of the m1-m4 steps
# do the calculations
m_m0 = m0[i]["total_m"]
m_m1 = m1[i]["total_m"]
m_m2 = m2[i]["total_m"]
m_m3 = m3[i]["total_m"]
m_m4 = m4[i]["total_m"]
if (m_m2 == None) or (m_m3 == None) or (m_m4 == None):
Q_DB.append([name, H_lab, (m_m1 - m_m0) / m_m0])
Q_DSC.append([name, None, None])
mu_ds.append([name, None])
H_max.append([name, None])
H_est.append([name, None])
Eps_alt.append([name, None])
Eps_alt_abs.append([name, None])
Err_alt.append([name, None])
Err_ds.append([name, None])
Err_total.append([name, None])
else: # calculate also the Q_DSC ratio and all the parameters
Q_DB.append([name, H_lab, (m_m1 - m_m0) / m_m0
]) # no corrected version, only "normal version"
# check for the corrected version of un-corrected version for Q_DSC & prameter calculations
if (c_str == "_corr"):
m0M = [m0[i]["x"], m0[i]["y"], m0[i]["z"]]
m1M = [m1[i]["x"], m1[i]["y"], m1[i]["z"]]
m2M = [m2[i]["x"], m2[i]["y"], m2[i]["z"]]
m3M = [m3[i]["x"], m3[i]["y"], m3[i]["z"]]
m4M = [m4[i]["x"], m4[i]["y"], m4[i]["z"]]
NRMrem = helpers.list_mult_num(
helpers.list_plus_list(m1M, m2M), 0.5)
m1pTRM = helpers.list_min_list(m1M, NRMrem)
m2pTRM = helpers.list_min_list(m2M, NRMrem)
m3pTRM = helpers.list_min_list(m3M, NRMrem)
m4pTRM = helpers.list_min_list(m4M, NRMrem)
m_m0 = m0[i]["total_m"] # m_m0_corr
m_m1 = helpers.norm(NRMrem) + helpers.norm(
m1pTRM) # m_m1_corr
m_m2 = helpers.norm(NRMrem) - helpers.norm(
m2pTRM) # exception to the rule
m_m3 = helpers.norm(NRMrem) + helpers.norm(m3pTRM)
m_m4 = helpers.norm(NRMrem) + helpers.norm(m4pTRM)
Q_DSC.append([
name, H_lab,
2 * ((1 + alpha) * m_m1 - m_m0 - alpha * m_m3) /
(2 * m_m0 - m_m1 - m_m2)
])
mu_ds.append(
[name, (m_m1 - m_m3) / (m_m3 - 0.5 * (m_m1 + m_m2))])
H_max.append(
[name, (2 * m_m0 - m_m1 - m_m2) / (m_m1 - m_m2) * H_lab])
H_est.append([
name, (2 * m_m0 - m_m1 - m_m2) /
((1 + 2 * alpha) * m_m1 - 2 * alpha * m_m3 - m_m2) * H_lab
])
Eps = (m_m4 - m_m1) / m_m1
Eps_alt.append([name, Eps])
Eps_alt_abs.append([name, abs((m_m1 - m_m4) / m_m1)])
# calculate the error estimates
# nummerator & denominator
num = 2 * ((1 + alpha) * m_m1 - m_m0 - alpha * m_m3)
den = (2 * m_m0) - m_m1 - m_m2
# partial derivatives of Q_DSC
d_num_m1 = 2 * (1 + alpha)
d_num_m2 = 0
d_num_m3 = -2 * alpha
d_den_m1 = -1
d_den_m2 = -1
d_den_m3 = 0
# terms
Term_1 = m_m1 * ((den * d_num_m1) -
(num * d_den_m1)) / (den)**2
Term_2 = m_m2 * ((den * d_num_m2) -
(num * d_den_m2)) / (den)**2
Term_3 = m_m3 * ((den * d_num_m3) -
(num * d_den_m3)) / (den)**2
dQ_DSC_alt = Eps**2 * (Term_1**2 + Term_2**2 + Term_3**2)
dQ_DSC_ds = (((m_m3 - m_m1) / den)**2) / 3
Err_alt.append([name, dQ_DSC_alt])
Err_ds.append([name, dQ_DSC_ds])
Err_total.append([name, dQ_DSC_alt + dQ_DSC_ds])
return [
Q_DB, Q_DSC, mu_ds, H_max, H_est, Eps_alt, Eps_alt_abs, Err_alt,
Err_ds, Err_total
]
def train(tileSize=64, numPuzzles=23, num_permutations=10, batch_size=16):
# On server with PET and PCT in
image_dir = "/hepgpu3-data1/dmcsween/Data128/ResampleData/PlanningCT"
print("Load Data")
image_data, __image, __label = load.data_reader(image_dir, image_dir,
image_dir)
image_array, image_affine = image_data.get_data()
moving_array, moving_affine = __image.get_data()
dvf_array, dvf_affine = __label.get_data()
"""
list_avail_keys = help.get_moveable_keys(image_array)
hamming_set = pd.read_csv(
"hamming_set_PCT.txt", sep=",", header=None)
"""
avail_keys = pd.read_csv("avail_keys_both.txt", sep=",", header=None)
print("Len keys:", len(avail_keys))
list_avail_keys = [(avail_keys.loc[i, 0], avail_keys.loc[i, 1],
avail_keys.loc[i, 2]) for i in range(len(avail_keys))]
print(list_avail_keys)
# Get hamming set
print("Load hamming Set")
hamming_set = pd.read_csv("hamming_set.txt", sep=",", header=None)
#hamming_set = hamming_set.loc[:9]
print("Ham Len", len(hamming_set))
print(hamming_set)
fixed_array, moving_array, dvf_array = helper.shuffle_inplace(
image_array, moving_array, dvf_array)
# Ignore moving and dvf
validation_dataset, validation_moving, validation_dvf, train_dataset, train_moving, train_dvf = helper.split_data(
fixed_array, moving_array, dvf_array, split_ratio=0.15)
normalised_train = helper.norm(train_dataset)
normalised_val = helper.norm(validation_dataset)
# Output all data from a training session into a dated folder
outputPath = "./logs"
# hamming_list = [0, 1, 2, 3, 4]
# img_idx = [0, 1, 2, 3, 4]
# callbacks
checkpoint = ModelCheckpoint(outputPath + '/best_model.h5',
monitor='val_acc',
verbose=1,
save_best_only=True,
period=1)
reduce_lr_plateau = ReduceLROnPlateau(monitor='val_acc',
patience=10,
verbose=1)
# early_stop = EarlyStopping(monitor='val_acc', patience=5, verbose=1)
tensorboard = TrainValTensorBoard(write_graph=False)
callbacks = [checkpoint, reduce_lr_plateau, tensorboard]
# BUILD Model
model = createSharedAlexnet3D_onemodel()
# for layer in model.layers:
# print(layer.name, layer.output_shape)
opt = optimizers.SGD(lr=0.01)
plot_model(model, to_file='model.png')
print(model.summary())
model.compile(optimizer=opt,
loss='categorical_crossentropy',
metrics=['accuracy'])
model.fit_generator(generator=gen.generator(normalised_train,
list_avail_keys,
hamming_set,
batch_size=batch_size,
N=num_permutations),
epochs=1000,
verbose=1,
steps_per_epoch=normalised_train.shape[0] //
batch_size,
validation_data=gen.generator(normalised_val,
list_avail_keys,
hamming_set,
batch_size=batch_size,
N=num_permutations),
validation_steps=normalised_val.shape[0] // batch_size,
callbacks=callbacks,
shuffle=False)
model.save('model_best.h5')
