def pair_diff_PI(norm_shots, interp_rps, qs):
print("doing corr pairing...")
#dummy qs
num_phi = norm_shots.shape[-1]
eps = distance.cdist(interp_rps, interp_rps, metric='euclidean')
# do this so the diagonals are not the minimum, i.e. don't pair shot with itself
epsI = 1.1 * eps.max(1) * np.identity(eps.shape[0])
eps += epsI
shot_preference = np.roll(eps.argsort(1), 1, axis=1)
pref_dict = {str(E[0]): list(E[1:]) for E in shot_preference.astype(str)}
print("stable roommate pair....")
pairs_dict = stable.stableroomate(prefs=pref_dict)
pairing = np.array(MakeTagPairs._remove_duplicate_pairs(pairs_dict))
print("computing difference intensities...")
diff_norm = np.zeros(
(norm_shots.shape[0] / 2, qs.size, norm_shots.shape[-1]),
dtype=np.float64)
for index, pp in enumerate(pairing):
diff_norm[index] = norm_shots[pp[0]] - norm_shots[pp[1]]
return diff_norm, pairing
def pair_diff_PI(norm_shots, interp_rps, qs):
print("doing corr pairing...")
#dummy qs
num_phi=norm_shots.shape[-1]
eps = distance.cdist(interp_rps,interp_rps, metric='euclidean')
# do this so the diagonals are not the minimum, i.e. don't pair shot with itself
epsI = 1.1 * eps.max(1) * np.identity(eps.shape[0])
eps += epsI
shot_preference = np.roll(eps.argsort(1), 1, axis=1)
pref_dict = {str(E[0]): list(E[1:])
for E in shot_preference.astype(str)}
print("stable roommate pair....")
pairs_dict = stable.stableroomate(prefs=pref_dict)
pairing = np.array(MakeTagPairs._remove_duplicate_pairs(pairs_dict) )
print("computing difference intensities...")
diff_norm = np.zeros( (norm_shots.shape[0]/2,
qs.size,
norm_shots.shape[-1]),
dtype=np.float64 )
for index, pp in enumerate( pairing ):
diff_norm[index] = norm_shots[pp[0]]-norm_shots[pp[1]]
return diff_norm, pairing
def pair_shots(eps):
#epsI = 1.1*eps.max(1) * np.identity(eps.shape[0])
#eps += epsI
shot_preference = np.roll(eps.argsort(1), 1, axis=1)
pref_dict = {str(E[0]): list(E[1:])
for E in shot_preference.astype(str)}
pairs_dict = stable.stableroomate(prefs=pref_dict)
return pairs_dict
def _pair_group(self):
"""
Pairs exposures in a dataframe group according to the
similarity of their respective polynomials
"""
ngroup = len(self.df_g)
Px = np.arange(self.nphi)
Py = np.vstack([np.polynomial.chebyshev.chebval(Px, c)
for c in self.df_g.cheby_fit_pkremove])
# polynomial distance measure between shot i and shot j
print(" Calculating the distance matrix...")
eps = distance.cdist(Py, Py, metric='euclidean')
print(" Calculating the preference matrix...")
# Add the max to the diagonal, otherwise the diagonal will
# register as the minimum (each shot is closest to itself)
epsI = 1.1 * eps.max(1) * np.identity(ngroup)
eps += epsI
# column 0 is the shot, columns 1->N are the "closest"
# shots, 1 being the closest and N being the furthest
self._shot_preference = np.roll(eps.argsort(1), 1, axis=1)
print(" storing in hash table...")
pref_dict = {str(E[0]): list(E[1:])
for E in self._shot_preference.astype(str)}
# use the stable roommate (Irving's) algorithm to pair the shots
print(" Forming the pairings using Irving's algorthm...")
pairs_dict = stable.stableroomate(prefs=pref_dict)
pairs = self._remove_duplicate_pairs(pairs_dict)
tag_pairs = [(str(self.df_g.tag[i]), str(self.df_g.tag[j]))
for i, j in pairs]
self.tag_pairs.extend(tag_pairs)
pdists = [eps[i][j] for i, j in pairs]
score = np.mean(pdists) / np.mean(eps.ravel())
print(" Pairing score for group %d: %.3f" % (self.groupID, score))
self.outfile.create_dataset(
'group%d/pair_distances' %
self.groupID, data=pdists)
self.outfile.create_dataset(
'group%d/tag_pairs' %
self.groupID, data=tag_pairs)
self.outfile.create_dataset('group%d/pairs' % self.groupID, data=pairs)
self.outfile.create_dataset('group%d/score' % self.groupID, data=score)
self.outfile.create_dataset(
'group%d/distance_matrix' %
self.groupID, data=eps)
print ("Saved pairing data to %s!"%self.outfile.filename)
def corr_pair_diff_PI(norm_shots, mask_corr, qs,
qidx_pair = 25,
phi_offset=10):
"""
Pair polar intensities at one q value using the corr pair method
Correlations of individual intensities are computed. Shots with similar correlations are paired
Pairing metrics are euclidean distances between single-shot correlations
Pairing done by the Stable roommate method
norm_shots - numpy.array, Nshot*Nq*Nphi, normalized and zeroed polar intensities
mask_corr - numpy.array, Nq*Nphi, correlations of the mask used for the PI shots
qs - numpy.array, Nq, q values covered by the PI shots
qidx_pair - int, idx of the q values at which to pair shots
phi_offset - int, number of pixels in phi to ignore on the two extremes of the correlations
Ignoring these pixels means we are not looking at the very high values near 0 and pi
"""
print("doing corr pairing...")
num_phi=norm_shots.shape[-1]
dc = DiffCorr(norm_shots,
qs,0,pre_dif=True)
corr = dc.autocorr()
corr/=mask_corr
corr=corr[:,:,phi_offset:num_phi/2-phi_offset]
eps = distance.cdist(corr[:,qidx_pair],corr[:,qidx_pair], metric='euclidean')
# do this so the diagonals are not the minimum, i.e. don't pair shot with itself
epsI = 1.1 * eps.max(1) * np.identity(eps.shape[0])
eps += epsI
shot_preference = np.roll(eps.argsort(1), 1, axis=1)
pref_dict = {str(E[0]): list(E[1:])
for E in shot_preference.astype(str)}
print("stable roommate pair....")
pairs_dict = stable.stableroomate(prefs=pref_dict)
pairing = np.array(MakeTagPairs._remove_duplicate_pairs(pairs_dict) )
print("computing difference intensities...")
diff_norm = np.zeros( (norm_shots.shape[0]/2,
1,
norm_shots.shape[-1]),
dtype=np.float64 )
for index, pp in enumerate( pairing ):
diff_norm[index,0] = norm_shots[pp[0],qidx_pair]-norm_shots[pp[1],qidx_pair]
return diff_norm
def corr_pair_diff_PI(norm_shots, mask_corr, qs, qidx_pair=25, phi_offset=10):
"""
Pair polar intensities at one q value using the corr pair method
Correlations of individual intensities are computed. Shots with similar correlations are paired
Pairing metrics are euclidean distances between single-shot correlations
Pairing done by the Stable roommate method
norm_shots - numpy.array, Nshot*Nq*Nphi, normalized and zeroed polar intensities
mask_corr - numpy.array, Nq*Nphi, correlations of the mask used for the PI shots
qs - numpy.array, Nq, q values covered by the PI shots
qidx_pair - int, idx of the q values at which to pair shots
phi_offset - int, number of pixels in phi to ignore on the two extremes of the correlations
Ignoring these pixels means we are not looking at the very high values near 0 and pi
"""
print("doing corr pairing...")
num_phi = norm_shots.shape[-1]
dc = DiffCorr(norm_shots, qs, 0, pre_dif=True)
corr = dc.autocorr()
corr /= mask_corr
corr = corr[:, :, phi_offset:num_phi / 2 - phi_offset]
eps = distance.cdist(corr[:, qidx_pair],
corr[:, qidx_pair],
metric='euclidean')
# do this so the diagonals are not the minimum, i.e. don't pair shot with itself
epsI = 1.1 * eps.max(1) * np.identity(eps.shape[0])
eps += epsI
shot_preference = np.roll(eps.argsort(1), 1, axis=1)
pref_dict = {str(E[0]): list(E[1:]) for E in shot_preference.astype(str)}
print("stable roommate pair....")
pairs_dict = stable.stableroomate(prefs=pref_dict)
pairing = np.array(MakeTagPairs._remove_duplicate_pairs(pairs_dict))
print("computing difference intensities...")
diff_norm = np.zeros((norm_shots.shape[0] / 2, 1, norm_shots.shape[-1]),
dtype=np.float64)
for index, pp in enumerate(pairing):
diff_norm[index,
0] = norm_shots[pp[0], qidx_pair] - norm_shots[pp[1],
qidx_pair]
return diff_norm
def pair_diff_PI(max_pos_cluster_shots,
degree = 15,
qidx_pair = 25):
print("doing cheby pairing...")
if max_pos_cluster_shots.shape[0]%2>0:
max_pos_cluster_shots = max_pos_cluster_shots[:-1]
# cheby pair within each
# pairing with chebyfit polynomials
# I am going to use the qidx = 25 for pairing
print("fitting to polynomials....")
fits = np.zeros( (max_pos_cluster_shots.shape[0],
max_pos_cluster_shots.shape[-1])
,dtype = np.float64 )
for ii in range(max_pos_cluster_shots.shape[0]):
try:
_,_,yfit = fit_periodic(max_pos_cluster_shots[ii,qidx_pair].copy(),
mask=np.ones(max_pos_cluster_shots.shape[-1],dtype=bool),
deg=degree,overlap=0.1)
fits[ii] = yfit
except TypeError:
# print ii
# print max_pos_cluster_shots[ii,qidx_pair]
# sys.exit()
continue
eps = distance.cdist(fits, fits, metric='euclidean')
# do this so the diagonals are not the minimum, i.e. don't pair shot with itself
epsI = 1.1 * eps.max(1) * np.identity(eps.shape[0])
eps += epsI
shot_preference = np.roll(eps.argsort(1), 1, axis=1)
pref_dict = {str(E[0]): list(E[1:])
for E in shot_preference.astype(str)}
print("stable roommate pair....")
pairs_dict = stable.stableroomate(prefs=pref_dict)
pairing = np.array(MakeTagPairs._remove_duplicate_pairs(pairs_dict) )
print("computing difference intensities...")
diff_norm = np.zeros( (max_pos_cluster_shots.shape[0]/2,
max_pos_cluster_shots.shape[1],
max_pos_cluster_shots.shape[-1]),
dtype=np.float64 )
for index, pp in enumerate( pairing ):
diff_norm[index] = max_pos_cluster_shots[pp[0]]-max_pos_cluster_shots[pp[1]]
return diff_norm
def pair_diff_PI(max_pos_cluster_shots, degree=15, qidx_pair=25):
print("doing cheby pairing...")
if max_pos_cluster_shots.shape[0] % 2 > 0:
max_pos_cluster_shots = max_pos_cluster_shots[:-1]
# cheby pair within each
# pairing with chebyfit polynomials
# I am going to use the qidx = 25 for pairing
print("fitting to polynomials....")
fits = np.zeros(
(max_pos_cluster_shots.shape[0], max_pos_cluster_shots.shape[-1]),
dtype=np.float64)
for ii in range(max_pos_cluster_shots.shape[0]):
try:
_, _, yfit = fit_periodic(
max_pos_cluster_shots[ii, qidx_pair].copy(),
mask=np.ones(max_pos_cluster_shots.shape[-1], dtype=bool),
deg=degree,
overlap=0.1)
fits[ii] = yfit
except TypeError:
# print ii
# print max_pos_cluster_shots[ii,qidx_pair]
# sys.exit()
continue
eps = distance.cdist(fits, fits, metric='euclidean')
# do this so the diagonals are not the minimum, i.e. don't pair shot with itself
epsI = 1.1 * eps.max(1) * np.identity(eps.shape[0])
eps += epsI
shot_preference = np.roll(eps.argsort(1), 1, axis=1)
pref_dict = {str(E[0]): list(E[1:]) for E in shot_preference.astype(str)}
print("stable roommate pair....")
pairs_dict = stable.stableroomate(prefs=pref_dict)
pairing = np.array(MakeTagPairs._remove_duplicate_pairs(pairs_dict))
print("computing difference intensities...")
diff_norm = np.zeros(
(max_pos_cluster_shots.shape[0] / 2, max_pos_cluster_shots.shape[1],
max_pos_cluster_shots.shape[-1]),
dtype=np.float64)
for index, pp in enumerate(pairing):
diff_norm[index] = max_pos_cluster_shots[
pp[0]] - max_pos_cluster_shots[pp[1]]
return diff_norm
def pair_diff_PI(norm_shots,
mask_corr,
qs,
qidx_pair=25,
phi_offset=0,
pair_method='int'):
if pair_method == 'corr':
print("doing corr pairing...")
#dummy qs
num_phi = norm_shots.shape[-1]
dc = DiffCorr(norm_shots, qs, 0, pre_dif=True)
corr = dc.autocorr()
corr /= mask_corr
corr = corr[:, :, phi_offset:num_phi / 2 - phi_offset]
eps = distance.cdist(corr[:, qidx_pair],
corr[:, qidx_pair],
metric='euclidean')
if pair_method == 'int':
print "doing intensity pair..."
eps = distance.cdist(norm_shots[:, qidx_pair],
norm_shots[:, qidx_pair],
metric='euclidean')
# do this so the diagonals are not the minimum, i.e. don't pair shot with itself
epsI = 1.1 * eps.max(1) * np.identity(eps.shape[0])
eps += epsI
shot_preference = np.roll(eps.argsort(1), 1, axis=1)
pref_dict = {str(E[0]): list(E[1:]) for E in shot_preference.astype(str)}
print("stable roommate pair....")
pairs_dict = stable.stableroomate(prefs=pref_dict)
pairing = np.array(MakeTagPairs._remove_duplicate_pairs(pairs_dict))
print("computing difference intensities...")
diff_norm = np.zeros(
(norm_shots.shape[0] / 2, norm_shots.shape[1], norm_shots.shape[-1]),
dtype=np.float64)
for index, pp in enumerate(pairing):
diff_norm[index] = norm_shots[pp[0]] - norm_shots[pp[1]]
return diff_norm, pairing
def pair_diff_PI(norm_shots, mask,
step_size = 10,
delta=5,
qidx_pair = 25):
print("doing cheby pairing...")
if norm_shots.shape[0]%2>0:
norm_shots = norm_shots[:-1]
num_steps = int(np.ceil(norm_shots.shape[-1]/float(delta)))
fits = np.zeros([norm_shots.shape[0],num_steps])
for ss in range(norm_shots.shape[0]):
shot = norm_shots[ss,qidx_pair]
shot,_,_ = remove_peaks(shot,mask[qidx_pair])
reduce_shot = np.zeros(num_steps,dtype=np.float64)
for step in range(num_steps):
reduce_shot[step] = np.sum(shot[step*delta:(step*delta+step_size)])
fits[ss] = reduce_shot
eps = distance.cdist(fits, fits, metric='euclidean')
# do this so the diagonals are not the minimum, i.e. don't pair shot with itself
epsI = 1.1 * eps.max(1) * np.identity(eps.shape[0])
eps += epsI
shot_preference = np.roll(eps.argsort(1), 1, axis=1)
pref_dict = {str(E[0]): list(E[1:])
for E in shot_preference.astype(str)}
print("stable roommate pair....")
pairs_dict = stable.stableroomate(prefs=pref_dict)
pairing = np.array(MakeTagPairs._remove_duplicate_pairs(pairs_dict) )
print("computing difference intensities...")
diff_norm = np.zeros( (norm_shots.shape[0]/2,
norm_shots.shape[1],
norm_shots.shape[-1]),
dtype=np.float64 )
for index, pp in enumerate( pairing ):
diff_norm[index] = norm_shots[pp[0]]-norm_shots[pp[1]]
return diff_norm
def pair_diff_PI(norm_shots, mask_corr, qs,
qidx_pair = 25,
phi_offset=0,
pair_method='int'):
if pair_method=='corr':
print("doing corr pairing...")
#dummy qs
num_phi=norm_shots.shape[-1]
dc = DiffCorr(norm_shots,
qs,0,pre_dif=True)
corr = dc.autocorr()
corr/=mask_corr
corr=corr[:,:,phi_offset:num_phi/2-phi_offset]
eps = distance.cdist(corr[:,qidx_pair],corr[:,qidx_pair], metric='euclidean')
if pair_method=='int':
print "doing intensity pair..."
eps = distance.cdist(norm_shots[:,qidx_pair],norm_shots[:,qidx_pair], metric='euclidean')
# do this so the diagonals are not the minimum, i.e. don't pair shot with itself
epsI = 1.1 * eps.max(1) * np.identity(eps.shape[0])
eps += epsI
shot_preference = np.roll(eps.argsort(1), 1, axis=1)
pref_dict = {str(E[0]): list(E[1:])
for E in shot_preference.astype(str)}
print("stable roommate pair....")
pairs_dict = stable.stableroomate(prefs=pref_dict)
pairing = np.array(MakeTagPairs._remove_duplicate_pairs(pairs_dict) )
print("computing difference intensities...")
diff_norm = np.zeros( (norm_shots.shape[0]/2,
norm_shots.shape[1],
norm_shots.shape[-1]),
dtype=np.float64 )
for index, pp in enumerate( pairing ):
diff_norm[index] = norm_shots[pp[0]]-norm_shots[pp[1]]
return diff_norm, pairing
def pair_diff_PI(norm_shots, mask, step_size=10, delta=5, qidx_pair=25):
print("doing cheby pairing...")
if norm_shots.shape[0] % 2 > 0:
norm_shots = norm_shots[:-1]
num_steps = int(np.ceil(norm_shots.shape[-1] / float(delta)))
fits = np.zeros([norm_shots.shape[0], num_steps])
for ss in range(norm_shots.shape[0]):
shot = norm_shots[ss, qidx_pair]
shot, _, _ = remove_peaks(shot, mask[qidx_pair])
reduce_shot = np.zeros(num_steps, dtype=np.float64)
for step in range(num_steps):
reduce_shot[step] = np.sum(shot[step * delta:(step * delta +
step_size)])
fits[ss] = reduce_shot
eps = distance.cdist(fits, fits, metric='euclidean')
# do this so the diagonals are not the minimum, i.e. don't pair shot with itself
epsI = 1.1 * eps.max(1) * np.identity(eps.shape[0])
eps += epsI
shot_preference = np.roll(eps.argsort(1), 1, axis=1)
pref_dict = {str(E[0]): list(E[1:]) for E in shot_preference.astype(str)}
print("stable roommate pair....")
pairs_dict = stable.stableroomate(prefs=pref_dict)
pairing = np.array(MakeTagPairs._remove_duplicate_pairs(pairs_dict))
print("computing difference intensities...")
diff_norm = np.zeros(
(norm_shots.shape[0] / 2, norm_shots.shape[1], norm_shots.shape[-1]),
dtype=np.float64)
for index, pp in enumerate(pairing):
diff_norm[index] = norm_shots[pp[0]] - norm_shots[pp[1]]
return diff_norm
for ii in range(num_shots):
_,_,yfit = fit_periodic(shots_merge[ii,2].copy(),
mask=np.ones(shots_merge.shape[-1],dtype=bool),
deg=degree,overlap=0.1)
fits[ii] = yfit
eps = distance.cdist(fits, fits, metric='euclidean')
# do this so the diagonals are not the minimum, i.e. don't pair shot with itself
epsI = 1.1 * eps.max(1) * np.identity(eps.shape[0])
eps += epsI
shot_preference = np.roll(eps.argsort(1), 1, axis=1)
pref_dict = {str(E[0]): list(E[1:])
for E in shot_preference.astype(str)}
pairs_dict = stable.stableroomate(prefs=pref_dict)
pairing = np.array(MakeTagPairs._remove_duplicate_pairs(pairs_dict) )
# print pairing.shape
diff_PI = np.zeros( (num_shots/2, 3, shots_merge.shape[-1]),
dtype=np.float64 )
# print diff_PI.shape
# print shots_merge[pairing[0][0]].shape
for idx, pp in enumerate( pairing ):
diff_PI[idx] = shots_merge[pp[0]]-shots_merge[pp[1]]
# diff cor time
dc = DiffCorr(diff_PI, qvalues,
_, _, yfit = fit_periodic(shots_merge[ii, 2].copy(),
mask=np.ones(shots_merge.shape[-1],
dtype=bool),
deg=degree,
overlap=0.1)
fits[ii] = yfit
eps = distance.cdist(fits, fits, metric='euclidean')
# do this so the diagonals are not the minimum, i.e. don't pair shot with itself
epsI = 1.1 * eps.max(1) * np.identity(eps.shape[0])
eps += epsI
shot_preference = np.roll(eps.argsort(1), 1, axis=1)
pref_dict = {str(E[0]): list(E[1:]) for E in shot_preference.astype(str)}
pairs_dict = stable.stableroomate(prefs=pref_dict)
pairing = np.array(MakeTagPairs._remove_duplicate_pairs(pairs_dict))
# print pairing.shape
diff_PI = np.zeros((num_shots / 2, 3, shots_merge.shape[-1]),
dtype=np.float64)
# print diff_PI.shape
# print shots_merge[pairing[0][0]].shape
for idx, pp in enumerate(pairing):
diff_PI[idx] = shots_merge[pp[0]] - shots_merge[pp[1]]
# diff cor time
dc = DiffCorr(diff_PI, qvalues, k_beam, pre_dif=True)
corr = dc.autocorr().mean(0)