def get_contact(self, i, simulator):
'''
This function selects a proportion of indexes representing contagious
particles (exposed, infected, severe infected) based on their disease
transmittion rates and retrieves the contacted particles based on their IDs.
With accordance whether the app was installed, particles ids and the time state
are saved in the contact_cell.
'''
exposed_id = np.array(np.nonzero(self.states['exposed'])[0])
ind_end_exp = np.random.randint(
len(exposed_id),
size=int(
np.ceil(simulator.TRANSMISSION_RATE_EXPOSED *
len(exposed_id))))
infected_id = np.array(np.nonzero(self.states['infected'])[0])
sev_infected_id = np.array(np.nonzero(self.states['severe_inf'])[0])
ind_end_sev = np.random.randint(
len(sev_infected_id),
size=int(
np.ceil(simulator.TRANSMISSION_RATE_SEVERE_INFECT *
len(sev_infected_id))))
isolated_id = np.array(np.nonzero(self.states['true_iso'])[0])
ind_end_iso = np.random.randint(
len(isolated_id),
size=int(
np.ceil(simulator.TRANSMISSION_RATE_QUA * len(isolated_id))))
array = np.hstack(
(exposed_id[ind_end_exp], infected_id,
sev_infected_id[ind_end_sev], isolated_id[ind_end_iso])).ravel()
if len(array) > 1:
contacts = [
int(i)
for sublist in KDTree(self.x, leaf_size=2, metric="manhattan").
query_radius(self.x[array], r=simulator.init_cont_threshold)
for i in sublist
]
contact_ids = KDTree(self.x, leaf_size=2,
metric="manhattan").query_radius(
self.x[array],
r=simulator.init_cont_threshold,
count_only=False)
for k in range(len(contact_ids)):
if len(contact_ids[k]) <= 0:
continue
curr_ind = array[k]
if ((self.app[curr_ind] == 1) &
((self.epidemic_state[curr_ind] == 1) |
(self.epidemic_state[curr_ind] == 0) |
(self.epidemic_state[curr_ind] == 2) |
(self.epidemic_state[curr_ind] == 7))):
temp_cont = []
temp_time = []
for j in range(1, len(contact_ids[k])):
cont_ind = contact_ids[k][j]
if ((self.app[cont_ind] == 1) &
(((self.epidemic_state[cont_ind] == 1)) |
(self.epidemic_state[cont_ind] == 0) |
(self.epidemic_state[cont_ind] == 2) |
(self.epidemic_state[cont_ind] == 7))):
temp_cont.append(cont_ind)
temp_time.append(i * simulator.delta_t)
self.contact_cell[curr_ind, 0] = temp_cont
self.contact_cell[curr_ind, 1] = temp_time
self.temp[contacts] = 1
contact_ids = np.where((self.temp == 1)
& (self.epidemic_state == 0))
return contacts
KDTree_path_out = join(seq_path_out, 'KDTree')
os.makedirs(seq_path_out) if not exists(seq_path_out) else None
os.makedirs(pc_path_out) if not exists(pc_path_out) else None
os.makedirs(KDTree_path_out) if not exists(KDTree_path_out) else None
if int(seq_id) < 11:
label_path = join(seq_path, 'labels')
label_path_out = join(seq_path_out, 'labels')
os.makedirs(label_path_out) if not exists(label_path_out) else None
scan_list = np.sort(os.listdir(pc_path))
for scan_id in scan_list:
print(scan_id)
points = DP.load_pc_kitti(join(pc_path, scan_id))
labels = DP.load_label_kitti(join(label_path, str(scan_id[:-4]) + '.label'), remap_lut)
sub_points, sub_labels = DP.grid_sub_sampling(points, labels=labels, grid_size=grid_size)
search_tree = KDTree(sub_points)
KDTree_save = join(KDTree_path_out, str(scan_id[:-4]) + '.pkl')
np.save(join(pc_path_out, scan_id)[:-4], sub_points)
np.save(join(label_path_out, scan_id)[:-4], sub_labels)
with open(KDTree_save, 'wb') as f:
pickle.dump(search_tree, f)
if seq_id == '08':
proj_path = join(seq_path_out, 'proj')
os.makedirs(proj_path) if not exists(proj_path) else None
proj_inds = np.squeeze(search_tree.query(points, return_distance=False))
proj_inds = proj_inds.astype(np.int32)
proj_save = join(proj_path, str(scan_id[:-4]) + '_proj.pkl')
with open(proj_save, 'wb') as f:
pickle.dump([proj_inds], f)
else:
proj_path = join(seq_path_out, 'proj')
def nearest_neighboors(x, y, z, r):
D = np.array([x, y, z])
D = D.T
tree = KDTree(D, leaf_size=2500)
dist, ind = tree.query(r, k=33)
return dist[0], ind[0]
y = [X[h][1], X[n][1]]
z = [X[h][2], X[n][2]]
ax.plot_wireframe(x, y, z, color='black')
ax.set_xlim(minX, maxX)
ax.set_ylim(minY, maxY)
ax.set_zlim(minZ, maxZ)
ax.set_title(name + " - NaNE: " + str(num))
ax.view_init(30, 50)
# Save the figure and show
plt.tight_layout()
fig.savefig("tests/exp1/graphs/" + name + '/Graph - NaN: ' + str(num) + '.png')
print(name + " graph generated! NaN:", num)
tree = KDTree(X)
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
for k in ks:
for h, i in enumerate(X):
ns = tree.query([i], k=k+1, return_distance=False)
for n in list(ns[0])[1:]:
x = [X[h][0], X[n][0]]
y = [X[h][1], X[n][1]]
z = [X[h][2], X[n][2]]
ax.plot_wireframe(x, y, z, color='black')
ax.set_xlim(minX, maxX)
ax.set_ylim(minY, maxY)
ax.set_zlim(minZ, maxZ)
ax.set_title(name + " - kNN: " + str(k))
ax.view_init(30, 50)
t2_cpu = time.process_time()
print('APBS time: {}\n'.format(t2 - t1))
print('APBS cpu time: {}\n'.format(t2_cpu - t1_cpu))
iface = np.zeros(len(regular_mesh.vertices))
if 'compute_iface' in masif_opts and masif_opts['compute_iface']:
# Compute the surface of the entire complex and from that compute the interface.
v3, f3, _, _, _ = computeMSMS(pdb_filename,\
protonate=True)
# Regularize the mesh
mesh = pymesh.form_mesh(v3, f3)
full_regular_mesh = fix_mesh(mesh, masif_opts['mesh_res'])
# Find the vertices that are in the iface.
v3 = full_regular_mesh.vertices
# Find the distance between every vertex in regular_mesh.vertices and those in the full complex.
kdt = KDTree(v3)
d, r = kdt.query(regular_mesh.vertices)
d = np.square(
d) # Square d, because this is how it was in the pyflann version.
assert (len(d) == len(regular_mesh.vertices))
iface_v = np.where(d >= 2.0)[0]
iface[iface_v] = 1.0
# Convert to ply and save.
save_ply(out_filename1+".ply", regular_mesh.vertices,\
regular_mesh.faces, normals=vertex_normal, charges=vertex_charges,\
normalize_charges=True, hbond=vertex_hbond, hphob=vertex_hphobicity,\
iface=iface)
else:
# Convert to ply and save.
save_ply(out_filename1+".ply", regular_mesh.vertices,\
def Extend(data, limit=0.04, num_neighbours=18):
submission = CreateSubmission(0, data)
hits = data[['hit_id', 'x', 'y', 'z', 'volume_id', 'layer_id', 'module_id']]
df = submission.merge(hits, on=['hit_id'], how='left')
df = df.assign(d = np.sqrt( df.x**2 + df.y**2 + df.z**2 ))
df = df.assign(r = np.sqrt( df.x**2 + df.y**2))
df = df.assign(arctan2 = np.arctan2(df.z, df.r))
for angle in range(-90,90,1):
#df1 = df.loc[(df.arctan2>(angle-0.5)/180*np.pi) & (df.arctan2<(angle+0.5)/180*np.pi)]
df1 = df.loc[(df.arctan2>(angle-1.5)/180*np.pi) & (df.arctan2<(angle+1.5)/180*np.pi)]
min_num_neighbours = len(df1)
if min_num_neighbours<3: continue
hit_ids = df1.hit_id.values
x,y,z = df1[['x', 'y', 'z']].values.T
r = (x**2 + y**2)**0.5
r = r/1000
a = np.arctan2(y,x)
c = np.cos(a)
s = np.sin(a)
tree = KDTree(np.column_stack([c, s, r]), metric='euclidean')
track_ids = list(df1.track_id.unique())
num_track_ids = len(track_ids)
min_length=3
for i in range(num_track_ids):
p = track_ids[i]
if p==0: continue
idx = np.where(df1.track_id==p)[0]
if len(idx)<min_length: continue
if angle>0:
idx = idx[np.argsort( z[idx])]
else:
idx = idx[np.argsort(-z[idx])]
## start and end points ##
idx0,idx1 = idx[0],idx[-1]
a0 = a[idx0]
a1 = a[idx1]
r0 = r[idx0]
r1 = r[idx1]
c0 = c[idx0]
c1 = c[idx1]
s0 = s[idx0]
s1 = s[idx1]
da0 = a[idx[1]] - a[idx[0]] #direction
dr0 = r[idx[1]] - r[idx[0]]
direction0 = np.arctan2(dr0,da0)
da1 = a[idx[-1]] - a[idx[-2]]
dr1 = r[idx[-1]] - r[idx[-2]]
direction1 = np.arctan2(dr1,da1)
## extend start point
ns = tree.query([[c0, s0, r0]], k=min(num_neighbours, min_num_neighbours), return_distance=False)
ns = np.concatenate(ns)
direction = np.arctan2(r0 - r[ns], a0 - a[ns])
diff = 1 - np.cos(direction - direction0)
ns = ns[(r0 - r[ns] > 0.01) & (diff < (1 - np.cos(limit)))]
for n in ns: df.loc[df.hit_id == hit_ids[n], 'track_id'] = p
## extend end point
ns = tree.query([[c1, s1, r1]], k=min(num_neighbours, min_num_neighbours), return_distance=False)
ns = np.concatenate(ns)
direction = np.arctan2(r[ns] - r1, a[ns] - a1)
diff = 1 - np.cos(direction - direction1)
ns = ns[(r[ns] - r1 > 0.01) & (diff < (1 - np.cos(limit)))]
for n in ns: df.loc[df.hit_id == hit_ids[n], 'track_id'] = p
df = df[['hit_id', 'track_id']]
temp = data.merge(df, how='left', on='hit_id', suffixes=('_old', ''))
data.update(temp.track_id)
return data
def test_perigee_period_given_apogee():
# This test uses KNN to answer two BQL queries.
# SIMULATE perigee_km, period_minutes GIVEN apogee_km = 500;
# SIMULATE apogee_km, period_minutes;
# The outputs of the query are scattered on a plot.
rng = gu.gen_rng(1)
# Load the satellites dataset.
filename = os.path.join(os.path.dirname(__file__),
'graphical/resources/satellites.csv')
satellites = pd.read_csv(filename)
# Extract target columns of interest.
D = satellites[['Apogee_km', 'Perigee_km', 'Period_minutes']].dropna()
X = np.asarray(D)
# Extract the nearest neighbors given A=500.
tree = KDTree(X[:, 0].reshape(-1, 1))
_, neighbors = tree.query([[500]], k=20)
perigees = X[neighbors[0][:10], 1]
periods = X[neighbors[0][:10], 2]
# Learn the joint distribution by assuming P,T|A are independent.
perigees_ind = rng.normal(np.mean(perigees), np.std(perigees), size=20)
periods_ind = rng.normal(np.mean(periods), np.std(perigees), size=20)
# Create a KNN.
distargs = {
'outputs': {
'stattypes': ['numerical', 'numerical', 'numerical'],
'statargs': [{}, {}, {}]
}
}
knn = MultivariateKnn([0, 1, 2], None, distargs=distargs, K=30, rng=rng)
for i, row in enumerate(X):
knn.incorporate(i, dict(zip([0, 1, 2], row)))
# Sample from the dependent KNN.
samples_dep = knn.simulate(-1, [1, 2], {0: 500}, N=20)
logpdfs = [knn.logpdf(-1, s, {0: 500}) for s in samples_dep]
assert all(not np.isinf(l) for l in logpdfs)
# Create an axis.
_fig, ax = plt.subplots()
# Scatter the actual neighborhood.
ax.scatter(perigees, periods, color='b', label='Actual Satellites')
# Plot the independent knn.
ax.scatter(perigees_ind,
periods_ind,
color='r',
alpha=.5,
label='Independent KNN')
# Plot the dependent knn.
ax.scatter([s[1] for s in samples_dep], [s[2] for s in samples_dep],
color='g',
alpha=.5,
label='Dependent KNN')
# Prepare the axes.
ax.set_title('SIMULATE Perigee_km, Period_minutes GIVEN Apogee_km = 500',
fontweight='bold')
ax.set_xlabel('Perigee', fontweight='bold')
ax.set_ylabel('Period', fontweight='bold')
ax.grid()
ax.legend(framealpha=0, loc='upper left')
# Now simulate from the joint distributions of apogee, perigee.
samples_joint = knn.simulate(-1, [0, 2], N=100)
# Create an axis.
_fig, ax = plt.subplots()
# Scatter the actual data.
ax.scatter(X[:, 0], X[:, 2], color='b', label='Actual Satellites')
# Scatter the simulated data.
ax.scatter([s[0] for s in samples_joint], [s[2] for s in samples_joint],
color='r',
label='Dependent KNN')
# Prepare the axes.
ax.set_title('SIMULATE period_minutes, apogee_km LIMIT 500',
fontweight='bold')
ax.set_xlabel('Apogee', fontweight='bold')
ax.set_ylabel('Period', fontweight='bold')
ax.set_xlim([-500, 50000])
ax.set_ylim([-100, 1800])
ax.grid()
ax.legend(framealpha=0, loc='upper left')
# Reveal!
plt.close('all')
else:
int_features = np.vstack((data['vert_ind'], data['class'])).T
sub_points, sub_colors, sub_int_features = DP.grid_sub_sampling(
points,
features=colors,
labels=int_features,
grid_size=subsampling_parameter)
sub_colors = sub_colors / 255
if cloud_split == 'test':
sub_vert_inds = np.squeeze(sub_int_features)
sub_labels = None
else:
sub_vert_inds = sub_int_features[:, 0]
sub_labels = sub_int_features[:, 1]
search_tree = KDTree(sub_points, leaf_size=50)
with open(KDTree_file, 'wb') as f:
pickle.dump(search_tree, f)
if cloud_split == 'test':
write_ply(sub_ply_file, [sub_points, sub_colors, sub_vert_inds],
['x', 'y', 'z', 'red', 'green', 'blue', 'vert_ind'])
else:
write_ply(
sub_ply_file,
[sub_points, sub_colors, sub_labels, sub_vert_inds],
['x', 'y', 'z', 'red', 'green', 'blue', 'class', 'vert_ind'])
if isfile(proj_file):
print('{:s} proj_file already exists\n'.format(cloud_name))
else:
if labels_train[ii] == 1
]
#### initial visualization
plt.xlim(0.0, 1.0)
plt.ylim(0.0, 1.0)
plt.scatter(bumpy_fast, grade_fast, color="b", label="fast")
plt.scatter(grade_slow, bumpy_slow, color="r", label="slow")
plt.legend()
plt.xlabel("bumpiness")
plt.ylabel("grade")
plt.show()
################################################################################
### your code here! name your classifier object clf if you want the
### visualization code (prettyPicture) to show you the decision boundary
from sklearn.neighbors import KDTree
kdt = KDTree(features_train, leaf_size=30, metric='euclidean')
t0 = time()
kdt.query(features_train, k=2, return_distance=False)
print("training time:", round(time() - t0, 3), "s")
t0 = time()
print(kdt.score(features_test))
try:
# prettyPicture(clf, features_test, labels_test)
prettyPicture(kdt, features_test, labels_test)
except NameError:
pass
def periodic_voronoi(structure, logfile=sys.stdout):
"""
:param ASE.Atoms structure:
"""
pbcc = PBCCalculator(structure.cell)
# Make a 3x3x3 supercell
supercell = structure.repeat((3, 3, 3))
qhull_output = None
logfile.write("Qvoronoi ---")
# Run qhull
with tempfile.NamedTemporaryFile('w',
prefix = 'qvor',
suffix='.in', delete = False) as infile, \
tempfile.NamedTemporaryFile('r',
prefix = 'qvor',
suffix='.out',
delete=True) as outfile:
# -- Write input file --
infile.write("3\n") # num of dimensions
infile.write("%i\n" % len(supercell)) # num of points
np.savetxt(infile, supercell.get_positions(), fmt='%.16f')
infile.flush()
cmdline = [
"qvoronoi", "TI", infile.name, "FF", "Fv", "TO", outfile.name
]
process = subprocess.Popen(cmdline,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
retcode = process.wait()
logfile.write(process.stdout.read())
if retcode != 0:
raise RuntimeError("qvoronoi returned exit code %i" % retcode)
qhull_output = outfile.read()
facets_regex = re.compile(
"""
-[ \t](?P<facetkey>f[0-9]+) [\n]
[ \t]*-[ ]flags: .* [\n]
[ \t]*-[ ]normal: .* [\n]
[ \t]*-[ ]offset: .* [\n]
[ \t]*-[ ]center:(?P<center>([ ][\-]?[0-9]*[\.]?[0-9]*(e[-?[0-9]+)?){3}) [ \t] [\n]
[ \t]*-[ ]vertices:(?P<vertices>([ ]p[0-9]+\(v[0-9]+\))+) [ \t]? [\n]
[ \t]*-[ ]neighboring[ ]facets:(?P<neighbors>([ ]f[0-9]+)+)
""", re.X | re.M)
vertices_re = re.compile('(?<=p)[0-9]+')
# Allocate stuff
centers = []
vertices = []
facet_indexes_taken = set()
facet_index_to_our_index = {}
all_facets_centers = []
# ---- Read facets
facet_index = -1
next_our_index = 0
for facet_match in facets_regex.finditer(qhull_output):
center = np.asarray(map(float, facet_match.group('center').split()))
facet_index += 1
all_facets_centers.append(center)
if not pbcc.is_in_image_of_cell(center, (1, 1, 1)):
continue
verts = map(int, vertices_re.findall(facet_match.group('vertices')))
verts_in_main_cell = tuple(v % len(structure) for v in verts)
facet_indexes_taken.add(facet_index)
centers.append(center)
vertices.append(verts_in_main_cell)
facet_index_to_our_index[facet_index] = next_our_index
next_our_index += 1
end_of_facets = facet_match.end()
facet_count = facet_index + 1
logfile.write(" qhull gave %i vertices; kept %i" %
(facet_count, len(centers)))
# ---- Read ridges
qhull_output_after_facets = qhull_output[end_of_facets:].strip()
ridge_re = re.compile('^\d+ \d+ \d+(?P<verts>( \d+)+)$', re.M)
ridges = [[int(v) for v in match.group('verts').split()]
for match in ridge_re.finditer(qhull_output_after_facets)]
# only take ridges with at least 1 facet in main unit cell.
ridges = [r for r in ridges if any(f in facet_indexes_taken for f in r)]
# shift centers back into normal unit cell
centers -= np.sum(structure.cell, axis=0)
nearest_center = KDTree(centers)
ridges_in_main_cell = set()
threw_out = 0
for r in ridges:
ridge_centers = np.asarray(
[all_facets_centers[f] for f in r if f < len(all_facets_centers)])
if not pbcc.all_in_unit_cell(ridge_centers):
continue
pbcc.wrap_points(ridge_centers)
dists, ridge_centers_in_main = nearest_center.query(
ridge_centers, return_distance=True)
if np.any(dists > 0.00001):
threw_out += 1
continue
assert ridge_centers_in_main.shape == (
len(ridge_centers), 1), "%s" % ridge_centers_in_main.shape
ridge_centers_in_main = ridge_centers_in_main[:, 0]
ridges_in_main_cell.add(frozenset(ridge_centers_in_main))
logfile.write(" Threw out %i ridges" % threw_out)
logfile.flush()
return centers, vertices, ridges_in_main_cell
def compute(dim_map,
dim_x,
f,
n_niches=1000,
max_evals=1e5,
params=cm.default_params,
log_file=None,
variation_operator=cm.variation):
"""CVT MAP-Elites
Vassiliades V, Chatzilygeroudis K, Mouret JB. Using centroidal voronoi tessellations to scale up the multidimensional archive of phenotypic elites algorithm. IEEE Transactions on Evolutionary Computation. 2017 Aug 3;22(4):623-30.
Format of the logfile: evals archive_size max mean median 5%_percentile, 95%_percentile
"""
# setup the parallel processing pool
num_cores = multiprocessing.cpu_count()
pool = multiprocessing.Pool(num_cores)
# create the CVT
c = cm.cvt(n_niches, dim_map, params['cvt_samples'],
params['cvt_use_cache'])
kdt = KDTree(c, leaf_size=30, metric='euclidean')
cm.__write_centroids(c)
archive = {} # init archive (empty)
n_evals = 0 # number of evaluations since the beginning
b_evals = 0 # number evaluation since the last dump
# main loop
while (n_evals < max_evals):
to_evaluate = []
# random initialization
if len(archive) <= params['random_init'] * n_niches:
for i in range(0, params['random_init_batch']):
x = np.random.uniform(low=params['min'],
high=params['max'],
size=dim_x)
to_evaluate += [(x, f)]
else: # variation/selection loop
keys = list(archive.keys())
# we select all the parents at the same time because randint is slow
rand1 = np.random.randint(len(keys), size=params['batch_size'])
rand2 = np.random.randint(len(keys), size=params['batch_size'])
for n in range(0, params['batch_size']):
# parent selection
x = archive[keys[rand1[n]]]
y = archive[keys[rand2[n]]]
# copy & add variation
z = variation_operator(x.x, y.x, params)
to_evaluate += [(z, f)]
# evaluation of the fitness for to_evaluate
s_list = cm.parallel_eval(__evaluate, to_evaluate, pool, params)
# natural selection
for s in s_list:
__add_to_archive(s, s.desc, archive, kdt)
# count evals
n_evals += len(to_evaluate)
b_evals += len(to_evaluate)
# write archive
if b_evals >= params['dump_period'] and params['dump_period'] != -1:
print("[{}/{}]".format(n_evals, int(max_evals)),
end=" ",
flush=True)
cm.__save_archive(archive, n_evals)
b_evals = 0
# write log
if log_file != None:
fit_list = np.array([x.fitness for x in archive.values()])
log_file.write("{} {} {} {} {} {} {}\n".format(
n_evals, len(archive.keys()), fit_list.max(),
np.mean(fit_list), np.median(fit_list),
np.percentile(fit_list, 5), np.percentile(fit_list, 95)))
log_file.flush()
cm.__save_archive(archive, n_evals)
return archive
# ============================================================================= # End of Feature Creation # ============================================================================= #Now combine the feature vectors - use only doc2vec, tfidf for docs, and LDA for KNN all_txt_features=np.hstack([TSVD_reduced,lda_tx,docvec_array]) # ============================================================================= # Modeling # ============================================================================= #from sklearn.neighbors import KNeighborsClassifier #Try K-D Tree instead from sklearn.neighbors import KDTree # row_nb_list - use this for indexing kdt = KDTree(all_txt_features, leaf_size=30, metric='euclidean') #TODO: low-priority for now, consider updating to another distance metric (cosine) #Using KDTree to find articles similar to that of our original documents # returned_list=[] # for el in row_nb_list: dist, idx = kdt.query(all_txt_features,k=50) #TODO: if we want to get fancy, since there is document overlap in the retrieved #results, we could have a function that would dynamically run and retrieve # until we get the exact amount the user specifies #TODO: consider adding input upfront for user to specify exactly number of # articles they want to have retrieved #pickle and save
if (ext == ".nii" or ext == ".nrrd" or ext == ".nhdr"):
intact_volume = sitk.ReadImage(m_string3)
intact_array = sitk.GetArrayFromImage(intact_volume)
else:
intact_volume = RIM.dicom_series_reader(m_string3)
intact_array = itk.GetArrayFromImage(intact_volume)
# intact_volume=RIM.dicom_series_reader(str(unicode('\\\\samba.cs.ucalgary.ca\\fatemeh.yazdanbakhsh\Documents\Data_Sets\Calgary\TBone-2015\TBoneCBCT-2015-10\L2963L','utf-8')))
intact_array_original = intact_array
##########################################################################
data = intact_array
data = np.where(data == 255.0)
X = np.asarray(data).transpose()
tree = KDTree(X, leaf_size=10)
tree.kernel_density(X[0:15], h=0.1, kernel='gaussian')
# print(tree.query_radius(X[:1], r=0.3, count_only=True))
print(tree.query_radius(X[0:15], r=0.3))
# ax.scatter(data[0], data[1],data[2], c='b', **plot_kwds)
#
# # plt.show()
#
# import matplotlib.pyplot as plt
# import pandas as pd
#
# import numpy as np
# # import scipy.cluster.hierarchy as shc
# #
# # plt.figure(figsize=(10, 7))
# # plt.title("Customer Dendograms")
def generate_label_views(kzip_path,
ssd_version,
gt_type,
n_voting=40,
nb_views=2,
ws=(256, 128),
comp_window=8e3,
out_path=None,
verbose=False):
"""
Parameters
----------
kzip_path : str
gt_type : str
ssd_version : str
n_voting : int
Number of collected nodes during BFS for majority vote (label smoothing)
nb_views : int
ws: Tuple[int]
comp_window : float
initial_run : bool
if True, will copy SSV from default SSD to SSD with version=gt_type
out_path : str
If given, export mesh colored accoring to GT labels
verbose : bool
Print additional information
Returns
-------
Tuple[np.array]
raw, label and index views
"""
assert gt_type in ["axgt",
"spgt"], "Currently only spine and axon GT is supported"
n_labels = 3 if gt_type == "axgt" else 4
palette = generate_palette(n_labels)
sso_id = int(re.findall("/(\d+).", kzip_path)[0])
sso = SuperSegmentationObject(sso_id, version=ssd_version)
if initial_run: # use default SSD version
orig_sso = SuperSegmentationObject(sso_id)
orig_sso.copy2dir(dest_dir=sso.ssv_dir)
if not sso.attr_dict_exists:
msg = 'Attribute dict of original SSV was not copied successfully ' \
'to target SSD.'
raise ValueError(msg)
sso.load_attr_dict()
indices, vertices, normals = sso.mesh
# # Load mesh
vertices = vertices.reshape((-1, 3))
# load skeleton
skel = load_skeleton(kzip_path)["skeleton"]
skel_nodes = list(skel.getNodes())
node_coords = np.array(
[n.getCoordinate() * sso.scaling for n in skel_nodes])
node_labels = np.array(
[str2intconverter(n.getComment(), gt_type) for n in skel_nodes],
dtype=np.int)
node_coords = node_coords[(node_labels != -1)]
node_labels = node_labels[(node_labels != -1)]
# create KD tree from skeleton node coordinates
tree = KDTree(node_coords)
# transfer labels from skeleton to mesh
dist, ind = tree.query(vertices, k=1)
vertex_labels = node_labels[ind] # retrieving labels of vertices
if n_voting > 0:
vertex_labels = bfs_smoothing(vertices,
vertex_labels,
n_voting=n_voting)
color_array = palette[vertex_labels].astype(np.float32) / 255.
# for getting vertices of individual SSO
# np.save("/wholebrain/u/pschuber/spiness_skels/sso_%d_vertlabels.k.zip" % sso.id, vertex_labels)
# np.save("/wholebrain/u/pschuber/spiness_skels/sso_%d_verts.k.zip" % sso.id, vertices)
if out_path is not None:
if gt_type == 'spgt': #
colors = [[0.6, 0.6, 0.6, 1], [0.9, 0.2, 0.2, 1],
[0.1, 0.1, 0.1, 1], [0.05, 0.6, 0.6, 1],
[0.9, 0.9, 0.9, 1]]
else: # dendrite, axon, soma, background
colors = [[0.6, 0.6, 0.6, 1], [0.9, 0.2, 0.2, 1],
[0.1, 0.1, 0.1, 1], [0.9, 0.9, 0.9, 1]]
colors = (np.array(colors) * 255).astype(np.uint8)
color_array_mesh = colors[
vertex_labels][:,
0] # TODO: check why only first element, maybe colors introduces an additional axis
write_mesh2kzip("{}/sso_{}_gtlabels.k.zip".format(out_path, sso.id),
sso.mesh[0],
sso.mesh[1],
sso.mesh[2],
color_array_mesh,
ply_fname="gtlabels.ply")
# Initializing mesh object with ground truth coloring
mo = MeshObject("neuron", indices, vertices, color=color_array)
# use downsampled locations for view locations, only if they are close to a
# labeled skeleton node
locs = np.concatenate(sso.sample_locations(cache=False))
dist, ind = tree.query(locs)
locs = locs[dist[:, 0] < 2000] #[::3][:5] # TODO add as parameter
# # # To get view locations
# dest_folder = os.path.expanduser("~") + \
# "/spiness_skels/{}/view_imgs_{}/".format(sso_id, n_voting)
# if not os.path.isdir(dest_folder):
# os.makedirs(dest_folder)
# loc_text = ''
# for i, c in enumerate(locs):
# loc_text += str(i) + "\t" + str((c / np.array([10, 10, 20])).astype(np.int)) +'\n' #rescalling to the voxel grid
# with open("{}/viewcoords.txt".format(dest_folder), "w") as f:
# f.write(loc_text)
# # # DEBUG PART END
label_views, rot_mat = _render_mesh_coords(locs,
mo,
depth_map=False,
return_rot_matrices=True,
ws=ws,
smooth_shade=False,
nb_views=nb_views,
comp_window=comp_window,
verbose=verbose)
label_views = remap_rgb_labelviews(label_views, palette)[:, None]
index_views = render_sso_coords_index_views(sso,
locs,
rot_mat=rot_mat,
verbose=verbose,
nb_views=nb_views,
ws=ws,
comp_window=comp_window)
raw_views = render_sso_coords(sso,
locs,
nb_views=nb_views,
ws=ws,
comp_window=comp_window,
verbose=verbose,
rot_mat=rot_mat)
return raw_views, label_views, index_views
# a weekday, then False.
# Use weekday() function in pandas, and 0 is Monday, 5 is Saturday, and 6
# is Sunday
df['If_Weekend'] = df['DateTime'].map(
lambda x: x.weekday() == 5 or x.weekday() == 6)
# Add zip code column
# Use KDTree to map the latitude and longitude to the zipcode areas
def get_zipcode(tree, value, ziplist):
dist, ind = tree.query(value, k=3)
return str(int(ziplist[ind[0, 0]][0]))
ziplist = np.loadtxt("ExternalData/USZIPCODE.txt", delimiter=',')
tree = KDTree(ziplist[:, 1:], leaf_size=2)
a = np.concatenate((np.array(df['Latitude']).reshape(-1, 1),
np.array(df['Longitude']).reshape(-1, 1)), axis=1)
final_list = []
count = 1
for i in a:
count += 1
if count % 10000 == 0:
print(count)
final_list.append(get_zipcode(tree, i.reshape(1,-1), ziplist))
df['ZipCode'] = final_list
# Save the cleaned data file
df.to_csv("Cleaned_data_updated_zipcode.csv", index=False)
import numpy as np
import cv2
from sklearn.neighbors import KDTree
from kdTreeNNeighbors import createM
orb = cv2.ORB()
frame = cv2.imread('dp.png')
kp = orb.detect(frame, None)
kp, des1 = orb.compute(frame, kp)
frame = cv2.imread('dp2.png')
kp = orb.detect(frame, None)
kp, des2 = orb.compute(frame, kp)
X = np.array(des2)
tree = KDTree(X, leaf_size=2, metric='euclidean')
for each in des1:
dist, ind = tree.query(each, k=size(X))
print ind
knnProfileVars = [
"AGE", "YRS_CLIMBING", "HEIGHT", "APEINDEX", "WEIGHT", "BMI", "B_AVG",
"S_AVG"
]
knnData = normData[knnProfileVars]
knnData.describe()
# The reason I propose this variables is because it contains their current level, and descriptive variables that they cannot change, they simply describe their current phisique and experience. In order to create groups of climbers with similar phisiques, experience and current performance, regardless of the way they train, how often they climb, how they approach improvement, what they eat, etc. because those are the variables that the climber can actually change in order to produce a change in their performance.
#
# Now lets use a KDTree (the algorithm inside the KNN algorithm) to find the nearest neighbors of a random climber, lets say the 10th climber in the list.
# In[22]:
climberID = 155
randomClimber = knnData.loc[climberID, :]
tree = KDTree(knnData)
dist, ids = tree.query([randomClimber], k=int(len(knnData.index) / 3))
closestClimbers = knnData.loc[ids[0], :]
comparison = pd.DataFrame()
comparison["SUBJECT_CLIMBER"] = randomClimber
comparison["AVERAGE_CLIMBER"] = knnData.mean()
comparison["KNN_CLOSEST_AVG"] = closestClimbers.mean()
print("After finding the", str(int(len(knnData.index) / 3)),
"nearest neighbors we see this behavior in the data distribution")
#display(comparison)
# As you can see, the new group is conformed of climbers with a profile that is closer in similarity to the climber we care about. So now, learning the effect of particular actions on perforance makes more sense, since climbers with very similar bodies and experience intuitively would benefit from similar actions. So if a climber with a similar profile had certain benefit from an action, it would suggest that you probably should too.
#
def extend_sample(model,
labeled,
unlabeled,
val,
test,
augmentation=None,
times_augmentation=1,
batch_size=32):
model.train(False)
labeled_loader = DataLoader(labeled, batch_size=batch_size, shuffle=False)
labeled_X = create_embeddings(model, labeled_loader)
labeled_y = labeled.get_targets()
if augmentation is not None:
labeled.transform = augmentation
aug_labeled_X = []
aug_labeled_loader = DataLoader(labeled,
batch_size=batch_size,
shuffle=False)
for _ in range(times_augmentation):
aug_labeled_X.append(create_embeddings(model, aug_labeled_loader))
aug_labeled_y.append(labeled_y)
aug_labeled_X = np.vstack(aug_labeled_X)
aug_labeled_y = np.hstack(labeled_y)
unlabeled_loader = DataLoader(unlabeled,
batch_size=batch_size,
shuffle=False)
unlabeled_X = create_embeddings(model, unlabeled_loader)
extended_samples = {}
extended_samples[1] = {'X': labeled_X, 'y': labeled_y}
index = KDTree(unlabeled_X)
for k_neighbors in K_NEIGHBORS:
extended_indices = index.query(labeled_X,
return_distance=False,
k=k_neighbors)
extended_y = np.tile(labeled_y.reshape(-1, 1),
(1, k_neighbors)).ravel()
extended_X = []
for i in tqdm(extended_indices.ravel()):
extended_X.append(np.array(index.data[i]))
extended_X = np.stack(extended_X)
extended_X = np.vstack([labeled_X, extended_X])
extended_y = np.hstack([labeled_y, extended_y])
extended_samples[k_neighbors + 1] = {'X': extended_X, 'y': extended_y}
val_loader = DataLoader(val, batch_size=batch_size, shuffle=False)
val_X = create_embeddings(model, val_loader)
val_y = val.get_targets()
test_loader = DataLoader(test, batch_size=batch_size, shuffle=False)
test_X = create_embeddings(model, test_loader)
test_y = test.targets.cpu().data.numpy()
result = {
'extended': extended_samples,
'val': {
'X': val_X,
'y': val_y
},
'test': {
'X': test_X,
'y': test_y
}
}
return result
def icp_point_to_point_stochastic(data,
ref,
max_iter,
RMS_threshold,
sampling_limit,
final_overlap=1.):
'''
Iterative closest point algorithm with a point to point strategy.
Inputs :
data = (d x N_data) matrix where "N_data" is the number of points and "d" the dimension
ref = (d x N_ref) matrix where "N_ref" is the number of points and "d" the dimension
max_iter = stop condition on the number of iterations
RMS_threshold = stop condition on the distance
sampling_limit = maximum number of points to use to compute transformations
final_overlap = overlap parameter
Returns :
data_aligned = data aligned on reference cloud
R_list = list of the (d x d) rotation matrices found at each iteration
T_list = list of the (d x 1) translation vectors found at each iteration
neighbors_list = At each iteration, you search the nearest neighbors of each data point in
the ref cloud and this obtain a (1 x N_data) array of indices. This is the list of those
arrays at each iteration
'''
# Variable for aligned data
data_aligned = np.copy(data)
# Initiate lists
d, n = data.shape
R_list = [np.eye(d)]
T_list = [np.zeros((d, 1))]
neighbors_list = []
RMS_list = []
kdtree = KDTree(ref.T)
n_samples = min(n, sampling_limit)
n_overlap = int(final_overlap * n_samples)
for i in range(max_iter):
# Sampling points
data_idx = np.random.choice(n, n_samples, replace=False)
# Matching points
dist, neighbors = kdtree.query(data_aligned[:, data_idx].T, k=1)
neighbors = neighbors.squeeze()
if n_overlap != n_samples:
dist = dist.squeeze()
best_neighbors = np.argpartition(dist, n_overlap)[:n_overlap]
neighbors = neighbors[best_neighbors]
data_idx = data_idx[best_neighbors]
# Estimating the best transform
R, T = best_rigid_transform(data_aligned[:, data_idx], ref[:,
neighbors])
# Computing the full transform
T = R @ T_list[-1] + T
R = R @ R_list[-1]
# Store everything
T_list.append(T)
R_list.append(R)
neighbors_list.append(neighbors)
# Aligne the data
data_aligned = R @ data + T
# Check the RMS threshold
rms = RMS(data_aligned[:, data_idx], ref[:, neighbors].squeeze())
RMS_list.append(rms)
if rms < RMS_threshold:
return data_aligned, R_list[1:], T_list[
1:], neighbors_list, RMS_list
return data_aligned, R_list[1:], T_list[1:], neighbors_list, RMS_list
def load_subsampled_clouds(self, subsampling_parameter):
"""
Presubsample point clouds and load into memory (Load KDTree for neighbors searches)
"""
if 0 < subsampling_parameter <= 0.01:
raise ValueError('subsampling_parameter too low (should be over 1 cm')
# Create path for files
tree_path = os.path.join(self.path, 'input_{:.3f}'.format(subsampling_parameter))
if not os.path.exists(tree_path):
os.makedirs(tree_path)
# Initiate containers
self.input_trees = {'training': [], 'validation': []}
self.input_colors = {'training': [], 'validation': []}
self.input_labels = {'training': [], 'validation': []}
for i, file_path in enumerate(self.train_files):
# Restart timer
t0 = time.time()
# get cloud name and split
cloud_name = file_path.split('/')[-1][:-4]
if self.all_splits[i] == self.validation_split:
cloud_split = 'validation'
else:
cloud_split = 'training'
# Name of the input files
KDTree_file = os.path.join(tree_path, '{:s}_KDTree.pkl'.format(cloud_name))
sub_ply_file = os.path.join(tree_path, '{:s}.ply'.format(cloud_name))
# Check if inputs have already been computed
if os.path.isfile(KDTree_file):
print('\nFound KDTree for cloud {:s}, subsampled at {:.3f}'.format(cloud_name, subsampling_parameter))
# read ply with data
data = read_ply(sub_ply_file)
sub_colors = np.vstack((data['red'], data['green'], data['blue'])).T
sub_labels = data['class']
# Read pkl with search tree
with open(KDTree_file, 'rb') as f:
search_tree = pickle.load(f)
else:
print(
'\nPreparing KDTree for cloud {:s}, subsampled at {:.3f}'.format(cloud_name, subsampling_parameter))
# Read ply file
data = read_ply(file_path)
points = np.vstack((data['x'], data['y'], data['z'])).T
colors = np.vstack((data['red'], data['green'], data['blue'])).T
labels = data['class']
# Subsample cloud
sub_points, sub_colors, sub_labels = grid_subsampling(points,
features=colors,
labels=labels,
sampleDl=subsampling_parameter)
# Rescale float color and squeeze label
sub_colors = sub_colors / 255
sub_labels = np.squeeze(sub_labels)
# Get chosen neighborhoods
search_tree = KDTree(sub_points, leaf_size=50)
# Save KDTree
with open(KDTree_file, 'wb') as f:
pickle.dump(search_tree, f)
# Save ply
write_ply(sub_ply_file,
[sub_points, sub_colors, sub_labels],
['x', 'y', 'z', 'red', 'green', 'blue', 'class'])
# Fill data containers
self.input_trees[cloud_split] += [search_tree]
self.input_colors[cloud_split] += [sub_colors]
self.input_labels[cloud_split] += [sub_labels]
size = sub_colors.shape[0] * 4 * 7
print('{:.1f} MB loaded in {:.1f}s'.format(size * 1e-6, time.time() - t0))
print('\nPreparing reprojection indices for testing')
# Get number of clouds
self.num_training = len(self.input_trees['training'])
self.num_validation = len(self.input_trees['validation'])
# Get validation and test reprojection indices
self.validation_proj = []
self.validation_labels = []
i_val = 0
for i, file_path in enumerate(self.train_files):
# Restart timer
t0 = time.time()
# Get info on this cloud
cloud_name = file_path.split('/')[-1][:-4]
# Validation projection and labels
if self.all_splits[i] == self.validation_split:
proj_file = os.path.join(tree_path, '{:s}_proj.pkl'.format(cloud_name))
if os.path.isfile(proj_file):
with open(proj_file, 'rb') as f:
proj_inds, labels = pickle.load(f)
else:
data = read_ply(file_path)
points = np.vstack((data['x'], data['y'], data['z'])).T
labels = data['class']
# Compute projection inds
proj_inds = np.squeeze(self.input_trees['validation'][i_val].query(points, return_distance=False))
proj_inds = proj_inds.astype(np.int32)
# Save
with open(proj_file, 'wb') as f:
pickle.dump([proj_inds, labels], f)
self.validation_proj += [proj_inds]
self.validation_labels += [labels]
i_val += 1
print('{:s} done in {:.1f}s'.format(cloud_name, time.time() - t0))
return
def _get_local_region(self, X_test_norm):
""" Get local region for each test instance
Parameters
----------
X_test_norm : numpy array, shape (n_samples, n_features)
Normalized test data
Returns
-------
final_local_region_list : List of lists, shape [n_samples, [local_region]]
Indices of training samples in the local region of each test sample
"""
# Initialize the local region list
local_region_list = [[]] * X_test_norm.shape[0]
if self.local_max_features > 1.0:
warnings.warn(
"Local max features greater than 1.0, reducing to 1.0")
self.local_max_features = 1.0
if self.X_train_norm_.shape[1] * self.local_min_features < 1:
warnings.warn(
"Local min features smaller than 1, increasing to 1.0")
self.local_min_features = 1.0
# perform multiple iterations
for _ in range(self.local_region_iterations):
# if min and max are the same, then use all features
if self.local_max_features == self.local_min_features:
features = range(0, self.X_train_norm_.shape[1])
warnings.warn("Local min features equals local max features; "
"use all features instead.")
else:
# randomly generate feature subspaces
features = generate_bagging_indices(
self.random_state,
bootstrap_features=False,
n_features=self.X_train_norm_.shape[1],
min_features=int(self.X_train_norm_.shape[1] *
self.local_min_features),
max_features=int(self.X_train_norm_.shape[1] *
self.local_max_features))
# build KDTree out of training subspace
tree = KDTree(self.X_train_norm_[:, features])
# Find neighbors of each test instance
_, ind_arr = tree.query(X_test_norm[:, features],
k=self.local_region_size)
# add neighbors to local region list
for j in range(X_test_norm.shape[0]):
local_region_list[j] = local_region_list[j] + \
ind_arr[j, :].tolist()
# keep nearby points which occur at least local_region_threshold times
final_local_region_list = [[]] * X_test_norm.shape[0]
for j in range(X_test_norm.shape[0]):
final_local_region_list[j] = [
item for item, count in collections.Counter(
local_region_list[j]).items()
if count > self.local_region_threshold
]
return final_local_region_list
return False
f = open("test.ply", "r")
points = []
for line in f:
words = line.split()
if (isfloat(words[0])):
# print(str(words[0]))
points.append([float(words[0]), float(words[1]), float(words[2])])
f.close()
points = np.array(points)
tree = KDTree(points, leaf_size=50)
distance = []
for x in points:
# print x[0]
dist, ind = tree.query([x], k=50)
distance.append(np.average(dist))
# print 'the average distance for the first point is', np.average(dist)
distance = np.array(distance)
upper = np.percentile(distance, 80)
indToRemove = []
for x in range(len(distance)):
if (distance[x] > upper):
indToRemove.append(x)
# print indToRemove
def prune_pins(pins):
'''
prunes stop list, left_turns, right_turns
@param pins: list of points of interest pins to prune
@return [kept_stop_signs,kept_left, kept_right]
'''
stops = []
left = []
right = []
for x in pins:
if x[2] == 0:
stops.append(x)
if x[2] == 1:
left.append(x)
if x[2] == 2:
right.append(x)
#cluster points in each class
clustered_right = cluster_pins(right, 3)
clustered_left = cluster_pins(left, 3)
clustered_stop = cluster_pins(stops, 0, 1.0 / 75000.0)
#remove turn clusters with speed > 35MPH or other
kept_left = []
kept_right = []
for cluster in clustered_right:
if cluster[3] < 35:
kept_right.append(cluster)
for cluster in clustered_left:
if cluster[3] < 35:
kept_left.append(cluster)
#remove stop signs near tunrs
left_array = np.array(kept_left)[:][:, 0:3] #ignore speed dimension
left_array[:,
2] = left_array[:,
2] * 100 #scale the class so that you always get closest neighbor of desired class
right_array = np.array(kept_right)[:][:, 0:3] #ignore speed dimension
right_array[:,
2] = right_array[:,
2] * 100 #scale the class so that you always get closest neighbor of desired clas
inp = np.concatenate((left_array, right_array), axis=0) #input
kd_tree = KDTree(inp)
kept_stop_signs = []
for index, stop in enumerate(clustered_stop):
cur_loc = np.array(stop)[0:3]
cur_stop_coords = cur_loc[0:2]
left_index = cur_loc.copy()
left_index[
2] = 100 #scale the class so that you always get closest neighbor of desired class
right_index = cur_loc.copy()
right_index[2] = 200
#get closet turn in class by querying kdtree
right_dist, close_right_index = kd_tree.query(
right_index.reshape(1, -1)) #index of the closest right point
left_dist, close_left_index = kd_tree.query(left_index.reshape(
1, -1)) #index of the closest right point
closest_left_coords = inp[close_left_index[0]][0]
closest_right_coords = inp[close_right_index[0]][0]
#haversine more accurate than distance generated from tree query
closest_left = haversine(cur_stop_coords[0], cur_stop_coords[1],
closest_left_coords[0],
closest_left_coords[1])
closest_right = haversine(cur_stop_coords[0], cur_stop_coords[1],
closest_right_coords[0],
closest_right_coords[1])
if closest_left > 0.2 and closest_right > 0.2: #about a tenth of a mile
kept_stop_signs.append(clustered_stop[index])
return [kept_stop_signs, kept_left, kept_right]
def load_subsampled_clouds(self, subsampling_parameter):
"""
Presubsample point clouds and load into memory (Load KDTree for neighbors searches
"""
if 0 < subsampling_parameter <= 0.01:
raise ValueError(
'subsampling_parameter too low (should be over 1 cm')
# Create path for files
tree_path = join(self.path,
'input_{:.3f}'.format(subsampling_parameter))
if not exists(tree_path):
makedirs(tree_path)
# List of training files
self.train_files = np.sort([
join(self.train_path, f) for f in listdir(self.train_path)
if f[-4:] == '.ply'
])
# Add test files
self.test_files = np.sort([
join(self.test_path, f) for f in listdir(self.test_path)
if f[-4:] == '.ply'
])
files = np.hstack((self.train_files, self.test_files))
# Initiate containers
self.input_trees = {'training': [], 'validation': [], 'test': []}
self.input_colors = {'training': [], 'validation': [], 'test': []}
self.input_labels = {'training': [], 'validation': []}
# Advanced display
N = len(files)
progress_n = 30
fmt_str = '[{:<' + str(progress_n) + '}] {:5.1f}%'
print('\nPreparing KDTree for all scenes, subsampled at {:.3f}'.format(
subsampling_parameter))
for i, file_path in enumerate(files):
# Restart timer
t0 = time.time()
# get cloud name and split
cloud_name = file_path.split('/')[-1][:-4]
cloud_folder = file_path.split('/')[-2]
if 'train' in cloud_folder:
if self.all_splits[i] == self.validation_split:
cloud_split = 'validation'
else:
cloud_split = 'training'
else:
cloud_split = 'test'
if (cloud_split != 'test'
and self.load_test) or (cloud_split == 'test'
and not self.load_test):
continue
# Name of the input files
KDTree_file = join(tree_path, '{:s}_KDTree.pkl'.format(cloud_name))
sub_ply_file = join(tree_path, '{:s}.ply'.format(cloud_name))
# Check if inputs have already been computed
if isfile(KDTree_file):
# read ply with data
data = read_ply(sub_ply_file)
sub_reflectance = np.expand_dims(data['reflectance'], 1)
if cloud_split == 'test':
sub_labels = None
else:
sub_labels = data['class']
# Read pkl with search tree
with open(KDTree_file, 'rb') as f:
search_tree = pickle.load(f)
else:
# Read ply file
data = read_ply(file_path)
points = np.vstack(
(data['x'], data['y'], data['z'])).astype(np.float32).T
reflectance = np.expand_dims(data['reflectance'],
1).astype(np.float32)
if cloud_split == 'test':
int_features = None
else:
int_features = data['class']
# Saturate reflectance
reflectance = np.minimum(reflectance, 50.0)
# Subsample cloud
sub_data = grid_subsampling(points,
features=reflectance,
labels=int_features,
sampleDl=subsampling_parameter)
# Rescale and saturate float reflectance
sub_reflectance = sub_data[1] / 50.0
# Get chosen neighborhoods
search_tree = KDTree(sub_data[0], leaf_size=50)
# Save KDTree
with open(KDTree_file, 'wb') as f:
pickle.dump(search_tree, f)
# Save ply
if cloud_split == 'test':
sub_labels = None
write_ply(sub_ply_file, [sub_data[0], sub_reflectance],
['x', 'y', 'z', 'reflectance'])
else:
sub_labels = np.squeeze(sub_data[2])
write_ply(sub_ply_file,
[sub_data[0], sub_reflectance, sub_labels],
['x', 'y', 'z', 'reflectance', 'class'])
# Fill data containers
self.input_trees[cloud_split] += [search_tree]
self.input_colors[cloud_split] += [sub_reflectance]
if cloud_split in ['training', 'validation']:
self.input_labels[cloud_split] += [sub_labels]
print('', end='\r')
print(fmt_str.format('#' * (((i + 1) * progress_n) // N),
100 * (i + 1) / N),
end='',
flush=True)
# Get number of clouds
self.num_training = len(self.input_trees['training'])
self.num_validation = len(self.input_trees['validation'])
self.num_test = len(self.input_trees['test'])
# Get validation and test reprojection indices
self.validation_proj = []
self.validation_labels = []
self.test_proj = []
self.test_labels = []
i_val = 0
i_test = 0
# Advanced display
N = max(self.num_validation + self.num_test, 1)
print('', end='\r')
print(fmt_str.format('#' * progress_n, 100), flush=True)
print('\nPreparing reprojection indices for validation and test')
for i, file_path in enumerate(files):
# get cloud name and split
cloud_name = file_path.split('/')[-1][:-4]
cloud_folder = file_path.split('/')[-2]
# Validation projection and labels
if (not self.load_test
) and 'train' in cloud_folder and self.all_splits[
i] == self.validation_split:
proj_file = join(tree_path, '{:s}_proj.pkl'.format(cloud_name))
if isfile(proj_file):
with open(proj_file, 'rb') as f:
proj_inds, labels = pickle.load(f)
else:
# Get original points
data = read_ply(file_path)
points = np.vstack((data['x'], data['y'], data['z'])).T
labels = data['class']
# Compute projection inds
proj_inds = np.squeeze(
self.input_trees['validation'][i_val].query(
points, return_distance=False))
proj_inds = proj_inds.astype(np.int32)
# Save
with open(proj_file, 'wb') as f:
pickle.dump([proj_inds, labels], f)
self.validation_proj += [proj_inds]
self.validation_labels += [labels]
i_val += 1
# Test projection
if self.load_test and 'test' in cloud_folder:
proj_file = join(tree_path, '{:s}_proj.pkl'.format(cloud_name))
if isfile(proj_file):
with open(proj_file, 'rb') as f:
proj_inds = pickle.load(f)
else:
# Get original points
data = read_ply(file_path)
points = np.vstack((data['x'], data['y'], data['z'])).T
# Compute projection inds
proj_inds = np.squeeze(
self.input_trees['test'][i_test].query(
points, return_distance=False))
proj_inds = proj_inds.astype(np.int32)
# Save
with open(proj_file, 'wb') as f:
pickle.dump(proj_inds, f)
self.test_proj += [proj_inds]
self.test_labels += [np.zeros(0, dtype=np.int32)]
i_test += 1
print('', end='\r')
print(fmt_str.format('#' * (((i_val + i_test) * progress_n) // N),
100 * (i_val + i_test) / N),
end='',
flush=True)
print('\n')
return
def extrapolate_racmo_firn(base_dir,
EPSG,
MODEL,
tdec,
X,
Y,
SEARCH='BallTree',
NN=10,
POWER=2.0,
SIGMA=1.5,
VARIABLE='zs',
FILL_VALUE=None,
REFERENCE=False):
#-- set parameters based on input model
FIRN_FILE = {}
if (MODEL == 'FGRN11'):
#-- filename and directory for input FGRN11 file
FIRN_FILE['zs'] = 'FDM_zs_FGRN11_1960-2016.nc'
FIRN_FILE['FirnAir'] = 'FDM_FirnAir_FGRN11_1960-2016.nc'
FIRN_DIRECTORY = ['RACMO', 'FGRN11_1960-2016']
elif (MODEL == 'FGRN055'):
#-- filename and directory for input FGRN055 file
FIRN_FILE['zs'] = 'FDM_zs_FGRN055_1960-2017_interpol.nc'
FIRN_FILE['FirnAir'] = 'FDM_FirnAir_FGRN055_1960-2017_interpol.nc'
FIRN_DIRECTORY = ['RACMO', 'FGRN055_1960-2017']
elif (MODEL == 'XANT27'):
#-- filename and directory for input XANT27 file
FIRN_FILE['zs'] = 'FDM_zs_ANT27_1979-2016.nc'
FIRN_FILE['FirnAir'] = 'FDM_FirnAir_ANT27_1979-2016.nc'
FIRN_DIRECTORY = ['RACMO', 'XANT27_1979-2016']
elif (MODEL == 'ASE055'):
#-- filename and directory for input ASE055 file
FIRN_FILE['zs'] = 'FDM_zs_ASE055_1979-2015.nc'
FIRN_FILE['FirnAir'] = 'FDM_FirnAir_ASE055_1979-2015.nc'
FIRN_DIRECTORY = ['RACMO', 'ASE055_1979-2015']
elif (MODEL == 'XPEN055'):
#-- filename and directory for input XPEN055 file
FIRN_FILE['zs'] = 'FDM_zs_XPEN055_1979-2016.nc'
FIRN_FILE['FirnAir'] = 'FDM_FirnAir_XPEN055_1979-2016.nc'
FIRN_DIRECTORY = ['RACMO', 'XPEN055_1979-2016']
#-- Open the RACMO NetCDF file for reading
ddir = os.path.join(base_dir, *FIRN_DIRECTORY)
fileID = netCDF4.Dataset(os.path.join(ddir, FIRN_FILE[VARIABLE]), 'r')
#-- Get data from each netCDF variable and remove singleton dimensions
fd = {}
fd[VARIABLE] = np.squeeze(fileID.variables[VARIABLE][:].copy())
fd['lon'] = fileID.variables['lon'][:, :].copy()
fd['lat'] = fileID.variables['lat'][:, :].copy()
fd['time'] = fileID.variables['time'][:].copy()
#-- invalid data value
fv = np.float(fileID.variables[VARIABLE]._FillValue)
#-- input shape of RACMO firn data
nt, ny, nx = np.shape(fd[VARIABLE])
#-- close the NetCDF files
fileID.close()
#-- indices of specified ice mask
i, j = np.nonzero(fd[VARIABLE][0, :, :] != fv)
#-- use a gaussian filter to smooth mask
gs = {}
gs['mask'] = scipy.ndimage.gaussian_filter(fd['mask'],
SIGMA,
mode='constant',
cval=0)
#-- indices of smoothed ice mask
ii, jj = np.nonzero(np.ceil(gs['mask']) == 1.0)
#-- use a gaussian filter to smooth each firn field
gs[VARIABLE] = np.ma.zeros((nt, ny, nx), fill_value=fv)
gs[VARIABLE].mask = np.ma.zeros((nt, ny, nx), dtype=np.bool)
for t in range(nt):
#-- replace fill values before smoothing data
temp1 = np.zeros((ny, nx))
#-- reference to first firn field
if REFERENCE:
temp1[i, j] = fd[VARIABLE][t, i, j] - fd[VARIABLE][0, i, j]
else:
temp1[i, j] = fd[VARIABLE][t, i, j].copy()
#-- smooth firn field
temp2 = scipy.ndimage.gaussian_filter(temp1,
SIGMA,
mode='constant',
cval=0)
#-- scale output smoothed firn field
gs[VARIABLE][t, ii, jj] = temp2[ii, jj] / gs['mask'][ii, jj]
#-- replace valid firn values with original
gs[VARIABLE][t, i, j] = temp1[i, j]
#-- set mask variables for time
gs[VARIABLE].mask[t, :, :] = (gs['mask'] == 0.0)
#-- convert RACMO latitude and longitude to input coordinates (EPSG)
proj1 = pyproj.Proj("+init={0}".format(EPSG))
proj2 = pyproj.Proj("+init=EPSG:{0:d}".format(4326))
xg, yg = pyproj.transform(proj2, proj1, fd['lon'], fd['lat'])
#-- construct search tree from original points
#-- can use either BallTree or KDTree algorithms
xy1 = np.concatenate((xg[ii, jj, None], yg[ii, jj, None]), axis=1)
tree = BallTree(xy1) if (SEARCH == 'BallTree') else KDTree(xy1)
#-- output interpolated arrays of firn variable (height or firn air content)
npts = len(tdec)
extrap_data = np.ma.zeros((npts), fill_value=fv, dtype=np.float)
extrap_data.data[:] = extrap_data.fill_value
extrap_data.mask = np.zeros((npts), dtype=np.bool)
#-- type designating algorithm used (1:interpolate, 2:backward, 3:forward)
extrap_data.interpolation = np.zeros((npts), dtype=np.uint8)
#-- find days that can be interpolated
if np.any((tdec >= fd['time'].min()) & (tdec < fd['time'].max())):
#-- indices of dates for interpolated days
ind, = np.nonzero((tdec >= fd['time'].min())
& (tdec < fd['time'].max()))
#-- reduce x, y and t coordinates
xind, yind, tind = (X[ind], Y[ind], tdec[ind])
#-- find indices for linearly interpolating in time
f = scipy.interpolate.interp1d(fd['time'],
np.arange(nt),
kind='linear')
date_indice = f(tind).astype(np.int)
#-- for each unique firn date
#-- linearly interpolate in time between two firn maps
#-- then then inverse distance weighting to extrapolate in space
for k in np.unique(date_indice):
kk, = np.nonzero(date_indice == k)
count = np.count_nonzero(date_indice == k)
#-- query the search tree to find the NN closest points
xy2 = np.concatenate((xind[kk, None], yind[kk, None]), axis=1)
dist, indices = tree.query(xy2, k=NN, return_distance=True)
#-- normalized weights if POWER > 0 (typically between 1 and 3)
#-- in the inverse distance weighting
power_inverse_distance = dist**(-POWER)
s = np.sum(power_inverse_distance, axis=1)
w = power_inverse_distance / np.broadcast_to(
s[:, None], (count, NN))
#-- firn height or air content for times before and after tdec
firn1 = gs[VARIABLE][k, ii, jj]
firn2 = gs[VARIABLE][k + 1, ii, jj]
#-- linearly interpolate to date
dt = (tind[kk] - fd['time'][k]) / (fd['time'][k + 1] -
fd['time'][k])
#-- spatially extrapolate using inverse distance weighting
extrap_data[kk] = (1.0-dt)*np.sum(w*firn1[indices],axis=1) + \
dt*np.sum(w*firn2[indices], axis=1)
#-- set interpolation type (1: interpolated in time)
extrap_data.interpolation[ind] = 1
#-- check if needing to extrapolate backwards in time
count = np.count_nonzero(tdec < fd['time'].min())
if (count > 0):
#-- indices of dates before firn model
ind, = np.nonzero(tdec < fd['time'].min())
#-- query the search tree to find the NN closest points
xy2 = np.concatenate((X[ind, None], Y[ind, None]), axis=1)
dist, indices = tree.query(xy2, k=NN, return_distance=True)
#-- normalized weights if POWER > 0 (typically between 1 and 3)
#-- in the inverse distance weighting
power_inverse_distance = dist**(-POWER)
s = np.sum(power_inverse_distance, axis=1)
w = power_inverse_distance / np.broadcast_to(s[:, None], (count, NN))
#-- calculate a regression model for calculating values
#-- read first 10 years of data to create regression model
N = 365
#-- spatially interpolate firn elevation or air content to coordinates
FIRN = np.zeros((count, N))
T = np.zeros((N))
#-- create interpolated time series for calculating regression model
for k in range(N):
#-- time at k
T[k] = gs['time'][k]
#-- spatially extrapolate firn elevation or air content
firn1 = fd[VARIABLE][k, ii, jj]
FIRN[:, k] = np.sum(w * firn1[indices], axis=1)
#-- calculate regression model
for n, v in enumerate(ind):
extrap_data[v] = regress_model(
T,
FIRN[n, :],
tdec[v],
ORDER=2,
CYCLES=[0.25, 0.5, 1.0, 2.0, 4.0, 5.0],
RELATIVE=T[0])
#-- set interpolation type (2: extrapolated backwards in time)
extrap_data.interpolation[ind] = 2
#-- check if needing to extrapolate forward in time
count = np.count_nonzero(tdec >= fd['time'].max())
if (count > 0):
#-- indices of dates after firn model
ind, = np.nonzero(tdec >= fd['time'].max())
#-- query the search tree to find the NN closest points
xy2 = np.concatenate((X[ind, None], Y[ind, None]), axis=1)
dist, indices = tree.query(xy2, k=NN, return_distance=True)
#-- normalized weights if POWER > 0 (typically between 1 and 3)
#-- in the inverse distance weighting
power_inverse_distance = dist**(-POWER)
s = np.sum(power_inverse_distance, axis=1)
w = power_inverse_distance / np.broadcast_to(s[:, None], (count, NN))
#-- calculate a regression model for calculating values
#-- read last 10 years of data to create regression model
N = 365
#-- spatially interpolate firn elevation or air content to coordinates
FIRN = np.zeros((count, N))
T = np.zeros((N))
#-- create interpolated time series for calculating regression model
for k in range(N):
kk = nt - N + k
#-- time at k
T[k] = fd['time'][kk]
#-- spatially extrapolate firn elevation or air content
firn1 = gs[VARIABLE][kk, ii, jj]
FIRN[:, k] = np.sum(w * firn1[indices], axis=1)
#-- calculate regression model
for n, v in enumerate(ind):
extrap_data[v] = regress_model(
T,
FIRN[n, :],
tdec[v],
ORDER=2,
CYCLES=[0.25, 0.5, 1.0, 2.0, 4.0, 5.0],
RELATIVE=T[-1])
#-- set interpolation type (3: extrapolated forward in time)
extrap_data.interpolation[ind] = 3
#-- complete mask if any invalid in data
invalid, = np.nonzero(extrap_data.data == extrap_data.fill_value)
extrap_data.mask[invalid] = True
#-- replace fill value if specified
if FILL_VALUE:
extrap_data.fill_value = FILL_VALUE
extrap_data.data[extrap_data.mask] = extrap_data.fill_value
#-- return the interpolated values
return extrap_data
def update_top_activations(self, features, label, l_points, input_points, radius, max_computed=60):
top_num = self.top_features.shape[0]
# Compute top indice for each feature
max_indices = np.argmax(features, axis=0)
# get top_point neighborhoods
for features_i, idx in enumerate(max_indices[:max_computed]):
if features[idx, features_i] <= self.top_features[-1, features_i]:
continue
# if label in self.top_classes[:, features_i]:
# ind0 = np.where(self.top_classes[:, features_i] == label)[0][0]
# if features[idx, features_i] <= self.top_features[ind0, features_i]:
# continue
# elif ind0 < top_num - 1:
# self.top_features[ind0:-1, features_i] = self.top_features[ind0 + 1:, features_i]
# self.top_classes[ind0:-1, features_i] = self.top_classes[ind0 + 1:, features_i]
# for next_i in range(ind0 + 1, top_num):
# old_f = join(self.visu_path, self.fmt_str.format(features_i, next_i + 1))
# new_f = join(self.visu_path, self.fmt_str.format(features_i, next_i))
# if exists(old_f):
# if exists(new_f):
# remove(new_f)
# rename(old_f, new_f)
# Find indice where new top should be placed
top_i = np.where(features[idx, features_i] > self.top_features[:, features_i])[0][0]
# # Update top features
# if top_i < top_num - 1:
# self.top_features[top_i + 1:, features_i] = self.top_features[top_i:-1, features_i]
# self.top_features[top_i, features_i] = features[idx, features_i]
# self.top_classes[top_i + 1:, features_i] = self.top_classes[top_i:-1, features_i]
# self.top_classes[top_i, features_i] = label
# Find in which batch lays the point
if self.saving:
# Get inputs
l_features = features[:, features_i]
point = l_points[idx, :]
dist = np.linalg.norm(input_points - point, axis=1)
influence = (radius - dist) / radius
# Project response on input cloud
if l_points.shape[0] == input_points.shape[0]:
responses = l_features
else:
tree = KDTree(l_points, leaf_size=50)
nn_k = min(l_points.shape[0], 10)
interp_dists, interp_inds = tree.query(input_points, nn_k, return_distance=True)
tukeys = np.square(1 - np.square(interp_dists / radius))
tukeys[interp_dists > radius] = 0
responses = np.sum(l_features[interp_inds] * tukeys, axis=1)
# Handle last examples
for next_i in range(top_num - 1, top_i, -1):
old_f = join(self.visu_path, self.fmt_str.format(features_i, next_i))
new_f = join(self.visu_path, self.fmt_str.format(features_i, next_i + 1))
if exists(old_f):
if exists(new_f):
remove(new_f)
rename(old_f, new_f)
# Save
filename = join(self.visu_path, self.fmt_str.format(features_i, top_i + 1))
write_ply(filename,
[input_points, influence, responses],
['x', 'y', 'z', 'influence', 'responses'])
""" from DOI 10.1007/s10851-009-0161-2, #4 """
dist = np.zeros((len(a), len(b)))
for bi in prange(len(b)):
dist[:, bi] = 1 - np.abs(np.dot(a, b[bi]))
return dist
""" use sklearn KDTree for reduction of points for query (euclidean) """
from sklearn.neighbors import KDTree
qgrid_pos = np.copy(qgrid)
qgrid_pos[qgrid_pos[:, 0] < 0] *= -1
tree = KDTree(qgrid_pos)
# rad = ( 1 - np.cos(theta) ) / 2
# euc_rad = 4*np.sin(theta)**2
rad = np.sqrt(2 * (1 - np.cos(0.5 * theta)))
euc_rad = np.sqrt(4 * np.sin(0.25 * theta)**2)
fibre_marc = {}
def calcFibre(symHKL, yset, qgrid, phi, rad, tree, euc_rad, quatSymOps):
cphi = np.cos(phi / 2)
sphi = np.sin(phi / 2)
def show_deformable_kernels(self, model, dataset, deform_idx=0):
##########################################
# First choose the visualized deformations
##########################################
# List all deformation ops
all_ops = [op for op in tf.get_default_graph().get_operations() if op.name.startswith('KernelPointNetwork')
and op.name.endswith('deformed_KP')]
if len(all_ops) > 0:
print('\nPossible deformed indices:')
for i, t in enumerate(all_ops):
print(i, ': ', t.name)
else:
raise ValueError('No deformable convolution found in this network')
# Chosen deformations
deformed_KP_tensor = all_ops[deform_idx].outputs[0]
# Layer index
layer_idx = int(all_ops[deform_idx].name.split('/')[1].split('_')[-1])
# Original kernel point positions
KP_vars = [v for v in tf.global_variables() if 'kernel_points' in v.name]
tmp = np.array(all_ops[deform_idx].name.split('/'))
test = []
for v in KP_vars:
cmp = np.array(v.name.split('/'))
l = min(len(cmp), len(tmp))
cmp = cmp[:l]
tmp = tmp[:l]
test += [np.sum(cmp == tmp)]
chosen_KP = np.argmax(test)
print('You chose to visualize the output of operation named: ' + all_ops[deform_idx].name)
print('\n****************************************************************************')
# Run model on all test examples
# ******************************
# Initialise iterator with test data
self.sess.run(dataset.test_init_op)
count = 0
while True:
try:
# Run one step of the model
t = [time.time()]
ops = (deformed_KP_tensor,
model.inputs['points'],
model.inputs['features'],
model.inputs['pools'],
model.inputs['in_batches'],
KP_vars)
stacked_deformed_KP, \
all_points, \
all_colors, \
all_pools, \
in_batches, \
original_KPs = self.sess.run(ops, {model.dropout_prob: 1.0})
t += [time.time()]
count += in_batches.shape[0]
# Stack all batches
max_ind = np.max(in_batches)
stacked_batches = []
for b_i, b in enumerate(in_batches):
stacked_batches += [b[b < max_ind - 0.5] * 0 + b_i]
stacked_batches = np.hstack(stacked_batches)
# Find batches at wanted layer
for l in range(model.config.num_layers - 1):
if l >= layer_idx:
break
stacked_batches = stacked_batches[all_pools[l][:, 0]]
# Get each example and update top_activations
in_points = []
in_colors = []
deformed_KP = []
points = []
lookuptrees = []
for b_i, b in enumerate(in_batches):
b = b[b < max_ind - 0.5]
in_points += [all_points[0][b]]
deformed_KP += [stacked_deformed_KP[stacked_batches == b_i]]
points += [all_points[layer_idx][stacked_batches == b_i]]
lookuptrees += [KDTree(points[-1])]
if all_colors.shape[1] == 4:
in_colors += [all_colors[b, 1:]]
else:
in_colors += [None]
print('New batch size : ', len(in_batches))
###########################
# Interactive visualization
###########################
# Create figure for features
fig1 = mlab.figure('Features', bgcolor=(1.0, 1.0, 1.0), size=(1280, 920))
fig1.scene.parallel_projection = False
# Indices
global obj_i, point_i, plots, offsets, p_scale, show_in_p, aim_point
p_scale = 0.03
obj_i = 0
point_i = 0
plots = {}
offsets = False
show_in_p = 2
aim_point = np.zeros((1, 3))
def picker_callback(picker):
""" Picker callback: this get called when on pick events.
"""
global plots, aim_point
if 'in_points' in plots:
if plots['in_points'].actor.actor._vtk_obj in [o._vtk_obj for o in picker.actors]:
point_rez = \
plots['in_points'].glyph.glyph_source.glyph_source.output.points.to_array().shape[0]
new_point_i = int(np.floor(picker.point_id / point_rez))
if new_point_i < len(plots['in_points'].mlab_source.points):
# Get closest point in the layer we are interested in
aim_point = plots['in_points'].mlab_source.points[new_point_i:new_point_i + 1]
update_scene()
if 'points' in plots:
if plots['points'].actor.actor._vtk_obj in [o._vtk_obj for o in picker.actors]:
point_rez = plots['points'].glyph.glyph_source.glyph_source.output.points.to_array().shape[
0]
new_point_i = int(np.floor(picker.point_id / point_rez))
if new_point_i < len(plots['points'].mlab_source.points):
# Get closest point in the layer we are interested in
aim_point = plots['points'].mlab_source.points[new_point_i:new_point_i + 1]
update_scene()
def update_scene():
global plots, offsets, p_scale, show_in_p, aim_point, point_i
# Get the current view
v = mlab.view()
roll = mlab.roll()
# clear figure
for key in plots.keys():
plots[key].remove()
plots = {}
# Plot new data feature
p = points[obj_i]
# Rescale points for visu
p = (p * 1.5 / model.config.in_radius)
# Show point cloud
if show_in_p <= 1:
plots['points'] = mlab.points3d(p[:, 0],
p[:, 1],
p[:, 2],
resolution=8,
scale_factor=p_scale,
scale_mode='none',
color=(0, 1, 1),
figure=fig1)
if show_in_p >= 1:
# Get points and colors
in_p = in_points[obj_i]
in_p = (in_p * 1.5 / model.config.in_radius)
# Color point cloud if possible
in_c = in_colors[obj_i]
if in_c is not None:
# Primitives
scalars = np.arange(len(in_p)) # Key point: set an integer for each point
# Define color table (including alpha), which must be uint8 and [0,255]
colors = np.hstack((in_c, np.ones_like(in_c[:, :1])))
colors = (colors * 255).astype(np.uint8)
plots['in_points'] = mlab.points3d(in_p[:, 0],
in_p[:, 1],
in_p[:, 2],
scalars,
resolution=8,
scale_factor=p_scale * 0.8,
scale_mode='none',
color=(0.8, 0.8, 0.8),
figure=fig1)
# plots['in_points'].module_manager.scalar_lut_manager.lut.table = colors
else:
plots['in_points'] = mlab.points3d(in_p[:, 0],
in_p[:, 1],
in_p[:, 2],
resolution=8,
scale_factor=p_scale * 0.8,
scale_mode='none',
figure=fig1)
# Get KP locations
rescaled_aim_point = aim_point * model.config.in_radius / 1.5
point_i = lookuptrees[obj_i].query(rescaled_aim_point, return_distance=False)[0][0]
if offsets:
KP = points[obj_i][point_i] + deformed_KP[obj_i][point_i]
scals = np.ones_like(KP[:, 0])
else:
KP = points[obj_i][point_i] + original_KPs[chosen_KP]
scals = np.zeros_like(KP[:, 0])
KP = (KP * 1.5 / model.config.in_radius)
plots['KP'] = mlab.points3d(KP[:, 0],
KP[:, 1],
KP[:, 2],
scals,
colormap='autumn',
resolution=8,
scale_factor=1.2 * p_scale,
scale_mode='none',
vmin=0,
vmax=1,
figure=fig1)
if True:
plots['center'] = mlab.points3d(p[point_i, 0],
p[point_i, 1],
p[point_i, 2],
scale_factor=1.1 * p_scale,
scale_mode='none',
color=(0, 1, 0),
figure=fig1)
# New title
plots['title'] = mlab.title(str(obj_i), color=(0, 0, 0), size=0.3, height=0.01)
text = '<--- (press g for previous)' + 50 * ' ' + '(press h for next) --->'
plots['text'] = mlab.text(0.01, 0.01, text, color=(0, 0, 0), width=0.98)
plots['orient'] = mlab.orientation_axes()
# Set the saved view
mlab.view(*v)
mlab.roll(roll)
return
def animate_kernel():
global plots, offsets, p_scale, show_in_p
# Get KP locations
KP_def = points[obj_i][point_i] + deformed_KP[obj_i][point_i]
KP_def = (KP_def * 1.5 / model.config.in_radius)
KP_def_color = (1, 0, 0)
KP_rigid = points[obj_i][point_i] + original_KPs[chosen_KP]
KP_rigid = (KP_rigid * 1.5 / model.config.in_radius)
KP_rigid_color = (1, 0.7, 0)
if offsets:
t_list = np.linspace(0, 1, 150, dtype=np.float32)
else:
t_list = np.linspace(1, 0, 150, dtype=np.float32)
@mlab.animate(delay=10)
def anim():
for t in t_list:
plots['KP'].mlab_source.set(x=t * KP_def[:, 0] + (1 - t) * KP_rigid[:, 0],
y=t * KP_def[:, 1] + (1 - t) * KP_rigid[:, 1],
z=t * KP_def[:, 2] + (1 - t) * KP_rigid[:, 2],
scalars=t * np.ones_like(KP_def[:, 0]))
yield
anim()
return
def keyboard_callback(vtk_obj, event):
global obj_i, point_i, offsets, p_scale, show_in_p
if vtk_obj.GetKeyCode() in ['b', 'B']:
p_scale /= 1.5
update_scene()
elif vtk_obj.GetKeyCode() in ['n', 'N']:
p_scale *= 1.5
update_scene()
if vtk_obj.GetKeyCode() in ['g', 'G']:
obj_i = (obj_i - 1) % len(deformed_KP)
point_i = 0
update_scene()
elif vtk_obj.GetKeyCode() in ['h', 'H']:
obj_i = (obj_i + 1) % len(deformed_KP)
point_i = 0
update_scene()
elif vtk_obj.GetKeyCode() in ['k', 'K']:
offsets = not offsets
animate_kernel()
elif vtk_obj.GetKeyCode() in ['z', 'Z']:
show_in_p = (show_in_p + 1) % 3
update_scene()
elif vtk_obj.GetKeyCode() in ['0']:
print('Saving')
# Find a new name
file_i = 0
file_name = 'KP_{:03d}.ply'.format(file_i)
files = [f for f in listdir('KP_clouds') if f.endswith('.ply')]
while file_name in files:
file_i += 1
file_name = 'KP_{:03d}.ply'.format(file_i)
KP_deform = points[obj_i][point_i] + deformed_KP[obj_i][point_i]
KP_normal = points[obj_i][point_i] + original_KPs[chosen_KP]
# Save
write_ply(join('KP_clouds', file_name),
[in_points[obj_i], in_colors[obj_i]],
['x', 'y', 'z', 'red', 'green', 'blue'])
write_ply(join('KP_clouds', 'KP_{:03d}_deform.ply'.format(file_i)),
[KP_deform],
['x', 'y', 'z'])
write_ply(join('KP_clouds', 'KP_{:03d}_normal.ply'.format(file_i)),
[KP_normal],
['x', 'y', 'z'])
print('OK')
return
# Draw a first plot
pick_func = fig1.on_mouse_pick(picker_callback)
pick_func.tolerance = 0.01
update_scene()
fig1.scene.interactor.add_observer('KeyPressEvent', keyboard_callback)
mlab.show()
except tf.errors.OutOfRangeError:
break
points_target_dawnsample.normals)
points_source_iss = iss(data=points_source_dawnsample_numpy,
radius=original_voxel_size * 2.5,
nms_radius=original_voxel_size * 10)
points_target_iss = iss(data=points_target_dawnsample_numpy,
radius=original_voxel_size * 2.5,
nms_radius=original_voxel_size * 10)
print('source iss shape:', points_source_iss.shape)
print('target iss shape:', points_target_iss.shape)
#pointCloudShow(points_source_dawnsample_numpy,points_source_dawnsample_numpy[points_source_iss])
#pointCloudShow(points_target_dawnsample_numpy,points_target_dawnsample_numpy[points_target_iss])
# step4 build kdtree and compute RNN
leaf_size = 4
radius = original_voxel_size * 4
source_search_tree = KDTree(points_source_dawnsample_numpy, leaf_size)
target_search_tree = KDTree(points_target_dawnsample_numpy, leaf_size)
source_nearest_idx = source_search_tree.query_radius(
points_source_dawnsample_numpy, radius) # 求解每个点的最邻近点
target_nearest_idx = target_search_tree.query_radius(
points_target_dawnsample_numpy, radius) # 求解每个点的最邻近点
# step5 description：FPFH
Bin = 5
points_source_fpfh = np.asarray([
describe(points_source_dawnsample_numpy,
points_source_dawnsample_numpy_normal, source_nearest_idx,
keypoint_id, radius, Bin)
for keypoint_id in points_source_iss
])
points_target_fpfh = np.asarray([
rms = np.sqrt(total_sqrms.sum() / len(total_sqrms))
return rms
def read_pts(file_path, dim=3):
my_file = read_ply(file_path)
my_pts = np.vstack(
(my_file['x'], my_file['y'], my_file['z'])) if dim == 3 else np.vstack(
(my_file['x'], my_file['y']))
return my_pts
if __name__ == '__main__':
file_path_ref = "../data/Notre_Dame_Des_Champs_1.ply"
file_path_data = "../data/NDDC_transform.ply"
pts_ref = read_pts(file_path_ref)
pts_data = read_pts(file_path_data)
ref_tree = KDTree(pts_ref.T, leaf_size=10)
idx = np.squeeze(ref_tree.query(pts_data.T, k=1, return_distance=False))
print("RMS on whole point cloud: %f" % RMS(pts_data, pts_ref[:, idx]))
print("RMS on 90%% point cloud: %f" % RMS(pts_data, pts_ref[:, idx], 0.9))
print("RMS on 70%% point cloud: %f" % RMS(pts_data, pts_ref[:, idx], 0.7))
print("RMS on 50%% point cloud: %f" % RMS(pts_data, pts_ref[:, idx], 0.5))
