def calc_mms(path_mms, out_dir_mms, method):
list_cells = os.listdir(out_dir_mms)
for fn_cell in list_cells:
list_ply = os.listdir(out_dir_mms + fn_cell)
print fn_cell, len(list_ply)
if len(list_ply) == 1:
fn_ply = list_ply[0]
fn = out_dir_mms + fn_cell + '\\' + fn_ply
data = read_bin(fn, 7)
d = rasterize(data, 0.1, dim=2)
list_keys = d.keys()
new_points = []
data = np.array(data)
for key in list_keys:
list_xyz = data[d[key]]
new_points.append(list(method(list_xyz, axis=0)))
write_points(new_points, path_mms + fn_cell + '.ply')
del new_points
else:
list_ply = os.listdir(out_dir_mms + fn_cell)
list_data = []
list_d = []
list_keys = []
for fn_ply in list_ply:
fn = out_dir_mms + fn_cell + '\\' + fn_ply
data = read_bin(fn, 7)
d = rasterize(data, 0.1, dim=2)
list_data.append(np.array(data))
list_d.append(d)
list_keys.extend(d.keys())
del data, d
list_keys = list(set(list_keys))
new_points = []
for key in list_keys:
list_xyz = np.array([0, 0, 0])
for i in xrange(len(list_d)):
data = list_data[i]
d = list_d[i]
if d.has_key(key):
list_xyz = mergecloud(list_xyz, data[d[key]])
list_xyz = list_xyz[1:]
new_points.append(list(method(list_xyz, axis=0)))
write_points(new_points, path_mms + fn_cell + '.ply')
del new_points
def ground_filter(data_mms, radius, res_list, r,
cleaning=[1.0, 0.5, 0.25, 0.1, 0.05], is_plot = False):
save_img = None
for res in res_list:
reduced = []
radius += 1
d_mms1 = rasterize(data_mms, res, dim = 2)
if len(data_mms) == 0:
break
threshold = 2 * res
img, d_mms1 = read_points_to_image(data_mms, d_mms1, res, r)
if res == 0.1:
save_img = img
if res in cleaning: #[1.0, 0.5, 0.25, 0.1, 0.05]: #[0.25, 0.1]: 0.05 is necessary for reflection case
d_mms1 = remove_isolated_seg(img, d_mms1, is_plot = is_plot)
indexs_res, new_data = update_points(data_mms, d_mms1, img, threshold, res, radius, r)
reduced = data_mms[indexs_res]
data_mms = data_mms[new_data]
return reduced, save_img
def split_runid(in_dir_update, out_dir_update, start_index):
## # Kaffeezimmer IKG
## x_offset = 548495
## y_offset = 5804458
## r = 25
list_runids = os.listdir(in_dir_update)
for fn_runid in list_runids[start_index:]:
runid = fn_runid[:9]
data = read_bin_double(in_dir_update + fn_runid, 7)
data = np.array(data) - [x_offset, y_offset, 0]
d = rasterize(data, r, dim=2)
for cell_idx in d.keys():
x, y = [int(idx) for idx in cell_idx.split('+')]
ply_name, cell_name = coord_fn_from_cell_index(x, y, runid)
subdata = data[d[cell_idx]] - [x * r, y * r, 0]
check_and_create(out_dir_update + cell_name)
write_points(subdata,
out_dir_update + cell_name + '\\' + ply_name + '.ply')
del subdata
del data, d
print fn_runid
def calculate_heigh_difference(data_mms, data_ref, res_ref, r):
# Spatial indexing
d_ref = rasterize(data_ref, res_ref, dim=2)
d_mms = rasterize(data_mms, res_ref, dim=2)
# Calculate difference image
delta = []
img = np.zeros((int(r / res_ref), int(r / res_ref)))
for key, value in d_mms.iteritems():
# Get image coordinates
x, y = [int(k) for k in key.split('+')]
z_mms = data_mms[value, 2]
mean_mms = np.mean(z_mms)
if key in d_ref:
mean_ref = np.mean(data_ref[d_ref[key], 2])
delta.append(mean_mms - mean_ref)
img[x, y] = mean_mms - mean_ref
return delta, img
def down_sampling(data_mms, d_mms1, res, r):
d_mms1 = rasterize(data_mms, res, dim=2)
img, d_mms1 = read_points_to_image(data_mms, d_mms1, res, r)
indexs_low_res = []
for key in list(set(d_mms1.keys())):
i, j = [int(k) for k in key.split('+')]
if i == int(r / res) or j == int(
r / res): # Ignore the out of bound data
print(i, j, int(r / res))
continue
index_res = d_mms1[key][np.argmin(data_mms[d_mms1[key], 2])]
indexs_low_res.append(index_res)
return indexs_low_res
def local_to_UTM_update_ref_core(fn, args):
in_dir, out_dir, r, x_offset, y_offset = args
m, n = fn[:17].split('_')
[mm, nn] = coord(m, n, r, x_offset, y_offset)
list_duplicate = os.listdir(in_dir + fn)
if len(list_duplicate) == 1:
new_pointcloud = np.array(
read_bin(in_dir + fn + '\\' + list_duplicate[0], 7))
write_points_double(
np.array(new_pointcloud) + [mm, nn, 0], out_dir + fn)
else:
pointcloud = np.array([0, 0, 0])
for fn_under in list_duplicate:
pointcloud = mergecloud(
pointcloud, np.array(read_bin(in_dir + fn + '//' + fn_under,
7)))
pointcloud = pointcloud[1:]
d = rasterize(pointcloud, 0.5, dim=2)
new_pointcloud = []
for key in d.keys():
new_pointcloud.append(np.mean(pointcloud[d[key]], axis=0))
## if np.std(pointcloud[d[key]][:,2])>0.1:
## print fn, np.std(pointcloud[d[key]][:,2]), np.mean( pointcloud[d[key]], axis = 0)
write_points_double(
np.array(new_pointcloud) + [mm, nn, 0], out_dir + fn)
def shiftvalue(data_mms, res_ref, data_ref, d_ref, reject_threshold, r):
d_mms = rasterize(data_mms, res_ref, dim=2)
img = np.zeros((int(r / res_ref), int(r / res_ref)))
delta = []
for key, value in d_mms.iteritems():
x, y = [int(k) for k in key.split('+')]
z_mms = data_mms[value, 2]
mean_mms = np.mean(z_mms)
l = len(z_mms)
if l > 5:
mean_ref = np.mean(data_ref[d_ref[key], 2])
delta.append(mean_mms - mean_ref)
img[x, y] = mean_mms - mean_ref
if len(delta) > 1:
np.place(img, img == 0, None)
delta = np.array(delta)
#alldelta = np.copy(delta)
ind = reject_outliers(delta, reject_threshold)
## np.place(delta, ind==False, None)
## import matplotlib.pyplot as plt
## plt.figure()
## plt.plot(range(len(alldelta)),alldelta,'bo')
## plt.plot(range(len(delta)),delta,'r+')
## plt.show()
return np.mean(delta[ind]), img
else:
print("Too sparse points")
return None, None
def split_ref_to_tiles(ref_path, ref_out_dir, r, x_offset, y_offset, res_ref):
check_and_create(ref_out_dir)
list_ref_ply = os.listdir(ref_path)
print("Loading")
num_point_per_kacheln = int(pow(r / res_ref, 2))
imcomplete = []
for fn_ref in list_ref_ply:
data = read_bin_double(ref_path + fn_ref, 9)
data = np.array(data) - [x_offset, y_offset, 0]
d = rasterize(data, r, dim=2)
check_and_create(ref_out_dir + fn_ref)
for cell_idx in d.keys():
x, y = [int(idx) for idx in cell_idx.split('+')]
ply_name, cell_name = coord_fn_from_cell_index(x, y, '')
subdata = data[d[cell_idx]] - [x * r, y * r, 0]
output_fn = ref_out_dir + fn_ref + '\\' + cell_name + '.ply'
write_points(subdata, output_fn)
if len(d[cell_idx]) < num_point_per_kacheln:
imcomplete.append(cell_name + '.ply')
del subdata
del data, d
print("Load finished")
list_ref_tiles = os.listdir(ref_out_dir)
removed = []
for i in xrange(len(list_ref_tiles) - 1):
list_left = ref_out_dir + list_ref_tiles[i]
list_right = ref_out_dir + list_ref_tiles[i + 1]
left = os.listdir(list_left)
right = os.listdir(list_right)
intersect = set(left).intersection(right)
for fn in intersect:
data_left = read_bin(list_left + '\\' + fn, 7)
data_right = read_bin(list_right + '\\' + fn, 7)
data_new = mergecloud(data_left, data_right)
if len(data_new) == num_point_per_kacheln:
os.remove(list_left + '\\' + fn)
os.remove(list_right + '\\' + fn)
write_points(data_new, ref_out_dir + fn)
removed.append(fn)
intersection = list(set(imcomplete) - set(removed))
check_and_create(ref_out_dir + 'reduced\\')
for fn_list in list_ref_tiles:
path = ref_out_dir + fn_list + '\\'
list_path = os.listdir(path)
for fn_file in list_path:
if fn_file in intersection:
shutil.move(path + fn_file,
ref_out_dir + 'reduced\\' + fn_file)
else:
shutil.move(path + fn_file, ref_out_dir + fn_file)
[os.rmdir(ref_out_dir + fn_list) for fn_list in list_ref_tiles]
print 'Finish spliting'
def cal_ref_duplicate(full_list, out_dir_update, path):
list_cells = os.listdir(out_dir_update)
for fn_cell in list_cells:
list_ply = os.listdir(out_dir_update + fn_cell)
if len(list_ply) == 1:
copytoDst(out_dir_update + fn_cell, list_ply[0], path)
else:
list_count = []
for fn_ply in list_ply:
fn = out_dir_update + fn_cell + '\\' + fn_ply
data = read_bin(fn, 7)
list_count.append(len(data))
if np.mean(list_count) == 2500:
copytoDst(out_dir_update + fn_cell, list_ply[0], path)
else:
list_count = np.array(list_count)
new_count = list_count[list_count != 2500]
new_list = np.array(list_ply)[list_count != 2500]
if len(new_count) == 1:
copytoDst(out_dir_update + fn_cell, new_list[0], path)
else:
union = []
list_data = []
list_d = []
for fn_ply in new_list:
fn = out_dir_update + fn_cell + '\\' + fn_ply
data = read_bin(fn, 7)
d = rasterize(data, 0.5, dim=2)
list_data.append(data)
list_d.append(d)
del_list = set(full_list) - set(d.keys())
union.extend(del_list)
res = list(set(full_list) - set(union))
new_points = []
for key in res:
list_z = []
x = 0
y = 0
for i in xrange(len(list_d)):
data = list_data[i]
d = list_d[i]
list_z.append(data[d[key][0]][2])
x, y = data[d[key][0]][0:2]
new_points.append([x, y, np.median(list_z)])
if len(res) > 0:
write_points(new_points, path + fn_cell + '.ply')
if np.std(list_z) > 0.5:
print fn_cell, len(res), np.mean(list_z), np.median(
list_z), np.std(list_z)
def update_dtm(list_shift_img, raster_size, radius, ref_cut_dir,
ref_update_dir, shift, res_ref, list_pointcloud_ref,
ref_out_dir):
# nonvalue
nonvalue = -999.0
check_and_create(ref_cut_dir)
check_and_create(ref_update_dir)
for fn in tqdm(list_shift_img.keys()):
img = list_shift_img[fn] - shift
single_len = img.shape[0]
new_size = (2 * radius + 1) * img.shape[0]
neighbour = np.zeros((new_size, new_size))
m, n = fn[:17].split('_')
int_m = Hexa2Decimal(m)
int_n = Hexa2Decimal(n)
combi = np.array(
list((product(range(-radius, radius + 1),
range(-radius, radius + 1)))))
combi_global = combi + [int_m, int_n]
neigh_list = [
coord_fn_from_cell_index(m, n, '')[1] + '.ply'
for m, n in combi_global
]
not_in_list = []
for neigh, loc in zip(neigh_list, combi):
if neigh in list_shift_img.keys():
a, b = (loc + radius) * single_len
neighbour[a:a + single_len,
b:b + single_len] = list_shift_img[neigh] - shift
else:
if neigh in list_pointcloud_ref:
a, b = (loc + radius) * single_len
not_in_list.append([neigh, (a, b)])
print fn, not_in_list
img = neighbour
img = np.nan_to_num(img)
filtered, boundbuffer, mask = apply_gaussian(img, 0, 0, nonvalue,
'linear')
boundbuffer = np.nan_to_num(boundbuffer)
a, b = (np.array([0, 0]) + radius) * single_len
update = boundbuffer[a:a + single_len, b:b + single_len]
upmask = mask[a:a + single_len, b:b + single_len]
data_ref = read_bin(ref_out_dir + fn, 7)
d_ref = rasterize(data_ref, res_ref, dim=2)
data_ref = np.array(data_ref)
raster_size = single_len
data_output = []
for i in xrange(0, raster_size):
for j in xrange(0, raster_size):
string = str.join('+', [str(i), str(j)])
index = d_ref[string]
if upmask[i, j] == 0:
data_output.append(data_ref[index][0] +
[0, 0, update[i, j]])
write_points(data_output, ref_cut_dir + fn)
for fn_not, (a, b) in not_in_list:
update = boundbuffer[a:a + single_len, b:b + single_len]
print np.sum(update)
if abs(np.sum(update)) > 0.01:
data_ref = read_bin(ref_out_dir + fn_not, 7)
d_ref = rasterize(data_ref, res_ref, dim=2)
data_ref = np.array(data_ref)
data_output = []
for i in xrange(0, raster_size):
for j in xrange(0, raster_size):
string = str.join('+', [str(i), str(j)])
index = d_ref[string]
data_output.append(data_ref[index][0] +
[0, 0, update[i, j]])
check_and_create(ref_update_dir + fn_not)
write_points(
data_output,
ref_update_dir + fn_not + '//' + fn_not + '_from_' + fn)