return result
if __name__ == "__main__":
radius = 3
res_list = [1.0, 0.5, 0.25, 0.1, 0.05]
#args = [pointcloud_path, ground_filtering_out_dir,
# r, x_offset, y_offset, geoid, sigma_geoid]
args = [pts_dir, ref_dir, tmp_dir, gfilter_out_dir,
r, x_offset, y_offset, geoid, sigma_geoid, radius, res_list]
list_pointcloud = os.listdir(pts_dir)
check_and_create(gfilter_out_dir)
print("Total number of tiles:", len(list_pointcloud))
list_tiles = sorted(set(list_pointcloud) - set(os.listdir(gfilter_out_dir)))
print("Number of tiles still need filtering:", len(list_tiles))
## Process all with 6 cores
#random.shuffle(list_tiles)
#result = ground_filter_multicore(list_tiles, args)
#print('process finished', sum(result), '/', len(result))
from tqdm import tqdm
result = []
for fn in tqdm(list_pointcloud):
r = ground_filter_aligned_data(fn, args)
result.append(r)
if np.isnan(value):
invalid.append(key)
# Remove the invalid from dictionary
for key in invalid:
list_shift_img.pop(key)
dict_shift_value.pop(key)
shift = generate_report(list_shift_value, list_shift_img, dict_shift_value,
out_path, r, x_offset,y_offset, res_ref)
print 'report generated'
raster_size = r/res_ref #30
radius = 1
check_and_create(geo_ground_filtering_out_dir)
check_and_create(final_dir)
check_and_create(rest_dir)
if 0:
update_dtm(list_shift_img, raster_size, radius, ref_cut_dir, ref_update_dir,
shift, res_ref, list_pointcloud_ref, ref_dir)
print 'updates generated'
#shift = 42.9317700613 # Hannover - 42.9316920581
#shift = 43.5341477257 # Hildesheim - After reject outlier
#shift = 43.5346042674 # Hildesheim - Before reject outlier
#shift = 43.4262356488 # SEhi
# Process the mms and update of ref together
local_to_UTM(mms_dir, dict_shift_value.keys(), geo_ground_filtering_out_dir,
from environmentSetting import gupdate_out_dir, ref_dir, tmp_dir
from environmentSetting import r, x_offset, y_offset, geoid, sigma_geoid
shift = 43.53326697865561 # Report0
is_plot_flag = True #False
# If points changed based on runid h
if os.path.isdir(correct_out_dir):
mms_dir = correct_out_dir
else:
mms_dir = gfilter_out_dir
list_mms = os.listdir(mms_dir)
# tmp foldersfor intermediat result
check_and_create(gupdate_out_dir)
out_dir = gupdate_out_dir
args = [
mms_dir, ref_dir, tmp_dir, gupdate_out_dir, r, x_offset, y_offset,
geoid, sigma_geoid, res_ref, res_update
]
# Load the shift values of each tile according to the output of calc_diff
dict_shift_value = np.load(tmp_dir + project_name + "_shift.npy",
allow_pickle=True)[0]
dict_shift_value = shifts_cleaning(dict_shift_value, list_mms)
if 1:
from tqdm import tqdm
from lib.util import local2global
###############################################################################
# Last step - collect all the tiles into larger area
###############################################################################
import os
from tqdm import tqdm
from environmentSetting import *
from lib.util import check_and_create
from lib.ply import write_points, write_points_double, read_bin, read_bin_double
out_dtm_dir = out_path + 'finalout_all\\'
check_and_create(out_dtm_dir)
pathA = geo_ground_filtering_out_dir
pathB = final_dir
pathC = rest_dir
listA = os.listdir(pathA)
listB = os.listdir(pathB)
listC = os.listdir(pathC)
#fns_dtm = ['ffffff9', 'ffffffa', 'ffffffb', 'ffffffc', 'ffffffd', 'ffffffe', 'fffffff',
# '0000000', '0000001', '0000002', '0000003', '0000004', '0000005', '0000006']
#fns_dtm = ['ffffffe', 'fffffff',
# '0000000', '0000001', '0000002', '0000003', '0000004', '0000005', '0000006']
fns_dtm = ['ffffffc', 'ffffffd', 'ffffffe']
for fn_dtm in fns_dtm:
###############################################################################
# This step is only necessary, when one of the runid are changed and save at
# Folder
#
# Yu Feng, 25.03.2020
###############################################################################
import os
from shutil import copyfile
from environmentSetting import correct_out_dir, gfilter_out_dir
from lib.util import check_and_create
# Target
check_and_create(correct_out_dir)
# Source
correct_runid_out_dir = "D:/_data/_mms/Hildesheim_updated/"
"""Caution!!! Copy operation here!"""
if 0:
for fn in os.listdir(gfilter_out_dir):
if fn not in os.listdir(correct_out_dir):
copyfile(gfilter_out_dir + fn, correct_out_dir + fn)
# Copy to new folder and rename
for runid in ["1150893024", "1150892210"]:
for fn in os.listdir(correct_runid_out_dir):
fn_runid = fn[:10]
if fn_runid == runid:
base = fn[-21:]
shift = generate_report(list_shift_value, list_shift_img, dict_shift_value,
out_path, r, x_offset, y_offset, res_ref)
print 'report generated'
if 1:
raster_size = r / res_ref #30
radius = 1
from environmentSetting import merged_dir, final_dir, rest_dir
from environmentSetting import ref_cut_dir, ref_update_dir
from lib.util import check_and_create
from lib.update import update_dtm
from lib.produceDTM import local_to_UTM_tqdm, local_to_UTM_update_ref, local_to_UTM_rest_ref
check_and_create(merged_dir)
check_and_create(final_dir)
check_and_create(rest_dir)
list_pointcloud_ref = [fn for fn in os.listdir(ref_dir) if '.ply' in fn]
update_dtm(list_shift_img, raster_size, radius, ref_cut_dir,
ref_update_dir, shift, res_ref, list_pointcloud_ref, ref_dir)
print 'updates generated'
if 1:
#shift = 42.9317700613 # Hannover - 42.9316920581
#shift = 43.5341477257 # Hildesheim - After reject outlier
#shift = 43.5346042674 # Hildesheim - Before reject outlier
# Process the mms and update of ref together
CCPath : cloudcompare executive file location
file_path : point cloud file location
Returns
----------
ply file with features
"""
import os
from lib.util import check_and_create
# data path and out path
out_path = 'D:/_data/_mms/20200325Hildesheim/'
mms_dir = out_path + 'aligned_Corr/'
mms_local_dir = out_path + 'aligned_Corr_local/'
check_and_create(mms_local_dir)
sfeat_dir = out_path + 'surface_feat/'
check_and_create(sfeat_dir)
list_mms = os.listdir(mms_dir)
if len(os.listdir(mms_local_dir)) == 0:
# Move to local coordinate system
from tqdm import tqdm
from environmentSetting import r, x_offset, y_offset
from lib.cell2world import coord
from lib.load import load_mms
from lib.ply import write_points_double
for fn in tqdm(list_mms):
m, n = fn[:17].split('_')
out_path = 'D:/_data/_mms/20200325Hildesheim/'
gfilter_out_dir = out_path + 'aligned_GF/'
correct_out_dir = out_path + 'aligned_Corr/'
# Outputs
merged_dir = out_path + 'aligned_a/'
final_dir = out_path + 'aligned_b/'
rest_dir = out_path + 'aligned_c/'
gr_utm_dir = out_path + 'aligned_GR/'
pts_utm_dir = out_path + 'aligned_MMS/'
updates_dir = out_path + 'aligned_update_values/'
ref_update_dir = out_path + 'aligned_ref_update/'
ref_cut_dir = out_path + 'aligned_ref_update_cut/'
from lib.util import check_and_create
check_and_create(merged_dir)
check_and_create(final_dir)
check_and_create(rest_dir)
check_and_create(gr_utm_dir)
check_and_create(pts_utm_dir)
check_and_create(updates_dir)
check_and_create(ref_update_dir)
check_and_create(ref_cut_dir)
# If points corrected based on runid exist
if os.path.isdir(correct_out_dir) and len(os.listdir(correct_out_dir)) >0 :
mms_dir = correct_out_dir
else:
mms_dir = gfilter_out_dir
def generate_report(list_shift_value, list_shift_img, dict_shift_value,
out_path, r, x_offset, y_offset, res_ref):
report_path = out_path + 'report\\'
check_and_create(report_path)
ind = reject_outliers(list_shift_value, 20)
list_shift_value = np.array(list_shift_value)
values = np.array(list_shift_value)[ind]
max_value = max(values)
min_value = min(values)
dict_shift_value_cp = dict_shift_value.copy()
tobedeleted = []
for key, value in dict_shift_value_cp.iteritems():
if value < min_value or value > max_value:
tobedeleted.append(key)
[dict_shift_value_cp.pop(key, None) for key in tobedeleted]
plt.figure()
hist, bins = np.histogram(list_shift_value, bins=1000)
plt.bar(bins[:-1], hist, 0.001)
plt.savefig(report_path + 'without_rejection.png')
plt.figure()
hist, bins = np.histogram(list_shift_value[ind], bins=100)
plt.bar(bins[:-1], hist, 0.002)
plt.savefig(report_path + 'after_rejection.png')
shift = np.median(list_shift_value[ind])
minM, minN, lenM, lenN = get_range_from_fn_list(list_shift_img.keys(), r,
x_offset, y_offset)
nonvalue = -999.0
# generate_diff_image(list_shift_img, lenM, lenN, minM, minN,
# 1/res_ref, nonvalue, report_path,
# r, x_offset,y_offset, res_ref)
img = np.zeros((lenN, lenM))
for fn in list_shift_img.keys():
x, y = read_fn(fn[:17], r, x_offset, y_offset)
img[lenN - 1 - (y - minN) / r,
(x - minM) / r] = dict_shift_value[fn] - shift
plot_img(img)
plt.savefig(report_path + 'without_rejection_distribution.png')
img = np.zeros((lenN, lenM))
for fn in list_shift_img.keys():
if fn in dict_shift_value_cp:
x, y = read_fn(fn[:17], r, x_offset, y_offset)
img[lenN - 1 - (y - minN) / r,
(x - minM) / r] = dict_shift_value_cp[fn] - shift
plot_img(img)
plt.savefig(report_path + 'after_rejection_distribution.png')
with open(report_path + "Output.txt", "w") as text_file:
text_file.write("shift: %s \n" % shift)
text_file.write("std: %s \n" % np.std(list_shift_value[ind] - shift))
text_file.write("max: %s \n" % np.max(list_shift_value[ind] - shift))
text_file.write("min: %s \n" % np.min(list_shift_value[ind] - shift))
plt.show()
print shift, np.std(list_shift_value[ind] - shift)
print np.max(list_shift_value[ind] - shift), np.min(list_shift_value[ind] -
shift)
return shift