def get_complete_ignore_grid_ids():
id_list = []
if os.path.isfile(grid_complete_list_txt):
completed_id_list = [
int(item)
for item in io_function.read_list_from_txt(grid_complete_list_txt)
]
id_list.extend(completed_id_list)
# get manual excluded ids
if os.path.isfile(grid_excluded_list_txt):
exclude_id_list = [
int(item)
for item in io_function.read_list_from_txt(grid_excluded_list_txt)
]
id_list.extend(exclude_id_list)
# get ids that don't have DEM
if os.path.isfile(grid_no_dem_txt):
nodem_id_list = [
int(item)
for item in io_function.read_list_from_txt(grid_no_dem_txt)
]
id_list.extend(nodem_id_list)
# get ids that don't have DEM
if os.path.isfile(grid_no_valid_dem_txt):
no_valid_dem_id_list = [
int(item)
for item in io_function.read_list_from_txt(grid_no_valid_dem_txt)
]
id_list.extend(no_valid_dem_id_list)
return id_list
def b_exist_grid_dem_subsidence(id):
# if an grid don't have dem subsidence, then we think it's complete
if os.path.isfile(grid_no_subscidence_poly_txt):
grid_ids_no_subsidence = [
int(item) for item in io_function.read_list_from_txt(
grid_no_subscidence_poly_txt)
]
if id in grid_ids_no_subsidence:
return True
# if an id don't have enough dem for dem diff, then we think it's complete
if os.path.isfile(grid_dem_diff_less2dem_txt):
grid_ids_less_2dem = [
int(item) for item in io_function.read_list_from_txt(
grid_dem_diff_less2dem_txt)
]
if id in grid_ids_less_2dem:
return True
dem_subsidence_shps = io_function.get_file_list_by_pattern(
grid_dem_diffs_segment_dir,
'*_grid%d/*_grid%d_8bit_post.shp' % (id, id))
if len(dem_subsidence_shps) == 1:
return True
elif len(dem_subsidence_shps) > 1:
basic.outputlogMessage(
'warning, There are multiple DEM subsidence for grid: %d' % id)
for item in dem_subsidence_shps:
basic.outputlogMessage(item)
return True
else:
return False
def main(options, args):
# data_ini_dir = os.path.expanduser('~/Data/Arctic/canada_arctic/autoMapping/WR_multiDate_inis')
# training_root_dir = os.path.expanduser('~/Data/Arctic/canada_arctic/autoMapping/ray_results/tune_dataAug_para_tesia')
# template_dir = os.path.expanduser('~/Data/Arctic/canada_arctic/autoMapping/eval_new_data')
data_ini_dir = os.path.expanduser(
'~/Data/Arctic/canada_arctic/autoMapping/area_multiDate_inis')
training_root_dir = os.path.expanduser(
'~/Data/Arctic/canada_arctic/autoMapping/multiArea_deeplabV3+_8')
template_dir = os.path.expanduser(
'~/Data/Arctic/canada_arctic/autoMapping/multiArea_deeplabV3+_8')
if options.data_ini_dir_or_list is not None:
data_ini_dir = options.data_ini_dir_or_list
if options.training_root_dir is not None:
training_root_dir = options.training_root_dir
if options.template_dir is not None:
template_dir = options.template_dir
# get data list
if os.path.isdir(data_ini_dir):
area_ini_list = io_function.get_file_list_by_ext('.ini',
data_ini_dir,
bsub_folder=False)
else:
area_ini_list = io_function.read_list_from_txt(data_ini_dir)
# change to abslute path, since later, directory will be changed
area_ini_list = [os.path.abspath(item) for item in area_ini_list]
for idx, area_ini in enumerate(area_ini_list):
basic.outputlogMessage('%d/%d evaluation on %d areas' %
(idx, len(area_ini_list), idx))
run_evaluation_one_dataset(idx, area_ini, training_root_dir,
template_dir)
def get_train_val_sample_count(work_dir, para_file):
train_sample_txt = parameters.get_string_parameters(
para_file, 'training_sample_list_txt')
val_sample_txt = parameters.get_string_parameters(
para_file, 'validation_sample_list_txt')
train_list_txt = os.path.join(work_dir, 'list', train_sample_txt)
val_list_txt = os.path.join(work_dir, 'list', val_sample_txt)
train_lines = io_function.read_list_from_txt(train_list_txt)
val_lines = io_function.read_list_from_txt(val_list_txt)
basic.outputlogMessage(
'The count of training and validation samples are %d and %d' %
(len(train_lines), len(val_lines)))
return len(train_lines), len(val_lines)
def merge_polygons_patchBYpatch(patch_DN_range_txt, in_polygons, polygon_DNs, process_num=1):
patch_DN_range_list = [ int(item) for item in io_function.read_list_from_txt(patch_DN_range_txt)]
# print(patch_DN_range_list)
# divide polygons
patch_polygons = {}
range_count = len(patch_DN_range_list) - 1
for idx in range(range_count):
patch_polygons[idx] = []
for poly, dn in zip(in_polygons, polygon_DNs):
for idx in range(range_count):
if dn > patch_DN_range_list[idx] and dn <= patch_DN_range_list[idx+1]:
patch_polygons[idx].append(poly)
# merge polygon patch by patch
polygons_patch_merge = []
for idx in range(range_count):
print(timeTools.get_now_time_str(), 'will merge %d polygons for %d th patch'%(len(patch_polygons[idx]),idx))
if len(patch_polygons[idx]) < 2:
polygons_patch_merge.extend(patch_polygons[idx])
continue
adjacent_matrix = vector_gpd.build_adjacent_map_of_polygons(patch_polygons[idx], process_num=process_num)
if adjacent_matrix is False:
polygons_patch_merge.extend(patch_polygons[idx])
continue
merged_polygons = vector_features.merge_touched_polygons(patch_polygons[idx], adjacent_matrix)
polygons_patch_merge.extend(merged_polygons)
# merge polygon all
print(timeTools.get_now_time_str(), 'will merge %d polygons for the entire raster' % (len(polygons_patch_merge)))
adjacent_matrix = vector_gpd.build_adjacent_map_of_polygons(polygons_patch_merge, process_num=process_num)
if adjacent_matrix is False:
return polygons_patch_merge
last_merged_polygons = vector_features.merge_touched_polygons(polygons_patch_merge, adjacent_matrix)
return last_merged_polygons
def get_dem_diff_list_to_seg():
dem_diff_list = io_function.get_file_list_by_pattern(
dem_common.grid_dem_diffs_dir, '*DEM_diff_grid*.tif')
dem_diff_list_copy = dem_diff_list.copy()
if len(dem_diff_list) < 1:
print('No *DEM_diff*.tif in %s' % dem_common.grid_dem_diffs_dir)
return []
dem_diff_ids = [
int(re.findall('grid\d+', os.path.basename(item))[0][4:])
for item in dem_diff_list
]
dem_subsidence_shps = io_function.get_file_list_by_pattern(
dem_common.grid_dem_diffs_segment_dir, 'segment_result*/*_post.shp')
subsidence_ids = [
int(re.findall('grid\d+', os.path.basename(item))[0][4:])
for item in dem_subsidence_shps
]
no_subsidence_ids = []
if os.path.isfile(dem_common.grid_no_subscidence_poly_txt):
no_subsidence_ids = [
int(item) for item in io_function.read_list_from_txt(
dem_common.grid_no_subscidence_poly_txt)
]
subsidence_ids.extend(no_subsidence_ids)
# remove dem diff already been segmented or no subsidence
for id, dem_diff in zip(dem_diff_ids, dem_diff_list_copy):
if id in subsidence_ids:
dem_diff_list.remove(dem_diff)
return dem_diff_list
def update_complete_grid_list(grid_ids, task_list):
# based on some criteria, to check if results exist, then update grid_complete_list_txt
completed_id_list = []
if os.path.isfile(grid_complete_list_txt):
completed_id_list = [
int(item)
for item in io_function.read_list_from_txt(grid_complete_list_txt)
]
n_task = len(task_list)
if n_task < 1:
raise ValueError('No task in %s' % str(task_list))
for g_id in grid_ids:
if g_id in completed_id_list:
continue
# check if it has been completed based on multiple criteria
complete_count = 0
if 'dem_diff' in task_list and b_exist_gid_dem_diff(g_id):
complete_count += 1
if 'hillshade_headwall_line' in task_list and b_exist_dem_hillshade_newest_HWLine_grid(
g_id):
complete_count += 1
if 'dem_headwall_grid' in task_list and b_exist_grid_headwall_shp(
g_id):
complete_count += 1
if 'segment' in task_list and b_exist_grid_dem_subsidence(g_id):
complete_count += 1
# we may check more task results: segment, dem_headwall
if complete_count == n_task:
completed_id_list.append(g_id)
# save the txt
completed_id_list = [str(item) for item in completed_id_list]
io_function.save_list_to_txt(grid_complete_list_txt, completed_id_list)
def main():
# run in ~/Data/Arctic/canada_arctic/autoMapping/multiArea_sub_images on tesia
ini_list = io_function.get_file_list_by_pattern('./','area*.ini')
txt_list = io_function.get_file_list_by_pattern('./','area*.txt')
for txt in txt_list:
ini_s = io_function.read_list_from_txt(txt)
ini_list.extend(ini_s)
ini_list = [os.path.abspath(item) for item in ini_list]
file_names = [ io_function.get_name_no_ext(item) for item in ini_list ]
cur_dir = os.getcwd()
# show
[print(item) for item in ini_list]
time.sleep(3)
for name, area_ini in zip(file_names,ini_list):
word_dir = os.path.join(cur_dir,name)
io_function.mkdir(word_dir)
os.chdir(word_dir)
# copy and modify main_para.ini
io_function.copyfiletodir(os.path.join(cur_dir,'main_para.ini'),'./',overwrite=True)
io_function.copyfiletodir(os.path.join(cur_dir,'exe.sh'),'./',overwrite=True)
parameters.write_Parameters_file('main_para.ini','training_regions',area_ini)
# run exe.sh
res = os.system('./exe.sh')
if res !=0:
print(res)
sys.exit(1)
os.chdir(cur_dir)
def main(options, args):
save_dir = options.save_dir
dem_dir_or_txt = args[0]
ref_dem = options.ref_dem
dem_valid_per_txt = options.dem_valid_per_txt
process_num = options.process_num
if os.path.isfile(dem_dir_or_txt):
dem_list = io_function.read_list_from_txt(dem_dir_or_txt)
else:
dem_list = io_function.get_file_list_by_ext('.tif',
dem_dir_or_txt,
bsub_folder=False)
if dem_valid_per_txt is None:
dem_valid_per_txt = os.path.join(dem_dir_or_txt,
'dem_valid_percent.txt')
dem_count = len(dem_list)
if dem_count < 1:
raise ValueError('No input dem files in %s' % dem_dir_or_txt)
if ref_dem is None:
ref_dem = choose_reference_dem(dem_list, dem_valid_per_txt)
if ref_dem is None:
raise ValueError('Cannot find a reference DEM')
if ref_dem in dem_list:
dem_list.remove(ref_dem)
# co_registration_parallel(ref_dem,dem_list,save_dir,process_num)
co_registration_multi_process(ref_dem, dem_list, save_dir, process_num)
def main(options, args):
input = args[0]
if input.endswith('.txt'):
slope_tifs = io_function.read_list_from_txt(input)
elif os.path.isdir(input):
slope_tifs = io_function.get_file_list_by_ext('.tif',input, bsub_folder=True)
else:
slope_tifs = [ input]
process_num = options.process_num
working_dir = './'
save_dir = dem_headwall_shp_dir
if os.path.isdir(working_dir) is False:
io_function.mkdir(working_dir)
if os.path.isdir(save_dir) is False:
io_function.mkdir(save_dir)
failed_tifs = []
min_slope = options.min_slope
min_size = options.min_area
max_size = options.max_area
max_axis_width = options.max_axis_width
max_box_WH = options.max_box_WH
for idx, slope in enumerate(slope_tifs):
if extract_headwall_from_slope(idx, len(slope_tifs), slope,working_dir,save_dir, min_slope,min_size,max_size,max_axis_width,max_box_WH,process_num) is False:
failed_tifs.append(slope)
io_function.save_list_to_txt('extract_headwall_failed_tifs.txt',failed_tifs)
def merge_grid_ids_txt(task, fail_id_txt_list):
id_list = []
for txt in fail_id_txt_list:
id_list.extend(io_function.read_list_from_txt(txt))
id_list = list(set(id_list)) # remove redudant ones
save_path = '%s_fail_grid_ids.txt' % task
io_function.save_list_to_txt(save_path, id_list)
return save_path
def produce_dem_diff_grids(grid_polys, grid_ids, pre_name, reg_tifs,b_apply_matchtag, b_mosaic_id,b_mosaic_date,keep_dem_percent,o_res,process_num=4):
dem_ext_polys = get_dem_tif_ext_polygons(reg_tifs)
dem_diff_tifs = []
# mosaic and crop
if os.path.isfile(grid_dem_diff_less2dem_txt):
grid_id_less2dem_list = [ int(item) for item in io_function.read_list_from_txt(grid_dem_diff_less2dem_txt) ] # no need covert to int
else:
grid_id_less2dem_list = []
for grid_id, grid_poly in zip(grid_ids, grid_polys):
if grid_id in grid_id_less2dem_list:
basic.outputlogMessage('skip grid %d, previous processing shows that, the count of DEM is smaller than 2'%grid_id)
continue
save_dir = 'grid_%d_tmp_files'%grid_id
# check free disk space
work_dir = './'
free_GB = io_function.get_free_disk_space_GB(work_dir)
total_wait_time = 0
while free_GB < 50 and total_wait_time < 60*60*12:
basic.outputlogMessage(' The free disk space (%.4f) is less than 50 GB, wait 60 seconds'%free_GB)
time.sleep(60)
total_wait_time += 60
free_GB = io_function.get_free_disk_space_GB(work_dir)
# get subset of tifs
dem_poly_index = vector_gpd.get_poly_index_within_extent(dem_ext_polys, grid_poly)
if len(dem_poly_index) < 1:
basic.outputlogMessage('warning, no dem tifs within %d grid, skip' % grid_id)
save_id_grid_no_valid_dem(grid_id)
continue
dem_list_sub = [reg_tifs[index] for index in dem_poly_index]
# filter by month # cancel, because it removes many good data.
# dem_list_sub = filter_dem_by_month(dem_list_sub)
mosaic_tif_list = mosaic_crop_dem(dem_list_sub, save_dir, grid_id, grid_poly, b_mosaic_id, b_mosaic_date,
process_num, keep_dem_percent, o_res, pre_name, resample_method='average',b_mask_matchtag=b_apply_matchtag,
b_mask_stripDEM_outlier=b_mask_stripDEM_outlier,b_mask_surface_water=b_mask_surface_water,
b_mosaic_year=b_mosaic_year)
# dem co-registration (cancel, the result in not good with the default setting)
# dem differencing
save_dem_diff = os.path.join(grid_dem_diffs_dir, pre_name + '_DEM_diff_grid%d.tif'%grid_id)
save_date_diff = os.path.join(grid_dem_diffs_dir, pre_name + '_date_diff_grid%d.tif'%grid_id)
if dem_diff_newest_oldest(mosaic_tif_list, save_dem_diff, save_date_diff, process_num,
b_max_subsidence=b_max_subsidence,b_save_cm=True):
dem_diff_tifs.append(save_dem_diff)
else:
save_id_grid_dem_less_2(grid_id)
grid_id_less2dem_list.append(grid_id)
return dem_diff_tifs
def read_dem_diff_assigned_to_other_machine(job_list_pre):
dem_list_txts = io_function.get_file_list_by_pattern(
dem_common.process_log_dir, job_list_pre + '*.txt')
assign_dem_diff = []
for txt in dem_list_txts:
assign_dem_diff.extend(io_function.read_list_from_txt(txt))
return assign_dem_diff
def get_ext_shps():
# make sure in different machine, the extent file are in the same order, we create a list: ArcticDEM_subsets_list.txt
shps_list_txt = os.path.join(ext_shp_dir, 'ArcticDEM_subsets_list.txt')
shps_list = io_function.read_list_from_txt(shps_list_txt)
ext_shps = [os.path.join(ext_shp_dir, item) for item in shps_list]
# check existence
for ext in ext_shps:
io_function.is_file_exist(ext)
return ext_shps
def read_grid_ids_from_other_extent():
grid_txt_list = io_function.get_file_list_by_ext('.txt',
grid_ids_txt_dir,
bsub_folder=False)
other_grid_ids = []
for txt in grid_txt_list:
id_list = io_function.read_list_from_txt(txt)
other_grid_ids.extend(id_list)
other_grid_ids = [int(item) for item in other_grid_ids]
return other_grid_ids
def save_list_no_need_dem_files(file_name, file_list):
if len(file_list) < 1:
return True
# update the file list
save_list = []
if os.path.isfile(file_name):
save_list = io_function.read_list_from_txt(file_name)
for item in file_list:
if item in save_list:
continue
save_list.append(item)
return io_function.save_list_to_txt(file_name, save_list)
def sync_log_files(process_node, r_log_dir, process_log_dir):
# copy complete id list, dem info, grid_no_dem_ids.txt to remote machine
files_to_processNode = [
'strip_dem_cover_grids.txt', 'tile_dem_cover_grids.txt',
'grid_complete_ids.txt', 'grid_no_dem_ids.txt'
]
for file in files_to_processNode:
scp_communicate.copy_file_folder_to_remote_machine(
process_node, os.path.join(r_log_dir, file),
os.path.join(process_log_dir, file))
files_from_processNode = [
'grid_dem_diff_less2dem_ids.txt', 'grid_no_valid_dem_ids.txt',
'grid_no_headwall_ids.txt', 'grid_no_subscidence_poly_ids.txt',
'grid_no_watermask_ids.txt'
]
remote_name = process_node[1:].replace('_host',
'') # change $curc_host to curc
for file in files_from_processNode:
# copy the file, do not overwrite the local file
remote_file = os.path.join(
process_log_dir,
io_function.get_name_by_adding_tail(file, remote_name))
scp_communicate.copy_file_folder_from_remote_machine(
process_node, os.path.join(r_log_dir, file), remote_file)
# if they are new ids, then merged to "file"
local_file = os.path.join(process_log_dir, file)
remote_ids = io_function.read_list_from_txt(
remote_file) if os.path.isfile(remote_file) else [
] # no need, to int
local_ids = io_function.read_list_from_txt(
local_file) if os.path.isfile(local_file) else []
new_ids = [id for id in remote_ids if id not in local_ids]
if len(new_ids) < 1:
continue
else:
local_ids.extend(new_ids)
io_function.save_list_to_txt(local_file, local_ids)
def main(options, args):
# process_num = multiprocessing.cpu_count()
process_num = options.process_num
data_folder = arcticDEM_reg_tif_dir
if len(args) > 0:
data_folder = args[0]
tifs = io_function.get_file_list_by_pattern(
data_folder, '*.tif') # _dem_reg check all tifs
save_invalid_txt_path = os.path.basename(data_folder) + '_invalid_list.txt'
save_good_txt_path = os.path.basename(data_folder) + '_good_list.txt'
tif_count = len(tifs)
basic.outputlogMessage('get %d tif files in %s' % (tif_count, data_folder))
good_tifs = []
if os.path.isfile(save_good_txt_path):
good_tifs.extend(io_function.read_list_from_txt(save_good_txt_path))
invalid_tif = []
# remove good one for the list
if len(good_tifs) > 0:
tifs = [
item for item in tifs if os.path.basename(item) not in good_tifs
]
if process_num == 1:
for idx, tif in enumerate(tifs):
if check_one_tif(idx, tif_count, tif, good_tifs):
good_tifs.append(os.path.basename(tif))
else:
invalid_tif.append(os.path.basename(tif))
else:
theadPool = Pool(process_num) # multi processes
parameters_list = [(idx, tif_count, tif, good_tifs)
for idx, tif in enumerate(tifs)]
results = theadPool.starmap(check_one_tif,
parameters_list) # need python3
for tif, res in zip(tifs, results):
if res:
good_tifs.append(os.path.basename(tif))
else:
invalid_tif.append(os.path.basename(tif))
theadPool.close()
io_function.save_list_to_txt(save_invalid_txt_path, invalid_tif)
io_function.save_list_to_txt(save_good_txt_path, good_tifs)
def save_id_grid_no_subsidence(grid_id):
# grid_no_headwall_txt
if os.path.isdir(process_log_dir) is False:
io_function.mkdir(process_log_dir)
id_list = []
if os.path.isfile(grid_no_subscidence_poly_txt):
id_list = io_function.read_list_from_txt(grid_no_subscidence_poly_txt) # no need covert to int
id_str = str(grid_id)
if id_str in id_list:
return True
else:
# save by adding one line
with open(grid_no_subscidence_poly_txt,'a') as f_obj:
f_obj.writelines(str(grid_id) + '\n')
return True
def add_id_grid_to_txt(grid_id, txt_path):
# grid_no_valid_dem_ids.txt
if os.path.isdir(process_log_dir) is False:
io_function.mkdir(process_log_dir)
# update txt file
id_list = []
if os.path.isfile(txt_path):
id_list = io_function.read_list_from_txt(txt_path) # no need covert to int
id_str = str(grid_id)
if id_str in id_list:
return True
else:
# save by adding one line
with open(txt_path,'a') as f_obj:
f_obj.writelines(str(grid_id) + '\n')
return True
def check_one_extent(extent_shp):
print('start to check %s' % extent_shp)
# local_grid_id_txt is in the current dir
# log_grid_ids_txt, log_grid_ids_txt_done is in grid_ids_txt_dir
local_grid_id_txt, log_grid_ids_txt, log_grid_ids_txt_done = get_extent_grid_id_txt_done_files(
extent_shp)
if os.path.isfile(local_grid_id_txt) is False and os.path.isfile(
log_grid_ids_txt):
io_function.copy_file_to_dst(log_grid_ids_txt, local_grid_id_txt)
if os.path.isfile(local_grid_id_txt) is False:
print('the _grid_ids.txt for %s does not exist, maybe it has started' %
extent_shp)
return False
# check if it has been complete
if os.path.isfile(log_grid_ids_txt_done):
basic.outputlogMessage('Tasks for extent %s have been completed' %
extent_shp)
return True
grid_ids_to_process_txt = io_function.get_name_no_ext(
extent_shp) + '_' + 'grid_ids_to_process.txt'
# read from txt file directly
grid_ids = [
int(item) for item in io_function.read_list_from_txt(local_grid_id_txt)
]
update_complete_grid_list(grid_ids, task_list)
# check complete files, to see if it's done
# remove grids that has been complete or ignored
ignore_ids = get_complete_ignore_grid_ids()
num_grid_ids = save_grid_ids_need_to_process(
grid_ids, ignore_ids=ignore_ids, save_path=grid_ids_to_process_txt)
if num_grid_ids < 1:
print(datetime.now(), ' %s is marked as completed' % extent_shp)
make_note_all_task_done(extent_shp, curc_node)
else:
print(
datetime.now(),
' %s has not completed, %d grids to process, total: %d' %
(extent_shp, num_grid_ids, len(grid_ids)))
return True
def main(options, args):
# process_num = multiprocessing.cpu_count()
process_num = options.process_num
if len(args) > 0:
data_folder = args[0]
else:
data_folder = grid_dem_headwall_shp_dir
basic.outputlogMessage('check shapefiles in %s, with %d processes'%(data_folder,process_num))
vector_files = io_function.get_file_list_by_ext('.shp',data_folder ,bsub_folder=True)
save_invalid_txt_path = os.path.basename(data_folder) + '_incomplete_list.txt'
save_good_txt_path = os.path.basename(data_folder) + '_good_list.txt'
file_count = len(vector_files)
basic.outputlogMessage('The number of vector files: %d'%file_count)
good_files = []
if os.path.isfile(save_good_txt_path):
good_files.extend(io_function.read_list_from_txt(save_good_txt_path))
incomplete_files = []
# remove good one for the list
if len(good_files)>0:
vector_files = [item for item in vector_files if os.path.basename(item) not in good_files]
if process_num == 1:
# tifs = io_function.get_file_list_by_ext('.tif',arcticDEM_reg_tif_dir, bsub_folder=False)
for idx, tif in enumerate(vector_files):
if check_one_vector_file(idx, file_count, tif, good_files):
good_files.append(os.path.basename(tif))
else:
incomplete_files.append(os.path.basename(tif))
else:
theadPool = Pool(process_num) # multi processes
parameters_list = [(idx, file_count, tif, good_files) for idx, tif in enumerate(vector_files)]
results = theadPool.starmap(check_one_vector_file, parameters_list) # need python3
for tif, res in zip(vector_files, results):
if res:
good_files.append(os.path.basename(tif))
else:
incomplete_files.append(os.path.basename(tif))
theadPool.close()
io_function.save_list_to_txt(save_invalid_txt_path, incomplete_files)
io_function.save_list_to_txt(save_good_txt_path, good_files)
def monitor_process_failed_grids():
tasks = ['dem_diff', 'dem_headwall_grid']
while True:
all_fail_list = []
for task in tasks:
all_fail_list.extend(get_failed_grid_ids(task))
if len(all_fail_list) < 1:
print(
datetime.now(),
'there is no failed jobs for %s to process, wait 10 minutes' %
machine_name)
time.sleep(600)
continue
for task in tasks:
fail_id_txt_list = get_failed_grid_ids(task)
if len(fail_id_txt_list) < 1:
continue
print('\n', task, 'fail_id_txt_list:', fail_id_txt_list, '\n')
fail_ids_txt = merge_grid_ids_txt(task, fail_id_txt_list)
# start processing,
res = os.system('./run.sh %s %s' % (fail_ids_txt, task))
if res != 0:
sys.exit(1)
# wait all job done? Yes, in "parallel_processing_curc.py"
# check 'done.txt' in each folder
fail_ids = [
int(item)
for item in io_function.read_list_from_txt(fail_ids_txt)
]
for idx, grid_id in enumerate(fail_ids):
job_folder = '%s_%s' % (task_2_job_folder[task],
str(idx).zfill(5))
print('job_folder for grid_id: %d, is' % grid_id, job_folder)
if os.path.isfile(job_folder + '/done.txt'):
os.system('rm -r %s' % job_folder)
else:
sys.exit(1)
# remove fail txt anyway if all "done.txt" exists
for txt in fail_id_txt_list:
io_function.delete_file_or_dir(txt)
def save_id_grid_dem_less_2(grid_id):
# grid_dem_diff_less2dem_txt
if os.path.isdir(process_log_dir) is False:
io_function.mkdir(process_log_dir)
# update grid_dem_diff_less2dem_txt file
id_list = []
if os.path.isfile(grid_dem_diff_less2dem_txt):
id_list = io_function.read_list_from_txt(grid_dem_diff_less2dem_txt) # no need covert to int
id_str = str(grid_id)
if id_str in id_list:
return True
else:
# save by adding one line
with open(grid_dem_diff_less2dem_txt,'a') as f_obj:
f_obj.writelines(str(grid_id) + '\n')
# # save
# id_list.append(str(grid_id))
# io_function.save_list_to_txt(grid_dem_diff_less2dem_txt,id_list)
return True
def save_id_grid_no_dem(grid_id):
# grid_dem_diff_less2dem_txt
if os.path.isdir(process_log_dir) is False:
io_function.mkdir(process_log_dir)
# update grid_dem_diff_less2dem_txt file
id_list = []
if os.path.isfile(grid_no_dem_txt):
id_list = io_function.read_list_from_txt(
grid_no_dem_txt) # no need covert to int
id_str = str(grid_id)
if id_str in id_list:
return True
else:
# save
id_list.append(str(grid_id))
io_function.save_list_to_txt(grid_no_dem_txt, id_list)
basic.outputlogMessage('Save gird id (%d) to %s' %
(grid_id, grid_no_dem_txt))
return True
def get_not_completed_grids(grid_polys, grid_ids):
completed_id_list = []
if os.path.isfile(grid_complete_list_txt):
completed_id_list = [
int(item)
for item in io_function.read_list_from_txt(grid_complete_list_txt)
]
if len(completed_id_list) < 1:
return grid_polys, grid_ids
no_complete_polys = []
no_complete_ids = []
for poly, id in zip(grid_polys, grid_ids):
if id in completed_id_list:
pass
else:
no_complete_polys.append(poly)
no_complete_ids.append(id)
return no_complete_polys, no_complete_ids
def test_get_sub_images_pixel_json_files():
print('\n')
print('running test_get_sub_images_pixel_json_files')
dir = os.path.expanduser(
'~/Data/Arctic/alaska/autoMapping/alaskaNS_yolov4_3/multi_inf_results')
polygons_shp = os.path.join(
dir,
'I0/I0_alaska_north_slope_planet_rgb_2020_alaskaNS_yolov4_3_exp3.shp')
image_folder_or_path = io_function.read_list_from_txt(
os.path.join(dir, '0.txt'))[0]
process_num = 1
bufferSize = 300
dstnodata = 0
saved_dir = './'
b_rectangle = True
image_pattern = '.tif'
class_names = ['rts']
get_sub_images_pixel_json_files(polygons_shp, image_folder_or_path,
image_pattern, class_names, bufferSize,
dstnodata, saved_dir, b_rectangle,
process_num)
def b_exist_grid_headwall_shp(id):
# if a grid don't have headwall (due to really flat or other reasons), then we think it's complete
if os.path.isfile(grid_no_headwall_txt):
grid_ids_no_headwall = [
int(item)
for item in io_function.read_list_from_txt(grid_no_headwall_txt)
]
if id in grid_ids_no_headwall:
return True
headwall_shps_dir = io_function.get_file_list_by_pattern(
grid_dem_headwall_shp_dir, '*_grid%d' % id)
if len(headwall_shps_dir) == 1:
return True
elif len(headwall_shps_dir) > 1:
basic.outputlogMessage(
'warning, There are multiple headwall shps for grid: %d' % id)
for item in headwall_shps_dir:
basic.outputlogMessage(item)
return True
else:
return False
def b_exist_gid_dem_diff(id):
# if an id don't have enough dem for dem diff, then we think it's complete
if os.path.isfile(grid_dem_diff_less2dem_txt):
grid_ids_less_2dem = [
int(item) for item in io_function.read_list_from_txt(
grid_dem_diff_less2dem_txt)
]
if id in grid_ids_less_2dem:
return True
dem_diff_files = io_function.get_file_list_by_pattern(
grid_dem_diffs_dir, '*_DEM_diff_grid%d.tif' % id)
if len(dem_diff_files) == 1:
return True
elif len(dem_diff_files) > 1:
basic.outputlogMessage(
'warning, There are multiple DEM difference for grid: %d' % id)
for item in dem_diff_files:
basic.outputlogMessage(item)
return True
else:
return False
def b_exist_dem_hillshade_newest_HWLine_grid(id):
# if a grid don't have headwall (due to really flat or other reasons), then we think it's complete
if os.path.isfile(grid_no_headwall_txt):
grid_ids_no_headwall = [
int(item)
for item in io_function.read_list_from_txt(grid_no_headwall_txt)
]
if id in grid_ids_no_headwall:
return True
hillshade_newest_HDLine_tifs = io_function.get_file_list_by_pattern(
grid_hillshade_newest_HDLine_dir, '*_grid%d.tif' % id)
if len(hillshade_newest_HDLine_tifs) == 1:
return True
elif len(hillshade_newest_HDLine_tifs) > 1:
basic.outputlogMessage(
'warning, There are multiple hillshade (newest) HDLine tif for grid: %d'
% id)
for item in hillshade_newest_HDLine_tifs:
basic.outputlogMessage(item)
return True
else:
return False
