def main():
# run in ~/Data/Arctic/canada_arctic/autoMapping/multiArea_sub_images on tesia
ini_list = io_function.get_file_list_by_pattern('./','area*.ini')
txt_list = io_function.get_file_list_by_pattern('./','area*.txt')
for txt in txt_list:
ini_s = io_function.read_list_from_txt(txt)
ini_list.extend(ini_s)
ini_list = [os.path.abspath(item) for item in ini_list]
file_names = [ io_function.get_name_no_ext(item) for item in ini_list ]
cur_dir = os.getcwd()
# show
[print(item) for item in ini_list]
time.sleep(3)
for name, area_ini in zip(file_names,ini_list):
word_dir = os.path.join(cur_dir,name)
io_function.mkdir(word_dir)
os.chdir(word_dir)
# copy and modify main_para.ini
io_function.copyfiletodir(os.path.join(cur_dir,'main_para.ini'),'./',overwrite=True)
io_function.copyfiletodir(os.path.join(cur_dir,'exe.sh'),'./',overwrite=True)
parameters.write_Parameters_file('main_para.ini','training_regions',area_ini)
# run exe.sh
res = os.system('./exe.sh')
if res !=0:
print(res)
sys.exit(1)
os.chdir(cur_dir)
def get_dem_diff_list_to_seg():
dem_diff_list = io_function.get_file_list_by_pattern(
dem_common.grid_dem_diffs_dir, '*DEM_diff_grid*.tif')
dem_diff_list_copy = dem_diff_list.copy()
if len(dem_diff_list) < 1:
print('No *DEM_diff*.tif in %s' % dem_common.grid_dem_diffs_dir)
return []
dem_diff_ids = [
int(re.findall('grid\d+', os.path.basename(item))[0][4:])
for item in dem_diff_list
]
dem_subsidence_shps = io_function.get_file_list_by_pattern(
dem_common.grid_dem_diffs_segment_dir, 'segment_result*/*_post.shp')
subsidence_ids = [
int(re.findall('grid\d+', os.path.basename(item))[0][4:])
for item in dem_subsidence_shps
]
no_subsidence_ids = []
if os.path.isfile(dem_common.grid_no_subscidence_poly_txt):
no_subsidence_ids = [
int(item) for item in io_function.read_list_from_txt(
dem_common.grid_no_subscidence_poly_txt)
]
subsidence_ids.extend(no_subsidence_ids)
# remove dem diff already been segmented or no subsidence
for id, dem_diff in zip(dem_diff_ids, dem_diff_list_copy):
if id in subsidence_ids:
dem_diff_list.remove(dem_diff)
return dem_diff_list
def main(options, args):
root_dir = args[0]
if os.path.isdir(root_dir) is False:
raise ValueError('%s not exists'%root_dir)
folder_pattern = options.folder_pattern
folder_list = io_function.get_file_list_by_pattern(root_dir,folder_pattern)
folder_list = [item for item in folder_list if os.path.isdir(item) ]
folder_list.sort()
# ray may create a new folder if the previous one already exists
dupli_foldes = io_function.get_file_list_by_pattern(root_dir,folder_pattern + '_????')
dupli_foldes = [item for item in dupli_foldes if os.path.isdir(item) ]
dupli_foldes.sort()
folder_list.extend(dupli_foldes)
para_file = options.para_file
output_file = options.output
if output_file is None:
output_file = os.path.basename(root_dir) + '.xlsx'
train_output = {}
train_output['folder'] = []
for para in para_ini_list:
train_output[para] = []
train_output['train_class_0'] = []
train_output['train_class_1'] = []
train_output['val_class_0'] = []
train_output['val_class_1'] = []
train_output['class_1'] = []
train_output['overall'] = []
train_output['step'] = []
train_output['early_stopping'] = []
train_output['model_train_iter'] = []
for folder in folder_list:
print('read parameter and results for %s'%folder)
train_output['folder'].append(os.path.basename(folder))
read_para_values(folder,para_file,train_output)
get_training_image_patch_count(folder,train_output)
get_miou_of_overall_and_class_1_step(folder, para_file, train_output)
get_early_stopping_trained_iteration(folder, para_file, train_output)
# save to excel file
train_out_table_pd = pd.DataFrame(train_output)
with pd.ExcelWriter(output_file) as writer:
train_out_table_pd.to_excel(writer, sheet_name='training parameter and results')
# set format
workbook = writer.book
format = workbook.add_format({'num_format': '#0.000'})
train_out_table_sheet = writer.sheets['training parameter and results']
train_out_table_sheet.set_column('O:P',None,format)
pass
def read_a_meta_of_scene(scene_folder_or_geojson,scene_id_list):
# get scene id
if os.path.isfile(scene_folder_or_geojson): # geojson file
scene_id = os.path.splitext(os.path.basename(scene_folder_or_geojson))[0]
geojson_path = scene_folder_or_geojson
scene_folder = os.path.splitext(scene_folder_or_geojson)[0]
else:
# scene_folder
scene_id = os.path.basename(scene_folder_or_geojson)
geojson_path = scene_folder_or_geojson + '.geojson'
scene_folder = scene_folder_or_geojson
# if already exists
if scene_id in scene_id_list:
return None,None,None,None,None,None,None,None,None
print(scene_id)
# get metadata path
cloud_cover = 101
acquisitionDate = datetime(1970,1,1)
metadata_paths = io_function.get_file_list_by_pattern(scene_folder,'*metadata.xml')
if len(metadata_paths) < 1:
basic.outputlogMessage('warning, there is no metadata file in %s'%scene_folder)
elif len(metadata_paths) > 1:
basic.outputlogMessage('warning, there are more than one metadata files in %s' % scene_folder)
else:
# read metadata
metadata_path = metadata_paths[0]
cloud_cover = read_cloud_cover(metadata_path)
acquisitionDate = read_acquired_datetime(metadata_path)
assets = io_function.get_file_list_by_pattern(scene_folder,'*')
asset_count = len(assets)
asset_files = sorted([ os.path.basename(item) for item in assets])
asset_files =','.join(asset_files)
image_type = 'analytic' # 'analytic_sr' (surface reflectance) or 'analytic'
sr_tif = io_function.get_file_list_by_pattern(scene_folder,'*_SR.tif')
if len(sr_tif) == 1:
image_type = 'analytic_sr'
# consider as the downloading time
if os.path.isfile(geojson_path):
modified_time = io_function.get_file_modified_time(geojson_path)
else:
geojson_path = ''
modified_time = io_function.get_file_modified_time(scene_folder)
return scene_id,cloud_cover,acquisitionDate,geojson_path,scene_folder,asset_count,image_type,asset_files,modified_time
def get_Planet_SR_image_list_overlap_a_polygon(polygon,geojson_list, cloud_cover_thr, save_list_path=None):
'''
get planet surface reference (SR) list overlap a polygon (within or overlap part of the polygon)
:param polygon: polygon in the shapely format
:param geojson_list:
:param save_list_path: save the list to a txt file.
:return:
'''
image_path_list = []
cloud_cover_list = []
for geojson_file in geojson_list:
# print(geojson_file)
with open(geojson_file) as json_obj:
geom = json.load(json_obj)
img_ext = shape(geom)
inter = polygon.intersection(img_ext)
if inter.is_empty is False:
img_dir = os.path.splitext(geojson_file)[0]
sr_img_paths = io_function.get_file_list_by_pattern(img_dir,'*_SR.tif')
if len(sr_img_paths) < 1:
basic.outputlogMessage('warning, no SR image in %s, try to find Analytic images'%img_dir)
sr_img_paths = io_function.get_file_list_by_pattern(img_dir, '*AnalyticMS.tif')
meta_data_paths = io_function.get_file_list_by_pattern(img_dir,'*_metadata.xml')
if len(sr_img_paths) != len(meta_data_paths):
# raise ValueError('the count of metadata files and images is different')
basic.outputlogMessage('warning: the count of metadata files and images is different for %s'%img_dir)
continue
if len(sr_img_paths) < 1:
basic.outputlogMessage('warning, no Planet SR image in the %s'%img_dir)
elif len(sr_img_paths) > 1:
basic.outputlogMessage('warning, more than one Planet SR image in the %s'%img_dir)
else:
# check cloud cover
cloud_cover = read_cloud_cover(meta_data_paths[0])
if cloud_cover > cloud_cover_thr:
continue
# add image
image_path_list.append(sr_img_paths[0])
cloud_cover_list.append(cloud_cover)
if save_list_path is not None:
with open(save_list_path,'w') as f_obj:
for image_path in image_path_list:
f_obj.writelines(image_path + '\n')
return image_path_list, cloud_cover_list
def read_down_load_geometry(folder):
'''
read geojson files in a folder. geojson file stores the geometry of a file, and save to global varialbes
:param folder: the save folder
:return:
'''
global downloaded_scene_geometry
json_list = io_function.get_file_list_by_ext('.geojson',
folder,
bsub_folder=False)
for json_file in json_list:
# ignore the scenes in the excluded list
item_id = os.path.splitext(os.path.basename(json_file))[0]
if item_id in manually_excluded_scenes:
continue
scene_folder = os.path.splitext(json_file)[0]
asset_files = io_function.get_file_list_by_pattern(scene_folder, '*')
if len(asset_files) < 3:
basic.outputlogMessage(
'downloading of scene %s is not compelte, ignore it' % item_id)
continue
with open(json_file) as json_file:
data = json.load(json_file)
# p(data) # test
downloaded_scene_geometry.append(data)
def find_results_for_one_grid_id(root_dir,
folder,
pattern,
grid_id,
grid_poly,
description='dem_subsidence'):
shp_dir = os.path.join(root_dir, folder)
shps = io_function.get_file_list_by_pattern(shp_dir, pattern)
if len(shps) < 1:
print('warning, no results in folder: %s, with pattern: %s' %
(folder, pattern))
return None
if description == 'dem_subsidence':
if len(shps) == 1:
return shps[0]
# if it's subsidence from dem diff, should only have one shp
if len(shps) > 1:
raise ValueError(
'warning, more than one result in folder: %s, with pattern: %s'
% (folder, pattern))
if description == 'headwall':
return shps
def save_planet_images_to_excel(image_dir,save_xlsx):
if os.path.isfile(image_dir):
basic.outputlogMessage('Warning, Input %s is a file, expected a folder, skip it'%image_dir)
return False
# read save_xlsx if it exist
# may be not good to exclude these the scene id if we want to update some records.
# if os.path.isfile(save_xlsx):
# df = pd.read_excel(save_xlsx)
# scene_id_list.extend(df['scene_id'].to_list())
# get scene folders (idealy, the number of scene folder are the same to the one of geojson files)
scene_geojson_folders = io_function.get_file_list_by_pattern(image_dir,'????????_??????_*') # acquired date_time
if len(scene_geojson_folders) < 1:
raise ValueError('There is no scene folder or geojson in %s'%image_dir)
scene_table, scene_without_asset = get_meta_dict(scene_geojson_folders)
df = pd.DataFrame(scene_table)
with pd.ExcelWriter(save_xlsx) as writer:
df.to_excel(writer)
basic.outputlogMessage('write records of downloaded scenes to %s'%save_xlsx)
scene_folder_no_assets_txt = os.path.splitext(save_xlsx)[0] + '_scenes_noAsset.txt'
with open(scene_folder_no_assets_txt, 'w') as f_obj:
for scene_dir in scene_without_asset:
f_obj.writelines(scene_dir + '\n')
return True
def get_local_folder(local_dir, folder_pattern):
# get local dir
folder_list = io_function.get_file_list_by_pattern(local_dir,
folder_pattern)
folder_list = [item for item in folder_list if os.path.isdir(item)]
folder_list.sort()
return folder_list
def b_exist_grid_dem_subsidence(id):
# if an grid don't have dem subsidence, then we think it's complete
if os.path.isfile(grid_no_subscidence_poly_txt):
grid_ids_no_subsidence = [
int(item) for item in io_function.read_list_from_txt(
grid_no_subscidence_poly_txt)
]
if id in grid_ids_no_subsidence:
return True
# if an id don't have enough dem for dem diff, then we think it's complete
if os.path.isfile(grid_dem_diff_less2dem_txt):
grid_ids_less_2dem = [
int(item) for item in io_function.read_list_from_txt(
grid_dem_diff_less2dem_txt)
]
if id in grid_ids_less_2dem:
return True
dem_subsidence_shps = io_function.get_file_list_by_pattern(
grid_dem_diffs_segment_dir,
'*_grid%d/*_grid%d_8bit_post.shp' % (id, id))
if len(dem_subsidence_shps) == 1:
return True
elif len(dem_subsidence_shps) > 1:
basic.outputlogMessage(
'warning, There are multiple DEM subsidence for grid: %d' % id)
for item in dem_subsidence_shps:
basic.outputlogMessage(item)
return True
else:
return False
def main():
basic.setlogfile('log_to_relative_dem_8bit.txt')
if os.path.isdir(relative_dem_dir) is False:
io_function.mkdir(relative_dem_dir)
# 500 pixel by 500 pixel, that is 1 km by 1 km
patch_width = 500
patch_height = 500
process_num = 1
failed_tifs = []
dem_reg_list = io_function.get_file_list_by_pattern(
arcticDEM_reg_tif_dir, '*dem_reg.tif')
count = len(dem_reg_list)
for idx, tif in enumerate(dem_reg_list):
print('%d/%d convert %s to relative DEM (8bit)' %
(idx + 1, count, tif))
rel_dem_8bit = io_function.get_name_by_adding_tail(tif, 'relDEM8bit')
rel_dem_8bit = os.path.join(relative_dem_dir,
os.path.basename(rel_dem_8bit))
try:
dem_to_relative_dem(tif, rel_dem_8bit, patch_width, patch_height,
process_num)
except:
failed_tifs.append(tif)
with open('to_relative_dem_failed_cases.txt', 'w') as f_obj:
for item in failed_tifs:
f_obj.writelines(item + '\n')
pass
def get_existing_dem_diff(dem_diff_dir, grid_base_name, grid_ids):
existing_tif = []
grid_id_no_dem_tiff = []
for id in grid_ids:
# dem_diff = os.path.join(dem_diff_dir, grid_base_name + '_DEM_diff_grid%d.tif'%id)
# if os.path.isfile(dem_diff):
# existing_tif.append(dem_diff)
# continue
dem_diff_files = io_function.get_file_list_by_pattern(dem_diff_dir, '*_DEM_diff_grid%d.tif'%id)
if len(dem_diff_files) == 1:
existing_tif.append(dem_diff_files[0])
continue
elif len(dem_diff_files) > 1:
existing_tif.append(dem_diff_files[0])
basic.outputlogMessage('warning, There are multiple DEM_diff tif for grid: %d'%id)
for item in dem_diff_files: basic.outputlogMessage(item)
continue
else:
pass
grid_id_no_dem_tiff.append(id)
if len(existing_tif) > 0:
basic.outputlogMessage('%d existing grid dem diff files for the input grid_ids or extent'%len(existing_tif))
else:
basic.outputlogMessage('no existing grid dem diff files')
return existing_tif, grid_id_no_dem_tiff
def get_existing_grid_headwall_shp(headwall_shp_dir, grid_base_name, grid_ids):
existing_grid_headwall_shp = []
grid_id_no_headwall_shp = []
for id in grid_ids:
headwall_shps_dir = io_function.get_file_list_by_pattern(
headwall_shp_dir, '*_grid%d' % id)
if len(headwall_shps_dir) == 1:
existing_grid_headwall_shp.append(headwall_shps_dir[0])
continue
elif len(headwall_shps_dir) > 1:
existing_grid_headwall_shp.append(headwall_shps_dir[0])
basic.outputlogMessage(
'warning, There are multiple headwall shps for grid: %d' % id)
for item in headwall_shps_dir:
basic.outputlogMessage(item)
continue
else:
pass
grid_id_no_headwall_shp.append(id)
if len(existing_grid_headwall_shp) > 0:
basic.outputlogMessage(
'%d existing grid headwall shps for the input grid_ids or extent' %
len(existing_grid_headwall_shp))
else:
basic.outputlogMessage('no existing grid headwall shps')
return existing_grid_headwall_shp, grid_id_no_headwall_shp
def main(options, args):
t_polygons_shp = args[0]
image_folder = args[1] # folder for store image tile (many split block of a big image)
b_label_image = options.no_label_image
process_num = options.process_num
# check training polygons
assert io_function.is_file_exist(t_polygons_shp)
t_polygons_shp_all = options.all_training_polygons
if t_polygons_shp_all is None:
basic.outputlogMessage('Warning, the full set of training polygons is not assigned, '
'it will consider the one in input argument is the full set of training polygons')
t_polygons_shp_all = t_polygons_shp
else:
if get_projection_proj4(t_polygons_shp) != get_projection_proj4(t_polygons_shp_all):
raise ValueError('error, projection insistence between %s and %s'%(t_polygons_shp, t_polygons_shp_all))
assert io_function.is_file_exist(t_polygons_shp_all)
# get image tile list
# image_tile_list = io_function.get_file_list_by_ext(options.image_ext, image_folder, bsub_folder=False)
image_tile_list = io_function.get_file_list_by_pattern(image_folder,options.image_ext)
if len(image_tile_list) < 1:
raise IOError('error, failed to get image tiles in folder %s'%image_folder)
check_projection_rasters(image_tile_list) # it will raise errors if found problems
# comment out on June 18, 2021,
# check_1or3band_8bit(image_tile_list) # it will raise errors if found problems
#need to check: the shape file and raster should have the same projection.
if get_projection_proj4(t_polygons_shp) != get_projection_proj4(image_tile_list[0]):
raise ValueError('error, the input raster (e.g., %s) and vector (%s) files don\'t have the same projection'%(image_tile_list[0],t_polygons_shp))
# check these are EPSG:4326 projection
if get_projection_proj4(t_polygons_shp).strip() == '+proj=longlat +datum=WGS84 +no_defs':
bufferSize = meters_to_degress_onEarth(options.bufferSize)
else:
bufferSize = options.bufferSize
saved_dir = options.out_dir
# if os.system('mkdir -p ' + os.path.join(saved_dir,'subImages')) != 0:
# sys.exit(1)
# if os.system('mkdir -p ' + os.path.join(saved_dir,'subLabels')) !=0:
# sys.exit(1)
io_function.mkdir(os.path.join(saved_dir,'subImages'))
if b_label_image:
io_function.mkdir(os.path.join(saved_dir,'subLabels'))
dstnodata = options.dstnodata
if 'qtb_sentinel2' in image_tile_list[0]:
# for qtb_sentinel-2 mosaic
pre_name = '_'.join(os.path.splitext(os.path.basename(image_tile_list[0]))[0].split('_')[:4])
else:
pre_name = os.path.splitext(os.path.basename(image_tile_list[0]))[0]
get_sub_images_and_labels(t_polygons_shp, t_polygons_shp_all, bufferSize, image_tile_list,
saved_dir, pre_name, dstnodata, brectangle=options.rectangle, b_label=b_label_image,
proc_num=process_num)
def test_dem_tif_to_8bit():
dem_diff_list = io_function.get_file_list_by_pattern('./','*.tif')
count = len(dem_diff_list)
for idx, tif in enumerate(dem_diff_list):
print('%d/%d convert %s to 8 bit'%(idx+1, count, tif))
tif_8bit = io_function.get_name_by_adding_tail(tif, '8bit')
output = os.path.join(grid_dem_diffs_8bit_dir, os.path.basename(tif_8bit))
dem_tif_to_8bit(tif,output)
def read_dem_diff_assigned_to_other_machine(job_list_pre):
dem_list_txts = io_function.get_file_list_by_pattern(
dem_common.process_log_dir, job_list_pre + '*.txt')
assign_dem_diff = []
for txt in dem_list_txts:
assign_dem_diff.extend(io_function.read_list_from_txt(txt))
return assign_dem_diff
def main(options, args):
root_dir = args[0]
if os.path.isdir(root_dir) is False:
raise ValueError('%s not exists' % root_dir)
folder_pattern = options.folder_pattern
checked_folders = []
while True:
folder_list = io_function.get_file_list_by_pattern(
root_dir, folder_pattern)
folder_list = [item for item in folder_list if os.path.isdir(item)]
folder_list.sort()
# ray may create a new folder if the previous one already exists
dupli_foldes = io_function.get_file_list_by_pattern(
root_dir, folder_pattern + '_????')
dupli_foldes = [item for item in dupli_foldes if os.path.isdir(item)]
dupli_foldes.sort()
folder_list.extend(dupli_foldes)
print(str(datetime.now()),
'start moving or removing files or folders\n')
check_folder_count = 0
for folder in folder_list:
if folder in checked_folders:
continue
print('checking folder: %s' % folder)
if io_function.check_file_or_dir_is_old(folder, time_hour_thr):
print(
'%s is older than %f hours, will remove some files inside it'
% (folder, time_hour_thr))
remove_files(folder)
checked_folders.append(folder)
check_folder_count += 1
print(str(datetime.now()),
'removing files in %d folders' % check_folder_count)
time.sleep(60) # wait
pass
def get_loss_learning_rate_list(log_dir):
# add the tensorboard in the tf1x version
tf1x_dir = os.path.join(os.path.dirname(os.path.dirname(tf1x_python)),
'lib', 'python3.7', 'site-packages')
sys.path.insert(0, tf1x_dir)
from tensorboard.backend.event_processing.event_accumulator import EventAccumulator
tf_size_guidance = {
'compressedHistograms': 10,
'images': 0,
'scalars': 0, # set a 0, to load all scalars
'histograms': 1
}
events_files = io_function.get_file_list_by_pattern(log_dir, 'events*')
if len(events_files) < 1:
print('warning, No events file in %s' % log_dir)
return None
event_acc = EventAccumulator(log_dir, tf_size_guidance)
event_acc.Reload()
#Show all tags in the log file
# tag_dict = event_acc.Tags()
# io_function.save_dict_to_txt_json('event_acc.txt',tag_dict)
# "scalars": [
# "clone_0/Losses/clone_0//clone_loss",
# "total_loss_1",
# "learning_rate",
# "losses/clone_0/semantic_merged_logits/mul_1",
# "clone_0/Losses/regularization_loss",
# "global_step/sec" : how much time it takes for each step
# ],
loss_learnrate_dic = {}
total_loss_1_event = event_acc.Scalars('total_loss_1')
total_loss_list = [
item[2] for item in total_loss_1_event
] # item[0] is wall_time, item[1] is step, item [2] is the value
loss_learnrate_dic['total_loss'] = total_loss_list
step_list = [item[1] for item in total_loss_1_event]
wall_time_list = [
item[0] for item in total_loss_1_event
] # we can use datetime.fromtimestamp() to convert datetime
learning_rate_event = event_acc.Scalars('learning_rate')
learning_rate_list = [item[2] for item in learning_rate_event]
loss_learnrate_dic['learning_rate'] = learning_rate_list
loss_learnrate_dic['step'] = step_list
loss_learnrate_dic['wall_time'] = wall_time_list
io_function.save_dict_to_txt_json(
os.path.join(log_dir, 'loss_learning_rate.txt'), loss_learnrate_dic)
def check_align_folder(dem_tif):
# by default, dem_align.py save the results to where dem_tif is
res_dir = os.path.dirname(dem_tif)
align_folder = os.path.splitext(
os.path.basename(dem_tif))[0] + '_dem_align'
align_dir = os.path.join(res_dir, align_folder)
# after dem_align.py usually have 9 files
align_outputs = io_function.get_file_list_by_pattern(align_dir, '*')
# print(align_outputs)
return align_outputs
def get_sub_images_pixel_json_files(polygons_shp, image_folder_or_path,
image_pattern, class_names, bufferSize,
dstnodata, saved_dir, b_rectangle,
process_num):
# check training polygons
assert io_function.is_file_exist(polygons_shp)
# get image tile list
# image_tile_list = io_function.get_file_list_by_ext(options.image_ext, image_folder, bsub_folder=False)
if os.path.isdir(image_folder_or_path):
image_tile_list = io_function.get_file_list_by_pattern(
image_folder_or_path, image_pattern)
else:
assert io_function.is_file_exist(image_folder_or_path)
image_tile_list = [image_folder_or_path]
if len(image_tile_list) < 1:
raise IOError('error, failed to get image tiles in folder %s' %
image_folder_or_path)
get_subImages.check_projection_rasters(
image_tile_list) # it will raise errors if found problems
get_subImages.check_1or3band_8bit(
image_tile_list) # it will raise errors if found problems
# need to check: the shape file and raster should have the same projection.
if get_subImages.get_projection_proj4(
polygons_shp) != get_subImages.get_projection_proj4(
image_tile_list[0]):
raise ValueError(
'error, the input raster (e.g., %s) and vector (%s) files don\'t have the same projection'
% (image_tile_list[0], polygons_shp))
# check these are EPSG:4326 projection
if get_subImages.get_projection_proj4(
polygons_shp).strip() == '+proj=longlat +datum=WGS84 +no_defs':
bufferSize = get_subImages.meters_to_degress_onEarth(bufferSize)
pre_name = os.path.splitext(os.path.basename(image_tile_list[0]))[0]
saved_dir = os.path.join(saved_dir, pre_name + '_subImages')
if os.path.isdir(saved_dir) is False:
io_function.mkdir(saved_dir)
get_sub_images_and_json_files(polygons_shp,
class_names,
bufferSize,
image_tile_list,
saved_dir,
pre_name,
dstnodata,
brectangle=b_rectangle,
proc_num=process_num)
def tifs_to_png(image_dir):
tif_list = io_function.get_file_list_by_pattern(image_dir, '*/*.tif')
for idx, tif in enumerate(tif_list):
print('tif to png: %d/%d tif' % (idx + 1, len(tif_list)))
basename = io_function.get_name_no_ext(tif)
save_path = os.path.join(image_dir, basename + '.png')
if os.path.isfile(save_path):
print('%s exists, skip' % save_path)
continue
command_str = "gdal_translate -of PNG %s %s" % (tif, save_path)
basic.os_system_exit_code(command_str)
def main():
file_list = io_function.get_file_list_by_pattern(arcticDEM_reg_tif_dir,
'*_dem_reg.tif')
print('Get %d dem_reg.tif from %s' %
(len(file_list), arcticDEM_reg_tif_dir))
year_dates = [
timeTools.get_yeardate_yyyymmdd(os.path.basename(item),
pattern='[0-9]{8}_')
for item in file_list
]
month_list = [item.month for item in year_dates]
value_list = month_list
# save unique date to txt file
dates_unique = set(year_dates)
dates_unique = sorted(dates_unique)
dates_unique_str = [
timeTools.date2str(item, '%Y-%m-%d') for item in dates_unique
]
io_function.save_list_to_txt('dates_unique.txt', dates_unique_str)
# plot a histogram
# bin_count = 12
bins = np.arange(0, 12, 1)
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(8, 8))
n, bins, patches = ax.hist(value_list,
bins=bins,
alpha=0.75,
ec="black",
linewidth='1.5',
color='grey',
hatch='',
rwidth=1) # density = True, # label=labels,
# ax.legend(prop={'size': 12})
plt.xticks(bins)
ax.tick_params(axis='both',
which='both',
direction='out',
length=7,
labelsize=20) # ,width=50 #,
# if xlabelrotation is not None:
# ax.tick_params(axis='x', labelrotation=90)
# if ylim is not None:
# ax.set_ylim(ylim)
plt.gcf().subplots_adjust(bottom=0.15)
# plt.grid(True)
plt.savefig('ArcticDEM_strip_date_hist.jpg') #
def copy_subImages_labels_directly(subImage_dir, subLabel_dir, area_ini):
input_image_dir = parameters.get_directory_None_if_absence(
area_ini, 'input_image_dir')
# it is ok consider a file name as pattern and pass it the following functions to get file list
input_image_or_pattern = parameters.get_string_parameters(
area_ini, 'input_image_or_pattern')
# label raster folder
label_raster_dir = parameters.get_directory_None_if_absence(
area_ini, 'label_raster_dir')
sub_images_list = []
label_path_list = []
if os.path.isdir(subImage_dir) is False:
io_function.mkdir(subImage_dir)
if os.path.isdir(subLabel_dir) is False:
io_function.mkdir(subLabel_dir)
sub_images = io_function.get_file_list_by_pattern(input_image_dir,
input_image_or_pattern)
for sub_img in sub_images:
# find the corresponding label raster
label_name = io_function.get_name_by_adding_tail(
os.path.basename(sub_img), 'label')
label_path = os.path.join(label_raster_dir, label_name)
if os.path.isfile(label_path):
sub_images_list.append(sub_img)
label_path_list.append(label_path)
else:
print('Warning, cannot find label for %s in %s' %
(sub_img, label_raster_dir))
# copy sub-images, adding to txt files
with open('sub_images_labels_list.txt', 'a') as f_obj:
for tif_path, label_file in zip(sub_images_list, label_path_list):
if label_file is None:
continue
dst_subImg = os.path.join(subImage_dir, os.path.basename(tif_path))
# copy sub-images
io_function.copy_file_to_dst(tif_path, dst_subImg, overwrite=True)
dst_label_file = os.path.join(subLabel_dir,
os.path.basename(label_file))
io_function.copy_file_to_dst(label_file,
dst_label_file,
overwrite=True)
sub_image_label_str = dst_subImg + ":" + dst_label_file + '\n'
f_obj.writelines(sub_image_label_str)
def main():
reg_tif_dir = 'arcticdem_registration_tifs'
while True:
print(str(datetime.now()),
'start moving or removing files or folders\n\n')
reg_files = io_function.get_file_list_by_pattern(reg_tif_dir, '*')
print('registration file count: %d in %s' %
(len(reg_files), reg_tif_dir))
for file in reg_files:
if check_file_or_dir_is_old(file, time_hour_thr):
print(
'%s is older than %f hours, will be moved to archieved dir'
% (file, time_hour_thr))
io_function.movefiletodir(file,
arcticDEM_reg_tif_dir,
overwrite=True)
SETSM_dir = io_function.get_file_list_by_pattern(
'./', 'SETSM_*2m_v3.0')
print('SETSM folder count: %d in %s' % (len(SETSM_dir), './'))
for folder in SETSM_dir:
if check_file_or_dir_is_old(folder, time_hour_thr):
print('%s is older than %f hours, will be removed' %
(folder, time_hour_thr))
io_function.delete_file_or_dir(folder)
grid_tmp_dir = io_function.get_file_list_by_pattern('./', 'grid*files')
print('Grid tmp folder count: %d in %s' % (len(grid_tmp_dir), './'))
for folder in grid_tmp_dir:
if check_file_or_dir_is_old(folder, time_hour_thr):
print('%s is older than %f hours, will be removed' %
(folder, time_hour_thr))
io_function.delete_file_or_dir(folder)
time.sleep(60) # wait
pass
def check_files_existence(dir, pre_name):
file_pattern = ['*dem_reg.tif',
'*reg_dem.tif'] # Arctic strip and tile (mosaic) version
for pattern in file_pattern:
file_list = io_function.get_file_list_by_pattern(
dir, pre_name + pattern)
if len(file_list) > 0:
return True
else:
# check if in the archived dir
if os.path.isdir(arcticDEM_reg_tif_dir):
file_list_archived = io_function.get_file_list_by_pattern(
arcticDEM_reg_tif_dir, pre_name + pattern)
if len(file_list_archived) > 0:
return True
if os.path.isdir(arcticDEM_mosaic_reg_tif_dir):
file_list_archived = io_function.get_file_list_by_pattern(
arcticDEM_mosaic_reg_tif_dir, pre_name + pattern)
if len(file_list_archived) > 0:
return True
return False
def get_file_path_parameter(parafile, data_dir, data_name_or_pattern):
data_dir = parameters.get_directory_None_if_absence(parafile, data_dir)
data_name_or_pattern = parameters.get_string_parameters_None_if_absence(parafile, data_name_or_pattern)
if data_dir is None or data_name_or_pattern is None:
return None
file_list = io_function.get_file_list_by_pattern(data_dir,data_name_or_pattern)
if len(file_list) < 1:
raise IOError('NO file in %s with name or pattern: %s'%(data_dir, data_name_or_pattern))
if len(file_list) == 1:
return file_list[0]
else:
# return multiple files
return file_list
def main(options, args):
# process_num = multiprocessing.cpu_count()
process_num = options.process_num
data_folder = arcticDEM_reg_tif_dir
if len(args) > 0:
data_folder = args[0]
tifs = io_function.get_file_list_by_pattern(
data_folder, '*.tif') # _dem_reg check all tifs
save_invalid_txt_path = os.path.basename(data_folder) + '_invalid_list.txt'
save_good_txt_path = os.path.basename(data_folder) + '_good_list.txt'
tif_count = len(tifs)
basic.outputlogMessage('get %d tif files in %s' % (tif_count, data_folder))
good_tifs = []
if os.path.isfile(save_good_txt_path):
good_tifs.extend(io_function.read_list_from_txt(save_good_txt_path))
invalid_tif = []
# remove good one for the list
if len(good_tifs) > 0:
tifs = [
item for item in tifs if os.path.basename(item) not in good_tifs
]
if process_num == 1:
for idx, tif in enumerate(tifs):
if check_one_tif(idx, tif_count, tif, good_tifs):
good_tifs.append(os.path.basename(tif))
else:
invalid_tif.append(os.path.basename(tif))
else:
theadPool = Pool(process_num) # multi processes
parameters_list = [(idx, tif_count, tif, good_tifs)
for idx, tif in enumerate(tifs)]
results = theadPool.starmap(check_one_tif,
parameters_list) # need python3
for tif, res in zip(tifs, results):
if res:
good_tifs.append(os.path.basename(tif))
else:
invalid_tif.append(os.path.basename(tif))
theadPool.close()
io_function.save_list_to_txt(save_invalid_txt_path, invalid_tif)
io_function.save_list_to_txt(save_good_txt_path, good_tifs)
def select_polygons_by_ids_in_filenames(in_shp, folder, save_path,id_name='id'):
'''get ids from files name, then select polygons from the shapefile '''
if os.path.isfile(save_path):
basic.outputlogMessage('warning, %s exists, skip'%save_path)
return True
files = io_function.get_file_list_by_pattern(folder,'*')
manu_sel_ids = [get_id_from_filepath(item) for item in files]
## check if there are duplicated ones
check_duplicated_ids(manu_sel_ids)
pol_ids = vector_gpd.read_attribute_values_list(in_shp, id_name)
# find match index
select_idx = [ pol_ids.index(sel_id) for sel_id in manu_sel_ids ]
return vector_gpd.save_shapefile_subset_as(select_idx,in_shp,save_path)
def get_tifs_bounding_boxes(image_dir):
tif_list = io_function.get_file_list_by_pattern(image_dir, '*/*.tif')
for idx, tif in enumerate(tif_list):
print('get bounding box: %d/%d tif' % (idx + 1, len(tif_list)))
basename = io_function.get_name_no_ext(tif)
save_path = os.path.join(image_dir, basename + '_bound.geojson')
if os.path.isfile(save_path):
print('%s exists, skip' % save_path)
continue
command_str = imgExt + " %s -o tmp.gpkg" % tif
basic.os_system_exit_code(command_str)
command_str = "ogr2ogr -f GeoJSON -t_srs EPSG:3413 %s tmp.gpkg" % save_path # note: projection is EPSG:3413
basic.os_system_exit_code(command_str)
io_function.delete_file_or_dir('tmp.gpkg')
def main():
# for the user defined module in code_dir, need to be imported in functions
# sys.path.insert(0, code_dir)
# import parameters
# import basic_src.io_function as io_function
# import workflow.whole_procedure as whole_procedure
# from utility.eva_report_to_tables import read_accuracy_multi_reports
loc_dir = "./ray_results"
tune_name = "tune_dataAug_para_tesia"
augment_options = get_augment_options()
file_folders = io_function.get_file_list_by_pattern(
os.path.join(loc_dir, tune_name), '*')
if len(file_folders) > 1:
b_resume = True
else:
b_resume = False
analysis = tune.run(
training_function,
resources_per_trial={"gpu": 1}, # use one GPUs,
local_dir=loc_dir,
name=tune_name,
# fail_fast=True, # Stopping after the first failure
log_to_file=("stdout.log",
"stderr.log"), #Redirecting stdout and stderr to files
trial_name_creator=tune.function(trial_name_string),
trial_dirname_creator=tune.function(trial_dir_string),
resume=b_resume,
config={"data_augmentation": tune.grid_search(augment_options)})
print("Best config: ",
analysis.get_best_config(metric="overall_miou", mode="max"))
# Get a dataframe for analyzing trial results.
df = analysis.results_df
output_file = 'training_dataAug_ray_tune_%s.xlsx' % (
datetime.now().strftime("%Y%m%d_%H%M%S"))
with pd.ExcelWriter(output_file) as writer:
df.to_excel(writer) # , sheet_name='accuracy table'
# set format
# workbook = writer.book
# format = workbook.add_format({'num_format': '#0.000'})
# acc_talbe_sheet = writer.sheets['accuracy table']
# acc_talbe_sheet.set_column('G:I',None,format)
print('write trial results to %s' % output_file)
