def mask_nodata_regions_surface_water(ref_raster, in_raster, out_raster, ref_nodata=0, out_nodata=0):
if os.path.isfile(out_raster):
print('%s already exists'%out_raster)
return True
# get nodata mask from ref_raster
command_str = 'gdal_calc.py --calc="(A>%d)" --outfile=tmp.tif -A %s --NoDataValue 0 --type=Byte '%(ref_nodata,ref_raster)
res = os.system(command_str)
if res != 0:
print(res)
sys.exit(1)
# apply the mask to in_raster
# in the surface water, 1 is water, 0 are other, so mask water outside extent to zero as well, eventually, set 0 as nodata
command_str = 'gdal_calc.py --calc="B*A" --outfile=%s -A tmp.tif -B %s --NoDataValue %d --type=Byte ' % (out_raster,in_raster,out_nodata)
res = os.system(command_str)
if res != 0:
print(res)
sys.exit(1)
io_function.delete_file_or_dir('tmp.tif')
return True
def remove_previous_data_or_results(para_file):
# remove previous data or result if necessary
if os.path.isfile(time_txt):
io_function.delete_file_or_dir(time_txt)
# command_string = os.path.join(eo_dir, 'workflow', 'remove_previous_data.py') + ' ' + para_file
# basic.os_system_exit_code(command_string)
from remove_previous_data import remove_previous_data
return remove_previous_data(para_file)
def process_arcticDEM_tiles(tar_list,
save_dir,
inter_format,
resample_method,
o_res,
extent_poly,
extent_id,
pre_name,
b_rm_inter=True):
'''
process the mosaic (not multi-temporal) version of ArcticDEM
:param tar_list:
:param save_dir:
:param inter_format:
:param resample_method:
:param o_res: output resolution
:param extent_poly: extent polygons, in the same projection of ArcticDEM
:param extent_id: extent id
:param pre_name:
:param b_rm_inter:
:return:
'''
# unpackage and crop to extent
dem_tif_list, tar_folders = process_dem_tarball(tar_list,
save_dir,
inter_format,
o_res,
extent_poly=extent_poly,
poly_id=extent_id,
process_num=4)
if len(dem_tif_list) < 1:
raise ValueError('No DEM extracted from tarballs')
dem_name = os.path.basename(tar_folders[0])[-7:]
save_path = os.path.join(
save_dir,
pre_name + '_' + dem_name + '_ArcticTileDEM_sub_%d.tif' % extent_id)
RSImageProcess.mosaic_crop_images_gdalwarp(
dem_tif_list,
save_path,
resampling_method=resample_method,
o_format=inter_format,
xres=o_res,
yres=o_res,
compress='lzw',
tiled='yes',
bigtiff='if_safer')
# remove intermediate files
if b_rm_inter:
basic.outputlogMessage('remove intermediate files')
for folder in tar_folders:
io_function.delete_file_or_dir(folder)
return True
def move_files(save_dir, out_fig, out_hist_info):
if os.path.isdir(save_dir) is False:
io_function.mkdir(save_dir)
trim_fig = io_function.get_name_by_adding_tail(out_fig, 'trim')
os.system('convert -trim %s %s' % (out_fig, trim_fig))
io_function.movefiletodir(trim_fig, save_dir, overwrite=True)
io_function.delete_file_or_dir(out_fig)
# io_function.movefiletodir(out_fig,save_dir,overwrite=True)
io_function.movefiletodir(out_hist_info, save_dir, overwrite=True)
def postProcess_total_F1(minimum_area, min_slope, dem_diff_uplimit,
dem_diff_buffer_size, min_demD_area, IOU_threshold):
# when ray start a process, we need to add code_dir again and import user-defined modules
sys.path.insert(0, code_dir)
sys.path.insert(
0, os.path.join(code_dir, 'workflow')
) # require in ray when import other modules in workflow folder
import basic_src.io_function as io_function
import workflow.whole_procedure as whole_procedure
para_file = 'main_para.ini'
# ray tune will change current folder to its logdir, change it back
# os.chdir(curr_dir_before_ray)
print('\n\n\n current folder', os.getcwd(), '\n\n\n')
# allow ray to change current folder to its logdir, then we can run parallel
work_dir = os.getcwd()
copy_original_mapped_polygons(curr_dir_before_ray, work_dir)
# work_dir = curr_dir_before_ray
# create a training folder
inf_post_note = str(minimum_area) + '_' + str(min_slope) + '_' + str(
dem_diff_uplimit) + '_' + str(dem_diff_buffer_size) + '_' + str(
IOU_threshold)
# copy copy_ini_files
copy_ini_files(curr_dir_before_ray, work_dir)
# change para_file
modify_parameter(para_file, 'minimum_area', minimum_area)
modify_parameter(para_file, 'minimum_slope', min_slope)
modify_parameter(para_file, 'dem_difference_range',
'None,' + str(dem_diff_uplimit))
modify_parameter(para_file, 'buffer_size_dem_diff', dem_diff_buffer_size)
modify_parameter(para_file, 'minimum_dem_reduction_area', min_demD_area)
modify_parameter(para_file, 'IOU_threshold', IOU_threshold)
# run training
# whole_procedure.run_whole_procedure(para_file,working_dir=work_dir)
test_id = 'multiArea_deeplabV3+_5_exp6'
whole_procedure.post_processing_backup(para_file,
inf_post_note=inf_post_note,
b_skip_getshp=True,
test_id=test_id)
io_function.delete_file_or_dir(
'multi_inf_results') # remove a folder to save storage
# calculate the F1 score across all regions (total F1)
totalF1 = get_total_F1score(work_dir)
return totalF1
def get_tifs_bounding_boxes(image_dir):
tif_list = io_function.get_file_list_by_pattern(image_dir, '*/*.tif')
for idx, tif in enumerate(tif_list):
print('get bounding box: %d/%d tif' % (idx + 1, len(tif_list)))
basename = io_function.get_name_no_ext(tif)
save_path = os.path.join(image_dir, basename + '_bound.geojson')
if os.path.isfile(save_path):
print('%s exists, skip' % save_path)
continue
command_str = imgExt + " %s -o tmp.gpkg" % tif
basic.os_system_exit_code(command_str)
command_str = "ogr2ogr -f GeoJSON -t_srs EPSG:3413 %s tmp.gpkg" % save_path # note: projection is EPSG:3413
basic.os_system_exit_code(command_str)
io_function.delete_file_or_dir('tmp.gpkg')
def monitor_process_failed_grids():
tasks = ['dem_diff', 'dem_headwall_grid']
while True:
all_fail_list = []
for task in tasks:
all_fail_list.extend(get_failed_grid_ids(task))
if len(all_fail_list) < 1:
print(
datetime.now(),
'there is no failed jobs for %s to process, wait 10 minutes' %
machine_name)
time.sleep(600)
continue
for task in tasks:
fail_id_txt_list = get_failed_grid_ids(task)
if len(fail_id_txt_list) < 1:
continue
print('\n', task, 'fail_id_txt_list:', fail_id_txt_list, '\n')
fail_ids_txt = merge_grid_ids_txt(task, fail_id_txt_list)
# start processing,
res = os.system('./run.sh %s %s' % (fail_ids_txt, task))
if res != 0:
sys.exit(1)
# wait all job done? Yes, in "parallel_processing_curc.py"
# check 'done.txt' in each folder
fail_ids = [
int(item)
for item in io_function.read_list_from_txt(fail_ids_txt)
]
for idx, grid_id in enumerate(fail_ids):
job_folder = '%s_%s' % (task_2_job_folder[task],
str(idx).zfill(5))
print('job_folder for grid_id: %d, is' % grid_id, job_folder)
if os.path.isfile(job_folder + '/done.txt'):
os.system('rm -r %s' % job_folder)
else:
sys.exit(1)
# remove fail txt anyway if all "done.txt" exists
for txt in fail_id_txt_list:
io_function.delete_file_or_dir(txt)
def process_one_dem(idx, count, tif, product_list, arcticDEM_slope_dir,
arcticDEM_slope_8bit_dir, arcticDEM_hillshade_dir,
arcticDEM_tpi_8bit_dir):
print('%d/%d convert %s to slope (8bit) and hillshade' %
(idx + 1, count, tif))
try:
slope_file = os.path.basename(
io_function.get_name_by_adding_tail(tif, 'slope'))
slope_file_bak = os.path.join(arcticDEM_slope_dir,
os.path.basename(slope_file))
if 'slope' in product_list or 'slope_8bit' in product_list:
slope_out = dem_to_slope(tif, slope_file, slope_file_bak)
if slope_out is False:
pass
else:
if 'slope_8bit' in product_list:
slope_8bit = io_function.get_name_by_adding_tail(
tif, 'slope8bit')
slope_8bit = os.path.join(arcticDEM_slope_8bit_dir,
os.path.basename(slope_8bit))
slope_to_8bit(slope_file, slope_8bit)
# delete or move the slope file
if 'slope' in product_list:
io_function.move_file_to_dst(slope_file, slope_file_bak)
else:
io_function.delete_file_or_dir(slope_file)
if 'hillshade' in product_list:
hillshapde = io_function.get_name_by_adding_tail(tif, 'hillshade')
hillshapde = os.path.join(arcticDEM_hillshade_dir,
os.path.basename(hillshapde))
dem_to_hillshade(tif, hillshapde)
if 'tpi' in product_list:
tip_8bit = io_function.get_name_by_adding_tail(tif, 'TPI8bit')
tip_8bit = os.path.join(arcticDEM_tpi_8bit_dir,
os.path.basename(tip_8bit))
dem_to_tpi_save_8bit(tif, tip_8bit)
return True
except:
print('failed in process %s' % tif)
return tif
def main():
basic.setlogfile('scp_log.txt')
while True:
# get remote dir
basic.outputlogMessage('get remote folders')
remote_folders = get_remote_folder(remote_dir, folder_pattern)
basic.outputlogMessage("%d remote folders" % len(remote_folders))
folder_list = get_local_folder(local_dir, folder_pattern)
basic.outputlogMessage("%d local folders" % len(folder_list))
folder_name_list = [os.path.basename(item) for item in folder_list]
for idx, r_folders in enumerate(remote_folders):
folder_name = os.path.basename(r_folders)
if folder_name in folder_name_list:
continue
basic.outputlogMessage('copy trained folder in %s' % folder_name)
command_str = 'scp -r ${tesia_host}:%s %s/%s' % (
r_folders, local_dir, folder_name)
print(command_str)
status, result = basic.getstatusoutput(command_str)
if status != 0:
sys.exit(1)
folder_list = get_local_folder(local_dir,
folder_pattern) # update local folder
# reomve incomplete folders
for folder in folder_list:
res_json = os.path.join(folder, 'result.json')
if os.path.isfile(res_json) and os.path.getsize(res_json) > 0:
continue
else:
basic.outputlogMessage('remove incomplete folder: %s' %
os.path.basename(folder))
io_function.delete_file_or_dir(folder)
basic.outputlogMessage('wait five hours')
time.sleep(3600 * 5) # wait five hours
pass
class TestdeeplabTrainclass():
if os.path.isdir('split_images'):
io_function.delete_file_or_dir('split_images')
if os.path.isdir('split_labels'):
io_function.delete_file_or_dir('split_labels')
io_function.mkdir('split_images')
io_function.mkdir('split_labels')
def test_split_a_pair_sub_image_label(self):
### split the training image to many small patch (480*480)
patch_w= 160 # parameters.get_string_parameters(para_file,'train_patch_width')
patch_h= 160 #parameters.get_string_parameters(para_file,'train_patch_height')
# notes
# set overlay as 80, then width or height of patches range from 240 to 320.
# so it will generate more patches than 160 ones
# overlay_x= 80 # parameters.get_string_parameters(para_file,'train_pixel_overlay_x')
# overlay_y= 80 #parameters.get_string_parameters(para_file,'train_pixel_overlay_y')
overlay_x= 160 # parameters.get_string_parameters(para_file,'train_pixel_overlay_x')
overlay_y= 160 #parameters.get_string_parameters(para_file,'train_pixel_overlay_y')
split_image_format= '.png' # parameters.get_string_parameters(para_file,'split_image_format')
trainImg_dir= 'subImages' # parameters.get_string_parameters(para_file,'input_train_dir')
labelImg_dir= 'subLabels' # parameters.get_string_parameters(para_file,'input_label_dir')
if os.path.isdir(trainImg_dir) is False:
raise IOError('%s not in the current folder, please get subImages first'%trainImg_dir)
if os.path.isdir(labelImg_dir) is False:
raise IOError('%s not in the current folder, please get subImages first'%labelImg_dir)
# sub_img_label_txt = 'sub_images_labels_list_test.txt'
sub_img_label_txt = 'sub_images_labels_list_1.txt'
if os.path.isfile(sub_img_label_txt) is False:
raise IOError('%s not in the current folder, please get subImages first' % sub_img_label_txt)
with open(sub_img_label_txt) as txt_obj:
line_list = [name.strip() for name in txt_obj.readlines()]
for line in line_list:
split_sub_images.split_a_pair_sub_image_label(line, patch_w, patch_h, overlay_x, overlay_y, split_image_format)
def main(options, args):
img_path = args[0]
# if nodata is not set, it will try to read from images.
img_nodata = options.nodata
output_shp = options.output_shp
if output_shp is None:
output_shp = os.path.splitext(os.path.basename(img_path))[0] + '_valid.shp'
#
out_mask = options.output_mask
valid_mask = get_valid_pixel_mask(img_path,img_nodata,out_mask=out_mask)
# to shapefile
raster_io.raster2shapefile(valid_mask,out_shp=output_shp,nodata=0) # the nodata for valid_mask is 0
if valid_mask=='tmp.tif':
io_function.delete_file_or_dir(valid_mask)
def predict_remoteSensing_image(para_file, image_path, save_dir,model, config_file, yolo_data, batch_size=1, b_python_api=True):
'''
run prediction of a remote sensing using yolov4
:param image_path:
:param model:
:param config_file:
:param yolo_data:
:param batch_size:
:param b_python_api: if true, use the python API of yolo
:return:
'''
patch_w = parameters.get_digit_parameters(para_file, "inf_patch_width", 'int')
patch_h = parameters.get_digit_parameters(para_file, "inf_patch_height", 'int')
overlay_x = parameters.get_digit_parameters(para_file, "inf_pixel_overlay_x", 'int')
overlay_y = parameters.get_digit_parameters(para_file, "inf_pixel_overlay_y", 'int')
if b_python_api:
# using the python API
predict_rs_image_yolo_poythonAPI(image_path, save_dir, model, config_file, yolo_data,
patch_w, patch_h, overlay_x, overlay_y, batch_size=batch_size)
# for each patch has a json file, may end up with a lot of json files, affect I/O
# try to merge them to one json file.
res_json_files = io_function.get_file_list_by_ext('.json', save_dir, bsub_folder=False)
merge_josn_path = os.path.join(save_dir,'all_patches.json')
merge_patch_json_files_to_one(res_json_files,merge_josn_path)
for f_json in res_json_files:
io_function.delete_file_or_dir(f_json)
else:
# divide image the many patches, then run prediction.
patch_list_txt = split_an_image(para_file,image_path,save_dir,patch_w,patch_h,overlay_x,overlay_y)
if patch_list_txt is None:
return False
result_json = save_dir + '_result.json'
commond_str = 'darknet detector test ' + yolo_data + ' ' + config_file + ' ' + model + ' -dont_show '
commond_str += ' -ext_output -out ' + result_json + ' < ' + patch_list_txt
print(commond_str)
res = os.system(commond_str)
if res !=0:
sys.exit(1)
def dem_to_tpi_save_8bit(input, output):
if os.path.isfile(output):
basic.outputlogMessage('%s exists, skip' % output)
return True
if os.path.isfile(input) is False:
basic.outputlogMessage('Waring, %s does not exist' % input)
return False
# Topographic Position Index
tpi_file = os.path.basename(
io_function.get_name_by_adding_tail(input, 'tpi'))
command_str = 'gdaldem TPI %s %s -of GTiff -co compress=lzw -co tiled=yes -co bigtiff=if_safer -b 1 ' % (
input, tpi_file)
basic.os_system_exit_code(command_str)
# to 8bit
if tpi_to_8bit(tpi_file, output) is True:
io_function.delete_file_or_dir(tpi_file)
return True
def zonal_stats_one_polygon(idx, polygon, image_tiles, img_tile_polygons, stats, nodata=None,range=None,
band = 1,all_touched=True):
overlap_index = vector_gpd.get_poly_index_within_extent(img_tile_polygons, polygon)
image_list = [image_tiles[item] for item in overlap_index]
if len(image_list) == 1:
out_image, out_tran,nodata = raster_io.read_raster_in_polygons_mask(image_list[0], polygon, nodata=nodata,
all_touched=all_touched,bands=band)
elif len(image_list) > 1:
# for the case it overlap more than one raster, need to produce a mosaic
tmp_saved_files = []
for k_img, image_path in enumerate(image_list):
# print(image_path)
tmp_save_path = os.path.splitext(os.path.basename(image_path))[0] + '_subset_poly%d'%idx +'.tif'
_, _,nodata = raster_io.read_raster_in_polygons_mask(image_path, polygon,all_touched=all_touched,nodata=nodata,
bands=band, save_path=tmp_save_path)
tmp_saved_files.append(tmp_save_path)
# mosaic files in tmp_saved_files
save_path = 'raster_for_poly%d.tif'%idx
mosaic_args_list = ['gdal_merge.py', '-o', save_path,'-n',str(nodata),'-a_nodata',str(nodata)]
mosaic_args_list.extend(tmp_saved_files)
if basic.exec_command_args_list_one_file(mosaic_args_list,save_path) is False:
raise IOError('error, obtain a mosaic (%s) failed'%save_path)
# read the raster
out_image, out_nodata = raster_io.read_raster_one_band_np(save_path,band=band)
# remove temporal raster
tmp_saved_files.append(save_path)
for item in tmp_saved_files:
io_function.delete_file_or_dir(item)
else:
basic.outputlogMessage('warning, cannot find raster for %d (start=0) polygon'%idx)
return None
# do calculation
return array_stats(out_image, stats, nodata,range=range)
def main():
reg_tif_dir = 'arcticdem_registration_tifs'
while True:
print(str(datetime.now()),
'start moving or removing files or folders\n\n')
reg_files = io_function.get_file_list_by_pattern(reg_tif_dir, '*')
print('registration file count: %d in %s' %
(len(reg_files), reg_tif_dir))
for file in reg_files:
if check_file_or_dir_is_old(file, time_hour_thr):
print(
'%s is older than %f hours, will be moved to archieved dir'
% (file, time_hour_thr))
io_function.movefiletodir(file,
arcticDEM_reg_tif_dir,
overwrite=True)
SETSM_dir = io_function.get_file_list_by_pattern(
'./', 'SETSM_*2m_v3.0')
print('SETSM folder count: %d in %s' % (len(SETSM_dir), './'))
for folder in SETSM_dir:
if check_file_or_dir_is_old(folder, time_hour_thr):
print('%s is older than %f hours, will be removed' %
(folder, time_hour_thr))
io_function.delete_file_or_dir(folder)
grid_tmp_dir = io_function.get_file_list_by_pattern('./', 'grid*files')
print('Grid tmp folder count: %d in %s' % (len(grid_tmp_dir), './'))
for folder in grid_tmp_dir:
if check_file_or_dir_is_old(folder, time_hour_thr):
print('%s is older than %f hours, will be removed' %
(folder, time_hour_thr))
io_function.delete_file_or_dir(folder)
time.sleep(60) # wait
pass
def get_sub_images_multi_regions(para_file):
print(
"extract sub-images and sub-labels for a given shape file (training polygons)"
)
if os.path.isfile(para_file) is False:
raise IOError('File %s not exists in current folder: %s' %
(para_file, os.getcwd()))
get_subImage_script = os.path.join(code_dir, 'datasets',
'get_subImages.py')
SECONDS = time.time()
# get name of training areas
multi_training_regions = parameters.get_string_list_parameters_None_if_absence(
para_file, 'training_regions')
if multi_training_regions is None or len(multi_training_regions) < 1:
raise ValueError('No training area is set in %s' % para_file)
# multi_training_files = parameters.get_string_parameters_None_if_absence(para_file, 'multi_training_files')
dstnodata = parameters.get_string_parameters(para_file, 'dst_nodata')
buffersize = parameters.get_string_parameters(para_file, 'buffer_size')
rectangle_ext = parameters.get_string_parameters(para_file,
'b_use_rectangle')
process_num = parameters.get_digit_parameters(para_file, 'process_num',
'int')
b_no_label_image = parameters.get_bool_parameters_None_if_absence(
para_file, 'b_no_label_image')
if os.path.isfile('sub_images_labels_list.txt'):
io_function.delete_file_or_dir('sub_images_labels_list.txt')
subImage_dir = parameters.get_string_parameters_None_if_absence(
para_file, 'input_train_dir')
subLabel_dir = parameters.get_string_parameters_None_if_absence(
para_file, 'input_label_dir')
# loop each training regions
for idx, area_ini in enumerate(multi_training_regions):
input_image_dir = parameters.get_directory_None_if_absence(
area_ini, 'input_image_dir')
# it is ok consider a file name as pattern and pass it the following functions to get file list
input_image_or_pattern = parameters.get_string_parameters(
area_ini, 'input_image_or_pattern')
b_sub_images_json = parameters.get_bool_parameters(
area_ini, 'b_sub_images_json')
if b_sub_images_json is True:
# copy sub-images, then covert json files to label images.
object_names = parameters.get_string_list_parameters(
para_file, 'object_names')
get_subImages_json.get_subimages_label_josn(
input_image_dir,
input_image_or_pattern,
subImage_dir,
subLabel_dir,
object_names,
b_no_label_image=b_no_label_image,
process_num=process_num)
pass
else:
all_train_shp = parameters.get_file_path_parameters_None_if_absence(
area_ini, 'training_polygons')
train_shp = parameters.get_string_parameters(
area_ini, 'training_polygons_sub')
# get subImage and subLabel for one training polygons
print(
'extract training data from image folder (%s) and polgyons (%s)'
% (input_image_dir, train_shp))
if b_no_label_image is True:
get_subImage_one_shp(get_subImage_script,
all_train_shp,
buffersize,
dstnodata,
rectangle_ext,
train_shp,
input_image_dir,
file_pattern=input_image_or_pattern,
process_num=process_num)
else:
get_subImage_subLabel_one_shp(
get_subImage_script,
all_train_shp,
buffersize,
dstnodata,
rectangle_ext,
train_shp,
input_image_dir,
file_pattern=input_image_or_pattern,
process_num=process_num)
# check black sub-images or most part of the sub-images is black (nodata)
new_sub_image_label_list = []
delete_sub_image_label_list = []
subImage_dir_delete = subImage_dir + '_delete'
subLabel_dir_delete = subLabel_dir + '_delete'
io_function.mkdir(subImage_dir_delete)
if b_no_label_image is None or b_no_label_image is False:
io_function.mkdir(subLabel_dir_delete)
get_valid_percent_entropy.plot_valid_entropy(subImage_dir)
with open('sub_images_labels_list.txt', 'r') as f_obj:
lines = f_obj.readlines()
for line in lines:
image_path, label_path = line.strip().split(':')
# valid_per = raster_io.get_valid_pixel_percentage(image_path)
valid_per, entropy = raster_io.get_valid_percent_shannon_entropy(
image_path) # base=10
if valid_per > 60 and entropy >= 0.5:
new_sub_image_label_list.append(line)
else:
delete_sub_image_label_list.append(line)
io_function.movefiletodir(image_path, subImage_dir_delete)
if os.path.isfile(label_path):
io_function.movefiletodir(label_path, subLabel_dir_delete)
if len(delete_sub_image_label_list) > 0:
with open('sub_images_labels_list.txt', 'w') as f_obj:
for line in new_sub_image_label_list:
f_obj.writelines(line)
# check weather they have the same subImage and subLabel
if b_no_label_image is None or b_no_label_image is False:
sub_image_list = io_function.get_file_list_by_pattern(
subImage_dir, '*.tif')
sub_label_list = io_function.get_file_list_by_pattern(
subLabel_dir, '*.tif')
if len(sub_image_list) != len(sub_label_list):
raise ValueError(
'the count of subImage (%d) and subLabel (%d) is different' %
(len(sub_image_list), len(sub_label_list)))
# save brief information of sub-images
height_list = []
width_list = []
band_count = 0
dtype = 'unknown'
for line in new_sub_image_label_list:
image_path, label_path = line.strip().split(':')
height, width, band_count, dtype = raster_io.get_height_width_bandnum_dtype(
image_path)
height_list.append(height)
width_list.append(width)
# save info to file, if it exists, it will be overwritten
img_count = len(new_sub_image_label_list)
with open('sub_images_patches_info.txt', 'w') as f_obj:
f_obj.writelines('information of sub-images: \n')
f_obj.writelines('number of sub-images : %d \n' % img_count)
f_obj.writelines('band count : %d \n' % band_count)
f_obj.writelines('data type : %s \n' % dtype)
f_obj.writelines('maximum width and height: %d, %d \n' %
(max(width_list), max(height_list)))
f_obj.writelines('minimum width and height: %d, %d \n' %
(min(width_list), min(height_list)))
f_obj.writelines(
'mean width and height: %.2f, %.2f \n\n' %
(sum(width_list) / img_count, sum(height_list) / img_count))
duration = time.time() - SECONDS
os.system(
'echo "$(date): time cost of getting sub images and labels: %.2f seconds">>time_cost.txt'
% duration)
def segment_subsidence_grey_image(dem_diff_grey_8bit, dem_diff, save_dir,process_num, subsidence_thr_m=-0.5, min_area=40, max_area=100000000,
b_rm_files=False):
'''
segment subsidence areas based on 8bit dem difference
:param dem_diff_grey_8bit:
:param dem_diff:
:param save_dir:
:param process_num:
:param subsidence_thr_m: mean value less than this one consider as subsidence (in meter)
:param min_area: min size in m^2 (defualt is 40 m^2, 10 pixels on ArcticDEM)
:param max_area: min size in m^2 (default is 10km by 10 km)
:return:
'''
io_function.is_file_exist(dem_diff_grey_8bit)
out_pre = os.path.splitext(os.path.basename(dem_diff_grey_8bit))[0]
segment_shp_path = os.path.join(save_dir, out_pre + '.shp')
# check if the final result exist
final_shp_path = io_function.get_name_by_adding_tail(segment_shp_path, 'post')
if os.path.isfile(final_shp_path):
basic.outputlogMessage('Warning, Final results (%s) of subsidence shapefile exists, skip'%final_shp_path)
return True
# get initial polygons
# because the label from segmentation for superpixels are not unique, so we may need to get mean dem diff based on polygons, set org_raster=None
label_path = segment_a_grey_image(dem_diff_grey_8bit,save_dir,process_num, org_raster=None)
if os.path.isfile(segment_shp_path) and vector_gpd.is_field_name_in_shp(segment_shp_path,'demD_mean'):
basic.outputlogMessage('%s exists, skip'%segment_shp_path)
else:
# remove segment_shp_path if it exist, but don't have demD_mean
if os.path.isfile(segment_shp_path):
io_function.delete_shape_file(segment_shp_path)
# remove nodato (it was copy from the input image)
command_str = 'gdal_edit.py -unsetnodata ' + label_path
basic.os_system_exit_code(command_str)
# convert the label to shapefile # remove -8 (to use 4 connectedness.)
command_string = 'gdal_polygonize.py %s -b 1 -f "ESRI Shapefile" %s' % (label_path, segment_shp_path)
res = os.system(command_string)
if res != 0:
sys.exit(1)
# get dem elevation information for each polygon
raster_statistic.zonal_stats_multiRasters(segment_shp_path, dem_diff, tile_min_overlap=tile_min_overlap,
stats=['mean', 'std','count'], prefix='demD',process_num=process_num)
# get DEM diff information for each polygon.
dem_diff_shp = get_dem_subscidence_polygons(segment_shp_path, dem_diff, dem_diff_thread_m=subsidence_thr_m,
min_area=min_area, max_area=max_area, process_num=process_num,b_rm_files=b_rm_files)
if dem_diff_shp is None:
id_str = re.findall('grid\d+', os.path.basename(dem_diff))[0][4:]
if len(id_str) > 1:
grid_id = int(id_str)
save_id_grid_no_subsidence(grid_id)
else:
basic.outputlogMessage('obtain elevation reduction polygons: %s'%dem_diff_shp)
## remove files, only keep the final results.
if b_rm_files:
io_function.delete_file_or_dir(label_path)
IDrange_txt = os.path.splitext(label_path)[0] + '_IDrange.txt'
io_function.delete_file_or_dir(IDrange_txt)
io_function.delete_shape_file(segment_shp_path)
# other intermediate files
other_shp_names = ['merged','surrounding','rmreldemD','rmshapeinfo','rmslope']
for name in other_shp_names:
rm_shp = io_function.get_name_by_adding_tail(segment_shp_path, name)
io_function.delete_shape_file(rm_shp)
return True
def train_kfold_cross_val(multi_training_files_allPolygons,
multi_training_files, k_value, test_num):
##################################################################
# get subset of polygons
training_shp_all = []
with open(multi_training_files_allPolygons, 'r') as f_obj:
training_lines = f_obj.readlines()
for line in training_lines:
line = line.strip()
training_shp_all.append(
line.split(':')[-1]) # the last one is the shape file
for training_shpAll in training_shp_all:
dir = os.path.dirname(training_shpAll)
file_name = os.path.basename(training_shpAll)
file_name_no_ext = os.path.splitext(file_name)[0]
dir_sub = os.path.join(
dir,
'%s_%d-fold_cross_val_t%d' % (file_name_no_ext, k_value, test_num))
if os.path.isdir(dir_sub) is False:
# will save to dir_sub}
io_function.mkdir(dir_sub)
create_shp_subset_polygons(dir_sub, training_shpAll, file_name,
k_value)
else:
# check shape file existence
sub_shps = io_function.get_file_list_by_pattern(dir_sub, '*.shp')
if len(sub_shps) == k_value:
print2file(
log,
"subset of shapefile already exist, skip creating new")
else:
create_shp_subset_polygons(dir_sub, training_shpAll, file_name,
k_value)
##################################################################
# training on k subset
for idx in range(1, k_value + 1):
# remove previous trained model (the setting are the same to exp10)
if os.path.isdir(trained_model_dir):
io_function.delete_file_or_dir(trained_model_dir)
print2file(log, "run training and inference of the %d_th fold" % idx)
# replace shape file path in "multi_training_files"
io_function.copy_file_to_dst(multi_training_files_allPolygons,
multi_training_files,
overwrite=True)
# replace shape file path in multi_training_files
for training_shpAll in training_shp_all:
dir = os.path.dirname(training_shpAll)
file_name_no_ext = os.path.splitext(
os.path.basename(training_shpAll))[0]
dir_sub = os.path.join(
dir, '%s_%d-fold_cross_val_t%d' %
(file_name_no_ext, k_value, test_num))
new_shp_path = os.path.join(
dir_sub, '%s_%dfold_%d.shp' % (file_name_no_ext, k_value, idx))
repalce_string_in_file(multi_training_files, training_shpAll,
new_shp_path)
# modify exe.sh
io_function.copy_file_to_dst('exe_template_kfold.sh',
'exe_qtp.sh',
overwrite=True)
new_line = '%dfold_%d_t%d' % (k_value, idx, test_num)
repalce_string_in_file('exe_qtp.sh', 'x_test_num', new_line)
# check results existence
result_shp = io_function.get_file_list_by_pattern(
'result_backup', '*' + new_line + '*/*.shp')
if len(result_shp) > 0:
print2file(log,
"results of test: %s already exist, skip" % new_line)
else:
# run training
print2file(log, "start: test:%d the %d_th fold" % (test_num, idx))
argslist = ['./exe_qtp.sh']
return_code = basic.exec_command_args_list(argslist)
# exit code is not 0, means something wrong, then quit
if return_code != 0:
sys.exit(return_code)
pass
def remove_no_need_dem_files(b_remove=True):
# if os.path.isfile(grid_complete_list_txt):
# completed_id_list = [int(item) for item in io_function.read_list_from_txt(grid_complete_list_txt)]
# else:
# print(datetime.now(), 'no complete grids')
# return True
#
# if os.path.isfile(grid_excluded_list_txt):
# exclude_id_list = [int(item) for item in io_function.read_list_from_txt(grid_excluded_list_txt)]
# completed_id_list.extend(exclude_id_list)
completed_id_list = get_complete_ignore_grid_ids()
if len(completed_id_list) < 1:
print(datetime.now(), 'no complete grids')
return True
if len(completed_id_list) < 1:
return True
completed_id_set = set(completed_id_list)
# check four folders: arcticDEM_tile_tarball_dir,arcticDEM_tile_reg_tif_dir,tarball_dir,arcticDEM_reg_tif_dir
strip_dem_cover_grids = io_function.read_dict_from_txt_json(
strip_dem_cover_grids_txt)
strip_no_need_list = [
strip for strip in strip_dem_cover_grids.keys()
if set(strip_dem_cover_grids[strip]).issubset(completed_id_set)
]
tile_dem_cover_grids = io_function.read_dict_from_txt_json(
tile_dem_cover_grids_txt)
tile_no_need_list = [
tile for tile in tile_dem_cover_grids.keys()
if set(tile_dem_cover_grids[tile]).issubset(completed_id_set)
]
if b_remove is False:
save_list_no_need_dem_files('no_need_ArcticDEM_strip_names.txt',
strip_no_need_list)
save_list_no_need_dem_files('no_need_ArcticDEM_mosaic_names.txt',
tile_no_need_list)
else:
# remove
basic.outputlogMessage(
'there are %d no need strip DEM, downloaded files will be or have been removed'
% len(strip_no_need_list))
for strip in strip_no_need_list:
file_list = io_function.get_file_list_by_pattern(
tarball_dir, strip + '*')
file_list_2 = io_function.get_file_list_by_pattern(
arcticDEM_reg_tif_dir, strip + '*')
file_list.extend(file_list_2)
if len(file_list) > 0:
for path in file_list:
basic.outputlogMessage('removing %s' % path)
io_function.delete_file_or_dir(path)
basic.outputlogMessage(
'there are %d no need tile DEM, downloaded files will be or have been removed'
% len(tile_no_need_list))
for tile in tile_no_need_list:
file_list = io_function.get_file_list_by_pattern(
arcticDEM_tile_tarball_dir, tile + '*')
file_list_2 = io_function.get_file_list_by_pattern(
arcticDEM_tile_reg_tif_dir, tile + '*')
file_list.extend(file_list_2)
# remove slope file derived ArcticDEM (mosaic)
file_list_3 = io_function.get_file_list_by_pattern(
arcticDEM_tile_slope_dir, tile + '*')
file_list.extend(file_list_3)
if len(file_list) > 0:
for path in file_list:
basic.outputlogMessage('removing %s' % path)
io_function.delete_file_or_dir(path)
def convert_planet_to_rgb_images(tif_path,
save_dir='RGB_images',
sr_min=0,
sr_max=3000,
save_org_dir=None,
sharpen=True,
rgb_nodata=0):
#if multiple processes try to derive the same rgb images, it may have problem.
# save output to 'RGB_images' + processID
if os.path.isdir(save_dir) is False:
io_function.mkdir(save_dir)
if save_org_dir is not None and os.path.isdir(save_org_dir) is False:
io_function.mkdir(save_org_dir)
if save_org_dir is not None:
copied_org_img_path = os.path.join(save_org_dir,
os.path.basename(tif_path))
io_function.copy_file_to_dst(tif_path, copied_org_img_path)
# filename_no_ext
output = os.path.splitext(os.path.basename(tif_path))[0]
if sharpen:
fin_output = os.path.join(save_dir, output + '_8bit_rgb_sharpen.tif')
else:
fin_output = os.path.join(save_dir, output + '_8bit_rgb.tif')
if os.path.isfile(fin_output):
basic.outputlogMessage(
"Skip, because File %s exists in current folder: %s" %
(fin_output, os.getcwd()))
return fin_output
# use fix min and max to make the color be consistent to sentinel-images
src_min = sr_min
src_max = sr_max
dst_min = 1 # 0 is the nodata, so set as 1
dst_max = 255
# gdal_translate -ot Byte -scale ${src_min} ${src_max} ${dst_min} ${dst_max} ${image_path} ${output}_8bit.tif
if 'SR.tif' in tif_path:
cmd_str = 'gdal_translate -ot Byte -scale %d %d %d %d -of VRT %s %s_8bit.tif' % (
src_min, src_max, dst_min, dst_max, tif_path, output)
else:
# gdal_contrast_stretch -percentile-range 0.01 0.99 ${output}.tif ${output}_8bit.tif
cmd_str = 'gdal_contrast_stretch -percentile-range 0.01 0.99 %s %s_8bit.tif' % (
tif_path, output)
status, result = basic.exec_command_string(cmd_str)
if status != 0:
print(result)
sys.exit(status)
# the third band is red, second is green, and first is blue
#gdal_translate -b 3 -b 2 -b 1 ${output}_8bit.tif ${output}_8bit_rgb.tif
cmd_str = 'gdal_translate -b 3 -b 2 -b 1 -of VRT %s_8bit.tif %s_8bit_rgb.tif' % (
output, output)
status, result = basic.exec_command_string(cmd_str)
if status != 0:
print(result)
sys.exit(status)
# python ${code_dir}/planetScripts/prePlanetImage.py ${output}_8bit_rgb.tif ${fin_output}
if sharpen:
cmd_str = 'python %s %s_8bit_rgb.tif %s' % (prePlanetImage, output,
fin_output)
else:
# convert from VRT format to tif format
cmd_str = 'gdal_translate -of GTiff %s_8bit_rgb.tif %s' % (output,
fin_output)
status, result = basic.exec_command_string(cmd_str)
if status != 0:
print(result)
sys.exit(status)
# set nodata
# gdal_edit.py -a_nodata 0 ${fin_output}
cmd_str = 'gdal_edit.py -a_nodata %d %s' % (rgb_nodata, fin_output)
status, result = basic.exec_command_string(cmd_str)
if status != 0:
print(result)
sys.exit(status)
io_function.delete_file_or_dir('%s_8bit.tif' % output)
io_function.delete_file_or_dir('%s_8bit_rgb.tif' % output)
return fin_output
def get_sub_image(idx,selected_polygon, image_tile_list, image_tile_bounds, save_path, dstnodata, brectangle ):
'''
get a mask image based on a selected polygon, it may cross two image tiles
:param selected_polygon: selected polygons
:param image_tile_list: image list
:param image_tile_bounds: the boxes of images in the list
:param save_path: save path
:param brectangle: if brectangle is True, crop the raster using bounds, else, use the polygon
:return: True is successful, False otherwise
'''
# find the images which the center polygon overlap (one or two images)
img_index = get_overlap_image_index([selected_polygon], image_tile_bounds)
if len(img_index) < 1:
basic.outputlogMessage(
'Warning, %dth polygon do not overlap any image tile, please check ' #and its buffer area
'(1) the shape file and raster have the same projection'
'and (2) this polygon is in the extent of images' % idx)
return False
image_list = [image_tile_list[item] for item in img_index]
# check it cross two or more images
if len(image_list) == 1:
# for the case that the polygon only overlap one raster
with rasterio.open(image_list[0]) as src:
polygon_json = mapping(selected_polygon)
# not necessary
# overlap_win = rasterio.features.geometry_window(src, [polygon_json], pad_x=0, pad_y=0, north_up=True, rotated=False,
# pixel_precision=3)
if brectangle:
# polygon_box = selected_polygon.bounds
polygon_json = mapping(selected_polygon.envelope) #shapely.geometry.Polygon([polygon_box])
# crop image and saved to disk
out_image, out_transform = mask(src, [polygon_json], nodata=dstnodata, all_touched=True, crop=True)
# test: save it to disk
out_meta = src.meta.copy()
out_meta.update({"driver": "GTiff",
"height": out_image.shape[1],
"width": out_image.shape[2],
"transform": out_transform}) # note that, the saved image have a small offset compared to the original ones (~0.5 pixel)
with rasterio.open(save_path, "w", **out_meta) as dest:
dest.write(out_image)
pass
else:
# for the case it overlap more than one raster, need to produce a mosaic
tmp_saved_files = []
for k_img,image_path in enumerate(image_list):
with rasterio.open(image_path) as src:
polygon_json = mapping(selected_polygon)
if brectangle:
# polygon_box = selected_polygon.bounds
polygon_json = mapping(selected_polygon.envelope) # shapely.geometry.Polygon([polygon_box])
# crop image and saved to disk
out_image, out_transform = mask(src, [polygon_json], nodata=dstnodata, all_touched=True, crop=True)
tmp_saved = os.path.splitext(save_path)[0] +'_%d'%k_img + os.path.splitext(save_path)[1]
# test: save it to disk
out_meta = src.meta.copy()
out_meta.update({"driver": "GTiff",
"height": out_image.shape[1],
"width": out_image.shape[2],
"transform": out_transform}) # note that, the saved image have a small offset compared to the original ones (~0.5 pixel)
with rasterio.open(tmp_saved, "w", **out_meta) as dest:
dest.write(out_image)
tmp_saved_files.append(tmp_saved)
# mosaic files in tmp_saved_files
mosaic_args_list = ['gdal_merge.py', '-o', save_path,'-n',str(dstnodata),'-a_nodata',str(dstnodata)]
mosaic_args_list.extend(tmp_saved_files)
if basic.exec_command_args_list_one_file(mosaic_args_list,save_path) is False:
raise IOError('error, obtain a mosaic (%s) failed'%save_path)
# # for test
# if idx==13:
# raise ValueError('for test')
# remove the tmp files
for tmp_file in tmp_saved_files:
io_function.delete_file_or_dir(tmp_file)
# if it will output a very large image (10000 by 10000 pixels), then raise a error
return True
basic.setlogfile('parallel_predict_rtsLog.txt')
predict_script = HOME + '/codes/PycharmProjects/Landuse_DL/sentinelScripts/predict_rts_oneImg.sh'
import GPUtil
import datetime
from multiprocessing import Process
machine_name = os.uname()[1]
start_time = datetime.datetime.now()
# remove previous results
outdir = 'multi_inf_results'
if os.path.isdir(outdir) and 'chpc' not in machine_name: # on ITSC service, need to manually deleted previous results
io_function.delete_file_or_dir(outdir)
os.system('mkdir -p '+ outdir)
# get GPU information on the machine
# https://github.com/anderskm/gputil
# deviceIDs = GPUtil.getAvailable(order = 'first', limit = 100, maxLoad = 0.5,
# maxMemory = 0.5, includeNan=False, excludeID=[], excludeUUID=[])
# print('available GPUs:',deviceIDs)
with open('inf_image_list.txt','r') as inf_obj:
inf_img_list = [name.strip() for name in inf_obj.readlines()]
img_count = len(inf_img_list)
if img_count < 1:
def remove_previous_data(para_file):
print("remove previous data or results to run again")
if os.path.isfile(para_file) is False:
raise IOError('File %s does not exists in current folder: %s' %
(para_file, os.getcwd()))
subImage_dir = parameters.get_string_parameters_None_if_absence(
para_file, 'input_train_dir')
subLabel_dir = parameters.get_string_parameters_None_if_absence(
para_file, 'input_label_dir')
if os.path.isdir(subImage_dir):
io_function.delete_file_or_dir(subImage_dir)
print('remove %s' % subImage_dir)
if os.path.isdir(subLabel_dir):
io_function.delete_file_or_dir(subLabel_dir)
print('remove %s' % subLabel_dir)
subImage_dir_delete = subImage_dir + '_delete'
subLabel_dir_delete = subLabel_dir + '_delete'
if os.path.isdir(subImage_dir_delete):
io_function.delete_file_or_dir(subImage_dir_delete)
print('remove %s' % subImage_dir_delete)
if os.path.isdir(subLabel_dir_delete):
io_function.delete_file_or_dir(subLabel_dir_delete)
print('remove %s ' % subLabel_dir_delete)
if os.path.isdir('split_images'):
io_function.delete_file_or_dir('split_images')
print('remove %s ' % 'split_images')
if os.path.isdir('split_labels'):
io_function.delete_file_or_dir('split_labels')
print('remove %s ' % 'split_labels')
images_including_aug = os.path.join('list', 'images_including_aug.txt')
if os.path.isfile(images_including_aug):
io_function.delete_file_or_dir(images_including_aug)
print('remove %s ' % 'list/images_including_aug.txt')
if os.path.isdir('tfrecord'):
io_function.delete_file_or_dir('tfrecord')
print('remove %s ' % 'tfrecord')
def split_sub_images(para_file):
print("split sub-images and sub-labels")
if os.path.isfile(para_file) is False:
raise IOError('File %s not exists in current folder: %s'%(para_file, os.getcwd()))
SECONDS = time.time()
if os.path.isdir('split_images'):
io_function.delete_file_or_dir('split_images')
if os.path.isdir('split_labels'):
io_function.delete_file_or_dir('split_labels')
io_function.mkdir('split_images')
### split the training image to many small patch (480*480)
patch_w=parameters.get_string_parameters(para_file,'train_patch_width')
patch_h=parameters.get_string_parameters(para_file,'train_patch_height')
overlay_x=parameters.get_string_parameters(para_file,'train_pixel_overlay_x')
overlay_y=parameters.get_string_parameters(para_file,'train_pixel_overlay_y')
split_image_format=parameters.get_string_parameters(para_file,'split_image_format')
trainImg_dir=parameters.get_string_parameters(para_file,'input_train_dir')
labelImg_dir=parameters.get_string_parameters(para_file,'input_label_dir')
proc_num = parameters.get_digit_parameters(para_file,'process_num','int')
if os.path.isdir(trainImg_dir) is False:
raise IOError('%s not in the current folder, please get subImages first'%trainImg_dir)
if os.path.isdir(labelImg_dir) is False:
print('warning, %s not in the current folder'%labelImg_dir)
else:
io_function.mkdir('split_labels')
sub_img_label_txt = 'sub_images_labels_list.txt'
if os.path.isfile(sub_img_label_txt) is False:
raise IOError('%s not in the current folder, please get subImages first' % sub_img_label_txt)
with open(sub_img_label_txt) as txt_obj:
line_list = [name.strip() for name in txt_obj.readlines()]
# for line in line_list:
# sub_image, sub_label = line.split(':')
#
# # split sub image
# split_to_patches(sub_image, 'split_images', patch_w, patch_h, overlay, overlay, split_image_format)
#
# # split sub label (change the file name to be the same as sub_image name)
# pre_name = os.path.splitext(os.path.basename(sub_image))[0]
# split_to_patches(sub_label, 'split_labels', patch_w, patch_h, overlay, overlay, split_image_format, file_pre_name=pre_name)
parameters_list = [(line, patch_w, patch_h, overlay_x, overlay_y, split_image_format) for line in line_list]
theadPool = Pool(proc_num) # multi processes
results = theadPool.starmap(split_a_pair_sub_image_label, parameters_list) # need python3
# output trainval.txt and val.txt file
files_list = io_function.get_file_list_by_ext(split_image_format, 'split_images',bsub_folder=False)
io_function.mkdir('list')
trainval = os.path.join('list','trainval.txt')
val = os.path.join('list','val.txt')
with open(trainval,'w') as w_obj:
for file_name in files_list:
w_obj.writelines(os.path.splitext(os.path.basename(file_name))[0] + '\n')
io_function.copy_file_to_dst(trainval,val,overwrite=True)
split_train_val.get_image_with_height_list(trainval, split_image_format, info_type='(no data augmentation)')
duration= time.time() - SECONDS
os.system('echo "$(date): time cost of splitting sub images and labels: %.2f seconds">>time_cost.txt'%duration)
if os.path.isfile(para_file) is False:
raise IOError('File %s not exists in current folder: %s'%(para_file, os.getcwd()))
deeplabRS=os.path.expanduser('~/codes/PycharmProjects/DeeplabforRS')
sys.path.insert(0, deeplabRS)
import parameters
import basic_src.io_function as io_function
eo_dir=os.path.expanduser("~/codes/PycharmProjects/Landuse_DL")
# split_image_script=os.path.join(eo_dir,'grss_data_fusion', 'split_image.py')
sys.path.insert(0, eo_dir)
import grss_data_fusion.split_image as split_image
if os.path.isdir('split_images'):
io_function.delete_file_or_dir('split_images')
if os.path.isdir('split_labels'):
io_function.delete_file_or_dir('split_labels')
io_function.mkdir('split_images')
io_function.mkdir('split_labels')
### split the training image to many small patch (480*480)
patch_w=parameters.get_string_parameters(para_file,'train_patch_width')
patch_h=parameters.get_string_parameters(para_file,'train_patch_height')
overlay=parameters.get_string_parameters(para_file,'train_pixel_overlay_x')
split_image_format=parameters.get_string_parameters(para_file,'split_image_format')
trainImg_dir=parameters.get_string_parameters(para_file,'input_train_dir')
labelImg_dir=parameters.get_string_parameters(para_file,'input_label_dir')
def produce_products_dem_subsidence(b_remove_job_folder=True):
# run segment jobs in local workstations.
task = 'segment'
max_list_count = 20
if 'donostia' in machine_name:
max_list_count = 8 # donostia is really slow, assigined less task to it
job_list_pre = 'job_seg_dem_diff_list_'
if os.path.isdir(dem_common.process_log_dir) is False:
io_function.mkdir(dem_common.process_log_dir)
dem_list_txt = os.path.join(dem_common.process_log_dir,
job_list_pre + machine_name + '.txt')
# when submit segment of dem_diff, no need ext_shp
ext_shp = "monitor_fail_segment_jobs"
while True:
dem_diff_list = get_dem_diff_list_to_seg()
# only handle file are old enough
dem_diff_list = get_dem_diff_old_enough(dem_diff_list)
dem_diff_ids = [get_grid_id_from_path(item) for item in dem_diff_list]
print('dem_diff_ids')
print(dem_diff_ids)
# remove dem_diff already assigined for other machine
if os.path.isfile(dem_list_txt):
io_function.delete_file_or_dir(dem_list_txt)
dem_diff_assigned = read_dem_diff_assigned_to_other_machine(
job_list_pre)
assigned_ids = [
get_grid_id_from_path(item) for item in dem_diff_assigned
]
print('assigned_ids')
print(assigned_ids)
keep_idx = [
idx for idx, id in enumerate(dem_diff_ids)
if id not in assigned_ids
]
dem_diff_list = [dem_diff_list[item] for item in keep_idx]
if len(dem_diff_list) < 1:
print(
datetime.now(),
'there is no DEM_diff for %s to seg, wait 10 minutes' %
machine_name)
time.sleep(600) # wait 10 min
continue
# save some of them to txt, for "parallel_processing_curc.py"
dem_diff_list = dem_diff_list[:max_list_count]
save_ids = [get_grid_id_from_path(item) for item in dem_diff_list]
print('save_ids')
print(save_ids)
io_function.save_list_to_txt(dem_list_txt, dem_diff_list)
res = os.system('./run.sh %s %s' % (ext_shp, task))
if res != 0:
sys.exit(1)
copy_segment_result_to_curc(save_ids)
if b_remove_job_folder:
os.system('rm -r seg_dem_diff_*')
io_function.delete_file_or_dir(dem_list_txt)
def process_dem_tarball(tar_list,
work_dir,
save_dir,
remove_inter_data=False,
rm_tarball=False,
apply_registration=False):
'''
process dem tarball, unpack, apply registration
:param tar_list:
:param work_dir:
:param save_dir:
:param remove_inter_data:
:param apply_registration:
:return:
'''
if os.path.isdir(save_dir) is False:
io_function.mkdir(save_dir)
if os.path.isfile('no_registration_strips.txt'):
no_registration_strips = io_function.read_list_from_txt(
'no_registration_strips.txt')
else:
no_registration_strips = []
out_dir_list = []
out_reg_tifs = []
for idx, targz in enumerate(tar_list):
tar_base = os.path.basename(targz)[:-7]
# check if no registraion information for this tarball
if './' + tar_base in no_registration_strips:
continue
if check_files_existence(save_dir, tar_base):
print("registration result of %s already exists, skip" % targz)
continue
# check free disk space
free_GB = io_function.get_free_disk_space_GB(work_dir)
total_wait_time = 0
while free_GB < 50 and total_wait_time < 60 * 60 * 12:
basic.outputlogMessage(
' The free disk space (%.4f) is less than 50 GB, wait 60 seconds'
% free_GB)
time.sleep(60)
total_wait_time += 60
free_GB = io_function.get_free_disk_space_GB(work_dir)
out_tif, out_dir = process_dem_one_tarball(targz, work_dir,
apply_registration)
if out_tif is None:
if rm_tarball:
io_function.delete_file_or_dir(targz)
continue
out_dir_list.append(out_dir)
# move file to save_dir
io_function.movefiletodir(out_tif, save_dir)
dem_log = os.path.join(out_dir, tar_base + '_dem.log')
if os.path.isfile(dem_log):
io_function.movefiletodir(dem_log, save_dir)
matchtag_tif = os.path.join(out_dir, tar_base + '_matchtag_reg.tif')
if os.path.isfile(matchtag_tif):
io_function.movefiletodir(matchtag_tif, save_dir)
matchtag_tif_log = os.path.join(out_dir, tar_base + '_matchtag.log')
if os.path.isfile(matchtag_tif_log):
io_function.movefiletodir(matchtag_tif_log, save_dir)
out_reg_tifs.append(os.path.join(save_dir, os.path.basename(out_tif)))
# remove folder
if remove_inter_data:
io_function.delete_file_or_dir(out_dir)
if rm_tarball:
io_function.delete_file_or_dir(targz)
# remove folder (in case failed in the previous step)
if remove_inter_data:
for dir in out_dir_list:
if os.path.isdir(dir):
io_function.delete_file_or_dir(dir)
return out_reg_tifs
def proc_ArcticDEM_strip_one_grid_polygon(tar_dir,
dem_polygons,
dem_urls,
o_res,
save_dir,
inter_format,
b_mosaic_id,
b_mosaic_date,
b_rm_inter,
b_dem_diff,
extent_poly,
extent_id,
keep_dem_percent,
process_num,
pre_name,
resample_method='average',
same_extent=False):
if check_dem_diff_results(save_dir, pre_name, extent_id):
return True
# get file in the tar_dir
tar_list = get_tar_list_sub(tar_dir, dem_polygons, dem_urls, extent_poly)
if len(tar_list) < 1:
basic.outputlogMessage(
'Warning, no tarball for the extent (id=%d) in %s' %
(extent_id, tar_dir))
return False
# unpackage and crop to extent
b_apply_registration = True
dem_tif_list, tar_folders = process_dem_tarball(
tar_list,
save_dir,
inter_format,
o_res,
extent_poly=extent_poly,
poly_id=extent_id,
same_extent=same_extent,
process_num=process_num,
apply_registration=b_apply_registration)
if len(dem_tif_list) < 1:
raise ValueError('No DEM extracted from tarballs')
proc_dem_mosaic_diff(dem_tif_list,
save_dir,
extent_id,
extent_poly,
b_mosaic_id,
b_mosaic_date,
process_num,
keep_dem_percent,
o_res,
b_dem_diff,
pre_name,
b_rm_inter,
resample_method=resample_method)
# remove intermediate files
if b_rm_inter:
basic.outputlogMessage('remove intermediate files')
for folder in tar_folders:
io_function.delete_file_or_dir(folder)
eo_dir = os.path.expanduser("~/codes/PycharmProjects/Landuse_DL")
get_subImage_script = os.path.join(eo_dir, 'sentinelScripts',
'get_subImages.py')
multi_training_files = parameters.get_string_parameters_None_if_absence(
para_file, 'multi_training_files')
input_image_dir = parameters.get_string_parameters(para_file,
'input_image_dir')
dstnodata = parameters.get_string_parameters(para_file, 'dst_nodata')
buffersize = parameters.get_string_parameters(para_file, 'buffer_size')
rectangle_ext = parameters.get_string_parameters(para_file, 'b_use_rectangle')
if os.path.isfile('sub_images_labels_list.txt'):
io_function.delete_file_or_dir('sub_images_labels_list.txt')
def get_subImage_subLabel_one_shp(all_train_shp,
buffersize,
dstnodata,
rectangle_ext,
train_shp,
input_image_dir,
file_pattern=None):
if file_pattern is None:
file_pattern = '*.tif'
command_string = get_subImage_script + ' -f ' + all_train_shp + ' -b ' + str(buffersize) + ' -e ' + file_pattern + \
' -o ' + os.getcwd() + ' -n ' + str(dstnodata) + ' ' + rectangle_ext + ' ' + train_shp + ' '+ input_image_dir
def create_moasic_of_each_grid_polygon(id,
polygon,
polygon_latlon,
out_res,
cloud_cover_thr,
geojson_list,
save_dir,
new_prj_wkt=None,
new_prj_proj4=None,
sr_min=0,
sr_max=3000,
to_rgb=True,
nodata=0,
save_org_dir=None,
resampling_method='min'):
'''
create mosaic for Planet images within a grid
:param polygon:
:param polygon_latlon:
:param out_res:
:param cloud_cover_thr:
:param geojson_list:
:param save_dir:
:param new_prj_wkt:
:param new_prj_proj4:
:param sr_min:
:param sr_max:
:param to_rgb:
:param nodata:
:return:
'''
time0 = time.time()
file_name = os.path.basename(save_dir)
fin_out = os.path.join(save_dir, file_name + '_sub_%d.tif' % id)
if os.path.isfile(fin_out):
basic.outputlogMessage(
'Warning, skip %s because it already exists, remove it if want to regenerate it'
% fin_out)
return fin_out
# get image list and cloud cover
planet_img_list, cloud_covers = get_Planet_SR_image_list_overlap_a_polygon(
polygon_latlon, geojson_list, cloud_cover_thr)
if len(planet_img_list) < 1:
basic.outputlogMessage('warning, no images within %d grid' % id)
return False
io_function.mkdir(save_dir)
print('images and their cloud cover for %dth grid:' % id)
for img, cloud_cover in zip(planet_img_list, cloud_covers):
print(img, cloud_cover)
proc_id = multiprocessing.current_process().pid
# convert to RGB images (for Planet)
rgb_image_list = []
rgb_dir = 'RGB_images_' + str(proc_id)
if to_rgb:
for tif_path in planet_img_list:
rgb_img = convert_planet_to_rgb_images(tif_path,
save_dir=rgb_dir,
save_org_dir=save_org_dir,
sr_min=sr_min,
sr_max=sr_max)
rgb_image_list.append(rgb_img)
if len(rgb_image_list) > 0:
planet_img_list = rgb_image_list
reproj_img_list = []
# reproject if necessary
reproj_dir = 'planet_images_reproj_' + str(proc_id)
if new_prj_wkt != None and new_prj_proj4 != None:
for tif_path in planet_img_list:
prj_out = reproject_planet_image(tif_path,
new_prj_wkt,
new_prj_proj4,
save_dir=reproj_dir)
# replace the image
if prj_out is not False and os.path.isfile(prj_out):
reproj_img_list.append(prj_out)
else:
# if not reproject, then append the original image.
reproj_img_list.append(tif_path)
if len(reproj_img_list) > 0:
planet_img_list = reproj_img_list
# create mosaic using gdal_merge.py
# because in gdal_merge.py, a later image will replace one, so we put image with largest cloud cover first
out = os.path.join(save_dir, file_name + '_sub_%d_tmp.tif' % id)
if os.path.isfile(out):
io_function.delete_file_or_dir(out)
# reverse=True to make it in descending order
img_cloud_list = [
(img_path, cloud)
for cloud, img_path in sorted(zip(cloud_covers, planet_img_list),
key=lambda pair: pair[0],
reverse=True)
]
# for checking
print('Image and its cloud after sorting:')
for (img_path, cloud) in img_cloud_list:
print(img_path, cloud)
tifs = [img_path for (img_path, cloud) in img_cloud_list]
tifs_str = ' '.join(tifs)
# cmd_str = 'gdal_merge.py -o %s -n %d -init %d -ps %d %d %s'%(out,nodata,nodata,out_res,out_res,tifs_str)
cmd_str = 'gdalbuildvrt -resolution user -tr %d %d -srcnodata %d -vrtnodata %d %s %s' % (
out_res, out_res, nodata, nodata, out, tifs_str)
status, result = basic.exec_command_string(cmd_str)
if status != 0:
print(result)
sys.exit(status)
# # # polygon.exterior.coords
# minx, miny, maxx, maxy = polygon.bounds # (minx, miny, maxx, maxy)
# print(minx, miny, maxx, maxy)
# results = RSImageProcess.subset_image_projwin(fin_out,out,minx, maxy, maxx, miny, xres=out_res,yres=out_res)
# print(results)
results = RSImageProcess.subset_image_by_polygon_box_image_min(
fin_out,
out,
polygon,
xres=out_res,
yres=out_res,
compress='lzw',
tiled='yes',
bigtiff='if_safer')
if results is False:
basic.outputlogMessage(
'Warning, Crop %s failed, keep the one without cropping' % out)
io_function.move_file_to_dst(out, fin_out)
else:
io_function.delete_file_or_dir(out)
# ## mosaic and crop at the same time together
# minx, miny, maxx, maxy = polygon.bounds # (minx, miny, maxx, maxy)
# print(minx, miny, maxx, maxy)
# results = RSImageProcess.mosaic_crop_images_gdalwarp(tifs,fin_out,src_nodata=nodata,min_x=minx,min_y=miny,max_x=maxx,max_y=maxy,
# xres=out_res,yres=out_res,resampling_method=resampling_method)
#
# if results is False:
# basic.outputlogMessage('Warning, create %s failed' % fin_out)
# return False
# sys.exit(0)
cost_time_sec = time.time() - time0
basic.outputlogMessage(
'finished creating %s cost %.2f seconds (%.2f minutes)' %
(fin_out, cost_time_sec, cost_time_sec / 60))
return fin_out
