def build_TFrecord_tf1x(para_file):
if os.path.isdir('tfrecord'):
basic.outputlogMessage(
'warning, tfrecord exists, skip build_TFrecord_tf1x')
return
command_string = os.path.join(eo_dir, 'workflow',
'build_TFrecord_tf1x.py') + ' ' + para_file
basic.os_system_exit_code(command_string)
def inference(para_file):
outdir = parameters.get_directory(para_file, 'inf_output_dir')
# don't remove it automatically
# if os.path.isdir(outdir):
# io_function.delete_file_or_dir(outdir)
# the script will check whether each image has been predicted
command_string = os.path.join(eo_dir, 'workflow',
'parallel_prediction.py') + ' ' + para_file
basic.os_system_exit_code(command_string)
def combine_hillshade_slope_tip(hillshade, slope, tpi, save_path):
if os.path.isfile(save_path):
print('%s exists, skip' % save_path)
return
# combine them.
command_str = 'gdal_merge.py -o %s -separate -a_nodata 255 -of GTiff -co compress=lzw -co tiled=yes -co bigtiff=if_safer %s %s %s' % (
save_path, hillshade, slope, tpi)
basic.os_system_exit_code(command_str)
def tifs_to_png(image_dir):
tif_list = io_function.get_file_list_by_pattern(image_dir, '*/*.tif')
for idx, tif in enumerate(tif_list):
print('tif to png: %d/%d tif' % (idx + 1, len(tif_list)))
basename = io_function.get_name_no_ext(tif)
save_path = os.path.join(image_dir, basename + '.png')
if os.path.isfile(save_path):
print('%s exists, skip' % save_path)
continue
command_str = "gdal_translate -of PNG %s %s" % (tif, save_path)
basic.os_system_exit_code(command_str)
def arcticDEM_strip_registration(strip_dir):
command_str = 'python ' + reg_py + ' ' + strip_dir
basic.os_system_exit_code(command_str)
end = '_dem_reg.tif'
reg_tif = os.path.join(strip_dir, os.path.basename(strip_dir) + end)
if os.path.isfile(reg_tif):
return reg_tif
else:
# with open('no_registration_strips.txt','a') as f_obj:
# f_obj.writelines('%s\n'%strip_dir)
f_obj = open('no_registration_strips.txt', 'a')
f_obj.writelines('%s\n' % strip_dir)
f_obj.close(
) # close file manually, to avoid dead lock when using multiple threads
return None
def tpi_to_8bit(input, output):
dst_nodat = 255
hist_max_percent = 0.98
hist_min_percent = 0.02
MIN_MAX_VALUE = '-1 1' # tpi range from -1 to 1
command_str = py8bit + ' ' + input + ' ' + output
command_str += ' -n ' + str(dst_nodat)
command_str += ' -u ' + str(hist_max_percent) + ' -l ' + str(
hist_min_percent)
command_str += ' -m ' + MIN_MAX_VALUE
# print(command_str)
basic.os_system_exit_code(command_str)
return True
def dem_to_hillshade(input, output):
if os.path.isfile(output):
basic.outputlogMessage('%s exists, skip' % output)
return True
if os.path.isfile(input) is False:
basic.outputlogMessage('Waring, %s does not exist' % input)
return False
# use the default setting in QGIS
# gdaldem hillshade ${dem} ${hillshade} -of GTiff -b 1 -z 1.0 -s 1.0 -az 315.0 -alt 45.0
command_str = 'gdaldem hillshade %s %s -of GTiff -co compress=lzw -co tiled=yes -co bigtiff=if_safer -b 1 -z 1.0 -s 1.0 -az 315.0 -alt 45.0' % (
input, output)
basic.os_system_exit_code(command_str)
return True
def get_tifs_bounding_boxes(image_dir):
tif_list = io_function.get_file_list_by_pattern(image_dir, '*/*.tif')
for idx, tif in enumerate(tif_list):
print('get bounding box: %d/%d tif' % (idx + 1, len(tif_list)))
basename = io_function.get_name_no_ext(tif)
save_path = os.path.join(image_dir, basename + '_bound.geojson')
if os.path.isfile(save_path):
print('%s exists, skip' % save_path)
continue
command_str = imgExt + " %s -o tmp.gpkg" % tif
basic.os_system_exit_code(command_str)
command_str = "ogr2ogr -f GeoJSON -t_srs EPSG:3413 %s tmp.gpkg" % save_path # note: projection is EPSG:3413
basic.os_system_exit_code(command_str)
io_function.delete_file_or_dir('tmp.gpkg')
def dem_to_slope(input, output, slope_file_bak):
if os.path.isfile(output):
basic.outputlogMessage('%s exists, skip' % output)
return output
if os.path.isfile(slope_file_bak):
basic.outputlogMessage('%s exists, skip' % slope_file_bak)
return slope_file_bak
if os.path.isfile(input) is False:
basic.outputlogMessage('Waring, %s does not exist' % input)
return False
# # use the default setting in QGIS
command_str = 'gdaldem slope %s %s -of GTiff -co compress=lzw -co tiled=yes -co bigtiff=if_safer -b 1 -s 1.0' % (
input, output)
basic.os_system_exit_code(command_str)
return output
def dem_to_tpi_save_8bit(input, output):
if os.path.isfile(output):
basic.outputlogMessage('%s exists, skip' % output)
return True
if os.path.isfile(input) is False:
basic.outputlogMessage('Waring, %s does not exist' % input)
return False
# Topographic Position Index
tpi_file = os.path.basename(
io_function.get_name_by_adding_tail(input, 'tpi'))
command_str = 'gdaldem TPI %s %s -of GTiff -co compress=lzw -co tiled=yes -co bigtiff=if_safer -b 1 ' % (
input, tpi_file)
basic.os_system_exit_code(command_str)
# to 8bit
if tpi_to_8bit(tpi_file, output) is True:
io_function.delete_file_or_dir(tpi_file)
return True
def training(para_file, gpu_num):
# if gpus is not None:
# gpu_num = len(gpus)
# if 'CUDA_VISIBLE_DEVICES' in os.environ.keys():
# gpus_str = [ item.strip() for item in os.environ['CUDA_VISIBLE_DEVICES'].split(',')]
# if len(gpus_str) == 1 and gpus_str[0] == '':
# gpu_num = 1 # no gpu, set a 1
# else:
# gpu_num = len(gpus)
# else:
# # find available gpus
# deviceIDs = GPUtil.getAvailable(order='first', limit=100, maxLoad=0.5,
# maxMemory=0.5, includeNan=False, excludeID=[], excludeUUID=[])
# gpu_num = len(deviceIDs)
# the script can check the trained iteration and decide to train or not
# command_string = os.path.join(eo_dir, 'workflow', 'deeplab_train.py') + ' ' + para_file + ' ' + str(gpu_num)
# basic.os_system_exit_code(command_string)
# import deeplab_train
# deeplab_train.deeplab_train_main(para_file, gpu_num)
# command_string = os.path.join(eo_dir, 'yolov4_dir', 'predict_yolo.py') + ' ' + para_file
command_string = 'darknet detector train data/obj.data yolov4_obj.cfg yolov4.conv.137 -gpus 0,1 -dont_show -map'
basic.os_system_exit_code(command_string)
def export_model(para_file):
# the script can check the trained model and decide to export or not
command_string = os.path.join(eo_dir, 'workflow',
'export_graph.py') + ' ' + para_file
basic.os_system_exit_code(command_string)
def segment_subsidence_grey_image(dem_diff_grey_8bit, dem_diff, save_dir,process_num, subsidence_thr_m=-0.5, min_area=40, max_area=100000000,
b_rm_files=False):
'''
segment subsidence areas based on 8bit dem difference
:param dem_diff_grey_8bit:
:param dem_diff:
:param save_dir:
:param process_num:
:param subsidence_thr_m: mean value less than this one consider as subsidence (in meter)
:param min_area: min size in m^2 (defualt is 40 m^2, 10 pixels on ArcticDEM)
:param max_area: min size in m^2 (default is 10km by 10 km)
:return:
'''
io_function.is_file_exist(dem_diff_grey_8bit)
out_pre = os.path.splitext(os.path.basename(dem_diff_grey_8bit))[0]
segment_shp_path = os.path.join(save_dir, out_pre + '.shp')
# check if the final result exist
final_shp_path = io_function.get_name_by_adding_tail(segment_shp_path, 'post')
if os.path.isfile(final_shp_path):
basic.outputlogMessage('Warning, Final results (%s) of subsidence shapefile exists, skip'%final_shp_path)
return True
# get initial polygons
# because the label from segmentation for superpixels are not unique, so we may need to get mean dem diff based on polygons, set org_raster=None
label_path = segment_a_grey_image(dem_diff_grey_8bit,save_dir,process_num, org_raster=None)
if os.path.isfile(segment_shp_path) and vector_gpd.is_field_name_in_shp(segment_shp_path,'demD_mean'):
basic.outputlogMessage('%s exists, skip'%segment_shp_path)
else:
# remove segment_shp_path if it exist, but don't have demD_mean
if os.path.isfile(segment_shp_path):
io_function.delete_shape_file(segment_shp_path)
# remove nodato (it was copy from the input image)
command_str = 'gdal_edit.py -unsetnodata ' + label_path
basic.os_system_exit_code(command_str)
# convert the label to shapefile # remove -8 (to use 4 connectedness.)
command_string = 'gdal_polygonize.py %s -b 1 -f "ESRI Shapefile" %s' % (label_path, segment_shp_path)
res = os.system(command_string)
if res != 0:
sys.exit(1)
# get dem elevation information for each polygon
raster_statistic.zonal_stats_multiRasters(segment_shp_path, dem_diff, tile_min_overlap=tile_min_overlap,
stats=['mean', 'std','count'], prefix='demD',process_num=process_num)
# get DEM diff information for each polygon.
dem_diff_shp = get_dem_subscidence_polygons(segment_shp_path, dem_diff, dem_diff_thread_m=subsidence_thr_m,
min_area=min_area, max_area=max_area, process_num=process_num,b_rm_files=b_rm_files)
if dem_diff_shp is None:
id_str = re.findall('grid\d+', os.path.basename(dem_diff))[0][4:]
if len(id_str) > 1:
grid_id = int(id_str)
save_id_grid_no_subsidence(grid_id)
else:
basic.outputlogMessage('obtain elevation reduction polygons: %s'%dem_diff_shp)
## remove files, only keep the final results.
if b_rm_files:
io_function.delete_file_or_dir(label_path)
IDrange_txt = os.path.splitext(label_path)[0] + '_IDrange.txt'
io_function.delete_file_or_dir(IDrange_txt)
io_function.delete_shape_file(segment_shp_path)
# other intermediate files
other_shp_names = ['merged','surrounding','rmreldemD','rmshapeinfo','rmslope']
for name in other_shp_names:
rm_shp = io_function.get_name_by_adding_tail(segment_shp_path, name)
io_function.delete_shape_file(rm_shp)
return True
def polygons2geojson(shp_path, save_dir):
command_str = poly2geojson + ' ' + shp_path + ' ' + save_dir
print(command_str)
basic.os_system_exit_code(command_str)
