def build_TFrecord_tf1x(para_file): if os.path.isdir('tfrecord'): basic.outputlogMessage( 'warning, tfrecord exists, skip build_TFrecord_tf1x') return command_string = os.path.join(eo_dir, 'workflow', 'build_TFrecord_tf1x.py') + ' ' + para_file basic.os_system_exit_code(command_string)
def inference(para_file): outdir = parameters.get_directory(para_file, 'inf_output_dir') # don't remove it automatically # if os.path.isdir(outdir): # io_function.delete_file_or_dir(outdir) # the script will check whether each image has been predicted command_string = os.path.join(eo_dir, 'workflow', 'parallel_prediction.py') + ' ' + para_file basic.os_system_exit_code(command_string)
def combine_hillshade_slope_tip(hillshade, slope, tpi, save_path): if os.path.isfile(save_path): print('%s exists, skip' % save_path) return # combine them. command_str = 'gdal_merge.py -o %s -separate -a_nodata 255 -of GTiff -co compress=lzw -co tiled=yes -co bigtiff=if_safer %s %s %s' % ( save_path, hillshade, slope, tpi) basic.os_system_exit_code(command_str)
def tifs_to_png(image_dir): tif_list = io_function.get_file_list_by_pattern(image_dir, '*/*.tif') for idx, tif in enumerate(tif_list): print('tif to png: %d/%d tif' % (idx + 1, len(tif_list))) basename = io_function.get_name_no_ext(tif) save_path = os.path.join(image_dir, basename + '.png') if os.path.isfile(save_path): print('%s exists, skip' % save_path) continue command_str = "gdal_translate -of PNG %s %s" % (tif, save_path) basic.os_system_exit_code(command_str)
def arcticDEM_strip_registration(strip_dir): command_str = 'python ' + reg_py + ' ' + strip_dir basic.os_system_exit_code(command_str) end = '_dem_reg.tif' reg_tif = os.path.join(strip_dir, os.path.basename(strip_dir) + end) if os.path.isfile(reg_tif): return reg_tif else: # with open('no_registration_strips.txt','a') as f_obj: # f_obj.writelines('%s\n'%strip_dir) f_obj = open('no_registration_strips.txt', 'a') f_obj.writelines('%s\n' % strip_dir) f_obj.close( ) # close file manually, to avoid dead lock when using multiple threads return None
def tpi_to_8bit(input, output): dst_nodat = 255 hist_max_percent = 0.98 hist_min_percent = 0.02 MIN_MAX_VALUE = '-1 1' # tpi range from -1 to 1 command_str = py8bit + ' ' + input + ' ' + output command_str += ' -n ' + str(dst_nodat) command_str += ' -u ' + str(hist_max_percent) + ' -l ' + str( hist_min_percent) command_str += ' -m ' + MIN_MAX_VALUE # print(command_str) basic.os_system_exit_code(command_str) return True
def dem_to_hillshade(input, output): if os.path.isfile(output): basic.outputlogMessage('%s exists, skip' % output) return True if os.path.isfile(input) is False: basic.outputlogMessage('Waring, %s does not exist' % input) return False # use the default setting in QGIS # gdaldem hillshade ${dem} ${hillshade} -of GTiff -b 1 -z 1.0 -s 1.0 -az 315.0 -alt 45.0 command_str = 'gdaldem hillshade %s %s -of GTiff -co compress=lzw -co tiled=yes -co bigtiff=if_safer -b 1 -z 1.0 -s 1.0 -az 315.0 -alt 45.0' % ( input, output) basic.os_system_exit_code(command_str) return True
def get_tifs_bounding_boxes(image_dir): tif_list = io_function.get_file_list_by_pattern(image_dir, '*/*.tif') for idx, tif in enumerate(tif_list): print('get bounding box: %d/%d tif' % (idx + 1, len(tif_list))) basename = io_function.get_name_no_ext(tif) save_path = os.path.join(image_dir, basename + '_bound.geojson') if os.path.isfile(save_path): print('%s exists, skip' % save_path) continue command_str = imgExt + " %s -o tmp.gpkg" % tif basic.os_system_exit_code(command_str) command_str = "ogr2ogr -f GeoJSON -t_srs EPSG:3413 %s tmp.gpkg" % save_path # note: projection is EPSG:3413 basic.os_system_exit_code(command_str) io_function.delete_file_or_dir('tmp.gpkg')
def dem_to_slope(input, output, slope_file_bak): if os.path.isfile(output): basic.outputlogMessage('%s exists, skip' % output) return output if os.path.isfile(slope_file_bak): basic.outputlogMessage('%s exists, skip' % slope_file_bak) return slope_file_bak if os.path.isfile(input) is False: basic.outputlogMessage('Waring, %s does not exist' % input) return False # # use the default setting in QGIS command_str = 'gdaldem slope %s %s -of GTiff -co compress=lzw -co tiled=yes -co bigtiff=if_safer -b 1 -s 1.0' % ( input, output) basic.os_system_exit_code(command_str) return output
def dem_to_tpi_save_8bit(input, output): if os.path.isfile(output): basic.outputlogMessage('%s exists, skip' % output) return True if os.path.isfile(input) is False: basic.outputlogMessage('Waring, %s does not exist' % input) return False # Topographic Position Index tpi_file = os.path.basename( io_function.get_name_by_adding_tail(input, 'tpi')) command_str = 'gdaldem TPI %s %s -of GTiff -co compress=lzw -co tiled=yes -co bigtiff=if_safer -b 1 ' % ( input, tpi_file) basic.os_system_exit_code(command_str) # to 8bit if tpi_to_8bit(tpi_file, output) is True: io_function.delete_file_or_dir(tpi_file) return True
def training(para_file, gpu_num): # if gpus is not None: # gpu_num = len(gpus) # if 'CUDA_VISIBLE_DEVICES' in os.environ.keys(): # gpus_str = [ item.strip() for item in os.environ['CUDA_VISIBLE_DEVICES'].split(',')] # if len(gpus_str) == 1 and gpus_str[0] == '': # gpu_num = 1 # no gpu, set a 1 # else: # gpu_num = len(gpus) # else: # # find available gpus # deviceIDs = GPUtil.getAvailable(order='first', limit=100, maxLoad=0.5, # maxMemory=0.5, includeNan=False, excludeID=[], excludeUUID=[]) # gpu_num = len(deviceIDs) # the script can check the trained iteration and decide to train or not # command_string = os.path.join(eo_dir, 'workflow', 'deeplab_train.py') + ' ' + para_file + ' ' + str(gpu_num) # basic.os_system_exit_code(command_string) # import deeplab_train # deeplab_train.deeplab_train_main(para_file, gpu_num) # command_string = os.path.join(eo_dir, 'yolov4_dir', 'predict_yolo.py') + ' ' + para_file command_string = 'darknet detector train data/obj.data yolov4_obj.cfg yolov4.conv.137 -gpus 0,1 -dont_show -map' basic.os_system_exit_code(command_string)
def export_model(para_file): # the script can check the trained model and decide to export or not command_string = os.path.join(eo_dir, 'workflow', 'export_graph.py') + ' ' + para_file basic.os_system_exit_code(command_string)
def segment_subsidence_grey_image(dem_diff_grey_8bit, dem_diff, save_dir,process_num, subsidence_thr_m=-0.5, min_area=40, max_area=100000000, b_rm_files=False): ''' segment subsidence areas based on 8bit dem difference :param dem_diff_grey_8bit: :param dem_diff: :param save_dir: :param process_num: :param subsidence_thr_m: mean value less than this one consider as subsidence (in meter) :param min_area: min size in m^2 (defualt is 40 m^2, 10 pixels on ArcticDEM) :param max_area: min size in m^2 (default is 10km by 10 km) :return: ''' io_function.is_file_exist(dem_diff_grey_8bit) out_pre = os.path.splitext(os.path.basename(dem_diff_grey_8bit))[0] segment_shp_path = os.path.join(save_dir, out_pre + '.shp') # check if the final result exist final_shp_path = io_function.get_name_by_adding_tail(segment_shp_path, 'post') if os.path.isfile(final_shp_path): basic.outputlogMessage('Warning, Final results (%s) of subsidence shapefile exists, skip'%final_shp_path) return True # get initial polygons # because the label from segmentation for superpixels are not unique, so we may need to get mean dem diff based on polygons, set org_raster=None label_path = segment_a_grey_image(dem_diff_grey_8bit,save_dir,process_num, org_raster=None) if os.path.isfile(segment_shp_path) and vector_gpd.is_field_name_in_shp(segment_shp_path,'demD_mean'): basic.outputlogMessage('%s exists, skip'%segment_shp_path) else: # remove segment_shp_path if it exist, but don't have demD_mean if os.path.isfile(segment_shp_path): io_function.delete_shape_file(segment_shp_path) # remove nodato (it was copy from the input image) command_str = 'gdal_edit.py -unsetnodata ' + label_path basic.os_system_exit_code(command_str) # convert the label to shapefile # remove -8 (to use 4 connectedness.) command_string = 'gdal_polygonize.py %s -b 1 -f "ESRI Shapefile" %s' % (label_path, segment_shp_path) res = os.system(command_string) if res != 0: sys.exit(1) # get dem elevation information for each polygon raster_statistic.zonal_stats_multiRasters(segment_shp_path, dem_diff, tile_min_overlap=tile_min_overlap, stats=['mean', 'std','count'], prefix='demD',process_num=process_num) # get DEM diff information for each polygon. dem_diff_shp = get_dem_subscidence_polygons(segment_shp_path, dem_diff, dem_diff_thread_m=subsidence_thr_m, min_area=min_area, max_area=max_area, process_num=process_num,b_rm_files=b_rm_files) if dem_diff_shp is None: id_str = re.findall('grid\d+', os.path.basename(dem_diff))[0][4:] if len(id_str) > 1: grid_id = int(id_str) save_id_grid_no_subsidence(grid_id) else: basic.outputlogMessage('obtain elevation reduction polygons: %s'%dem_diff_shp) ## remove files, only keep the final results. if b_rm_files: io_function.delete_file_or_dir(label_path) IDrange_txt = os.path.splitext(label_path)[0] + '_IDrange.txt' io_function.delete_file_or_dir(IDrange_txt) io_function.delete_shape_file(segment_shp_path) # other intermediate files other_shp_names = ['merged','surrounding','rmreldemD','rmshapeinfo','rmslope'] for name in other_shp_names: rm_shp = io_function.get_name_by_adding_tail(segment_shp_path, name) io_function.delete_shape_file(rm_shp) return True
def polygons2geojson(shp_path, save_dir): command_str = poly2geojson + ' ' + shp_path + ' ' + save_dir print(command_str) basic.os_system_exit_code(command_str)