def draw_one_value_hist(shp_file, field_name, output, logfile, bin_min,
bin_max, bin_width, ylim):
basic.setlogfile(logfile)
values = read_attribute(shp_file, field_name)
if field_name == 'INarea': # m^2 to ha
values = [item / 10000.0 for item in values]
xlabelrotation = None
if 'area' in field_name or 'INperimete' in field_name or 'circularit' in field_name or 'aspectLine' in field_name or \
'slo' in field_name or 'dem' in field_name or 'dem' in field_name:
xlabelrotation = 90
bins = np.arange(bin_min, bin_max, bin_width)
# plot histogram of slope values
# value_list,output,bins=None,labels=None,color=None,hatch=""
draw_one_list_histogram(values,
output,
bins=bins,
color=['grey'],
xlabelrotation=xlabelrotation,
ylim=ylim) # ,hatch='-'
# io_function.move_file_to_dst('processLog.txt', logfile, overwrite=True)
# io_function.move_file_to_dst('processLog.txt', os.path.join(out_dir, logfile), overwrite=True)
# io_function.move_file_to_dst(output, os.path.join(out_dir, output), overwrite=True)
basic.setlogfile('processLog.txt')
def main():
basic.setlogfile('log_to_relative_dem_8bit.txt')
if os.path.isdir(relative_dem_dir) is False:
io_function.mkdir(relative_dem_dir)
# 500 pixel by 500 pixel, that is 1 km by 1 km
patch_width = 500
patch_height = 500
process_num = 1
failed_tifs = []
dem_reg_list = io_function.get_file_list_by_pattern(
arcticDEM_reg_tif_dir, '*dem_reg.tif')
count = len(dem_reg_list)
for idx, tif in enumerate(dem_reg_list):
print('%d/%d convert %s to relative DEM (8bit)' %
(idx + 1, count, tif))
rel_dem_8bit = io_function.get_name_by_adding_tail(tif, 'relDEM8bit')
rel_dem_8bit = os.path.join(relative_dem_dir,
os.path.basename(rel_dem_8bit))
try:
dem_to_relative_dem(tif, rel_dem_8bit, patch_width, patch_height,
process_num)
except:
failed_tifs.append(tif)
with open('to_relative_dem_failed_cases.txt', 'w') as f_obj:
for item in failed_tifs:
f_obj.writelines(item + '\n')
pass
def segment_a_dem_diff(dem_diff_path,
process_num,
ele_diff_thr,
min_area,
max_area,
job_id=0):
basic.setlogfile('log_segment_dem_diff_job_%d.txt' % job_id)
# find 8bit one
tif_8bit = io_function.get_name_by_adding_tail(dem_diff_path, '8bit')
demD_8bit = os.path.join(dem_common.grid_dem_diffs_8bit_dir,
os.path.basename(tif_8bit))
if os.path.isfile(demD_8bit):
print('error, 8bit DEM diff not exists: %s ' % demD_8bit)
return False
grid_id = int(
re.findall('grid\d+', os.path.basename(dem_diff_path))[0][4:])
save_dir = os.path.join(dem_common.grid_dem_diffs_segment_dir,
'segment_result_grid%d' % grid_id)
return segment_subsidence_grey_image(demD_8bit,
dem_diff_path,
save_dir,
process_num,
subsidence_thr_m=ele_diff_thr,
min_area=min_area,
max_area=max_area)
def draw_two_values_hist(shp_file, field_name, raster_file, output, logfile,
bin_min, bin_max, bin_width, labels, ylim):
basic.setlogfile(logfile)
raster_values = read_oneband_image_to_1dArray(raster_file,
nodata=0,
ignore_small=bin_min)
bins = np.arange(bin_min, bin_max, bin_width)
# update
global global_bin_size
global_bin_size = bin_width
ylim = [item / (100.0 * bin_width) for item in ylim]
draw_two_list_histogram(shp_file,
field_name,
raster_values,
output,
bins=bins,
labels=labels,
color=['black', 'silver'],
ylim=ylim)
# io_function.move_file_to_dst('processLog.txt', logfile, overwrite=True)
# io_function.move_file_to_dst('processLog.txt', os.path.join(out_dir,logfile), overwrite=True)
# io_function.move_file_to_dst(output, os.path.join(out_dir,output), overwrite=True)
basic.setlogfile('processLog.txt')
def main(options, args):
extent_shp_or_ids_txt = args[0]
process_num = options.process_num
o_res = options.out_res
if os.path.isdir(grid_matchtag_sum_dir) is False:
io_function.mkdir(grid_matchtag_sum_dir)
basic.setlogfile('produce_matchtag_sum_ArcticDEM_log_%s.txt' %
timeTools.get_now_time_str())
# read grids and ids
time0 = time.time()
all_grid_polys, all_ids = vector_gpd.read_polygons_attributes_list(
grid_20_shp, 'id')
print('time cost of read polygons and attributes', time.time() - time0)
# get grid ids based on input extent
grid_base_name = os.path.splitext(
os.path.basename(extent_shp_or_ids_txt))[0]
grid_polys, grid_ids = get_grid_20(extent_shp_or_ids_txt, all_grid_polys,
all_ids)
# check dem difference existence
grid_dem_tifs, grid_ids_no_sum = get_existing_matchtag_sum(
grid_matchtag_sum_dir, grid_base_name, grid_ids)
if len(grid_ids_no_sum) > 0:
# refine grid_polys
if len(grid_ids) > len(grid_ids_no_sum):
id_index = [grid_ids.index(id) for id in grid_ids_no_sum]
grid_polys = [grid_polys[idx] for idx in id_index]
# # download ArcticDEM and applying registration
# tarballs, reg_tifs = download_dem_tarball(dem_strip_shp, grid_polys, arcticDEM_tarball_dir, grid_base_name,
# reg_tif_dir=arcticDEM_reg_tif_dir, poly_ids=grid_ids_no_demDiff)
#
# # unpack and applying registration
# if len(tarballs) > 0:
# basic.outputlogMessage('Processs %d dem tarballs'%len(tarballs))
# out_reg_tifs = process_dem_tarball(tarballs,'./',arcticDEM_reg_tif_dir,remove_inter_data=True, apply_registration=True)
# basic.outputlogMessage('Get %d new registration dem tifs' % len(out_reg_tifs))
# reg_tifs.extend(out_reg_tifs)
reg_tifs = io_function.get_file_list_by_ext('.tif',
arcticDEM_reg_tif_dir,
bsub_folder=False)
matchtag_tifs = [tif for tif in reg_tifs
if 'matchtag' in tif] # only keep matchtag
# crop, sum
out_dem_diffs = produce_matchtag_sum_grids(grid_polys,
grid_ids_no_sum,
grid_base_name,
matchtag_tifs,
o_res,
process_num=process_num)
def main(options, args):
extent_shp_or_ids_txt = args[0]
process_num = options.process_num
keep_dem_percent = options.keep_dem_percent
o_res = options.out_res
basic.setlogfile('produce_headwall_shp_ArcticDEM_log_%s.txt' %
timeTools.get_now_time_str())
if os.path.isdir(grid_dem_headwall_shp_dir) is False:
io_function.mkdir(grid_dem_headwall_shp_dir)
# read grids and ids
time0 = time.time()
all_grid_polys, all_ids = vector_gpd.read_polygons_attributes_list(
grid_20_shp, 'id')
print('time cost of read polygons and attributes', time.time() - time0)
# get grid ids based on input extent
grid_base_name = os.path.splitext(
os.path.basename(extent_shp_or_ids_txt))[0]
grid_polys, grid_ids = get_grid_20(extent_shp_or_ids_txt, all_grid_polys,
all_ids)
# check dem difference existence
grid_headwall_shps, grid_id_no_headwall_shp = get_existing_grid_headwall_shp(
grid_dem_headwall_shp_dir, grid_base_name, grid_ids)
if len(grid_id_no_headwall_shp) > 0:
# refine grid_polys
if len(grid_ids) > len(grid_id_no_headwall_shp):
id_index = [grid_ids.index(id) for id in grid_id_no_headwall_shp]
grid_polys = [grid_polys[idx] for idx in id_index]
reg_tifs = io_function.get_file_list_by_ext('.tif',
arcticDEM_reg_tif_dir,
bsub_folder=False)
reg_tifs = [tif for tif in reg_tifs
if 'matchtag' not in tif] # remove matchtag
#
headwall_shp_folders = extract_headwall_grids(grid_polys,
grid_id_no_headwall_shp,
grid_base_name,
reg_tifs,
b_mosaic_id,
b_mosaic_date,
keep_dem_percent,
o_res,
process_num=process_num)
def main():
basic.setlogfile('log_convert_dem_diff_to8bit.txt')
if os.path.isdir(grid_dem_diffs_8bit_dir) is False:
io_function.mkdir(grid_dem_diffs_8bit_dir)
dem_diff_list = io_function.get_file_list_by_pattern(grid_dem_diffs_dir,'*DEM_diff_grid*.tif')
count = len(dem_diff_list)
failed_tifs = []
for idx, tif in enumerate(dem_diff_list):
print('%d/%d convert %s to 8 bit'%(idx+1, count, tif))
tif_8bit = io_function.get_name_by_adding_tail(tif, '8bit')
output = os.path.join(grid_dem_diffs_8bit_dir, os.path.basename(tif_8bit))
if dem_tif_to_8bit(tif,output) is False:
failed_tifs.append(tif)
if len(failed_tifs)>0:
io_function.save_list_to_txt('failed_dem_diff_to8bit.txt',failed_tifs)
def main():
basic.setlogfile('scp_log.txt')
while True:
# get remote dir
basic.outputlogMessage('get remote folders')
remote_folders = get_remote_folder(remote_dir, folder_pattern)
basic.outputlogMessage("%d remote folders" % len(remote_folders))
folder_list = get_local_folder(local_dir, folder_pattern)
basic.outputlogMessage("%d local folders" % len(folder_list))
folder_name_list = [os.path.basename(item) for item in folder_list]
for idx, r_folders in enumerate(remote_folders):
folder_name = os.path.basename(r_folders)
if folder_name in folder_name_list:
continue
basic.outputlogMessage('copy trained folder in %s' % folder_name)
command_str = 'scp -r ${tesia_host}:%s %s/%s' % (
r_folders, local_dir, folder_name)
print(command_str)
status, result = basic.getstatusoutput(command_str)
if status != 0:
sys.exit(1)
folder_list = get_local_folder(local_dir,
folder_pattern) # update local folder
# reomve incomplete folders
for folder in folder_list:
res_json = os.path.join(folder, 'result.json')
if os.path.isfile(res_json) and os.path.getsize(res_json) > 0:
continue
else:
basic.outputlogMessage('remove incomplete folder: %s' %
os.path.basename(folder))
io_function.delete_file_or_dir(folder)
basic.outputlogMessage('wait five hours')
time.sleep(3600 * 5) # wait five hours
pass
def plot_valid_entropy(in_folder, save_file_pre=None):
if save_file_pre is None:
save_file_pre = os.path.basename(in_folder)
logfile = basic.logfile
basic.setlogfile(save_file_pre + 'hist_info.txt')
image_paths = io_function.get_file_list_by_ext('.tif',
in_folder,
bsub_folder=True)
if len(image_paths) < 1:
raise IOError('no tif files in %s' % in_folder)
valid_per_list = []
entropy_list = []
img_count = len(image_paths)
for idx, img_path in enumerate(image_paths):
print('%d/%d' % (idx + 1, img_count))
valid_per, entropy = raster_io.get_valid_percent_shannon_entropy(
img_path, log_base=10)
valid_per_list.append(valid_per)
entropy_list.append(entropy)
per_entropy_txt = save_file_pre + '_' + 'valid_per_entropy.txt'
save_hist_path = save_file_pre + '_' + 'hist.jpg'
with open(per_entropy_txt, 'w') as f_obj:
for path, per, entropy in zip(image_paths, valid_per_list,
entropy_list):
f_obj.writelines(
os.path.basename(path) + ' %.4f %.6f \n' % (per, entropy))
# plot the histogram
fig = plt.figure(figsize=(6, 4)) #
ax1 = fig.add_subplot(111)
n, bins, patches = plt.hist(x=entropy_list,
bins=50,
color='b',
rwidth=0.85)
# print(n, bins, patches)
plt.savefig(save_hist_path, dpi=200) # 300
histogram2logfile(entropy_list, bins, hist_tag=save_hist_path)
basic.setlogfile(logfile) # change log file name back
return save_hist_path
import numpy as np
from distutils.version import LooseVersion
import tqdm
if LooseVersion(tqdm.__version__) < LooseVersion("4.34"):
raise EnvironmentError('tqdm version should be newer (>= 4.34) for url download, run pip install tqdm -U')
from tqdm import tqdm
HOME = os.path.expanduser('~')
# path of DeeplabforRS
codes_dir2 = HOME +'/codes/PycharmProjects/DeeplabforRS'
sys.path.insert(0, codes_dir2)
import basic_src.basic as basic
basic.setlogfile('pytorch_deeplab_train.log')
from mypath import Path
from dataloaders import make_data_loader
from modeling.sync_batchnorm.replicate import patch_replication_callback
from modeling.deeplab import *
from utils.loss import SegmentationLosses
from utils.calculate_weights import calculate_weigths_labels
from utils.lr_scheduler import LR_Scheduler
from utils.saver import Saver
from utils.summaries import TensorboardSummary
from utils.metrics import Evaluator
class Trainer(object):
def __init__(self, args):
self.args = args
import sys, os
from optparse import OptionParser
import cv2
import numpy as np
HOME = os.path.expanduser('~')
# path of DeeplabforRS
codes_dir2 = HOME + '/codes/PycharmProjects/DeeplabforRS'
sys.path.insert(0, codes_dir2)
import parameters
import basic_src.basic as basic
basic.setlogfile('check_label_patches.txt')
def main(options, args):
input_file = args[0]
img_patch_dir = options.image_dir
label_patch_dir = options.label_dir
basic.outputlogMessage(
'check split image patches and label patches in %s, especially after data augmentation'
% input_file)
b_diff = False
num_classes_noBG = parameters.get_digit_parameters(options.para_file,
'NUM_CLASSES_noBG',
"--to_rgb",
action="store_true",
dest="to_rgb",
default=False,
help="true to convert all images to 8 bit rgb")
parser.add_option(
"-g",
"--group_date",
action="store_true",
dest="group_date",
default=False,
help="true to group image if their acquisition date is the same")
# parser.add_option("-i", "--item_types",
# action="store", dest="item_types",default='PSScene4Band',
# help="the item types, e.g., PSScene4Band,PSOrthoTile")
# parser.add_option("-a", "--planet_account",
# action="store", dest="planet_account",default='*****@*****.**',
# help="planet email account, e.g., [email protected]")
(options, args) = parser.parse_args()
# print(options.to_rgb)
if len(sys.argv) < 2 or len(args) < 1:
parser.print_help()
sys.exit(2)
basic.setlogfile('mosaic_images_crop_grid_%s.log' %
str(datetime.date(datetime.now())))
main(options, args)
add time: 24 September, 2019
"""
# note: it seems the codes cannot run on multi-nodes on ITSC services. So I have to submit jobs using different separately
import os, sys
import time
HOME = os.path.expanduser('~')
codes_dir2 = HOME +'/codes/PycharmProjects/DeeplabforRS'
sys.path.insert(0, codes_dir2)
import basic_src.basic as basic
import basic_src.io_function as io_function
basic.setlogfile('parallel_predict_rtsLog.txt')
predict_script = HOME + '/codes/PycharmProjects/Landuse_DL/sentinelScripts/predict_rts_oneImg.sh'
import GPUtil
import datetime
from multiprocessing import Process
machine_name = os.uname()[1]
start_time = datetime.datetime.now()
# remove previous results
outdir = 'multi_inf_results'
if os.path.isdir(outdir) and 'chpc' not in machine_name: # on ITSC service, need to manually deleted previous results
io_function.delete_file_or_dir(outdir)
parser.add_option("",
"--extent_shp",
action="store",
dest="extent_shp",
help="the area extent")
parser.add_option(
"-n",
"--region_name",
action="store",
dest="region_name",
help="the name of the area, which we download snow cover for")
# parser.add_option("", "--extent_xy",
# action="store", dest="extent_xy",
# help="a list of xy points")
# parser.add_option("-b", "--buffer_size",
# action="store", dest="buffer_size", type=int, default = 3000,
# help="the buffer size to crop image in meters")
(options, args) = parser.parse_args()
if len(sys.argv) < 2 or len(args) < 1:
parser.print_help()
sys.exit(2)
basic.setlogfile('get_timelapse_img_gee_%s.log' %
str(datetime.date(datetime.now())))
main(options, args)
# dem_path = os.path.join(dem_file_dir,'WR_extent_2m_v3.0_ArcticTileDEM_sub_1_prj.tif')
# dem patches
dem_file_dir = os.path.expanduser(
'~/Data/Arctic/canada_arctic/DEM/WR_dem_ArcticDEM_mosaic/dem_patches')
dem_list = io_function.get_file_list_by_ext('.tif',
dem_file_dir,
bsub_folder=False)
save_shp = os.path.basename(
io_function.get_name_by_adding_tail(shp, 'multi_raster_stats'))
io_function.copy_shape_file(shp, save_shp)
zonal_stats_multiRasters(save_shp,
dem_list,
nodata=None,
band=1,
stats=None,
prefix='dem',
range=None,
all_touched=True,
process_num=4)
def main():
test_zonal_stats_multiRasters()
pass
if __name__ == '__main__':
basic.setlogfile('raster_statistic.log')
main()
def image_augment_main(para_file, img_list_txt, save_list, img_dir, out_dir,
extension, is_ground_truth, proc_num):
basic.setlogfile('log_data_augmentation.txt')
if os.path.isfile(img_list_txt) is False:
raise IOError("File %s not exist" % img_list_txt)
if os.path.isdir(out_dir) is False:
os.makedirs(out_dir)
if img_dir != out_dir:
raise ValueError(
'set image dir and output dir be the same, making it easy to update image list'
)
# print(options.para_file)
augmentation = parameters.get_string_list_parameters_None_if_absence(
para_file, 'data_augmentation')
if augmentation is None or len(augmentation) < 1:
basic.outputlogMessage(
'No input augmentation requirement (e.g. flip), skip data augmentation'
)
return True
# number of classes
num_classes_noBG = parameters.get_digit_parameters_None_if_absence(
para_file, 'NUM_CLASSES_noBG', 'int')
global num_classes
num_classes = num_classes_noBG + 1
# ignored classes
ignore_classes = parameters.get_string_list_parameters_None_if_absence(
para_file, 'data_aug_ignore_classes')
with open(img_list_txt, 'r') as f_obj:
files_list = f_obj.readlines()
file_count = len(files_list)
index = 1
# for line in files_list:
#
# # ignore_classes
# if ignore_classes is not None and len(ignore_classes)>0:
# found_class = [ line.find(ignore_class) >= 0 for ignore_class in ignore_classes ]
# if True in found_class:
# continue
#
# file_path = line.strip()
# file_path = os.path.join(img_dir,file_path+extension)
# print ("Augmentation of image (%d / %d)"%(index,file_count))
# if image_augment(file_path,out_dir,is_ground_truth,augment=augmentation) is False:
# print ('Error, Failed in image augmentation')
# return False
# index += 1
parameters_list = [(index + 1, line, ignore_classes, file_count, img_dir,
extension, out_dir, is_ground_truth, augmentation)
for index, line in enumerate(files_list)]
theadPool = Pool(proc_num) # multi processes
results = theadPool.starmap(augment_one_line,
parameters_list) # need python3
augmented = [1 for item in results if item is True]
# print(sum(augmented))
if sum(augmented) < file_count:
basic.outputlogMessage('Some of the images belong to %s are ignored' %
','.join(ignore_classes))
# update img_list_txt (img_dir is the same as out_dir)
new_files = io_function.get_file_list_by_ext(extension,
out_dir,
bsub_folder=False)
new_files_noext = [
os.path.splitext(os.path.basename(item))[0] + '\n'
for item in new_files
]
basic.outputlogMessage('save new file list to %s' % save_list)
with open(save_list, 'w') as f_obj:
f_obj.writelines(new_files_noext)
usage = "usage: %prog [options] shapefile or shapefiles"
parser = OptionParser(usage=usage, version="1.0 2017-10-28")
parser.description = 'Introduction: plot accuracies of the results '
parser.add_option("-p", "--para",
action="store", dest="para_file",default='para.ini',
help="the parameters file")
parser.add_option("-o", "--output",
action="store", dest="output",default='P_R.jpg',
help="the parameters file")
(options, args) = parser.parse_args()
if len(sys.argv) < 2 or len(args) < 1:
parser.print_help()
sys.exit(2)
# set parameters files, mandatory for the path of ground truth polygons
if options.para_file is None:
print('error, no parameters file')
parser.print_help()
sys.exit(2)
else:
parameters.set_saved_parafile_path(options.para_file)
basic.setlogfile('accuracies_log.txt')
main(options, args)
pass
"--para",
action="store",
dest="para_file",
default='para.ini',
help="the parameters file")
parser.add_option("-i",
"--iou_threshold",
action="store",
dest="iou_threshold",
type=float,
default=0.01,
help="the iou threshold")
(options, args) = parser.parse_args()
if len(sys.argv) < 2 or len(args) < 1:
parser.print_help()
sys.exit(2)
# set parameters files, mandatory for the path of ground truth polygons
if options.para_file is None:
print('error, no parameters file')
parser.print_help()
sys.exit(2)
else:
parameters.set_saved_parafile_path(options.para_file)
basic.setlogfile('get_trueFN_log.txt')
main(options, args)
pass
def main(options, args):
process_num = options.process_num
buffer_size = options.buffer_size
# perform the selection grid by grid
basic.setlogfile('select_RTS_YOLO_demDiff_headwall_%s.txt' %
timeTools.get_now_time_str())
b_grid = options.b_grid
if b_grid:
# process the selection grid by grid
extent_shp_or_ids_txt = args[0]
yolo_result_dir = os.path.expanduser(
'~/Data/Arctic/alaska/autoMapping/alaskaNS_yolov4_1')
dem_subsidence_dir = grid_dem_diffs_segment_dir
grid_headwall_dir = grid_dem_headwall_shp_dir
# read grids and ids
time0 = time.time()
all_grid_polys, all_ids = vector_gpd.read_polygons_attributes_list(
grid_20_shp, 'id')
print('time cost of read polygons and attributes', time.time() - time0)
# get grid ids based on input extent
grid_base_name = os.path.splitext(
os.path.basename(extent_shp_or_ids_txt))[0]
grid_polys, grid_ids = get_grid_20(extent_shp_or_ids_txt,
all_grid_polys, all_ids)
# check dem difference existence
grid_rts_shps, grid_id_no_rts_shp = get_existing_select_grid_rts(
grid_rts_shp_dir, grid_base_name, grid_ids)
if len(grid_id_no_rts_shp) > 0:
# refine grid_polys
if len(grid_ids) > len(grid_id_no_rts_shp):
id_index = [grid_ids.index(id) for id in grid_id_no_rts_shp]
grid_polys = [grid_polys[idx] for idx in id_index]
#
rts_shp_folders = select_rts_map_demDiff_headwall_grids(
yolo_result_dir,
dem_subsidence_dir,
grid_headwall_dir,
grid_polys,
grid_id_no_rts_shp,
grid_base_name,
process_num=process_num)
else:
# processing the selection for two input shapefile
yolo_box_shp = args[0]
dem_subsidence_shp = args[1]
print('polygon group 1:', yolo_box_shp)
print('polygon group 2:', dem_subsidence_shp)
if options.save_path is not None:
save_path = options.save_path
else:
save_path = io_function.get_name_by_adding_tail(
yolo_box_shp, 'select')
select_polygons_overlap_others_in_group2(yolo_box_shp,
dem_subsidence_shp,
save_path,
buffer_size=buffer_size,
process_num=process_num)
pass
def main(options, args):
b_mosaic_ArcticDEM = options.b_mosaic_ArcticDEM
process_num = options.process_num
product_list = args # subset of [slope, slope_8bit, hillshade, tpi]
print('Will produce products includes: %s' % (product_list))
if b_mosaic_ArcticDEM:
print('Input is the mosaic version of AricticDEM')
from dem_common import arcticDEM_tile_reg_tif_dir,arcticDEM_tile_hillshade_dir,arcticDEM_tile_slope_8bit_dir,\
arcticDEM_tile_slope_dir,arcticDEM_tile_tpi_8bit_dir
arcticDEM_reg_tif_dir = arcticDEM_tile_reg_tif_dir
arcticDEM_hillshade_dir = arcticDEM_tile_hillshade_dir
arcticDEM_slope_8bit_dir = arcticDEM_tile_slope_8bit_dir
arcticDEM_slope_dir = arcticDEM_tile_slope_dir
arcticDEM_tpi_8bit_dir = arcticDEM_tile_tpi_8bit_dir
dem_pattern = '*reg_dem.tif'
else:
from dem_common import arcticDEM_reg_tif_dir, dem_hillshade_dir, dem_slope_dir, dem_slope_8bit_dir, dem_tpi_8bit_dir
arcticDEM_reg_tif_dir = arcticDEM_reg_tif_dir
arcticDEM_hillshade_dir = dem_hillshade_dir
arcticDEM_slope_8bit_dir = dem_slope_8bit_dir
arcticDEM_slope_dir = dem_slope_dir
arcticDEM_tpi_8bit_dir = dem_tpi_8bit_dir
dem_pattern = '*dem_reg.tif'
basic.setlogfile('log_dem_to_slope8bit_hillshade.txt')
if os.path.isdir(arcticDEM_slope_8bit_dir) is False:
io_function.mkdir(arcticDEM_slope_8bit_dir)
if os.path.isdir(arcticDEM_hillshade_dir) is False:
io_function.mkdir(arcticDEM_hillshade_dir)
if os.path.isdir(arcticDEM_tpi_8bit_dir) is False:
io_function.mkdir(arcticDEM_tpi_8bit_dir)
if os.path.isdir(arcticDEM_slope_dir) is False:
io_function.mkdir(arcticDEM_slope_dir)
failed_tifs = []
dem_reg_list = io_function.get_file_list_by_pattern(
arcticDEM_reg_tif_dir, dem_pattern)
count = len(dem_reg_list)
print('Find %d DEM in %s, with pattern: %s' %
(count, arcticDEM_reg_tif_dir, dem_pattern))
if process_num == 1:
for idx, tif in enumerate(dem_reg_list):
res = process_one_dem(idx, count, tif, product_list,
arcticDEM_slope_dir,
arcticDEM_slope_8bit_dir,
arcticDEM_hillshade_dir,
arcticDEM_tpi_8bit_dir)
if res is not True:
failed_tifs.append(res)
else:
parameters_list = [(idx, count, tif, product_list, arcticDEM_slope_dir,
arcticDEM_slope_8bit_dir, arcticDEM_hillshade_dir,
arcticDEM_tpi_8bit_dir)
for idx, tif in enumerate(dem_reg_list)]
theadPool = Pool(process_num)
results = theadPool.starmap(process_one_dem, parameters_list)
for res in results:
if res is not True:
failed_tifs.append(res)
theadPool.close()
with open('to_hillshade_slope8bit_failed_cases.txt', 'w') as f_obj:
for item in failed_tifs:
f_obj.writelines(item + '\n')
def main(options, args):
b_mosaic_ArcticDEM = options.b_mosaic_ArcticDEM
process_num = options.process_num
if b_mosaic_ArcticDEM:
print('Input is the mosaic version of AricticDEM')
arcticDEM_hillshade_dir = os.path.join(ArcticDEM_tmp_dir,
'arcticdem_mosaic_hillshade')
arcticDEM_slope_8bit_dir = os.path.join(ArcticDEM_tmp_dir,
'arcticdem_mosaic_slope_8bit')
arcticDEM_tpi_8bit_dir = os.path.join(ArcticDEM_tmp_dir,
'arcticdem_mosaic_tpi_8bit')
file_pattern = '*.tif'
arcticDEM_topo_rgb_dir = os.path.join(ArcticDEM_tmp_dir,
'arcticdem_mosaic_topoRGB_8bit')
else:
arcticDEM_hillshade_dir = os.path.join(ArcticDEM_tmp_dir,
'dem_hillshade')
arcticDEM_slope_8bit_dir = os.path.join(ArcticDEM_tmp_dir,
'dem_slope_8bit')
arcticDEM_tpi_8bit_dir = os.path.join(ArcticDEM_tmp_dir,
'dem_tpi_8bit')
file_pattern = '*.tif'
arcticDEM_topo_rgb_dir = os.path.join(ArcticDEM_tmp_dir,
'dem_topoRGB_8bit')
basic.setlogfile('log_compose_dem_products.txt')
if os.path.isdir(arcticDEM_topo_rgb_dir) is False:
io_function.mkdir(arcticDEM_topo_rgb_dir)
failed_tifs = []
hillshade_list = io_function.get_file_list_by_pattern(
arcticDEM_hillshade_dir, file_pattern)
count = len(hillshade_list)
print('Find %d Hillshade in %s, with pattern: %s' %
(count, arcticDEM_hillshade_dir, file_pattern))
if process_num == 1:
for idx, tif in enumerate(hillshade_list):
res = compose_rgb_from_hillshade_slope_tpi(
idx, count, tif, arcticDEM_slope_8bit_dir,
arcticDEM_tpi_8bit_dir, arcticDEM_topo_rgb_dir)
if res is not True:
failed_tifs.append(res)
else:
parameters_list = [(idx, count, tif, arcticDEM_slope_8bit_dir,
arcticDEM_tpi_8bit_dir, arcticDEM_topo_rgb_dir)
for idx, tif in enumerate(hillshade_list)]
theadPool = Pool(process_num)
results = theadPool.starmap(compose_rgb_from_hillshade_slope_tpi,
parameters_list)
for res in results:
if res is not True:
failed_tifs.append(res)
theadPool.close()
with open('to_rgb_from_hillshade_slope8bit_failed_cases.txt',
'w') as f_obj:
for item in failed_tifs:
f_obj.writelines(item + '\n')
"--cloud_cover",
action="store",
dest="cloud_cover",
type=float,
help=
"the could cover threshold, only accept images with cloud cover less than the threshold"
)
parser.add_option("-i",
"--item_types",
action="store",
dest="item_types",
default='PSScene4Band',
help="the item types, e.g., PSScene4Band,PSOrthoTile")
parser.add_option(
"-a",
"--planet_account",
action="store",
dest="planet_account",
default='*****@*****.**',
help="planet email account, e.g., [email protected]")
(options, args) = parser.parse_args()
if len(sys.argv) < 2 or len(args) < 1:
parser.print_help()
sys.exit(2)
basic.setlogfile('download_planet_images_%s.log' %
str(datetime.date(datetime.now())))
main(options, args)
)
parser.add_option(
"-t",
"--istraining",
action="store_true",
dest="istraining",
default=False,
help="to indicate the script will perform training process")
parser.add_option("-s",
"--shape_train",
action="store",
dest="polygon_train",
help="the shape file containing polygons for training")
parser.add_option("-o",
"--output",
action="store",
dest="output",
help="the output file path")
(options, args) = parser.parse_args()
if len(sys.argv) < 2:
parser.print_help()
sys.exit(2)
basic.setlogfile('planet_svm_log.txt')
main(options, args)
usage = "usage: %prog [options] input_image "
parser = OptionParser(usage=usage, version="1.0 2019-1-4")
parser.description = 'Introduction: pixel-based image classification based on random forest classifier'
parser.add_option("-p", "--ispreprocess",
action="store_true", dest="ispreprocess", default=False,
help="to indicate the script will perform pre-processing, if this set, istraining will be ignored")
parser.add_option("-t", "--istraining",
action="store_true", dest="istraining", default=False,
help="to indicate the script will perform training process")
parser.add_option("-s", "--shape_train",
action="store", dest="polygon_train",
help="the shape file containing polygons for training")
parser.add_option("-o", "--output",
action="store", dest="output",
help="the output file path")
(options, args) = parser.parse_args()
if len(sys.argv) < 2:
parser.print_help()
sys.exit(2)
basic.setlogfile('RandomForest_log.txt')
main(options, args)
def main(options, args):
print(
"%s : prediction using the trained model (run parallel if use multiple GPUs) "
% os.path.basename(sys.argv[0]))
machine_name = os.uname()[1]
start_time = datetime.datetime.now()
para_file = args[0]
if os.path.isfile(para_file) is False:
raise IOError('File %s not exists in current folder: %s' %
(para_file, os.getcwd()))
basic.setlogfile('parallel_predict_Log.txt')
deeplab_inf_script = os.path.join(code_dir, 'deeplabBased',
'deeplab_inference.py')
network_setting_ini = parameters.get_string_parameters(
para_file, 'network_setting_ini')
global tf1x_python
tf1x_python = parameters.get_file_path_parameters(network_setting_ini,
'tf1x_python')
trained_model = options.trained_model
outdir = parameters.get_directory(para_file, 'inf_output_dir')
# remove previous results (let user remove this folder manually or in exe.sh folder)
io_function.mkdir(outdir)
# get name of inference areas
multi_inf_regions = parameters.get_string_list_parameters(
para_file, 'inference_regions')
# max_parallel_inf_task = parameters.get_digit_parameters(para_file,'max_parallel_inf_task','int')
b_use_multiGPUs = parameters.get_bool_parameters(para_file,
'b_use_multiGPUs')
# loop each inference regions
sub_tasks = []
for area_idx, area_ini in enumerate(multi_inf_regions):
area_name = parameters.get_string_parameters(area_ini, 'area_name')
area_remark = parameters.get_string_parameters(area_ini, 'area_remark')
area_time = parameters.get_string_parameters(area_ini, 'area_time')
inf_image_dir = parameters.get_directory(area_ini, 'inf_image_dir')
# it is ok consider a file name as pattern and pass it the following functions to get file list
inf_image_or_pattern = parameters.get_string_parameters(
area_ini, 'inf_image_or_pattern')
inf_img_list = io_function.get_file_list_by_pattern(
inf_image_dir, inf_image_or_pattern)
img_count = len(inf_img_list)
if img_count < 1:
raise ValueError(
'No image for inference, please check inf_image_dir and inf_image_or_pattern in %s'
% area_ini)
area_save_dir = os.path.join(
outdir, area_name + '_' + area_remark + '_' + area_time)
io_function.mkdir(area_save_dir)
# parallel inference images for this area
CUDA_VISIBLE_DEVICES = []
if 'CUDA_VISIBLE_DEVICES' in os.environ.keys():
CUDA_VISIBLE_DEVICES = [
int(item.strip())
for item in os.environ['CUDA_VISIBLE_DEVICES'].split(',')
]
idx = 0
while idx < img_count:
if b_use_multiGPUs:
# get available GPUs # https://github.com/anderskm/gputil
deviceIDs = GPUtil.getAvailable(order='first',
limit=100,
maxLoad=0.5,
maxMemory=0.5,
includeNan=False,
excludeID=[],
excludeUUID=[])
# only use the one in CUDA_VISIBLE_DEVICES
if len(CUDA_VISIBLE_DEVICES) > 0:
deviceIDs = [
item for item in deviceIDs
if item in CUDA_VISIBLE_DEVICES
]
basic.outputlogMessage('on ' + machine_name +
', available GPUs:' +
str(deviceIDs) +
', among visible ones:' +
str(CUDA_VISIBLE_DEVICES))
else:
basic.outputlogMessage('on ' + machine_name +
', available GPUs:' +
str(deviceIDs))
if len(deviceIDs) < 1:
time.sleep(
60
) # wait one minute, then check the available GPUs again
continue
# set only the first available visible
gpuid = deviceIDs[0]
basic.outputlogMessage(
'%d: predict image %s on GPU %d of %s' %
(idx, inf_img_list[idx], gpuid, machine_name))
else:
gpuid = None
basic.outputlogMessage('%d: predict image %s on %s' %
(idx, inf_img_list[idx], machine_name))
# run inference
img_save_dir = os.path.join(area_save_dir, 'I%d' % idx)
inf_list_file = os.path.join(area_save_dir, '%d.txt' % idx)
# if it already exist, then skip
if os.path.isdir(img_save_dir) and is_file_exist_in_folder(
img_save_dir):
basic.outputlogMessage(
'folder of %dth image (%s) already exist, '
'it has been predicted or is being predicted' %
(idx, inf_img_list[idx]))
idx += 1
continue
with open(inf_list_file, 'w') as inf_obj:
inf_obj.writelines(inf_img_list[idx] + '\n')
sub_process = Process(target=predict_one_image_deeplab,
args=(deeplab_inf_script, para_file,
network_setting_ini, img_save_dir,
inf_list_file, gpuid, trained_model))
sub_process.start()
sub_tasks.append(sub_process)
if b_use_multiGPUs is False:
# wait until previous one finished
while sub_process.is_alive():
time.sleep(5)
idx += 1
# wait until predicted image patches exist or exceed 20 minutes
time0 = time.time()
elapsed_time = time.time() - time0
while elapsed_time < 20 * 60:
elapsed_time = time.time() - time0
file_exist = is_file_exist_in_folder(img_save_dir)
if file_exist is True or sub_process.is_alive() is False:
break
else:
time.sleep(5)
if sub_process.exitcode is not None and sub_process.exitcode != 0:
sys.exit(1)
# if 'chpc' in machine_name:
# time.sleep(60) # wait 60 second on ITSC services
# else:
# time.sleep(10)
# check all the tasks already finished
while b_all_task_finish(sub_tasks) is False:
basic.outputlogMessage('wait all tasks to finish')
time.sleep(60)
end_time = datetime.datetime.now()
diff_time = end_time - start_time
out_str = "%s: time cost of total parallel inference on %s: %d seconds" % (
str(end_time), machine_name, diff_time.seconds)
basic.outputlogMessage(out_str)
with open("time_cost.txt", 'a') as t_obj:
t_obj.writelines(out_str + '\n')
from skimage import io
# import skimage.transform
import numpy as np
HOME = os.path.expanduser('~')
# path of DeeplabforRS
codes_dir2 = HOME + '/codes/PycharmProjects/DeeplabforRS'
sys.path.insert(0, codes_dir2)
import parameters
import basic_src.basic as basic
import basic_src.io_function as io_function
basic.setlogfile('log_data_augmentation.txt')
# will be update in the main function
num_classes = 0
def remove_unexpected_ids(img_data, img_name):
'''
remove unexpected ids after augmentation, it will modify the numpy array
:param img_data: numpy array of image, should be one band
:param img_name: file name, help for debug
:return: True,
'''
unique_value = np.unique(img_data)
if len(unique_value) > num_classes:
# action="store", dest="end_date",
# help="the end date for inquiry, with format year-month-day, e.g., 2018-05-23")
parser.add_option("-e", "--extent_shp",
action="store", dest="extent_shp",
help="the path for extent, shapefile (EPSG:4326)")
parser.add_option("-c", "--cloud_cover",
action="store", dest="cloud_cover", type=float,default=0.3,
help="the could cover threshold, only accept images with cloud cover less than the threshold")
parser.add_option("-g", "--group_date",
action="store_true", dest="group_date",default=False,
help="true to group image if their acquisition date is the same")
parser.add_option("-x", "--save_xlsx_path",
action="store", dest="save_xlsx_path",
help="save the sence lists to xlsx file, if this is set, save_meta_shapefile will be ignored")
parser.add_option("-m", "--save_meta_shapefile",
action="store", dest="save_meta_shapefile",
help="the path for saving meta and extent of downloaded planet images")
(options, args) = parser.parse_args()
if len(sys.argv) < 2 or len(args) < 1:
parser.print_help()
sys.exit(2)
basic.setlogfile('get_planet_image_list_%s.log'%str(datetime.date(datetime.now())))
main(options, args)
basic.outputlogMessage(
'Finish removing polygons using multi-temporal mapping results')
if __name__ == "__main__":
usage = "usage: %prog [options] shape_file_dir file_pattern "
parser = OptionParser(usage=usage, version="1.0 2020-02-23")
parser.description = 'Introduction: remove polygons (e.g., non-active polygons) based on multi-temporal polygons '
parser.add_option("-p",
"--para",
action="store",
dest="para_file",
help="the parameters file")
(options, args) = parser.parse_args()
if len(sys.argv) < 2:
parser.print_help()
sys.exit(2)
# set parameters files
if options.para_file is None:
print('error, no parameters file')
parser.print_help()
sys.exit(2)
else:
parameters.set_saved_parafile_path(options.para_file)
basic.setlogfile('polygons_remove_nonActiveRTS.log')
main(options, args)
