def mosaic_images(img_list, output):
if os.path.isfile(output) is True:
print('warning, %s already exist, skip'%output)
return True
args_list=['gdal_merge.py','-init','0','-a_nodata','0','-o',output]
args_list.extend(img_list)
if basic.exec_command_args_list_one_file(args_list,output) is False:
raise IOError('output not exist')
return True
def convert_training_examples_from_shp_to_raster(shp_path, raster_path):
"""
convert training examples which stored in shape file to a raster file,
so these training examples can be shared by other shape file
:param shp_path: shape file path
:param raster_path: raster data path (the output data type is Byte)
:return:True if successful, False otherwise
"""
if io_function.is_file_exist(shp_path) is False:
return False
# convert class label (string) to class index (integer)
class_int_field = 'class_int'
class_label_field = 'class'
class_int_list = []
shp_operation_obj = shape_opeation()
if shp_operation_obj.has_field(shp_path, class_int_field) is False:
if shp_operation_obj.has_field(shp_path, class_label_field) is False:
basic.outputlogMessage('%s do not contain training examples' %
shp_path)
else:
# convert class label (string) to class index (integer) and create a new field name 'class_int'
attribute = ['class']
class_label_list = shp_operation_obj.get_shape_records_value(
shp_path, attribute)
classLabel.output_classLabel_to_txt('class_label_index.txt')
for label in class_label_list:
class_int_list.append(classLabel.get_class_index(label[0]))
shp_operation_obj.add_one_field_records_to_shapefile(
shp_path, class_int_list, class_int_field)
# check again whether there is 'class_int'
if shp_operation_obj.has_field(shp_path, class_int_field) is False:
basic.outputlogMessage(
"Error: There is not class_int field in the shape file")
assert False
# convert training example in shape file to raster
res = parameters.get_input_image_rescale()
layername = os.path.splitext(os.path.basename(shp_path))[0]
args_list = ['gdal_rasterize', '-a', class_int_field, '-ot', 'Byte', \
'-tr',str(res),str(res),'-l',layername,shp_path,raster_path]
result = basic.exec_command_args_list_one_file(args_list, raster_path)
if os.path.getsize(result) < 1:
return False
return True
def remove_polygons_outside_extent(input_shp, extent_shp, output):
'''
remove polygons not in the extent
:param input_shp:
:param extent_shp:
:param output:
:return:
'''
# check projection, must be the same
input_proj4 = map_projection.get_raster_or_vector_srs_info_proj4(input_shp)
extent_proj4 = map_projection.get_raster_or_vector_srs_info_proj4(extent_shp)
if input_proj4 != extent_proj4:
raise ValueError('error, projection insistence between %s and %s'%(input_shp, extent_shp))
## -progress: Only works if input layers have the "fast feature count" capability.
# ogr2ogr - progress - clipsrc ${extent_shp} ${save_shp} ${input_shp}
arg_list = ['ogr2ogr', '-progress', '-clipsrc', extent_shp, output, input_shp]
return basic.exec_command_args_list_one_file(arg_list, output)
def zonal_stats_one_polygon(idx, polygon, image_tiles, img_tile_polygons, stats, nodata=None,range=None,
band = 1,all_touched=True):
overlap_index = vector_gpd.get_poly_index_within_extent(img_tile_polygons, polygon)
image_list = [image_tiles[item] for item in overlap_index]
if len(image_list) == 1:
out_image, out_tran,nodata = raster_io.read_raster_in_polygons_mask(image_list[0], polygon, nodata=nodata,
all_touched=all_touched,bands=band)
elif len(image_list) > 1:
# for the case it overlap more than one raster, need to produce a mosaic
tmp_saved_files = []
for k_img, image_path in enumerate(image_list):
# print(image_path)
tmp_save_path = os.path.splitext(os.path.basename(image_path))[0] + '_subset_poly%d'%idx +'.tif'
_, _,nodata = raster_io.read_raster_in_polygons_mask(image_path, polygon,all_touched=all_touched,nodata=nodata,
bands=band, save_path=tmp_save_path)
tmp_saved_files.append(tmp_save_path)
# mosaic files in tmp_saved_files
save_path = 'raster_for_poly%d.tif'%idx
mosaic_args_list = ['gdal_merge.py', '-o', save_path,'-n',str(nodata),'-a_nodata',str(nodata)]
mosaic_args_list.extend(tmp_saved_files)
if basic.exec_command_args_list_one_file(mosaic_args_list,save_path) is False:
raise IOError('error, obtain a mosaic (%s) failed'%save_path)
# read the raster
out_image, out_nodata = raster_io.read_raster_one_band_np(save_path,band=band)
# remove temporal raster
tmp_saved_files.append(save_path)
for item in tmp_saved_files:
io_function.delete_file_or_dir(item)
else:
basic.outputlogMessage('warning, cannot find raster for %d (start=0) polygon'%idx)
return None
# do calculation
return array_stats(out_image, stats, nodata,range=range)
def get_polygon_shape_info(self, input_shp, out_box, bupdate=False):
"""
get Oriented minimum bounding box for a polygon shapefile,
and update the shape information based on oriented minimum bounding box to
the input shape file
:param input_shp: input polygon shape file
:param out_box: output Oriented minimum bounding box shape file
:param bupdate: indicate whether update the original input shapefile
:return:True is successful, False Otherwise
"""
if io_function.is_file_exist(input_shp) is False:
return False
args_list = ['qgis_function.py', input_shp, out_box]
if basic.exec_command_args_list_one_file(args_list, out_box) is False:
basic.outputlogMessage(
'get polygon shape information of %s failed' % input_shp)
return False
else:
basic.outputlogMessage(
'get polygon shape information of %s completed, output file is %s'
% (input_shp, out_box))
return True
def get_sub_image(idx,selected_polygon, image_tile_list, image_tile_bounds, save_path, dstnodata, brectangle ):
'''
get a mask image based on a selected polygon, it may cross two image tiles
:param selected_polygon: selected polygons
:param image_tile_list: image list
:param image_tile_bounds: the boxes of images in the list
:param save_path: save path
:param brectangle: if brectangle is True, crop the raster using bounds, else, use the polygon
:return: True is successful, False otherwise
'''
# find the images which the center polygon overlap (one or two images)
img_index = get_overlap_image_index([selected_polygon], image_tile_bounds)
if len(img_index) < 1:
basic.outputlogMessage(
'Warning, %dth polygon do not overlap any image tile, please check ' #and its buffer area
'(1) the shape file and raster have the same projection'
'and (2) this polygon is in the extent of images' % idx)
return False
image_list = [image_tile_list[item] for item in img_index]
# check it cross two or more images
if len(image_list) == 1:
# for the case that the polygon only overlap one raster
with rasterio.open(image_list[0]) as src:
polygon_json = mapping(selected_polygon)
# not necessary
# overlap_win = rasterio.features.geometry_window(src, [polygon_json], pad_x=0, pad_y=0, north_up=True, rotated=False,
# pixel_precision=3)
if brectangle:
# polygon_box = selected_polygon.bounds
polygon_json = mapping(selected_polygon.envelope) #shapely.geometry.Polygon([polygon_box])
# crop image and saved to disk
out_image, out_transform = mask(src, [polygon_json], nodata=dstnodata, all_touched=True, crop=True)
# test: save it to disk
out_meta = src.meta.copy()
out_meta.update({"driver": "GTiff",
"height": out_image.shape[1],
"width": out_image.shape[2],
"transform": out_transform}) # note that, the saved image have a small offset compared to the original ones (~0.5 pixel)
with rasterio.open(save_path, "w", **out_meta) as dest:
dest.write(out_image)
pass
else:
# for the case it overlap more than one raster, need to produce a mosaic
tmp_saved_files = []
for k_img,image_path in enumerate(image_list):
with rasterio.open(image_path) as src:
polygon_json = mapping(selected_polygon)
if brectangle:
# polygon_box = selected_polygon.bounds
polygon_json = mapping(selected_polygon.envelope) # shapely.geometry.Polygon([polygon_box])
# crop image and saved to disk
out_image, out_transform = mask(src, [polygon_json], nodata=dstnodata, all_touched=True, crop=True)
tmp_saved = os.path.splitext(save_path)[0] +'_%d'%k_img + os.path.splitext(save_path)[1]
# test: save it to disk
out_meta = src.meta.copy()
out_meta.update({"driver": "GTiff",
"height": out_image.shape[1],
"width": out_image.shape[2],
"transform": out_transform}) # note that, the saved image have a small offset compared to the original ones (~0.5 pixel)
with rasterio.open(tmp_saved, "w", **out_meta) as dest:
dest.write(out_image)
tmp_saved_files.append(tmp_saved)
# mosaic files in tmp_saved_files
mosaic_args_list = ['gdal_merge.py', '-o', save_path,'-n',str(dstnodata),'-a_nodata',str(dstnodata)]
mosaic_args_list.extend(tmp_saved_files)
if basic.exec_command_args_list_one_file(mosaic_args_list,save_path) is False:
raise IOError('error, obtain a mosaic (%s) failed'%save_path)
# # for test
# if idx==13:
# raise ValueError('for test')
# remove the tmp files
for tmp_file in tmp_saved_files:
io_function.delete_file_or_dir(tmp_file)
# if it will output a very large image (10000 by 10000 pixels), then raise a error
return True
def main(options, args):
# get multi-temporal shapefile list
para_file = options.para_file
b_remove = parameters.get_bool_parameters_None_if_absence(
para_file, 'b_remove_polygons_using_multitemporal_results')
# exit
if b_remove is None or b_remove is False:
basic.outputlogMessage(
'Warning, b_remove_polygons_using_multitemporal_results not set or is NO'
)
return True
shp_dir = args[0]
file_pattern = args[1]
polyon_shps_list = io_function.get_file_list_by_pattern(
shp_dir, file_pattern)
if len(polyon_shps_list) < 2:
raise ValueError(
'Error, less than two shapefiles, cannot conduct multi-polygon anlysis'
)
# make polyon_shps_list in order: I0 to In
polyon_shps_list.sort(
key=lambda x: int(re.findall('I\d+', os.path.basename(x))[0][1:]))
# print(polyon_shps_list)
# sys.exit()
# check projection of the shape file, should be the same
new_shp_proj4 = map_projection.get_raster_or_vector_srs_info_proj4(
polyon_shps_list[0])
for idx in range(len(polyon_shps_list) - 1):
shp_proj4 = map_projection.get_raster_or_vector_srs_info_proj4(
polyon_shps_list[idx + 1])
if shp_proj4 != new_shp_proj4:
raise ValueError('error, projection insistence between %s and %s' %
(new_shp_proj4, shp_proj4))
import remove_nonActive_thawSlumps
import polygons_change_analyze
# polygon change analysis
polygons_change_analyze.cal_multi_temporal_iou_and_occurrence(
polyon_shps_list, para_file)
# remove non active polygons
remove_nonActive_thawSlumps.remove_non_active_thaw_slumps(
polyon_shps_list, para_file)
# back up files and conduct evaluation
for idx, shp_path in enumerate(polyon_shps_list):
# evaluation files
shp_rmTimeiou = io_function.get_name_by_adding_tail(
shp_path, 'rmTimeiou')
basic.outputlogMessage('(%d/%d) evaluation of %s' %
(idx, len(polyon_shps_list), shp_rmTimeiou))
# evaluation
args_list = [
os.path.join(deeplabRS, 'evaluation_result.py'), '-p', para_file,
shp_rmTimeiou
]
if basic.exec_command_args_list_one_file(
args_list, 'evaluation_report.txt') is False:
return False
I_idx_str = re.findall('I\d+', os.path.basename(shp_rmTimeiou))
old_eva_report = io_function.get_file_list_by_pattern(
shp_dir, I_idx_str[0] + '*eva_report*' + '.txt')
old_eva_report = [
item for item in old_eva_report if 'rmTimeiou' not in item
]
old_eva_report_name = old_eva_report[0]
eva_report_name = io_function.get_name_by_adding_tail(
old_eva_report_name, 'rmTimeiou')
# io_function.move_file_to_dst(old_eva_report,backup_eva_report)
# io_function.move_file_to_dst('evaluation_report.txt', old_eva_report)
io_function.move_file_to_dst('evaluation_report.txt',
eva_report_name,
overwrite=True)
# back up the shape files (no_need)
basic.outputlogMessage(
'Finish removing polygons using multi-temporal mapping results')