def merge_polygon_for_demDiff_headwall_grids(dem_subsidence_shp,
headwall_shp_list,
output_dir,
buffer_size=50):
output = os.path.join(
output_dir,
os.path.basename(
io_function.get_name_by_adding_tail(dem_subsidence_shp,
'HW_select')))
if os.path.isfile(output):
print('warning, %s already exists' % output)
return output
# check projections?
subsidence_list = vector_gpd.read_polygons_gpd(dem_subsidence_shp)
headwall_all_list = []
for shp in headwall_shp_list:
headwall_list = vector_gpd.read_polygons_gpd(shp)
headwall_all_list.extend(headwall_list)
# # https://shapely.readthedocs.io/en/stable/manual.html#str-packed-r-tree
tree = STRtree(headwall_all_list)
subsidence_buff_list = [
item.buffer(buffer_size) for item in subsidence_list
]
select_idx = []
for idx, subsi_buff in enumerate(subsidence_buff_list):
adjacent_polygons = [
item for item in tree.query(subsi_buff)
if item.intersects(subsi_buff) or item.touches(subsi_buff)
]
if len(adjacent_polygons) > 0:
select_idx.append(idx)
basic.outputlogMessage('Select %d polygons from %d ones' %
(len(select_idx), len(subsidence_list)))
if len(select_idx) < 1:
return None
# save to subset of shaepfile
if vector_gpd.save_shapefile_subset_as(select_idx, dem_subsidence_shp,
output) is True:
return output
def read_polygons_from_small_patch(in_shp, in_raster):
'''read polygons and seperate to touch and not touch image edge groups'''
print(datetime.now(),
'reading polygons to touch and not touch image edge group')
polygons = vector_gpd.read_polygons_gpd(in_shp,
b_fix_invalid_polygon=False)
img_bound = raster_io.get_image_bound_box(in_raster)
img_resx, img_resy = raster_io.get_xres_yres_file(in_raster)
half_res = img_resx / 2.0
image_edge = vector_gpd.convert_image_bound_to_shapely_polygon(img_bound)
polygons_buff = [item.buffer(half_res)
for item in polygons] # buffer half pixel
# polygons_touch_img_edge_index = []
polygon_no_touch = []
polygons_touch = []
for idx, (polybuff, poly) in enumerate(zip(polygons_buff, polygons)):
if polybuff.within(image_edge):
polygon_no_touch.append(poly)
else:
# polygons_touch_img_edge_index.append(idx)
polygons_touch.append(poly)
# return polygons,polygons_touch_img_edge_index
return polygon_no_touch, polygons_touch
def calculate_distance_medial_axis(input_shp, out_shp, process_num=4, enlarge_m=20):
print('calculating polygon width based on medial axis')
code_dir = os.path.expanduser('~/codes/PycharmProjects/ChangeDet_DL/thawSlumpChangeDet')
sys.path.insert(0, code_dir)
# after test, found that when polygons are very narrow and irregular, cal_retreat_rate output wrong results.
# use buffer enlarge the polygons
polygons = vector_gpd.read_polygons_gpd(input_shp)
# for poly in polygons:
# if poly.geom_type == 'MultiPolygon':
# print(poly.geom_type,poly)
# cal_retreat_rate only use exterior, fill hole for buffer
# polygon_large = [ vector_gpd.fill_holes_in_a_polygon(item) for item in polygons]
polygon_large = polygons
# buffer
polygon_large = [item.buffer(enlarge_m) for item in polygon_large]
wkt = map_projection.get_raster_or_vector_srs_info_wkt(input_shp)
# save_large_shp = io_function.get_name_by_adding_tail(input_shp,'larger')
save_pd = pd.DataFrame({'Polygon':polygon_large})
vector_gpd.save_polygons_to_files(save_pd,'Polygon',wkt,out_shp)
# calculate width based on expanding areas
import cal_retreat_rate
if cal_retreat_rate.cal_expand_area_distance(out_shp, proc_num=process_num,save_medial_axis=True):
os.system('rm save_medial_axis_radius*.txt out_polygon_vertices_*.txt')
return out_shp
def remove_based_on_area(slope_bin_shp,min_area, max_area,wkt, rm_area_shp):
polygons = vector_gpd.read_polygons_gpd(slope_bin_shp,b_fix_invalid_polygon=False)
remain_polygons = []
# remove relative large but narrow ones.
remove_count = 0
for idx, poly in enumerate(polygons):
# remove quite large or too small ones
if poly.area > max_area or poly.area < min_area:
remove_count += 1
continue
remain_polygons.append(poly)
basic.outputlogMessage('remove %d polygons based on area, remain %d ones saving to %s' %
(remove_count, len(remain_polygons), rm_area_shp))
polyons_noMulti = [vector_gpd.MultiPolygon_to_polygons(idx, poly) for idx, poly in enumerate(remain_polygons)]
remain_polygons = []
for polys in polyons_noMulti:
polys = [poly for poly in polys if poly.area > min_area] # remove tiny polygon
remain_polygons.extend(polys)
print('convert MultiPolygon to polygons and remove tiny polgyons, remain %d' % (len(remain_polygons)))
if len(remain_polygons) < 1:
return False
save_pd = pd.DataFrame({'Polygon':remain_polygons})
vector_gpd.save_polygons_to_files(save_pd,'Polygon',wkt,rm_area_shp)
return rm_area_shp
def main(options, args):
extent_shp = args[0]
img_path = args[1]
save_dir = options.save_dir
#check projection
extent_prj = map_projection.get_raster_or_vector_srs_info_proj4(extent_shp)
img_prj = map_projection.get_raster_or_vector_srs_info_proj4(img_path)
if img_prj != extent_prj:
raise ValueError('Project of %s and %s is different' %
(extent_shp, img_path))
out_img = io_function.get_name_by_adding_tail(img_path, 'sub')
out_img = os.path.join(save_dir, os.path.basename(out_img))
extent_polys = vector_gpd.read_polygons_gpd(extent_shp)
if len(extent_polys) != 1:
raise ValueError('current only support one polygon')
for ext_poly in extent_polys:
subset_image_by_polygon_min(img_path,
out_img,
ext_poly,
resample_m='bilinear',
o_format='GTiff',
out_res=None)
pass
def main():
# grid polygons
grid_20km = os.path.join(shp_dir,'grid_20km.shp')
ArcticDEM_coverage = os.path.join(shp_dir,'tiles.shp')
# qtp_grid_50km and qtb_main_perma_area_simp have the same projection
grid_prj = map_projection.get_raster_or_vector_srs_info_proj4(grid_20km)
perma_area_prj = map_projection.get_raster_or_vector_srs_info_proj4(ArcticDEM_coverage)
if grid_prj != perma_area_prj:
raise ValueError('%s and %s do not have the same projection'%(grid_prj,perma_area_prj))
grids = vector_gpd.read_polygons_gpd(grid_20km)
DEM_areas = vector_gpd.read_polygons_gpd(ArcticDEM_coverage)
keep_grids = []
keep_grid_ids = []
id = 0
for idx, grid in enumerate(grids):
print(' processing %dth grid'%idx)
for dem_area in DEM_areas:
inte_res = dem_area.intersection(grid)
if inte_res.is_empty is False:
if inte_res.area < 100*100: # if it's too small, ignore it
continue
keep_grids.append(grid)
keep_grid_ids.append(id)
id += 1
# save
save_path = os.path.join(shp_dir,'ArcticDEM_grid_20km.shp')
save_polyons_attributes = {'id':keep_grid_ids, "Polygons":keep_grids}
# wkt_string = map_projection.get_raster_or_vector_srs_info_wkt(qtp_main_perma_area_simp)
wkt_string = map_projection.get_raster_or_vector_srs_info_proj4(grid_20km)
polygon_df = pd.DataFrame(save_polyons_attributes)
vector_gpd.save_polygons_to_files(polygon_df, 'Polygons', wkt_string, save_path)
pass
def rasterize_polygons(poly_path,
burn_value,
attribute_name,
xres,
yres,
save_path,
datatype='Byte'):
'''
:param poly_path:
:param burn_value:
:param attribute_name:
:param xres:
:param yres:
:param save_path:
:param datatype:
:return:
'''
import raster_io
import vector_gpd
import basic_src.map_projection as map_projection
if attribute_name is not None:
polygons, values = vector_gpd.read_polygons_attributes_list(
poly_path, attribute_name, b_fix_invalid_polygon=False)
burn_value = values
else:
polygons = vector_gpd.read_polygons_gpd(poly_path,
b_fix_invalid_polygon=False)
if datatype == 'Byte':
dtype = 'uint8'
elif datatype == 'UInt16':
dtype = 'uint16'
else:
dtype = 'int32'
extent = vector_gpd.get_vector_file_bounding_box(poly_path)
wkt_string = map_projection.get_raster_or_vector_srs_info_proj4(poly_path)
return raster_io.burn_polygons_to_a_raster(None,
polygons,
burn_value,
save_path,
dtype,
xres=xres,
yres=yres,
extent=extent,
ref_prj=wkt_string)
def read_extent_shapefile_epgs4326(exent_shp):
'''
read extent polygon, need in latlon projections
:param exent_shp:
:return:
'''
if exent_shp is None:
return None
shp_prj = map_projection.get_raster_or_vector_srs_info_proj4(exent_shp).strip()
if shp_prj != '+proj=longlat +datum=WGS84 +no_defs':
raise ValueError('only support the projections of longlat (EPSG:4326)')
extent_polygons = vector_gpd.read_polygons_gpd(exent_shp)
return extent_polygons
def test_build_adjacent_matrix_pygeo():
vector_path = os.path.expanduser(
'~/Data/dem_processing/grid_9053_tmp_files/20140701_dem_slope_bin_patches_all_polyTouchEdge.gpkg'
)
polygons = vector_gpd.read_polygons_gpd(vector_path,
b_fix_invalid_polygon=False)
print('read %d polygons' % len(polygons))
# polygons = [item.buffer(1.0) for item in polygons] # for complex polygons, buffer take a long time.
from pygeos import area, intersection, from_shapely, touches
import numpy as np
geo_polygons = from_shapely(polygons)
print(geo_polygons)
# inter_matrix = intersection(geo_polygons[:, np.newaxis], geo_polygons[np.newaxis, :])
# print(inter_matrix)
# print(touches(geo_polygons[:, np.newaxis], geo_polygons[np.newaxis, :]))
print(touches(geo_polygons, geo_polygons))
def main(options, args):
extent_shp = args[0]
dem_index_shp = args[1]
pre_name = os.path.splitext(os.path.basename(extent_shp))[0]
pre_name += '_Tile' if 'Tile' in os.path.basename(
dem_index_shp) else '_Strip'
xlsx_size_path = os.path.splitext(
os.path.basename(dem_index_shp))[0] + '_fileSize.xlsx'
print('xlsx file for saving file size', xlsx_size_path)
# extent polygons and projection (proj4)
extent_shp_prj = map_projection.get_raster_or_vector_srs_info_proj4(
extent_shp)
dem_shp_prj = map_projection.get_raster_or_vector_srs_info_proj4(
dem_index_shp)
if extent_shp_prj != dem_shp_prj:
basic.outputlogMessage(
'%s and %s do not have the same projection, will reproject %s' %
(extent_shp, dem_index_shp, os.path.basename(extent_shp)))
epsg = map_projection.get_raster_or_vector_srs_info_epsg(dem_index_shp)
# print(epsg)
# extent_polys = vector_gpd.read_shape_gpd_to_NewPrj(extent_shp,dem_shp_prj.strip())
extent_polys = vector_gpd.read_shape_gpd_to_NewPrj(extent_shp, epsg)
else:
extent_polys = vector_gpd.read_polygons_gpd(extent_shp)
# read 'grid_id' if the extent shp is from grid shp file, if not, grid_id_list will be None
grid_id_list = vector_gpd.read_attribute_values_list(extent_shp, 'grid_id')
if len(extent_polys) < 1:
raise ValueError('No polygons in %s' % extent_shp)
else:
basic.outputlogMessage('%d extent polygons in %s' %
(len(extent_polys), extent_shp))
get_file_size_dem_tarball(dem_index_shp,
extent_polys,
pre_name,
xlsx_size_path,
poly_ids=grid_id_list)
def remove_polygons_based_shapeinfo(in_shp, output_shp,area_limit,circularit_limit, holes_count ):
# remove polygons if they are larege (> area_limit) and narrow (<circularit_limit)
# if too many holse, may consider remove them as well.
# remove based on: INarea, INperimete,circularit,ratio_w_h, hole_count
if os.path.isfile(output_shp):
basic.outputlogMessage('%s exists, skip'%output_shp)
return output_shp
polygons = vector_gpd.read_polygons_gpd(in_shp)
# add some shape info
shape_info_list = [ vector_gpd.calculate_polygon_shape_info(item) for item in polygons]
shapeinfo_all_dict = vector_gpd.list_to_dict(shape_info_list)
vector_gpd.add_attributes_to_shp(in_shp,shapeinfo_all_dict)
shapefile = gpd.read_file(in_shp)
# remove relative large but narrow ones.
remove_count = 0
for idx, row in shapefile.iterrows():
shape_info = shape_info_list[idx]
# remove quite large but narrow ones
if shape_info['INarea'] > area_limit and shape_info['circularit'] < circularit_limit :
shapefile.drop(idx, inplace=True)
remove_count += 1
continue
# remove holes
if shape_info['hole_count'] > holes_count:
shapefile.drop(idx, inplace=True)
remove_count += 1
continue
basic.outputlogMessage('remove %d polygons based on shapeinfo, remain %d ones saving to %s' %
(remove_count, len(shapefile.geometry.values), output_shp))
if len(shapefile.geometry.values) < 1:
basic.outputlogMessage('After removing based on shapeinfo, no polygon to save')
return None
# save results
shapefile.to_file(output_shp, driver='ESRI Shapefile')
return output_shp
def post_processing_subsidence(in_shp):
polygons = vector_gpd.read_polygons_gpd(in_shp)
# get shapeinfo
# poly_shapeinfo_list = []
save_polyons = []
for poly in polygons:
# get INarea, INperimete, WIDTH, HEIGHT, ratio_w_h, hole_count
# shapeinfo = vector_gpd.calculate_polygon_shape_info(poly) # error: 'MultiPolygon' object has no attribute 'interiors'
# poly_shapeinfo_list.append(shapeinfo)
# if shapeinfo['INarea'] < 40: # remove the one with area smaller than 40 m^2
if poly.area < 90: # remove the one with area smaller than 40 m^2
continue
save_polyons.append(poly)
save_pd = pd.DataFrame({'Polygon': save_polyons})
wkt = map_projection.get_raster_or_vector_srs_info_wkt(in_shp)
save_shp = io_function.get_name_by_adding_tail(in_shp,'post')
vector_gpd.save_polygons_to_files(save_pd,'Polygon',wkt,save_shp)
def add_boxes_attributes(input, output, para_file=None,data_para_file=None):
if io_function.is_file_exist(input) is False:
return False
# copy output
if io_function.copy_shape_file(input, output) is False:
raise IOError('copy shape file %s failed'%input)
# calculate area, perimeter of polygons
if cal_add_area_length_of_polygon(output) is False:
return False
# calculate the width, height, and width_height ratio
boxes = vector_gpd.read_polygons_gpd(input)
bounding_boxes = [ vector_gpd.get_polygon_bounding_box(item) for item in boxes ]
# bounding box (minx, miny, maxx, maxy)
width_list = [ bound[2] - bound[0] for bound in bounding_boxes]
height_list = [ bound[3] - bound[1] for bound in bounding_boxes]
ratio_w_h_list = []
for w, h in zip(width_list, height_list):
if w > h:
ratio = h / w
else:
ratio = w / h
ratio_w_h_list.append(ratio)
add_attributes = {'WIDTH':width_list,'HEIGHT':height_list, 'ratio_w_h':ratio_w_h_list}
vector_gpd.add_attributes_to_shp(output,add_attributes)
# add topography of each box (consider the box as a polygon)
if data_para_file is not None and para_file is not None:
io_function.is_file_exist(data_para_file)
io_function.is_file_exist(para_file)
dem_files, slope_files, aspect_files, dem_diff_files = get_topographic_files(data_para_file)
process_num = parameters.get_digit_parameters(para_file,'process_num','int')
if calculate_polygon_topography(output,para_file,dem_files,slope_files,aspect_files=aspect_files,dem_diffs=dem_diff_files,
process_num=process_num) is False:
basic.outputlogMessage('Warning: calculate information of topography failed')
# return False # don't return
return output
def check_one_vector_file(idx,total,file_path, good_file_list):
if os.path.basename(file_path) in good_file_list:
return True
try:
print('checking %d/%d' % (idx, total))
# src = raster_io.open_raster_read(tif_path)
geometries = vector_gpd.read_polygons_gpd(file_path,b_fix_invalid_polygon=False)
if len(geometries) < 1:
basic.outputlogMessage('No geometries in %s'%file_path)
return False
geometries_list = [ item for item in geometries ]
if None in geometries_list:
basic.outputlogMessage('None geometries in %s'%file_path)
return False
except:
basic.outputlogMessage('incomplete vector: %s' % file_path)
return False
return True
def merge_polygon_rasterize(ref_raster, in_polygons):
# rasterize to raster
save_raster = os.path.basename(io_function.get_name_by_adding_tail(ref_raster,'merge'))
raster_io.burn_polygons_to_a_raster(ref_raster,in_polygons,1,save_raster)
# set nodata
if raster_io.set_nodata_to_raster_metadata(save_raster,0) is False:
raise IOError('Set nodata failed for %s'%save_raster)
# polygonize
out_shp = vector_gpd.raster2shapefile(save_raster, connect8=True)
if out_shp is None:
raise IOError('polygonzied failed for %s' % save_raster)
# read polygon
merge_polygons = vector_gpd.read_polygons_gpd(out_shp,b_fix_invalid_polygon=False)
print(timeTools.get_now_time_str(),'Get %d merged polygons'%len(merge_polygons))
return merge_polygons
def refine_dem_reduction_polygons(input_shp, min_area, max_area):
rm_min_max_area = os.path.basename(
io_function.get_name_by_adding_tail(input_shp, 'MinMaxArea'))
vector_gpd.remove_polygons_not_in_range(input_shp, 'poly_area', min_area,
max_area, rm_min_max_area)
# rm_min_max_area = input_shp
# add some shape info
shapefile = gpd.read_file(rm_min_max_area)
polygons = vector_gpd.read_polygons_gpd(rm_min_max_area)
shape_info_list = [
vector_gpd.calculate_polygon_shape_info(item) for item in polygons
]
shapeinfo_all_dict = list_to_dict(shape_info_list)
vector_gpd.add_attributes_to_shp(rm_min_max_area, shapeinfo_all_dict)
# INarea, INperimete, circularit, hole_count, WIDTH & HEIGHT (minimum_rotated_rectangle), ratio_w_h
# remove based on shape info
output = os.path.basename(
io_function.get_name_by_adding_tail(input_shp, 'final'))
remove_count = 0
for idx, row in shapefile.iterrows():
shape_info = shape_info_list[idx]
# remove quite large but narrow ones
if shape_info['INarea'] > 10000 and shape_info['circularit'] < 0.1:
shapefile.drop(idx, inplace=True)
remove_count += 1
basic.outputlogMessage(
'remove %d polygons based on shapeinfo, remain %d ones saving to %s' %
(remove_count, len(shapefile.geometry.values), output))
# save results
shapefile.to_file(output, driver='ESRI Shapefile')
def add_boxes_attributes(input, output):
if io_function.is_file_exist(input) is False:
return False
# copy output
if io_function.copy_shape_file(input, output) is False:
raise IOError('copy shape file %s failed' % input)
# calculate area, perimeter of polygons
if cal_add_area_length_of_polygon(output) is False:
return False
# calculate the width, height, and width_height ratio
boxes = vector_gpd.read_polygons_gpd(input)
bounding_boxes = [
vector_gpd.get_polygon_bounding_box(item) for item in boxes
]
# bounding box (minx, miny, maxx, maxy)
width_list = [bound[2] - bound[0] for bound in bounding_boxes]
height_list = [bound[3] - bound[1] for bound in bounding_boxes]
ratio_w_h_list = []
for w, h in zip(width_list, height_list):
if w > h:
ratio = h / w
else:
ratio = w / h
ratio_w_h_list.append(ratio)
add_attributes = {
'WIDTH': width_list,
'HEIGHT': height_list,
'ratio_w_h': ratio_w_h_list
}
vector_gpd.add_attributes_to_shp(output, add_attributes)
return output
def main(options, args):
save_dir = options.save_dir
extent_shp = options.extent_shp
process_num = options.process_num
dem_dir_or_txt = args[0]
if os.path.isfile(dem_dir_or_txt):
dem_list = io_function.read_list_from_txt(dem_dir_or_txt)
else:
dem_list = io_function.get_file_list_by_ext('.tif',
dem_dir_or_txt,
bsub_folder=False)
dem_count = len(dem_list)
if dem_count < 1:
raise ValueError('No input dem files in %s' % dem_dir_or_txt)
if extent_shp is not None:
pre_name = os.path.splitext(os.path.basename(extent_shp))[0]
else:
pre_name = os.path.basename(os.path.abspath(save_dir))
save_dem_diff = os.path.join(save_dir, pre_name + '_DEM_diff.tif')
save_date_diff = os.path.join(save_dir, pre_name + '_date_diff.tif')
if os.path.isfile(save_dem_diff) and os.path.isfile(save_date_diff):
print('%s and %s exists, skip' % (save_dem_diff, save_date_diff))
return
if extent_shp is not None:
# crop the DEM before differencing
extent_shp_base = os.path.splitext(os.path.basename(extent_shp))[0]
dem_prj = map_projection.get_raster_or_vector_srs_info_epsg(
dem_list[0])
extent_prj = map_projection.get_raster_or_vector_srs_info_epsg(
extent_shp)
if dem_prj != extent_prj:
raise ValueError(
'The projection of extent file (%s) and dem tifs is different'
% extent_shp)
extent_polys = vector_gpd.read_polygons_gpd(extent_shp)
if len(extent_polys) != 1:
raise ValueError('Only allow one polygon in %s' % extent_shp)
extPolys_ids = vector_gpd.read_attribute_values_list(extent_shp, 'id')
if extPolys_ids is None or None in extPolys_ids:
basic.outputlogMessage(
'Warning, field: id is not in %s, will create default ID for each grid'
% extent_shp)
extPolys_ids = [id + 1 for id in range(len(extent_polys))]
# crop
for idx, ext_poly in zip(extPolys_ids, extent_polys):
basic.outputlogMessage(
'crop and differnce DEM for the %d th extent (%d in total)' %
(idx, len(extent_polys)))
crop_dem_list = crop_to_same_exent_for_diff(
dem_list, save_dir, idx, ext_poly, process_num)
dem_list = crop_dem_list
dem_diff_newest_oldest(dem_list,
save_dem_diff,
save_date_diff,
process_num,
b_max_subsidence=options.max_subsidence)
def main(options, args):
extent_shp = args[0]
# ext_shp_prj = map_projection.get_raster_or_vector_srs_info_epsg(extent_shp)
# reproject if necessary, it seems that the gdalwarp can handle different projection
# if ext_shp_prj != 'EPSG:3413': # EPSG:3413 is the projection ArcticDEM used
# extent_shp_reprj = io_function.get_name_by_adding_tail(extent_shp,'3413')
# vector_gpd.reproject_shapefile(extent_shp,'EPSG:3413',extent_shp_reprj)
# extent_shp = extent_shp_reprj
tar_dir = options.ArcticDEM_dir
save_dir = options.save_dir
b_mosaic_id = options.create_mosaic_id
b_mosaic_date = options.create_mosaic_date
b_rm_inter = options.remove_inter_data
keep_dem_percent = options.keep_dem_percent
inter_format = options.format
arcticDEM_shp = options.arcticDEM_shp
o_res = options.out_res
b_dem_diff = options.create_dem_diff
dem_list_txt = options.dem_list_txt
# create mosaic is time consuming, but it also takes a lot memory. For a region of 50 km by 50 km, it may take 10 to 50 GB memory
process_num = options.process_num
basic.outputlogMessage(
'The number of processes for creating the mosaic is: %d' % process_num)
extent_shp_base = os.path.splitext(os.path.basename(extent_shp))[0]
extent_prj = map_projection.get_raster_or_vector_srs_info_epsg(extent_shp)
b_ArcticDEM_tar = False
dem_tif_list = []
if tar_dir is not None and arcticDEM_shp is not None:
b_ArcticDEM_tar = True
else:
dem_tif_list = io_function.read_list_from_txt(dem_list_txt)
# check projection
for dem_tif in dem_tif_list:
dem_prj = map_projection.get_raster_or_vector_srs_info_epsg(
dem_tif)
if dem_prj != extent_prj:
raise ValueError('The projection of %s is different from %s' %
(dem_prj, extent_prj))
b_ArcticDEM_tiles = False
if b_ArcticDEM_tar:
arcdem_prj = map_projection.get_raster_or_vector_srs_info_epsg(
arcticDEM_shp)
# read dem polygons and url
time0 = time.time()
dem_polygons, dem_urls = vector_gpd.read_polygons_attributes_list(
arcticDEM_shp, 'fileurl', b_fix_invalid_polygon=False)
print('time cost of read polygons and attributes', time.time() - time0)
basic.outputlogMessage('%d dem polygons in %s' %
(len(dem_polygons), extent_shp))
# get tarball list
tar_list = io_function.get_file_list_by_ext('.gz',
tar_dir,
bsub_folder=False)
if len(tar_list) < 1:
raise ValueError('No input tar.gz files in %s' % tar_dir)
if is_ArcticDEM_tiles(tar_list):
basic.outputlogMessage('Input is the mosaic version of ArcticDEM')
b_ArcticDEM_tiles = True
if extent_prj == arcdem_prj:
extent_polys = vector_gpd.read_polygons_gpd(extent_shp)
else:
extent_polys = vector_gpd.read_shape_gpd_to_NewPrj(
extent_shp, arcdem_prj)
else:
extent_polys = vector_gpd.read_polygons_gpd(extent_shp)
if len(extent_polys) < 1:
raise ValueError('No polygons in %s' % extent_shp)
else:
basic.outputlogMessage('%d extent polygons in %s' %
(len(extent_polys), extent_shp))
extPolys_ids = vector_gpd.read_attribute_values_list(extent_shp, 'id')
if extPolys_ids is None or None in extPolys_ids:
basic.outputlogMessage(
'Warning, field: id is not in %s, will create default ID for each grid'
% extent_shp)
extPolys_ids = [id + 1 for id in range(len(extent_polys))]
# print('extPolys_ids, count',extPolys_ids, len(extent_polys))
same_extent = False
if b_dem_diff:
# crop each one to the same extent, easy for DEM differnce.
same_extent = True
for idx, ext_poly in zip(extPolys_ids, extent_polys):
basic.outputlogMessage('get data for the %d th extent (%d in total)' %
(idx, len(extent_polys)))
if b_ArcticDEM_tar:
if b_ArcticDEM_tiles:
proc_ArcticDEM_tile_one_grid_polygon(tar_dir, dem_polygons,
dem_urls, o_res, save_dir,
inter_format, b_rm_inter,
ext_poly, idx,
extent_shp_base)
else:
proc_ArcticDEM_strip_one_grid_polygon(
tar_dir,
dem_polygons,
dem_urls,
o_res,
save_dir,
inter_format,
b_mosaic_id,
b_mosaic_date,
b_rm_inter,
b_dem_diff,
ext_poly,
idx,
keep_dem_percent,
process_num,
extent_shp_base,
resample_method='average',
same_extent=same_extent)
else:
proc_dem_mosaic_diff(dem_tif_list,
save_dir,
idx,
ext_poly,
b_mosaic_id,
b_mosaic_date,
process_num,
keep_dem_percent,
o_res,
b_dem_diff,
extent_shp_base,
b_rm_inter,
resample_method='average')
# extent polygons and projection (proj4)
extent_shp_prj = map_projection.get_raster_or_vector_srs_info_proj4(
extent_shp_or_ids_txt)
grid_shp_prj = map_projection.get_raster_or_vector_srs_info_proj4(
grid_20_shp)
if extent_shp_prj != grid_shp_prj:
basic.outputlogMessage(
'%s and %s do not have the same projection, will reproject %s' %
(extent_shp_or_ids_txt, grid_20_shp,
os.path.basename(extent_shp_or_ids_txt)))
epsg = map_projection.get_raster_or_vector_srs_info_epsg(grid_20_shp)
extent_polys = vector_gpd.read_shape_gpd_to_NewPrj(
extent_shp_or_ids_txt, epsg)
else:
extent_polys = vector_gpd.read_polygons_gpd(extent_shp_or_ids_txt)
crop_mosaic_reproject_dem_diff(grid_dem_tifs,
pre_name,
extent_poly,
o_res,
new_prj,
b_mosaic=False)
def main(options, args):
pass
if __name__ == '__main__':
usage = "usage: %prog [options] extent_shp or grid_id_list.txt "
def zonal_stats_multiRasters(in_shp,
raster_file_or_files,
nodata=None,
band=1,
stats=None,
prefix='',
range=None,
all_touched=True,
process_num=1):
'''
zonal statistic based on vectors, along multiple rasters (image tiles)
Args:
in_shp: input vector file
raster_file_or_files: a raster file or multiple rasters
nodata:
band: band
stats: like [mean, std, max, min]
range: interested values [min, max], None means infinity
all_touched:
process_num: process number for calculation
Returns:
'''
io_function.is_file_exist(in_shp)
if stats is None:
basic.outputlogMessage('warning, No input stats, set to ["mean"])')
stats = ['mean']
if isinstance(raster_file_or_files, str):
io_function.is_file_exist(raster_file_or_files)
image_tiles = [raster_file_or_files]
elif isinstance(raster_file_or_files, list):
image_tiles = raster_file_or_files
else:
raise ValueError('unsupport type for %s' % str(raster_file_or_files))
# check projection (assume we have the same projection), check them outside this function
# get image box
img_tile_boxes = [
raster_io.get_image_bound_box(tile) for tile in image_tiles
]
img_tile_polygons = [
vector_gpd.convert_image_bound_to_shapely_polygon(box)
for box in img_tile_boxes
]
polygons = vector_gpd.read_polygons_gpd(in_shp)
if len(polygons) < 1:
basic.outputlogMessage('No polygons in %s' % in_shp)
return False
# polygons_json = [mapping(item) for item in polygons] # no need when use new verion of rasterio
# process polygons one by one polygons and the corresponding image tiles (parallel and save memory)
# also to avoid error: daemonic processes are not allowed to have children
if process_num == 1:
stats_res_list = []
for idx, polygon in enumerate(polygons):
out_stats = zonal_stats_one_polygon(idx,
polygon,
image_tiles,
img_tile_polygons,
stats,
nodata=nodata,
range=range,
band=band,
all_touched=all_touched)
stats_res_list.append(out_stats)
elif process_num > 1:
threadpool = Pool(process_num)
para_list = [(idx, polygon, image_tiles, img_tile_polygons, stats,
nodata, range, band, all_touched)
for idx, polygon in enumerate(polygons)]
stats_res_list = threadpool.starmap(zonal_stats_one_polygon, para_list)
else:
raise ValueError('Wrong process number: %s ' % str(process_num))
# save to shapefile
add_attributes = {}
new_key_list = [prefix + '_' + key for key in stats_res_list[0].keys()]
for new_ley in new_key_list:
add_attributes[new_ley] = []
for stats_result in stats_res_list:
for key in stats_result.keys():
add_attributes[prefix + '_' + key].append(stats_result[key])
vector_gpd.add_attributes_to_shp(in_shp, add_attributes)
pass
def main(options, args):
extent_shp = args[0]
dem_index_shp = args[1]
b_arcticDEM_tile = False
global max_task_count
max_task_count = options.max_process_num
if 'Tile' in os.path.basename(dem_index_shp):
save_folder = arcticDEM_tile_tarball_dir
reg_tif_dir = arcticDEM_tile_reg_tif_dir
b_arcticDEM_tile = True
else:
save_folder = tarball_dir
reg_tif_dir = arcticDEM_reg_tif_dir
# use the user specific save_dir for saving downloaded tarballs
if options.save_dir is not None:
save_folder = options.save_dir
if os.path.isdir(save_folder) is False:
io_function.mkdir(save_folder)
save_folder = os.path.abspath(save_folder) # change to absolute path
pre_name = os.path.splitext(os.path.basename(extent_shp))[0]
pre_name += '_Tile' if 'Tile' in os.path.basename(
dem_index_shp) else '_Strip'
# extent polygons and projection (proj4)
extent_shp_prj = map_projection.get_raster_or_vector_srs_info_proj4(
extent_shp)
dem_shp_prj = map_projection.get_raster_or_vector_srs_info_proj4(
dem_index_shp)
if extent_shp_prj != dem_shp_prj:
basic.outputlogMessage(
'%s and %s do not have the same projection, will reproject %s' %
(extent_shp, dem_index_shp, os.path.basename(extent_shp)))
epsg = map_projection.get_raster_or_vector_srs_info_epsg(dem_index_shp)
# print(epsg)
# extent_polys = vector_gpd.read_shape_gpd_to_NewPrj(extent_shp,dem_shp_prj.strip())
extent_polys = vector_gpd.read_shape_gpd_to_NewPrj(extent_shp, epsg)
else:
extent_polys = vector_gpd.read_polygons_gpd(extent_shp)
# read 'grid_id' if the extent shp is from grid shp file, if not, grid_id_list will be None
grid_id_list = vector_gpd.read_attribute_values_list(extent_shp, 'grid_id')
if len(extent_polys) < 1:
raise ValueError('No polygons in %s' % extent_shp)
else:
basic.outputlogMessage(
'read %d extent polygons in %s for downloading and proc' %
(len(extent_polys), extent_shp))
download_dem_tarball(dem_index_shp,
extent_polys,
save_folder,
pre_name,
reg_tif_dir=reg_tif_dir,
poly_ids=grid_id_list,
b_arcticDEM_tile=b_arcticDEM_tile)
def get_extent_xy_from_shp(input_shp):
polygons = vector_gpd.read_polygons_gpd(input_shp)
if len(polygons) != 1:
raise ValueError('The extent only support one polygon')
x, y = vector_gpd.get_polygon_envelope_xy(polygons[0])
return x, y
def zonal_stats_multiRasters(in_shp, raster_file_or_files, nodata=None, band = 1, stats = None, prefix='',
range=None,buffer=None, all_touched=True, process_num=1):
'''
zonal statistic based on vectors, along multiple rasters (image tiles)
Args:
in_shp: input vector file
raster_file_or_files: a raster file or multiple rasters
nodata:
band: band
stats: like [mean, std, max, min]
range: interested values [min, max], None means infinity
buffer: expand polygon with buffer (meter) before the statistic
all_touched:
process_num: process number for calculation
Returns:
'''
io_function.is_file_exist(in_shp)
if stats is None:
basic.outputlogMessage('warning, No input stats, set to ["mean"])')
stats = ['mean']
stats_backup = stats.copy()
if 'area' in stats:
stats.remove('area')
if 'count' not in stats:
stats.append('count')
if isinstance(raster_file_or_files,str):
io_function.is_file_exist(raster_file_or_files)
image_tiles = [raster_file_or_files]
elif isinstance(raster_file_or_files,list):
image_tiles = raster_file_or_files
else:
raise ValueError('unsupport type for %s'%str(raster_file_or_files))
# check projection (assume we have the same projection), check them outside this function
# get image box
img_tile_boxes = [raster_io.get_image_bound_box(tile) for tile in image_tiles]
img_tile_polygons = [vector_gpd.convert_image_bound_to_shapely_polygon(box) for box in img_tile_boxes]
polygons = vector_gpd.read_polygons_gpd(in_shp)
if len(polygons) < 1:
basic.outputlogMessage('No polygons in %s'%in_shp)
return False
# polygons_json = [mapping(item) for item in polygons] # no need when use new verion of rasterio
if buffer is not None:
polygons = [ poly.buffer(buffer) for poly in polygons]
# process polygons one by one polygons and the corresponding image tiles (parallel and save memory)
stats_res_list = []
for idx, polygon in enumerate(polygons):
out_stats = zonal_stats_one_polygon(idx, polygon, image_tiles, img_tile_polygons, stats, nodata=nodata, range=range,
band=band, all_touched=all_touched)
stats_res_list.append(out_stats)
# threadpool = Pool(process_num)
# para_list = [ (idx, polygon, image_tiles, img_tile_polygons, stats, nodata, range,band, all_touched)
# for idx, polygon in enumerate(polygons)]
# stats_res_list = threadpool.starmap(zonal_stats_one_polygon,para_list)
# save to shapefile
add_attributes = {}
new_key_list = [ prefix + '_' + key for key in stats_res_list[0].keys()]
for new_ley in new_key_list:
add_attributes[new_ley] = []
for stats_result in stats_res_list:
for key in stats_result.keys():
add_attributes[prefix + '_' + key].append(stats_result[key])
if 'area' in stats_backup:
dx, dy = raster_io.get_xres_yres_file(image_tiles[0])
add_attributes[prefix + '_' + 'area'] = [ count*dx*dy for count in add_attributes[prefix + '_' + 'count'] ]
if 'count' not in stats_backup:
del add_attributes[prefix + '_' + 'count']
vector_gpd.add_attributes_to_shp(in_shp,add_attributes)
pass
def get_grid_20(extent_shp_or_id_txt, grid_polys, ids):
'''
get grid polygons and ids based on input extent (polygon in shpaefile) or ids (txt file)
if "file_name_base+'_grid_ids.txt'" exists, it will read id in this file directly.
:param extent_shp_or_id_txt:
:param grid_polys:
:param ids:
:return:
'''
io_function.is_file_exist(extent_shp_or_id_txt)
if extent_shp_or_id_txt.endswith('.txt'):
grid_ids = io_function.read_list_from_txt(extent_shp_or_id_txt)
grid_ids = [int(item) for item in grid_ids ]
else:
shp_corresponding_grid_ids_txt = get_corresponding_grid_ids_txt(extent_shp_or_id_txt)
if os.path.isfile(shp_corresponding_grid_ids_txt):
print('corresponding grid ids txt file for %s exists, read grid id from txt'%extent_shp_or_id_txt)
grid_ids = [ int(item) for item in io_function.read_list_from_txt(shp_corresponding_grid_ids_txt)]
basic.outputlogMessage('read %d grids within the extents (%s)'
% (len(grid_ids), os.path.basename(extent_shp_or_id_txt)))
else:
# extent polygons and projection (proj4)
extent_shp_prj = map_projection.get_raster_or_vector_srs_info_proj4(extent_shp_or_id_txt)
if extent_shp_prj == '':
raise ValueError('get proj4 of %s failed'%extent_shp_or_id_txt)
grid_shp_prj = map_projection.get_raster_or_vector_srs_info_proj4(grid_20_shp)
if grid_shp_prj=='':
raise ValueError('get proj4 of %s failed' % grid_20_shp)
if extent_shp_prj != grid_shp_prj:
basic.outputlogMessage('%s and %s do not have the same projection, will reproject %s'
% (extent_shp_or_id_txt, grid_20_shp, os.path.basename(extent_shp_or_id_txt)))
epsg = map_projection.get_raster_or_vector_srs_info_epsg(grid_20_shp)
# print(epsg)
# extent_polys = vector_gpd.read_shape_gpd_to_NewPrj(extent_shp,dem_shp_prj.strip())
extent_polys = vector_gpd.read_shape_gpd_to_NewPrj(extent_shp_or_id_txt, epsg)
else:
extent_polys = vector_gpd.read_polygons_gpd(extent_shp_or_id_txt)
ext_poly_count = len(extent_polys)
if ext_poly_count < 1:
raise ValueError('No polygons in %s'%extent_shp_or_id_txt)
grid_index = []
# if there are many polygons, this will take time.
for idx,ext_poly in enumerate(extent_polys):
print(timeTools.get_now_time_str(), 'get grids for extent idx', idx, 'total polygons:',ext_poly_count)
index = vector_gpd.get_poly_index_within_extent(grid_polys, ext_poly)
grid_index.extend(index)
grid_index = list(set(grid_index)) # remove duplicated ids
basic.outputlogMessage('find %d grids within the extents (%s)' % (len(grid_index), os.path.basename(extent_shp_or_id_txt)) )
grid_ids = [ ids[idx] for idx in grid_index]
grid_ids_str = [str(item) for item in grid_ids ]
io_function.save_list_to_txt(shp_corresponding_grid_ids_txt,grid_ids_str)
id_index = [ids.index(id) for id in grid_ids]
selected_grid_polys = [grid_polys[idx] for idx in id_index ]
return selected_grid_polys, grid_ids
def get_dem_subscidence_polygons(in_shp, dem_diff_tif, dem_diff_thread_m=-0.5, min_area=40, max_area=100000000, process_num=1,
b_rm_files=False):
save_shp = io_function.get_name_by_adding_tail(in_shp, 'post')
if os.path.isfile(save_shp):
basic.outputlogMessage('%s exists, skip'%save_shp)
return save_shp
demD_height, demD_width, demD_band_num, demD_date_type = raster_io.get_height_width_bandnum_dtype(dem_diff_tif)
# print(demD_date_type)
# # read mean elevation difference
# attributes_path = os.path.join(os.path.dirname(in_shp), shp_pre + '_attributes.txt')
#
# # for each seg lable [mean, std, pixel count], if dem_diff_tif is float 32, then in meters, if int16, then in centimeter
# poly_attributes = io_function.read_dict_from_txt_json(attributes_path)
# if int16, then it's in centimeter
if demD_date_type == 'int16':
dem_diff_thread_m = dem_diff_thread_m*100
# merge polygons touch each others
wkt = map_projection.get_raster_or_vector_srs_info_wkt(in_shp)
merged_shp = io_function.get_name_by_adding_tail(in_shp, 'merged')
if filter_merge_polygons(in_shp,merged_shp,wkt, min_area,max_area,dem_diff_tif,dem_diff_thread_m,process_num) is None:
return None
# in merge_polygons, it will remove some big polygons, convert MultiPolygon to Polygons, so neeed to update remain_polyons
remain_polyons = vector_gpd.read_polygons_gpd(merged_shp)
# check MultiPolygons again.
polyons_noMulti = [vector_gpd.MultiPolygon_to_polygons(idx, poly) for idx, poly in enumerate(remain_polyons)]
remain_polyons = []
for polys in polyons_noMulti:
polys = [poly for poly in polys if poly.area > min_area] # remove tiny polygon before buffer
remain_polyons.extend(polys)
print('convert MultiPolygon to polygons and remove small ones, remain %d' % (len(remain_polyons)))
# based on the merged polygons, surrounding polygons
buffer_surrounding = 20 # meters
surrounding_shp = io_function.get_name_by_adding_tail(in_shp, 'surrounding')
get_surrounding_polygons(remain_polyons, surrounding_shp, wkt, dem_diff_tif, buffer_surrounding, process_num)
rm_reldemD_shp = io_function.get_name_by_adding_tail(in_shp, 'rmreldemD')
if remove_polygons_based_relative_dem_diff(remain_polyons, merged_shp, surrounding_shp, wkt, rm_reldemD_shp, min_area,dem_diff_thread_m) is None:
return None
rm_shapeinfo_shp = io_function.get_name_by_adding_tail(in_shp, 'rmshapeinfo')
area_limit = 10000
circularit_limit = 0.1
holes_count = 20
if remove_polygons_based_shapeinfo(rm_reldemD_shp, rm_shapeinfo_shp, area_limit, circularit_limit, holes_count) is None:
return None
# remove based on slope
# use the slope derived from ArcitcDEM mosaic
slope_tif_list = io_function.get_file_list_by_ext('.tif',dem_common.arcticDEM_tile_slope_dir,bsub_folder=False)
basic.outputlogMessage('Find %d slope files in %s'%(len(slope_tif_list), dem_common.arcticDEM_tile_slope_dir))
rm_slope_shp = io_function.get_name_by_adding_tail(in_shp, 'rmslope')
max_slope = 20
if remove_based_slope(rm_shapeinfo_shp, rm_slope_shp,slope_tif_list, max_slope,process_num) is False:
return None
# copy
io_function.copy_shape_file(rm_slope_shp,save_shp)
# add date difference if they are available
date_diff_base = os.path.basename(dem_diff_tif).replace('DEM_diff','date_diff')
date_diff_tif = os.path.join(os.path.dirname(dem_diff_tif) , date_diff_base)
if os.path.isfile(date_diff_tif):
raster_statistic.zonal_stats_multiRasters(save_shp, date_diff_tif,tile_min_overlap=tile_min_overlap,
stats=['mean', 'std'], prefix='dateD',process_num=process_num)
return save_shp
def main(options, args):
save_dir = options.save_dir
extent_shp = options.extent_shp
process_num = options.process_num
o_res = options.out_res
b_mosaic_id = options.create_mosaic_id
b_mosaic_date = options.create_mosaic_date
keep_dem_percent = options.keep_dem_percent
dem_dir_or_txt = args[0]
if os.path.isfile(dem_dir_or_txt):
dem_list = io_function.read_list_from_txt(dem_dir_or_txt)
else:
dem_list = io_function.get_file_list_by_ext('.tif', dem_dir_or_txt, bsub_folder=False)
dem_list = [ tif for tif in dem_list if 'matchtag' not in tif ] # remove matchtag
dem_count = len(dem_list)
if dem_count < 1:
raise ValueError('No input dem files in %s' % dem_dir_or_txt)
resample_method= 'average'
if extent_shp is None:
# groups DEM based on the same strip ID
dem_groups = group_demTif_strip_pair_ID(dem_list)
# mosaic them direclty without consider the extent
mosaic_dir = os.path.join(save_dir, 'dem_stripID_mosaic' )
mosaic_dem_same_stripID(dem_groups, mosaic_dir, resample_method, process_num=process_num, save_source=True,
o_format='GTiff')
else:
extent_shp_base = os.path.splitext(os.path.basename(extent_shp))[0]
dem_prj = map_projection.get_raster_or_vector_srs_info_epsg(dem_list[0])
extent_prj = map_projection.get_raster_or_vector_srs_info_epsg(extent_shp)
# # check projection (time-consuming if there are many tif files)
# for dem_tif in dem_list:
# prj = map_projection.get_raster_or_vector_srs_info_epsg(dem_tif)
# if dem_prj != prj:
# raise ValueError('The projection inconsistent among dems (%s is different)'%dem_tif)
dem_ext_polys = get_dem_tif_ext_polygons(dem_list)
if extent_prj==dem_prj:
extent_polys = vector_gpd.read_polygons_gpd(extent_shp)
else:
extent_polys = vector_gpd.read_shape_gpd_to_NewPrj(extent_shp,dem_prj)
if len(extent_polys) < 1:
raise ValueError('No polygons in %s' % extent_shp)
else:
basic.outputlogMessage('%d extent polygons in %s' % (len(extent_polys), extent_shp))
extPolys_ids = vector_gpd.read_attribute_values_list(extent_shp, 'id')
if extPolys_ids is None or None in extPolys_ids:
basic.outputlogMessage('Warning, field: id is not in %s, will create default ID for each grid' % extent_shp)
extPolys_ids = [id + 1 for id in range(len(extent_polys))]
for idx, ext_poly in zip(extPolys_ids, extent_polys):
basic.outputlogMessage('mosaic and crop DEM for the %d th extent (%d in total)' % (idx, len(extent_polys)))
# get subset of DEM
dem_poly_ids = vector_gpd.get_poly_index_within_extent(dem_ext_polys, ext_poly)
if len(dem_poly_ids) < 1:
basic.outputlogMessage('no dem tifs within %d polygons'%idx)
continue
dem_list_sub = [dem_list[id] for id in dem_poly_ids]
mosaic_crop_dem(dem_list_sub, save_dir, idx, ext_poly, b_mosaic_id, b_mosaic_date,
process_num, keep_dem_percent, o_res, extent_shp_base, resample_method='average')
def main(options, args):
time0 = time.time()
image_dir = args[0]
geojson_list = io_function.get_file_list_by_ext('.geojson',
image_dir,
bsub_folder=False)
# remove some scenes, or maybe we should set bsub_folder=False
# geojson_list = [item for item in geojson_list if 'incomplete_scenes' not in item ] # remove those in "incomplete_scenes"
# geojson_list = [item for item in geojson_list if 'scenes_high_cloud_cover' not in item ] # remove those in "scenes_high_cloud_cover"
if len(geojson_list) < 1:
raise ValueError('There is no geojson files in %s' % image_dir)
basic.outputlogMessage('Image Dir: %s' % image_dir)
basic.outputlogMessage("Number of geojson files: %d" % len(geojson_list))
grid_polygon_shp = args[
1] # the polygon should be in projection Cartesian coordinate system (e.g., UTM )
basic.outputlogMessage('Image grid polygon shapefile: %s' %
grid_polygon_shp)
process_num = options.process_num
basic.outputlogMessage(
'The number of processes for creating the mosaic is: %d' % process_num)
# read grid polygons
grid_polygons = vector_gpd.read_polygons_gpd(grid_polygon_shp)
grid_ids = vector_gpd.read_attribute_values_list(grid_polygon_shp, 'id')
if grid_ids is None:
basic.outputlogMessage(
'Warning, field: id is not in %s, will create default ID for each grid'
% grid_polygon_shp)
grid_ids = [id + 1 for id in range(len(grid_polygons))]
shp_prj = map_projection.get_raster_or_vector_srs_info_proj4(
grid_polygon_shp).strip()
# print(shp_prj)
grid_polygons_latlon = grid_polygons
if shp_prj != '+proj=longlat +datum=WGS84 +no_defs':
# read polygons and reproject to 4326 projection
grid_polygons_latlon = vector_gpd.read_shape_gpd_to_NewPrj(
grid_polygon_shp, 'EPSG:4326')
# else:
# raise ValueError(' %s should be in projection of Cartesian coordinate system'%grid_polygon_shp)
shp_prj_wkt = map_projection.get_raster_or_vector_srs_info_wkt(
grid_polygon_shp)
max_sr = options.max_sr
min_sr = options.min_sr
original_img_copy_dir = options.original_img_copy_dir
b_to_rgb_8bit = options.to_rgb
basic.outputlogMessage('Convert to 8bit RGB images: %s' %
str(b_to_rgb_8bit))
# group planet image based on acquisition date
b_group_date = options.group_date
basic.outputlogMessage('Group Planet image based on acquisition date: %s' %
str(b_group_date))
if b_group_date:
# diff_days as 0, group images acquired at the same date
geojson_groups = group_planet_images_date(geojson_list, diff_days=0)
# sort based on yeardate in accending order : operator.itemgetter(0)
geojson_groups = dict(
sorted(geojson_groups.items(), key=operator.itemgetter(0)))
save_group_txt = 'geojson_groups_input_folder.txt'
basic.outputlogMessage(
'images are divided into %d groups, save to %s' %
(len(geojson_groups.keys()), save_group_txt))
io_function.save_dict_to_txt_json(save_group_txt, geojson_groups)
else:
geojson_groups = {'all': geojson_list}
# create mosaic of each grid
cloud_cover_thr = options.cloud_cover
cloud_cover_thr = cloud_cover_thr * 100 # for Planet image, it is percentage
out_res = options.out_res
cur_dir = os.getcwd()
resampling_method = options.merged_method
for key in geojson_groups.keys():
# # test
# if key != '20200701':
# continue
geojson_list = geojson_groups[key]
save_dir = os.path.basename(cur_dir) + '_mosaic_' + str(
out_res) + '_' + key
# print(save_dir)
if process_num == 1:
for id, polygon, poly_latlon in zip(grid_ids, grid_polygons,
grid_polygons_latlon):
# if id != 34:
# continue
create_moasic_of_each_grid_polygon(
id,
polygon,
poly_latlon,
out_res,
cloud_cover_thr,
geojson_list,
save_dir,
new_prj_wkt=shp_prj_wkt,
new_prj_proj4=shp_prj,
sr_min=min_sr,
sr_max=max_sr,
to_rgb=b_to_rgb_8bit,
save_org_dir=original_img_copy_dir,
resampling_method=resampling_method)
elif process_num > 1:
theadPool = Pool(process_num) # multi processes
parameters_list = [
(id, polygon, poly_latlon, out_res, cloud_cover_thr,
geojson_list, save_dir, shp_prj_wkt, shp_prj, min_sr, max_sr,
b_to_rgb_8bit, 0, original_img_copy_dir)
for id, polygon, poly_latlon in zip(grid_ids, grid_polygons,
grid_polygons_latlon)
]
results = theadPool.starmap(create_moasic_of_each_grid_polygon,
parameters_list) # need python3
theadPool.close()
else:
raise ValueError('incorrect process number: %d' % process_num)
cost_time_sec = time.time() - time0
basic.outputlogMessage(
'Done, total time cost %.2f seconds (%.2f minutes or %.2f hours)' %
(cost_time_sec, cost_time_sec / 60, cost_time_sec / 3600))
pass
def main():
# grid polygons
grid_50km = os.path.join(shp_dir,'PAMPA_outline_utm_50grid.shp')
# main permaforst areas based on permafrost map, has been pre-processed: remove small ones, simply the boundaries
main_area_simp = os.path.join(shp_dir,'PAMPA_outline_utm.shp')
# qtp_grid_50km and qtb_main_perma_area_simp have the same projection
grid_prj = map_projection.get_raster_or_vector_srs_info_proj4(grid_50km)
perma_area_prj = map_projection.get_raster_or_vector_srs_info_proj4(main_area_simp)
if grid_prj != perma_area_prj:
raise ValueError('%s and %s do not have the same projection'%(grid_prj,perma_area_prj))
grids = vector_gpd.read_polygons_gpd(grid_50km)
perma_areas = vector_gpd.read_polygons_gpd(main_area_simp)
# perma_size_list = vector_gpd.read_attribute_values_list(qtp_main_perma_area_simp,'Area_km2')
small_perma_areas_list = []
for idx, perma_poly in enumerate(perma_areas):
print(' processing %dth permafrost area'%idx)
# if the permafrost area is < 50*50 km^2, then do not split it to smaller ones.
# if size < 2500:
# perma_poly = vector_gpd.fill_holes_in_a_polygon(perma_poly)
# small_perma_areas_list.append(perma_poly)
# continue
# split the big permafrost area into many small ones
for grid in grids:
inte_res = perma_poly.intersection(grid)
if inte_res.is_empty is False:
inte_res_multi = vector_gpd.MultiPolygon_to_polygons(idx,inte_res)
for tmp in inte_res_multi:
# remove holes if they exist
small_ones = vector_gpd.fill_holes_in_a_polygon(tmp)
#################################
# we should remove some really small polygons (< 1 km^2)
small_perma_areas_list.append(small_ones)
##############################
# have to manually merge small polygons in QGIS to its adjacent ones.
# save
save_path = io_function.get_name_by_adding_tail(main_area_simp,'small')
save_path = os.path.join(shp_dir,os.path.basename(save_path))
save_polyons_attributes = {}
save_polyons_attributes["Polygons"] = small_perma_areas_list
# wkt_string = map_projection.get_raster_or_vector_srs_info_wkt(qtp_main_perma_area_simp)
wkt_string = map_projection.get_raster_or_vector_srs_info_proj4(main_area_simp)
polygon_df = pd.DataFrame(save_polyons_attributes)
vector_gpd.save_polygons_to_files(polygon_df, 'Polygons', wkt_string, save_path)
pass
def main():
dem_index_shp = os.path.expanduser(
'~/Data/Arctic/ArcticDEM/BROWSE_SERVER/indexes/ArcticDEM_Tile_Index_Rel7/ArcticDEM_Tile_Index_Rel7.shp'
)
# extent_shp = os.path.expanduser('~/Data/PDO/PDO_statistics_swatchs/swatch_bounding_boxes.shp')
extent_shp = os.path.expanduser(
'~/Data/PDO/extent_each_swatch/merge_all_qa_exent.shp')
# extent polygons and projection (proj4)
extent_shp_prj = map_projection.get_raster_or_vector_srs_info_proj4(
extent_shp)
dem_shp_prj = map_projection.get_raster_or_vector_srs_info_proj4(
dem_index_shp)
if extent_shp_prj != dem_shp_prj:
basic.outputlogMessage(
'%s and %s do not have the same projection, will reproject %s' %
(extent_shp, dem_index_shp, os.path.basename(extent_shp)))
epsg = map_projection.get_raster_or_vector_srs_info_epsg(dem_index_shp)
# print(epsg)
# extent_polys = vector_gpd.read_shape_gpd_to_NewPrj(extent_shp,dem_shp_prj.strip())
extent_polys = vector_gpd.read_shape_gpd_to_NewPrj(extent_shp, epsg)
else:
extent_polys = vector_gpd.read_polygons_gpd(extent_shp)
poly_ids = [idx for idx in range(len(extent_polys))]
if 'boxes' in os.path.basename(extent_shp):
nc_file_names = vector_gpd.read_attribute_values_list(
extent_shp, 'nc_file')
else:
nc_file_names = vector_gpd.read_attribute_values_list(
extent_shp, 'layer')
# read dem polygons and tile number
dem_polygons, dem_tiles = vector_gpd.read_polygons_attributes_list(
dem_index_shp, 'tile', b_fix_invalid_polygon=False)
for count, (idx, ext_poly) in enumerate(zip(poly_ids, extent_polys)):
basic.outputlogMessage('get data for the %d th extent (%d/%d)' %
(idx, count, len(extent_polys)))
save_txt_path = nc_file_names[idx] + '-' + 'dem_tiles_poly_%d.txt' % idx
if os.path.isfile(save_txt_path):
tiles = io_function.read_list_from_txt(save_txt_path)
basic.outputlogMessage('read %d dem tiles from %s' %
(len(tiles), save_txt_path))
else:
# get fileurl
dem_poly_ids = vector_gpd.get_poly_index_within_extent(
dem_polygons, ext_poly)
basic.outputlogMessage('find %d DEM within %d th extent' %
(len(dem_poly_ids), (idx)))
tiles = [dem_tiles[id] for id in dem_poly_ids]
# save to txt
io_function.save_list_to_txt(save_txt_path, tiles)
basic.outputlogMessage('save dem urls to %s' % save_txt_path)
# download and create a mosaic
url_head = 'https://data.pgc.umn.edu/elev/dem/setsm/ArcticDEM/mosaic/v3.0/32m/'
download_tarball_for_one_polygon(tarball_dir, dem_tif_dir, url_head,
tiles)
# create a mosaic
create_a_mosaic(nc_file_names[idx], idx, dem_eachSwatch_dir, ext_poly,
tiles)
bak_folder = 'small_tifs'
io_function.mkdir(bak_folder)
# remove small and duplicated ones
for file_name in nc_file_names:
crop_tifs = io_function.get_file_list_by_pattern(
dem_eachSwatch_dir, file_name + '*crop.tif')
if len(crop_tifs) == 1:
pass
elif len(crop_tifs) > 1:
# keep maximum one and move small ones
tif_files_size = [
io_function.get_file_size_bytes(item) for item in crop_tifs
]
max_size = max(tif_files_size)
max_index = tif_files_size.index(max_size)
del crop_tifs[max_index]
for tmp in crop_tifs:
io_function.movefiletodir(tmp, bak_folder)
tmp = tmp.replace('_crop', '')
io_function.movefiletodir(tmp, bak_folder)
else: # no tif
raise ValueError('Results for %s does not exist' % file_name)
