def merge_polygon_for_demDiff_headwall_grids(dem_subsidence_shp, headwall_shp_list, output_dir, buffer_size=50): output = os.path.join( output_dir, os.path.basename( io_function.get_name_by_adding_tail(dem_subsidence_shp, 'HW_select'))) if os.path.isfile(output): print('warning, %s already exists' % output) return output # check projections? subsidence_list = vector_gpd.read_polygons_gpd(dem_subsidence_shp) headwall_all_list = [] for shp in headwall_shp_list: headwall_list = vector_gpd.read_polygons_gpd(shp) headwall_all_list.extend(headwall_list) # # https://shapely.readthedocs.io/en/stable/manual.html#str-packed-r-tree tree = STRtree(headwall_all_list) subsidence_buff_list = [ item.buffer(buffer_size) for item in subsidence_list ] select_idx = [] for idx, subsi_buff in enumerate(subsidence_buff_list): adjacent_polygons = [ item for item in tree.query(subsi_buff) if item.intersects(subsi_buff) or item.touches(subsi_buff) ] if len(adjacent_polygons) > 0: select_idx.append(idx) basic.outputlogMessage('Select %d polygons from %d ones' % (len(select_idx), len(subsidence_list))) if len(select_idx) < 1: return None # save to subset of shaepfile if vector_gpd.save_shapefile_subset_as(select_idx, dem_subsidence_shp, output) is True: return output
def read_polygons_from_small_patch(in_shp, in_raster): '''read polygons and seperate to touch and not touch image edge groups''' print(datetime.now(), 'reading polygons to touch and not touch image edge group') polygons = vector_gpd.read_polygons_gpd(in_shp, b_fix_invalid_polygon=False) img_bound = raster_io.get_image_bound_box(in_raster) img_resx, img_resy = raster_io.get_xres_yres_file(in_raster) half_res = img_resx / 2.0 image_edge = vector_gpd.convert_image_bound_to_shapely_polygon(img_bound) polygons_buff = [item.buffer(half_res) for item in polygons] # buffer half pixel # polygons_touch_img_edge_index = [] polygon_no_touch = [] polygons_touch = [] for idx, (polybuff, poly) in enumerate(zip(polygons_buff, polygons)): if polybuff.within(image_edge): polygon_no_touch.append(poly) else: # polygons_touch_img_edge_index.append(idx) polygons_touch.append(poly) # return polygons,polygons_touch_img_edge_index return polygon_no_touch, polygons_touch
def calculate_distance_medial_axis(input_shp, out_shp, process_num=4, enlarge_m=20): print('calculating polygon width based on medial axis') code_dir = os.path.expanduser('~/codes/PycharmProjects/ChangeDet_DL/thawSlumpChangeDet') sys.path.insert(0, code_dir) # after test, found that when polygons are very narrow and irregular, cal_retreat_rate output wrong results. # use buffer enlarge the polygons polygons = vector_gpd.read_polygons_gpd(input_shp) # for poly in polygons: # if poly.geom_type == 'MultiPolygon': # print(poly.geom_type,poly) # cal_retreat_rate only use exterior, fill hole for buffer # polygon_large = [ vector_gpd.fill_holes_in_a_polygon(item) for item in polygons] polygon_large = polygons # buffer polygon_large = [item.buffer(enlarge_m) for item in polygon_large] wkt = map_projection.get_raster_or_vector_srs_info_wkt(input_shp) # save_large_shp = io_function.get_name_by_adding_tail(input_shp,'larger') save_pd = pd.DataFrame({'Polygon':polygon_large}) vector_gpd.save_polygons_to_files(save_pd,'Polygon',wkt,out_shp) # calculate width based on expanding areas import cal_retreat_rate if cal_retreat_rate.cal_expand_area_distance(out_shp, proc_num=process_num,save_medial_axis=True): os.system('rm save_medial_axis_radius*.txt out_polygon_vertices_*.txt') return out_shp
def remove_based_on_area(slope_bin_shp,min_area, max_area,wkt, rm_area_shp): polygons = vector_gpd.read_polygons_gpd(slope_bin_shp,b_fix_invalid_polygon=False) remain_polygons = [] # remove relative large but narrow ones. remove_count = 0 for idx, poly in enumerate(polygons): # remove quite large or too small ones if poly.area > max_area or poly.area < min_area: remove_count += 1 continue remain_polygons.append(poly) basic.outputlogMessage('remove %d polygons based on area, remain %d ones saving to %s' % (remove_count, len(remain_polygons), rm_area_shp)) polyons_noMulti = [vector_gpd.MultiPolygon_to_polygons(idx, poly) for idx, poly in enumerate(remain_polygons)] remain_polygons = [] for polys in polyons_noMulti: polys = [poly for poly in polys if poly.area > min_area] # remove tiny polygon remain_polygons.extend(polys) print('convert MultiPolygon to polygons and remove tiny polgyons, remain %d' % (len(remain_polygons))) if len(remain_polygons) < 1: return False save_pd = pd.DataFrame({'Polygon':remain_polygons}) vector_gpd.save_polygons_to_files(save_pd,'Polygon',wkt,rm_area_shp) return rm_area_shp
def main(options, args): extent_shp = args[0] img_path = args[1] save_dir = options.save_dir #check projection extent_prj = map_projection.get_raster_or_vector_srs_info_proj4(extent_shp) img_prj = map_projection.get_raster_or_vector_srs_info_proj4(img_path) if img_prj != extent_prj: raise ValueError('Project of %s and %s is different' % (extent_shp, img_path)) out_img = io_function.get_name_by_adding_tail(img_path, 'sub') out_img = os.path.join(save_dir, os.path.basename(out_img)) extent_polys = vector_gpd.read_polygons_gpd(extent_shp) if len(extent_polys) != 1: raise ValueError('current only support one polygon') for ext_poly in extent_polys: subset_image_by_polygon_min(img_path, out_img, ext_poly, resample_m='bilinear', o_format='GTiff', out_res=None) pass
def main(): # grid polygons grid_20km = os.path.join(shp_dir,'grid_20km.shp') ArcticDEM_coverage = os.path.join(shp_dir,'tiles.shp') # qtp_grid_50km and qtb_main_perma_area_simp have the same projection grid_prj = map_projection.get_raster_or_vector_srs_info_proj4(grid_20km) perma_area_prj = map_projection.get_raster_or_vector_srs_info_proj4(ArcticDEM_coverage) if grid_prj != perma_area_prj: raise ValueError('%s and %s do not have the same projection'%(grid_prj,perma_area_prj)) grids = vector_gpd.read_polygons_gpd(grid_20km) DEM_areas = vector_gpd.read_polygons_gpd(ArcticDEM_coverage) keep_grids = [] keep_grid_ids = [] id = 0 for idx, grid in enumerate(grids): print(' processing %dth grid'%idx) for dem_area in DEM_areas: inte_res = dem_area.intersection(grid) if inte_res.is_empty is False: if inte_res.area < 100*100: # if it's too small, ignore it continue keep_grids.append(grid) keep_grid_ids.append(id) id += 1 # save save_path = os.path.join(shp_dir,'ArcticDEM_grid_20km.shp') save_polyons_attributes = {'id':keep_grid_ids, "Polygons":keep_grids} # wkt_string = map_projection.get_raster_or_vector_srs_info_wkt(qtp_main_perma_area_simp) wkt_string = map_projection.get_raster_or_vector_srs_info_proj4(grid_20km) polygon_df = pd.DataFrame(save_polyons_attributes) vector_gpd.save_polygons_to_files(polygon_df, 'Polygons', wkt_string, save_path) pass
def rasterize_polygons(poly_path, burn_value, attribute_name, xres, yres, save_path, datatype='Byte'): ''' :param poly_path: :param burn_value: :param attribute_name: :param xres: :param yres: :param save_path: :param datatype: :return: ''' import raster_io import vector_gpd import basic_src.map_projection as map_projection if attribute_name is not None: polygons, values = vector_gpd.read_polygons_attributes_list( poly_path, attribute_name, b_fix_invalid_polygon=False) burn_value = values else: polygons = vector_gpd.read_polygons_gpd(poly_path, b_fix_invalid_polygon=False) if datatype == 'Byte': dtype = 'uint8' elif datatype == 'UInt16': dtype = 'uint16' else: dtype = 'int32' extent = vector_gpd.get_vector_file_bounding_box(poly_path) wkt_string = map_projection.get_raster_or_vector_srs_info_proj4(poly_path) return raster_io.burn_polygons_to_a_raster(None, polygons, burn_value, save_path, dtype, xres=xres, yres=yres, extent=extent, ref_prj=wkt_string)
def read_extent_shapefile_epgs4326(exent_shp): ''' read extent polygon, need in latlon projections :param exent_shp: :return: ''' if exent_shp is None: return None shp_prj = map_projection.get_raster_or_vector_srs_info_proj4(exent_shp).strip() if shp_prj != '+proj=longlat +datum=WGS84 +no_defs': raise ValueError('only support the projections of longlat (EPSG:4326)') extent_polygons = vector_gpd.read_polygons_gpd(exent_shp) return extent_polygons
def test_build_adjacent_matrix_pygeo(): vector_path = os.path.expanduser( '~/Data/dem_processing/grid_9053_tmp_files/20140701_dem_slope_bin_patches_all_polyTouchEdge.gpkg' ) polygons = vector_gpd.read_polygons_gpd(vector_path, b_fix_invalid_polygon=False) print('read %d polygons' % len(polygons)) # polygons = [item.buffer(1.0) for item in polygons] # for complex polygons, buffer take a long time. from pygeos import area, intersection, from_shapely, touches import numpy as np geo_polygons = from_shapely(polygons) print(geo_polygons) # inter_matrix = intersection(geo_polygons[:, np.newaxis], geo_polygons[np.newaxis, :]) # print(inter_matrix) # print(touches(geo_polygons[:, np.newaxis], geo_polygons[np.newaxis, :])) print(touches(geo_polygons, geo_polygons))
def main(options, args): extent_shp = args[0] dem_index_shp = args[1] pre_name = os.path.splitext(os.path.basename(extent_shp))[0] pre_name += '_Tile' if 'Tile' in os.path.basename( dem_index_shp) else '_Strip' xlsx_size_path = os.path.splitext( os.path.basename(dem_index_shp))[0] + '_fileSize.xlsx' print('xlsx file for saving file size', xlsx_size_path) # extent polygons and projection (proj4) extent_shp_prj = map_projection.get_raster_or_vector_srs_info_proj4( extent_shp) dem_shp_prj = map_projection.get_raster_or_vector_srs_info_proj4( dem_index_shp) if extent_shp_prj != dem_shp_prj: basic.outputlogMessage( '%s and %s do not have the same projection, will reproject %s' % (extent_shp, dem_index_shp, os.path.basename(extent_shp))) epsg = map_projection.get_raster_or_vector_srs_info_epsg(dem_index_shp) # print(epsg) # extent_polys = vector_gpd.read_shape_gpd_to_NewPrj(extent_shp,dem_shp_prj.strip()) extent_polys = vector_gpd.read_shape_gpd_to_NewPrj(extent_shp, epsg) else: extent_polys = vector_gpd.read_polygons_gpd(extent_shp) # read 'grid_id' if the extent shp is from grid shp file, if not, grid_id_list will be None grid_id_list = vector_gpd.read_attribute_values_list(extent_shp, 'grid_id') if len(extent_polys) < 1: raise ValueError('No polygons in %s' % extent_shp) else: basic.outputlogMessage('%d extent polygons in %s' % (len(extent_polys), extent_shp)) get_file_size_dem_tarball(dem_index_shp, extent_polys, pre_name, xlsx_size_path, poly_ids=grid_id_list)
def remove_polygons_based_shapeinfo(in_shp, output_shp,area_limit,circularit_limit, holes_count ): # remove polygons if they are larege (> area_limit) and narrow (<circularit_limit) # if too many holse, may consider remove them as well. # remove based on: INarea, INperimete,circularit,ratio_w_h, hole_count if os.path.isfile(output_shp): basic.outputlogMessage('%s exists, skip'%output_shp) return output_shp polygons = vector_gpd.read_polygons_gpd(in_shp) # add some shape info shape_info_list = [ vector_gpd.calculate_polygon_shape_info(item) for item in polygons] shapeinfo_all_dict = vector_gpd.list_to_dict(shape_info_list) vector_gpd.add_attributes_to_shp(in_shp,shapeinfo_all_dict) shapefile = gpd.read_file(in_shp) # remove relative large but narrow ones. remove_count = 0 for idx, row in shapefile.iterrows(): shape_info = shape_info_list[idx] # remove quite large but narrow ones if shape_info['INarea'] > area_limit and shape_info['circularit'] < circularit_limit : shapefile.drop(idx, inplace=True) remove_count += 1 continue # remove holes if shape_info['hole_count'] > holes_count: shapefile.drop(idx, inplace=True) remove_count += 1 continue basic.outputlogMessage('remove %d polygons based on shapeinfo, remain %d ones saving to %s' % (remove_count, len(shapefile.geometry.values), output_shp)) if len(shapefile.geometry.values) < 1: basic.outputlogMessage('After removing based on shapeinfo, no polygon to save') return None # save results shapefile.to_file(output_shp, driver='ESRI Shapefile') return output_shp
def post_processing_subsidence(in_shp): polygons = vector_gpd.read_polygons_gpd(in_shp) # get shapeinfo # poly_shapeinfo_list = [] save_polyons = [] for poly in polygons: # get INarea, INperimete, WIDTH, HEIGHT, ratio_w_h, hole_count # shapeinfo = vector_gpd.calculate_polygon_shape_info(poly) # error: 'MultiPolygon' object has no attribute 'interiors' # poly_shapeinfo_list.append(shapeinfo) # if shapeinfo['INarea'] < 40: # remove the one with area smaller than 40 m^2 if poly.area < 90: # remove the one with area smaller than 40 m^2 continue save_polyons.append(poly) save_pd = pd.DataFrame({'Polygon': save_polyons}) wkt = map_projection.get_raster_or_vector_srs_info_wkt(in_shp) save_shp = io_function.get_name_by_adding_tail(in_shp,'post') vector_gpd.save_polygons_to_files(save_pd,'Polygon',wkt,save_shp)
def add_boxes_attributes(input, output, para_file=None,data_para_file=None): if io_function.is_file_exist(input) is False: return False # copy output if io_function.copy_shape_file(input, output) is False: raise IOError('copy shape file %s failed'%input) # calculate area, perimeter of polygons if cal_add_area_length_of_polygon(output) is False: return False # calculate the width, height, and width_height ratio boxes = vector_gpd.read_polygons_gpd(input) bounding_boxes = [ vector_gpd.get_polygon_bounding_box(item) for item in boxes ] # bounding box (minx, miny, maxx, maxy) width_list = [ bound[2] - bound[0] for bound in bounding_boxes] height_list = [ bound[3] - bound[1] for bound in bounding_boxes] ratio_w_h_list = [] for w, h in zip(width_list, height_list): if w > h: ratio = h / w else: ratio = w / h ratio_w_h_list.append(ratio) add_attributes = {'WIDTH':width_list,'HEIGHT':height_list, 'ratio_w_h':ratio_w_h_list} vector_gpd.add_attributes_to_shp(output,add_attributes) # add topography of each box (consider the box as a polygon) if data_para_file is not None and para_file is not None: io_function.is_file_exist(data_para_file) io_function.is_file_exist(para_file) dem_files, slope_files, aspect_files, dem_diff_files = get_topographic_files(data_para_file) process_num = parameters.get_digit_parameters(para_file,'process_num','int') if calculate_polygon_topography(output,para_file,dem_files,slope_files,aspect_files=aspect_files,dem_diffs=dem_diff_files, process_num=process_num) is False: basic.outputlogMessage('Warning: calculate information of topography failed') # return False # don't return return output
def check_one_vector_file(idx,total,file_path, good_file_list): if os.path.basename(file_path) in good_file_list: return True try: print('checking %d/%d' % (idx, total)) # src = raster_io.open_raster_read(tif_path) geometries = vector_gpd.read_polygons_gpd(file_path,b_fix_invalid_polygon=False) if len(geometries) < 1: basic.outputlogMessage('No geometries in %s'%file_path) return False geometries_list = [ item for item in geometries ] if None in geometries_list: basic.outputlogMessage('None geometries in %s'%file_path) return False except: basic.outputlogMessage('incomplete vector: %s' % file_path) return False return True
def merge_polygon_rasterize(ref_raster, in_polygons): # rasterize to raster save_raster = os.path.basename(io_function.get_name_by_adding_tail(ref_raster,'merge')) raster_io.burn_polygons_to_a_raster(ref_raster,in_polygons,1,save_raster) # set nodata if raster_io.set_nodata_to_raster_metadata(save_raster,0) is False: raise IOError('Set nodata failed for %s'%save_raster) # polygonize out_shp = vector_gpd.raster2shapefile(save_raster, connect8=True) if out_shp is None: raise IOError('polygonzied failed for %s' % save_raster) # read polygon merge_polygons = vector_gpd.read_polygons_gpd(out_shp,b_fix_invalid_polygon=False) print(timeTools.get_now_time_str(),'Get %d merged polygons'%len(merge_polygons)) return merge_polygons
def refine_dem_reduction_polygons(input_shp, min_area, max_area): rm_min_max_area = os.path.basename( io_function.get_name_by_adding_tail(input_shp, 'MinMaxArea')) vector_gpd.remove_polygons_not_in_range(input_shp, 'poly_area', min_area, max_area, rm_min_max_area) # rm_min_max_area = input_shp # add some shape info shapefile = gpd.read_file(rm_min_max_area) polygons = vector_gpd.read_polygons_gpd(rm_min_max_area) shape_info_list = [ vector_gpd.calculate_polygon_shape_info(item) for item in polygons ] shapeinfo_all_dict = list_to_dict(shape_info_list) vector_gpd.add_attributes_to_shp(rm_min_max_area, shapeinfo_all_dict) # INarea, INperimete, circularit, hole_count, WIDTH & HEIGHT (minimum_rotated_rectangle), ratio_w_h # remove based on shape info output = os.path.basename( io_function.get_name_by_adding_tail(input_shp, 'final')) remove_count = 0 for idx, row in shapefile.iterrows(): shape_info = shape_info_list[idx] # remove quite large but narrow ones if shape_info['INarea'] > 10000 and shape_info['circularit'] < 0.1: shapefile.drop(idx, inplace=True) remove_count += 1 basic.outputlogMessage( 'remove %d polygons based on shapeinfo, remain %d ones saving to %s' % (remove_count, len(shapefile.geometry.values), output)) # save results shapefile.to_file(output, driver='ESRI Shapefile')
def add_boxes_attributes(input, output): if io_function.is_file_exist(input) is False: return False # copy output if io_function.copy_shape_file(input, output) is False: raise IOError('copy shape file %s failed' % input) # calculate area, perimeter of polygons if cal_add_area_length_of_polygon(output) is False: return False # calculate the width, height, and width_height ratio boxes = vector_gpd.read_polygons_gpd(input) bounding_boxes = [ vector_gpd.get_polygon_bounding_box(item) for item in boxes ] # bounding box (minx, miny, maxx, maxy) width_list = [bound[2] - bound[0] for bound in bounding_boxes] height_list = [bound[3] - bound[1] for bound in bounding_boxes] ratio_w_h_list = [] for w, h in zip(width_list, height_list): if w > h: ratio = h / w else: ratio = w / h ratio_w_h_list.append(ratio) add_attributes = { 'WIDTH': width_list, 'HEIGHT': height_list, 'ratio_w_h': ratio_w_h_list } vector_gpd.add_attributes_to_shp(output, add_attributes) return output
def main(options, args): save_dir = options.save_dir extent_shp = options.extent_shp process_num = options.process_num dem_dir_or_txt = args[0] if os.path.isfile(dem_dir_or_txt): dem_list = io_function.read_list_from_txt(dem_dir_or_txt) else: dem_list = io_function.get_file_list_by_ext('.tif', dem_dir_or_txt, bsub_folder=False) dem_count = len(dem_list) if dem_count < 1: raise ValueError('No input dem files in %s' % dem_dir_or_txt) if extent_shp is not None: pre_name = os.path.splitext(os.path.basename(extent_shp))[0] else: pre_name = os.path.basename(os.path.abspath(save_dir)) save_dem_diff = os.path.join(save_dir, pre_name + '_DEM_diff.tif') save_date_diff = os.path.join(save_dir, pre_name + '_date_diff.tif') if os.path.isfile(save_dem_diff) and os.path.isfile(save_date_diff): print('%s and %s exists, skip' % (save_dem_diff, save_date_diff)) return if extent_shp is not None: # crop the DEM before differencing extent_shp_base = os.path.splitext(os.path.basename(extent_shp))[0] dem_prj = map_projection.get_raster_or_vector_srs_info_epsg( dem_list[0]) extent_prj = map_projection.get_raster_or_vector_srs_info_epsg( extent_shp) if dem_prj != extent_prj: raise ValueError( 'The projection of extent file (%s) and dem tifs is different' % extent_shp) extent_polys = vector_gpd.read_polygons_gpd(extent_shp) if len(extent_polys) != 1: raise ValueError('Only allow one polygon in %s' % extent_shp) extPolys_ids = vector_gpd.read_attribute_values_list(extent_shp, 'id') if extPolys_ids is None or None in extPolys_ids: basic.outputlogMessage( 'Warning, field: id is not in %s, will create default ID for each grid' % extent_shp) extPolys_ids = [id + 1 for id in range(len(extent_polys))] # crop for idx, ext_poly in zip(extPolys_ids, extent_polys): basic.outputlogMessage( 'crop and differnce DEM for the %d th extent (%d in total)' % (idx, len(extent_polys))) crop_dem_list = crop_to_same_exent_for_diff( dem_list, save_dir, idx, ext_poly, process_num) dem_list = crop_dem_list dem_diff_newest_oldest(dem_list, save_dem_diff, save_date_diff, process_num, b_max_subsidence=options.max_subsidence)
def main(options, args): extent_shp = args[0] # ext_shp_prj = map_projection.get_raster_or_vector_srs_info_epsg(extent_shp) # reproject if necessary, it seems that the gdalwarp can handle different projection # if ext_shp_prj != 'EPSG:3413': # EPSG:3413 is the projection ArcticDEM used # extent_shp_reprj = io_function.get_name_by_adding_tail(extent_shp,'3413') # vector_gpd.reproject_shapefile(extent_shp,'EPSG:3413',extent_shp_reprj) # extent_shp = extent_shp_reprj tar_dir = options.ArcticDEM_dir save_dir = options.save_dir b_mosaic_id = options.create_mosaic_id b_mosaic_date = options.create_mosaic_date b_rm_inter = options.remove_inter_data keep_dem_percent = options.keep_dem_percent inter_format = options.format arcticDEM_shp = options.arcticDEM_shp o_res = options.out_res b_dem_diff = options.create_dem_diff dem_list_txt = options.dem_list_txt # create mosaic is time consuming, but it also takes a lot memory. For a region of 50 km by 50 km, it may take 10 to 50 GB memory process_num = options.process_num basic.outputlogMessage( 'The number of processes for creating the mosaic is: %d' % process_num) extent_shp_base = os.path.splitext(os.path.basename(extent_shp))[0] extent_prj = map_projection.get_raster_or_vector_srs_info_epsg(extent_shp) b_ArcticDEM_tar = False dem_tif_list = [] if tar_dir is not None and arcticDEM_shp is not None: b_ArcticDEM_tar = True else: dem_tif_list = io_function.read_list_from_txt(dem_list_txt) # check projection for dem_tif in dem_tif_list: dem_prj = map_projection.get_raster_or_vector_srs_info_epsg( dem_tif) if dem_prj != extent_prj: raise ValueError('The projection of %s is different from %s' % (dem_prj, extent_prj)) b_ArcticDEM_tiles = False if b_ArcticDEM_tar: arcdem_prj = map_projection.get_raster_or_vector_srs_info_epsg( arcticDEM_shp) # read dem polygons and url time0 = time.time() dem_polygons, dem_urls = vector_gpd.read_polygons_attributes_list( arcticDEM_shp, 'fileurl', b_fix_invalid_polygon=False) print('time cost of read polygons and attributes', time.time() - time0) basic.outputlogMessage('%d dem polygons in %s' % (len(dem_polygons), extent_shp)) # get tarball list tar_list = io_function.get_file_list_by_ext('.gz', tar_dir, bsub_folder=False) if len(tar_list) < 1: raise ValueError('No input tar.gz files in %s' % tar_dir) if is_ArcticDEM_tiles(tar_list): basic.outputlogMessage('Input is the mosaic version of ArcticDEM') b_ArcticDEM_tiles = True if extent_prj == arcdem_prj: extent_polys = vector_gpd.read_polygons_gpd(extent_shp) else: extent_polys = vector_gpd.read_shape_gpd_to_NewPrj( extent_shp, arcdem_prj) else: extent_polys = vector_gpd.read_polygons_gpd(extent_shp) if len(extent_polys) < 1: raise ValueError('No polygons in %s' % extent_shp) else: basic.outputlogMessage('%d extent polygons in %s' % (len(extent_polys), extent_shp)) extPolys_ids = vector_gpd.read_attribute_values_list(extent_shp, 'id') if extPolys_ids is None or None in extPolys_ids: basic.outputlogMessage( 'Warning, field: id is not in %s, will create default ID for each grid' % extent_shp) extPolys_ids = [id + 1 for id in range(len(extent_polys))] # print('extPolys_ids, count',extPolys_ids, len(extent_polys)) same_extent = False if b_dem_diff: # crop each one to the same extent, easy for DEM differnce. same_extent = True for idx, ext_poly in zip(extPolys_ids, extent_polys): basic.outputlogMessage('get data for the %d th extent (%d in total)' % (idx, len(extent_polys))) if b_ArcticDEM_tar: if b_ArcticDEM_tiles: proc_ArcticDEM_tile_one_grid_polygon(tar_dir, dem_polygons, dem_urls, o_res, save_dir, inter_format, b_rm_inter, ext_poly, idx, extent_shp_base) else: proc_ArcticDEM_strip_one_grid_polygon( tar_dir, dem_polygons, dem_urls, o_res, save_dir, inter_format, b_mosaic_id, b_mosaic_date, b_rm_inter, b_dem_diff, ext_poly, idx, keep_dem_percent, process_num, extent_shp_base, resample_method='average', same_extent=same_extent) else: proc_dem_mosaic_diff(dem_tif_list, save_dir, idx, ext_poly, b_mosaic_id, b_mosaic_date, process_num, keep_dem_percent, o_res, b_dem_diff, extent_shp_base, b_rm_inter, resample_method='average')
# extent polygons and projection (proj4) extent_shp_prj = map_projection.get_raster_or_vector_srs_info_proj4( extent_shp_or_ids_txt) grid_shp_prj = map_projection.get_raster_or_vector_srs_info_proj4( grid_20_shp) if extent_shp_prj != grid_shp_prj: basic.outputlogMessage( '%s and %s do not have the same projection, will reproject %s' % (extent_shp_or_ids_txt, grid_20_shp, os.path.basename(extent_shp_or_ids_txt))) epsg = map_projection.get_raster_or_vector_srs_info_epsg(grid_20_shp) extent_polys = vector_gpd.read_shape_gpd_to_NewPrj( extent_shp_or_ids_txt, epsg) else: extent_polys = vector_gpd.read_polygons_gpd(extent_shp_or_ids_txt) crop_mosaic_reproject_dem_diff(grid_dem_tifs, pre_name, extent_poly, o_res, new_prj, b_mosaic=False) def main(options, args): pass if __name__ == '__main__': usage = "usage: %prog [options] extent_shp or grid_id_list.txt "
def zonal_stats_multiRasters(in_shp, raster_file_or_files, nodata=None, band=1, stats=None, prefix='', range=None, all_touched=True, process_num=1): ''' zonal statistic based on vectors, along multiple rasters (image tiles) Args: in_shp: input vector file raster_file_or_files: a raster file or multiple rasters nodata: band: band stats: like [mean, std, max, min] range: interested values [min, max], None means infinity all_touched: process_num: process number for calculation Returns: ''' io_function.is_file_exist(in_shp) if stats is None: basic.outputlogMessage('warning, No input stats, set to ["mean"])') stats = ['mean'] if isinstance(raster_file_or_files, str): io_function.is_file_exist(raster_file_or_files) image_tiles = [raster_file_or_files] elif isinstance(raster_file_or_files, list): image_tiles = raster_file_or_files else: raise ValueError('unsupport type for %s' % str(raster_file_or_files)) # check projection (assume we have the same projection), check them outside this function # get image box img_tile_boxes = [ raster_io.get_image_bound_box(tile) for tile in image_tiles ] img_tile_polygons = [ vector_gpd.convert_image_bound_to_shapely_polygon(box) for box in img_tile_boxes ] polygons = vector_gpd.read_polygons_gpd(in_shp) if len(polygons) < 1: basic.outputlogMessage('No polygons in %s' % in_shp) return False # polygons_json = [mapping(item) for item in polygons] # no need when use new verion of rasterio # process polygons one by one polygons and the corresponding image tiles (parallel and save memory) # also to avoid error: daemonic processes are not allowed to have children if process_num == 1: stats_res_list = [] for idx, polygon in enumerate(polygons): out_stats = zonal_stats_one_polygon(idx, polygon, image_tiles, img_tile_polygons, stats, nodata=nodata, range=range, band=band, all_touched=all_touched) stats_res_list.append(out_stats) elif process_num > 1: threadpool = Pool(process_num) para_list = [(idx, polygon, image_tiles, img_tile_polygons, stats, nodata, range, band, all_touched) for idx, polygon in enumerate(polygons)] stats_res_list = threadpool.starmap(zonal_stats_one_polygon, para_list) else: raise ValueError('Wrong process number: %s ' % str(process_num)) # save to shapefile add_attributes = {} new_key_list = [prefix + '_' + key for key in stats_res_list[0].keys()] for new_ley in new_key_list: add_attributes[new_ley] = [] for stats_result in stats_res_list: for key in stats_result.keys(): add_attributes[prefix + '_' + key].append(stats_result[key]) vector_gpd.add_attributes_to_shp(in_shp, add_attributes) pass
def main(options, args): extent_shp = args[0] dem_index_shp = args[1] b_arcticDEM_tile = False global max_task_count max_task_count = options.max_process_num if 'Tile' in os.path.basename(dem_index_shp): save_folder = arcticDEM_tile_tarball_dir reg_tif_dir = arcticDEM_tile_reg_tif_dir b_arcticDEM_tile = True else: save_folder = tarball_dir reg_tif_dir = arcticDEM_reg_tif_dir # use the user specific save_dir for saving downloaded tarballs if options.save_dir is not None: save_folder = options.save_dir if os.path.isdir(save_folder) is False: io_function.mkdir(save_folder) save_folder = os.path.abspath(save_folder) # change to absolute path pre_name = os.path.splitext(os.path.basename(extent_shp))[0] pre_name += '_Tile' if 'Tile' in os.path.basename( dem_index_shp) else '_Strip' # extent polygons and projection (proj4) extent_shp_prj = map_projection.get_raster_or_vector_srs_info_proj4( extent_shp) dem_shp_prj = map_projection.get_raster_or_vector_srs_info_proj4( dem_index_shp) if extent_shp_prj != dem_shp_prj: basic.outputlogMessage( '%s and %s do not have the same projection, will reproject %s' % (extent_shp, dem_index_shp, os.path.basename(extent_shp))) epsg = map_projection.get_raster_or_vector_srs_info_epsg(dem_index_shp) # print(epsg) # extent_polys = vector_gpd.read_shape_gpd_to_NewPrj(extent_shp,dem_shp_prj.strip()) extent_polys = vector_gpd.read_shape_gpd_to_NewPrj(extent_shp, epsg) else: extent_polys = vector_gpd.read_polygons_gpd(extent_shp) # read 'grid_id' if the extent shp is from grid shp file, if not, grid_id_list will be None grid_id_list = vector_gpd.read_attribute_values_list(extent_shp, 'grid_id') if len(extent_polys) < 1: raise ValueError('No polygons in %s' % extent_shp) else: basic.outputlogMessage( 'read %d extent polygons in %s for downloading and proc' % (len(extent_polys), extent_shp)) download_dem_tarball(dem_index_shp, extent_polys, save_folder, pre_name, reg_tif_dir=reg_tif_dir, poly_ids=grid_id_list, b_arcticDEM_tile=b_arcticDEM_tile)
def get_extent_xy_from_shp(input_shp): polygons = vector_gpd.read_polygons_gpd(input_shp) if len(polygons) != 1: raise ValueError('The extent only support one polygon') x, y = vector_gpd.get_polygon_envelope_xy(polygons[0]) return x, y
def zonal_stats_multiRasters(in_shp, raster_file_or_files, nodata=None, band = 1, stats = None, prefix='', range=None,buffer=None, all_touched=True, process_num=1): ''' zonal statistic based on vectors, along multiple rasters (image tiles) Args: in_shp: input vector file raster_file_or_files: a raster file or multiple rasters nodata: band: band stats: like [mean, std, max, min] range: interested values [min, max], None means infinity buffer: expand polygon with buffer (meter) before the statistic all_touched: process_num: process number for calculation Returns: ''' io_function.is_file_exist(in_shp) if stats is None: basic.outputlogMessage('warning, No input stats, set to ["mean"])') stats = ['mean'] stats_backup = stats.copy() if 'area' in stats: stats.remove('area') if 'count' not in stats: stats.append('count') if isinstance(raster_file_or_files,str): io_function.is_file_exist(raster_file_or_files) image_tiles = [raster_file_or_files] elif isinstance(raster_file_or_files,list): image_tiles = raster_file_or_files else: raise ValueError('unsupport type for %s'%str(raster_file_or_files)) # check projection (assume we have the same projection), check them outside this function # get image box img_tile_boxes = [raster_io.get_image_bound_box(tile) for tile in image_tiles] img_tile_polygons = [vector_gpd.convert_image_bound_to_shapely_polygon(box) for box in img_tile_boxes] polygons = vector_gpd.read_polygons_gpd(in_shp) if len(polygons) < 1: basic.outputlogMessage('No polygons in %s'%in_shp) return False # polygons_json = [mapping(item) for item in polygons] # no need when use new verion of rasterio if buffer is not None: polygons = [ poly.buffer(buffer) for poly in polygons] # process polygons one by one polygons and the corresponding image tiles (parallel and save memory) stats_res_list = [] for idx, polygon in enumerate(polygons): out_stats = zonal_stats_one_polygon(idx, polygon, image_tiles, img_tile_polygons, stats, nodata=nodata, range=range, band=band, all_touched=all_touched) stats_res_list.append(out_stats) # threadpool = Pool(process_num) # para_list = [ (idx, polygon, image_tiles, img_tile_polygons, stats, nodata, range,band, all_touched) # for idx, polygon in enumerate(polygons)] # stats_res_list = threadpool.starmap(zonal_stats_one_polygon,para_list) # save to shapefile add_attributes = {} new_key_list = [ prefix + '_' + key for key in stats_res_list[0].keys()] for new_ley in new_key_list: add_attributes[new_ley] = [] for stats_result in stats_res_list: for key in stats_result.keys(): add_attributes[prefix + '_' + key].append(stats_result[key]) if 'area' in stats_backup: dx, dy = raster_io.get_xres_yres_file(image_tiles[0]) add_attributes[prefix + '_' + 'area'] = [ count*dx*dy for count in add_attributes[prefix + '_' + 'count'] ] if 'count' not in stats_backup: del add_attributes[prefix + '_' + 'count'] vector_gpd.add_attributes_to_shp(in_shp,add_attributes) pass
def get_grid_20(extent_shp_or_id_txt, grid_polys, ids): ''' get grid polygons and ids based on input extent (polygon in shpaefile) or ids (txt file) if "file_name_base+'_grid_ids.txt'" exists, it will read id in this file directly. :param extent_shp_or_id_txt: :param grid_polys: :param ids: :return: ''' io_function.is_file_exist(extent_shp_or_id_txt) if extent_shp_or_id_txt.endswith('.txt'): grid_ids = io_function.read_list_from_txt(extent_shp_or_id_txt) grid_ids = [int(item) for item in grid_ids ] else: shp_corresponding_grid_ids_txt = get_corresponding_grid_ids_txt(extent_shp_or_id_txt) if os.path.isfile(shp_corresponding_grid_ids_txt): print('corresponding grid ids txt file for %s exists, read grid id from txt'%extent_shp_or_id_txt) grid_ids = [ int(item) for item in io_function.read_list_from_txt(shp_corresponding_grid_ids_txt)] basic.outputlogMessage('read %d grids within the extents (%s)' % (len(grid_ids), os.path.basename(extent_shp_or_id_txt))) else: # extent polygons and projection (proj4) extent_shp_prj = map_projection.get_raster_or_vector_srs_info_proj4(extent_shp_or_id_txt) if extent_shp_prj == '': raise ValueError('get proj4 of %s failed'%extent_shp_or_id_txt) grid_shp_prj = map_projection.get_raster_or_vector_srs_info_proj4(grid_20_shp) if grid_shp_prj=='': raise ValueError('get proj4 of %s failed' % grid_20_shp) if extent_shp_prj != grid_shp_prj: basic.outputlogMessage('%s and %s do not have the same projection, will reproject %s' % (extent_shp_or_id_txt, grid_20_shp, os.path.basename(extent_shp_or_id_txt))) epsg = map_projection.get_raster_or_vector_srs_info_epsg(grid_20_shp) # print(epsg) # extent_polys = vector_gpd.read_shape_gpd_to_NewPrj(extent_shp,dem_shp_prj.strip()) extent_polys = vector_gpd.read_shape_gpd_to_NewPrj(extent_shp_or_id_txt, epsg) else: extent_polys = vector_gpd.read_polygons_gpd(extent_shp_or_id_txt) ext_poly_count = len(extent_polys) if ext_poly_count < 1: raise ValueError('No polygons in %s'%extent_shp_or_id_txt) grid_index = [] # if there are many polygons, this will take time. for idx,ext_poly in enumerate(extent_polys): print(timeTools.get_now_time_str(), 'get grids for extent idx', idx, 'total polygons:',ext_poly_count) index = vector_gpd.get_poly_index_within_extent(grid_polys, ext_poly) grid_index.extend(index) grid_index = list(set(grid_index)) # remove duplicated ids basic.outputlogMessage('find %d grids within the extents (%s)' % (len(grid_index), os.path.basename(extent_shp_or_id_txt)) ) grid_ids = [ ids[idx] for idx in grid_index] grid_ids_str = [str(item) for item in grid_ids ] io_function.save_list_to_txt(shp_corresponding_grid_ids_txt,grid_ids_str) id_index = [ids.index(id) for id in grid_ids] selected_grid_polys = [grid_polys[idx] for idx in id_index ] return selected_grid_polys, grid_ids
def get_dem_subscidence_polygons(in_shp, dem_diff_tif, dem_diff_thread_m=-0.5, min_area=40, max_area=100000000, process_num=1, b_rm_files=False): save_shp = io_function.get_name_by_adding_tail(in_shp, 'post') if os.path.isfile(save_shp): basic.outputlogMessage('%s exists, skip'%save_shp) return save_shp demD_height, demD_width, demD_band_num, demD_date_type = raster_io.get_height_width_bandnum_dtype(dem_diff_tif) # print(demD_date_type) # # read mean elevation difference # attributes_path = os.path.join(os.path.dirname(in_shp), shp_pre + '_attributes.txt') # # # for each seg lable [mean, std, pixel count], if dem_diff_tif is float 32, then in meters, if int16, then in centimeter # poly_attributes = io_function.read_dict_from_txt_json(attributes_path) # if int16, then it's in centimeter if demD_date_type == 'int16': dem_diff_thread_m = dem_diff_thread_m*100 # merge polygons touch each others wkt = map_projection.get_raster_or_vector_srs_info_wkt(in_shp) merged_shp = io_function.get_name_by_adding_tail(in_shp, 'merged') if filter_merge_polygons(in_shp,merged_shp,wkt, min_area,max_area,dem_diff_tif,dem_diff_thread_m,process_num) is None: return None # in merge_polygons, it will remove some big polygons, convert MultiPolygon to Polygons, so neeed to update remain_polyons remain_polyons = vector_gpd.read_polygons_gpd(merged_shp) # check MultiPolygons again. polyons_noMulti = [vector_gpd.MultiPolygon_to_polygons(idx, poly) for idx, poly in enumerate(remain_polyons)] remain_polyons = [] for polys in polyons_noMulti: polys = [poly for poly in polys if poly.area > min_area] # remove tiny polygon before buffer remain_polyons.extend(polys) print('convert MultiPolygon to polygons and remove small ones, remain %d' % (len(remain_polyons))) # based on the merged polygons, surrounding polygons buffer_surrounding = 20 # meters surrounding_shp = io_function.get_name_by_adding_tail(in_shp, 'surrounding') get_surrounding_polygons(remain_polyons, surrounding_shp, wkt, dem_diff_tif, buffer_surrounding, process_num) rm_reldemD_shp = io_function.get_name_by_adding_tail(in_shp, 'rmreldemD') if remove_polygons_based_relative_dem_diff(remain_polyons, merged_shp, surrounding_shp, wkt, rm_reldemD_shp, min_area,dem_diff_thread_m) is None: return None rm_shapeinfo_shp = io_function.get_name_by_adding_tail(in_shp, 'rmshapeinfo') area_limit = 10000 circularit_limit = 0.1 holes_count = 20 if remove_polygons_based_shapeinfo(rm_reldemD_shp, rm_shapeinfo_shp, area_limit, circularit_limit, holes_count) is None: return None # remove based on slope # use the slope derived from ArcitcDEM mosaic slope_tif_list = io_function.get_file_list_by_ext('.tif',dem_common.arcticDEM_tile_slope_dir,bsub_folder=False) basic.outputlogMessage('Find %d slope files in %s'%(len(slope_tif_list), dem_common.arcticDEM_tile_slope_dir)) rm_slope_shp = io_function.get_name_by_adding_tail(in_shp, 'rmslope') max_slope = 20 if remove_based_slope(rm_shapeinfo_shp, rm_slope_shp,slope_tif_list, max_slope,process_num) is False: return None # copy io_function.copy_shape_file(rm_slope_shp,save_shp) # add date difference if they are available date_diff_base = os.path.basename(dem_diff_tif).replace('DEM_diff','date_diff') date_diff_tif = os.path.join(os.path.dirname(dem_diff_tif) , date_diff_base) if os.path.isfile(date_diff_tif): raster_statistic.zonal_stats_multiRasters(save_shp, date_diff_tif,tile_min_overlap=tile_min_overlap, stats=['mean', 'std'], prefix='dateD',process_num=process_num) return save_shp
def main(options, args): save_dir = options.save_dir extent_shp = options.extent_shp process_num = options.process_num o_res = options.out_res b_mosaic_id = options.create_mosaic_id b_mosaic_date = options.create_mosaic_date keep_dem_percent = options.keep_dem_percent dem_dir_or_txt = args[0] if os.path.isfile(dem_dir_or_txt): dem_list = io_function.read_list_from_txt(dem_dir_or_txt) else: dem_list = io_function.get_file_list_by_ext('.tif', dem_dir_or_txt, bsub_folder=False) dem_list = [ tif for tif in dem_list if 'matchtag' not in tif ] # remove matchtag dem_count = len(dem_list) if dem_count < 1: raise ValueError('No input dem files in %s' % dem_dir_or_txt) resample_method= 'average' if extent_shp is None: # groups DEM based on the same strip ID dem_groups = group_demTif_strip_pair_ID(dem_list) # mosaic them direclty without consider the extent mosaic_dir = os.path.join(save_dir, 'dem_stripID_mosaic' ) mosaic_dem_same_stripID(dem_groups, mosaic_dir, resample_method, process_num=process_num, save_source=True, o_format='GTiff') else: extent_shp_base = os.path.splitext(os.path.basename(extent_shp))[0] dem_prj = map_projection.get_raster_or_vector_srs_info_epsg(dem_list[0]) extent_prj = map_projection.get_raster_or_vector_srs_info_epsg(extent_shp) # # check projection (time-consuming if there are many tif files) # for dem_tif in dem_list: # prj = map_projection.get_raster_or_vector_srs_info_epsg(dem_tif) # if dem_prj != prj: # raise ValueError('The projection inconsistent among dems (%s is different)'%dem_tif) dem_ext_polys = get_dem_tif_ext_polygons(dem_list) if extent_prj==dem_prj: extent_polys = vector_gpd.read_polygons_gpd(extent_shp) else: extent_polys = vector_gpd.read_shape_gpd_to_NewPrj(extent_shp,dem_prj) if len(extent_polys) < 1: raise ValueError('No polygons in %s' % extent_shp) else: basic.outputlogMessage('%d extent polygons in %s' % (len(extent_polys), extent_shp)) extPolys_ids = vector_gpd.read_attribute_values_list(extent_shp, 'id') if extPolys_ids is None or None in extPolys_ids: basic.outputlogMessage('Warning, field: id is not in %s, will create default ID for each grid' % extent_shp) extPolys_ids = [id + 1 for id in range(len(extent_polys))] for idx, ext_poly in zip(extPolys_ids, extent_polys): basic.outputlogMessage('mosaic and crop DEM for the %d th extent (%d in total)' % (idx, len(extent_polys))) # get subset of DEM dem_poly_ids = vector_gpd.get_poly_index_within_extent(dem_ext_polys, ext_poly) if len(dem_poly_ids) < 1: basic.outputlogMessage('no dem tifs within %d polygons'%idx) continue dem_list_sub = [dem_list[id] for id in dem_poly_ids] mosaic_crop_dem(dem_list_sub, save_dir, idx, ext_poly, b_mosaic_id, b_mosaic_date, process_num, keep_dem_percent, o_res, extent_shp_base, resample_method='average')
def main(options, args): time0 = time.time() image_dir = args[0] geojson_list = io_function.get_file_list_by_ext('.geojson', image_dir, bsub_folder=False) # remove some scenes, or maybe we should set bsub_folder=False # geojson_list = [item for item in geojson_list if 'incomplete_scenes' not in item ] # remove those in "incomplete_scenes" # geojson_list = [item for item in geojson_list if 'scenes_high_cloud_cover' not in item ] # remove those in "scenes_high_cloud_cover" if len(geojson_list) < 1: raise ValueError('There is no geojson files in %s' % image_dir) basic.outputlogMessage('Image Dir: %s' % image_dir) basic.outputlogMessage("Number of geojson files: %d" % len(geojson_list)) grid_polygon_shp = args[ 1] # the polygon should be in projection Cartesian coordinate system (e.g., UTM ) basic.outputlogMessage('Image grid polygon shapefile: %s' % grid_polygon_shp) process_num = options.process_num basic.outputlogMessage( 'The number of processes for creating the mosaic is: %d' % process_num) # read grid polygons grid_polygons = vector_gpd.read_polygons_gpd(grid_polygon_shp) grid_ids = vector_gpd.read_attribute_values_list(grid_polygon_shp, 'id') if grid_ids is None: basic.outputlogMessage( 'Warning, field: id is not in %s, will create default ID for each grid' % grid_polygon_shp) grid_ids = [id + 1 for id in range(len(grid_polygons))] shp_prj = map_projection.get_raster_or_vector_srs_info_proj4( grid_polygon_shp).strip() # print(shp_prj) grid_polygons_latlon = grid_polygons if shp_prj != '+proj=longlat +datum=WGS84 +no_defs': # read polygons and reproject to 4326 projection grid_polygons_latlon = vector_gpd.read_shape_gpd_to_NewPrj( grid_polygon_shp, 'EPSG:4326') # else: # raise ValueError(' %s should be in projection of Cartesian coordinate system'%grid_polygon_shp) shp_prj_wkt = map_projection.get_raster_or_vector_srs_info_wkt( grid_polygon_shp) max_sr = options.max_sr min_sr = options.min_sr original_img_copy_dir = options.original_img_copy_dir b_to_rgb_8bit = options.to_rgb basic.outputlogMessage('Convert to 8bit RGB images: %s' % str(b_to_rgb_8bit)) # group planet image based on acquisition date b_group_date = options.group_date basic.outputlogMessage('Group Planet image based on acquisition date: %s' % str(b_group_date)) if b_group_date: # diff_days as 0, group images acquired at the same date geojson_groups = group_planet_images_date(geojson_list, diff_days=0) # sort based on yeardate in accending order : operator.itemgetter(0) geojson_groups = dict( sorted(geojson_groups.items(), key=operator.itemgetter(0))) save_group_txt = 'geojson_groups_input_folder.txt' basic.outputlogMessage( 'images are divided into %d groups, save to %s' % (len(geojson_groups.keys()), save_group_txt)) io_function.save_dict_to_txt_json(save_group_txt, geojson_groups) else: geojson_groups = {'all': geojson_list} # create mosaic of each grid cloud_cover_thr = options.cloud_cover cloud_cover_thr = cloud_cover_thr * 100 # for Planet image, it is percentage out_res = options.out_res cur_dir = os.getcwd() resampling_method = options.merged_method for key in geojson_groups.keys(): # # test # if key != '20200701': # continue geojson_list = geojson_groups[key] save_dir = os.path.basename(cur_dir) + '_mosaic_' + str( out_res) + '_' + key # print(save_dir) if process_num == 1: for id, polygon, poly_latlon in zip(grid_ids, grid_polygons, grid_polygons_latlon): # if id != 34: # continue create_moasic_of_each_grid_polygon( id, polygon, poly_latlon, out_res, cloud_cover_thr, geojson_list, save_dir, new_prj_wkt=shp_prj_wkt, new_prj_proj4=shp_prj, sr_min=min_sr, sr_max=max_sr, to_rgb=b_to_rgb_8bit, save_org_dir=original_img_copy_dir, resampling_method=resampling_method) elif process_num > 1: theadPool = Pool(process_num) # multi processes parameters_list = [ (id, polygon, poly_latlon, out_res, cloud_cover_thr, geojson_list, save_dir, shp_prj_wkt, shp_prj, min_sr, max_sr, b_to_rgb_8bit, 0, original_img_copy_dir) for id, polygon, poly_latlon in zip(grid_ids, grid_polygons, grid_polygons_latlon) ] results = theadPool.starmap(create_moasic_of_each_grid_polygon, parameters_list) # need python3 theadPool.close() else: raise ValueError('incorrect process number: %d' % process_num) cost_time_sec = time.time() - time0 basic.outputlogMessage( 'Done, total time cost %.2f seconds (%.2f minutes or %.2f hours)' % (cost_time_sec, cost_time_sec / 60, cost_time_sec / 3600)) pass
def main(): # grid polygons grid_50km = os.path.join(shp_dir,'PAMPA_outline_utm_50grid.shp') # main permaforst areas based on permafrost map, has been pre-processed: remove small ones, simply the boundaries main_area_simp = os.path.join(shp_dir,'PAMPA_outline_utm.shp') # qtp_grid_50km and qtb_main_perma_area_simp have the same projection grid_prj = map_projection.get_raster_or_vector_srs_info_proj4(grid_50km) perma_area_prj = map_projection.get_raster_or_vector_srs_info_proj4(main_area_simp) if grid_prj != perma_area_prj: raise ValueError('%s and %s do not have the same projection'%(grid_prj,perma_area_prj)) grids = vector_gpd.read_polygons_gpd(grid_50km) perma_areas = vector_gpd.read_polygons_gpd(main_area_simp) # perma_size_list = vector_gpd.read_attribute_values_list(qtp_main_perma_area_simp,'Area_km2') small_perma_areas_list = [] for idx, perma_poly in enumerate(perma_areas): print(' processing %dth permafrost area'%idx) # if the permafrost area is < 50*50 km^2, then do not split it to smaller ones. # if size < 2500: # perma_poly = vector_gpd.fill_holes_in_a_polygon(perma_poly) # small_perma_areas_list.append(perma_poly) # continue # split the big permafrost area into many small ones for grid in grids: inte_res = perma_poly.intersection(grid) if inte_res.is_empty is False: inte_res_multi = vector_gpd.MultiPolygon_to_polygons(idx,inte_res) for tmp in inte_res_multi: # remove holes if they exist small_ones = vector_gpd.fill_holes_in_a_polygon(tmp) ################################# # we should remove some really small polygons (< 1 km^2) small_perma_areas_list.append(small_ones) ############################## # have to manually merge small polygons in QGIS to its adjacent ones. # save save_path = io_function.get_name_by_adding_tail(main_area_simp,'small') save_path = os.path.join(shp_dir,os.path.basename(save_path)) save_polyons_attributes = {} save_polyons_attributes["Polygons"] = small_perma_areas_list # wkt_string = map_projection.get_raster_or_vector_srs_info_wkt(qtp_main_perma_area_simp) wkt_string = map_projection.get_raster_or_vector_srs_info_proj4(main_area_simp) polygon_df = pd.DataFrame(save_polyons_attributes) vector_gpd.save_polygons_to_files(polygon_df, 'Polygons', wkt_string, save_path) pass
def main(): dem_index_shp = os.path.expanduser( '~/Data/Arctic/ArcticDEM/BROWSE_SERVER/indexes/ArcticDEM_Tile_Index_Rel7/ArcticDEM_Tile_Index_Rel7.shp' ) # extent_shp = os.path.expanduser('~/Data/PDO/PDO_statistics_swatchs/swatch_bounding_boxes.shp') extent_shp = os.path.expanduser( '~/Data/PDO/extent_each_swatch/merge_all_qa_exent.shp') # extent polygons and projection (proj4) extent_shp_prj = map_projection.get_raster_or_vector_srs_info_proj4( extent_shp) dem_shp_prj = map_projection.get_raster_or_vector_srs_info_proj4( dem_index_shp) if extent_shp_prj != dem_shp_prj: basic.outputlogMessage( '%s and %s do not have the same projection, will reproject %s' % (extent_shp, dem_index_shp, os.path.basename(extent_shp))) epsg = map_projection.get_raster_or_vector_srs_info_epsg(dem_index_shp) # print(epsg) # extent_polys = vector_gpd.read_shape_gpd_to_NewPrj(extent_shp,dem_shp_prj.strip()) extent_polys = vector_gpd.read_shape_gpd_to_NewPrj(extent_shp, epsg) else: extent_polys = vector_gpd.read_polygons_gpd(extent_shp) poly_ids = [idx for idx in range(len(extent_polys))] if 'boxes' in os.path.basename(extent_shp): nc_file_names = vector_gpd.read_attribute_values_list( extent_shp, 'nc_file') else: nc_file_names = vector_gpd.read_attribute_values_list( extent_shp, 'layer') # read dem polygons and tile number dem_polygons, dem_tiles = vector_gpd.read_polygons_attributes_list( dem_index_shp, 'tile', b_fix_invalid_polygon=False) for count, (idx, ext_poly) in enumerate(zip(poly_ids, extent_polys)): basic.outputlogMessage('get data for the %d th extent (%d/%d)' % (idx, count, len(extent_polys))) save_txt_path = nc_file_names[idx] + '-' + 'dem_tiles_poly_%d.txt' % idx if os.path.isfile(save_txt_path): tiles = io_function.read_list_from_txt(save_txt_path) basic.outputlogMessage('read %d dem tiles from %s' % (len(tiles), save_txt_path)) else: # get fileurl dem_poly_ids = vector_gpd.get_poly_index_within_extent( dem_polygons, ext_poly) basic.outputlogMessage('find %d DEM within %d th extent' % (len(dem_poly_ids), (idx))) tiles = [dem_tiles[id] for id in dem_poly_ids] # save to txt io_function.save_list_to_txt(save_txt_path, tiles) basic.outputlogMessage('save dem urls to %s' % save_txt_path) # download and create a mosaic url_head = 'https://data.pgc.umn.edu/elev/dem/setsm/ArcticDEM/mosaic/v3.0/32m/' download_tarball_for_one_polygon(tarball_dir, dem_tif_dir, url_head, tiles) # create a mosaic create_a_mosaic(nc_file_names[idx], idx, dem_eachSwatch_dir, ext_poly, tiles) bak_folder = 'small_tifs' io_function.mkdir(bak_folder) # remove small and duplicated ones for file_name in nc_file_names: crop_tifs = io_function.get_file_list_by_pattern( dem_eachSwatch_dir, file_name + '*crop.tif') if len(crop_tifs) == 1: pass elif len(crop_tifs) > 1: # keep maximum one and move small ones tif_files_size = [ io_function.get_file_size_bytes(item) for item in crop_tifs ] max_size = max(tif_files_size) max_index = tif_files_size.index(max_size) del crop_tifs[max_index] for tmp in crop_tifs: io_function.movefiletodir(tmp, bak_folder) tmp = tmp.replace('_crop', '') io_function.movefiletodir(tmp, bak_folder) else: # no tif raise ValueError('Results for %s does not exist' % file_name)