def get_overlap_image_index(polygons,image_boxes,min_overlap_area=1): ''' get the index of images that the polygons overlap :param polygons: a list of polygons :param image_boxes: the extent of the all the images :param min_overlap_area: minumum areas for checking the overlap, ignore the image if it's too small :return: ''' # find the images which the polygons overlap (one or two images) img_idx = [] # for a_poly in polygons: # a_poly_json = mapping(a_poly) # polygon_box = rasterio.features.bounds(a_poly_json) polygon_box = get_bounds_of_polygons(polygons) for idx, img_box in enumerate(image_boxes): if rasterio.coords.disjoint_bounds(img_box, polygon_box) is False: if idx not in img_idx: img_idx.append(idx) # check overlap for idx in img_idx: box_poly = convert_image_bound_to_shapely_polygon(image_boxes[idx]) poly_index = get_poly_index_within_extent(polygons,box_poly,min_overlap_area=min_overlap_area) # if no overlap, remove index if len(poly_index) < 1: img_idx.remove(idx) return img_idx
def read_polygons_from_small_patch(in_shp, in_raster): '''read polygons and seperate to touch and not touch image edge groups''' print(datetime.now(), 'reading polygons to touch and not touch image edge group') polygons = vector_gpd.read_polygons_gpd(in_shp, b_fix_invalid_polygon=False) img_bound = raster_io.get_image_bound_box(in_raster) img_resx, img_resy = raster_io.get_xres_yres_file(in_raster) half_res = img_resx / 2.0 image_edge = vector_gpd.convert_image_bound_to_shapely_polygon(img_bound) polygons_buff = [item.buffer(half_res) for item in polygons] # buffer half pixel # polygons_touch_img_edge_index = [] polygon_no_touch = [] polygons_touch = [] for idx, (polybuff, poly) in enumerate(zip(polygons_buff, polygons)): if polybuff.within(image_edge): polygon_no_touch.append(poly) else: # polygons_touch_img_edge_index.append(idx) polygons_touch.append(poly) # return polygons,polygons_touch_img_edge_index return polygon_no_touch, polygons_touch
def latlon_2_polygons(one_nc_info): print(one_nc_info) # .nc4 filename, top (max) latitude, bottom (min) latitude, left (min) longitude, right (max) longitude nc_filename = one_nc_info[0].strip() top_lat = float(one_nc_info[1].strip()) bottom_lat = float(one_nc_info[2].strip()) left_lon = float(one_nc_info[3].strip()) right_lon = float(one_nc_info[4].strip()) bounding_box = (left_lon, bottom_lat, right_lon, top_lat ) # box: (left, bottom, right, top) print(nc_filename, bounding_box) polygon = vector_gpd.convert_image_bound_to_shapely_polygon(bounding_box) return nc_filename, polygon
def main(options, args): img_dir = args[0] output = options.output if os.path.isdir(img_dir): img_pattern = options.image_pattern if output is None: output = os.path.basename(img_dir) + '_boxes.gpkg' img_list = io_function.get_file_list_by_pattern(img_dir, img_pattern) if len(img_list) < 1: raise ValueError('No images in %s with pattern: %s' % (img_dir, img_pattern)) else: # if it's a file img_list = [img_dir] if output is None: output = os.path.basename(img_dir) + '_bound.gpkg' print('Find %d rasters in %s' % (len(img_list), img_dir)) # check projections? prj4_1st = raster_io.get_projection(img_list[0], 'proj4') for idx in range(1, len(img_list)): prj4 = raster_io.get_projection(img_list[idx], 'proj4') if prj4_1st != prj4: raise ValueError('projection inconsistent between %s and %s ' % (img_list[0], img_list[idx])) img_boxes = [ raster_io.get_image_bound_box(img_path) for img_path in img_list ] img_box_polygons = [ vector_gpd.convert_image_bound_to_shapely_polygon(box) for box in img_boxes ] # save to file wkt = map_projection.get_raster_or_vector_srs_info_proj4(img_list[0]) save_pd = pd.DataFrame({'raster': img_list, 'Polygon': img_box_polygons}) vector_gpd.save_polygons_to_files(save_pd, 'Polygon', wkt, output, format='GPKG') print('save raster extents to %s' % output) return
def get_dem_tif_ext_polygons(dem_list): bounding_box_list = [ raster_io.get_image_bound_box(tif) for tif in dem_list ] ext_polys = [ vector_gpd.convert_image_bound_to_shapely_polygon(box) for box in bounding_box_list] return ext_polys
def zonal_stats_multiRasters(in_shp, raster_file_or_files, nodata=None, band=1, stats=None, prefix='', range=None, all_touched=True, process_num=1): ''' zonal statistic based on vectors, along multiple rasters (image tiles) Args: in_shp: input vector file raster_file_or_files: a raster file or multiple rasters nodata: band: band stats: like [mean, std, max, min] range: interested values [min, max], None means infinity all_touched: process_num: process number for calculation Returns: ''' io_function.is_file_exist(in_shp) if stats is None: basic.outputlogMessage('warning, No input stats, set to ["mean"])') stats = ['mean'] if isinstance(raster_file_or_files, str): io_function.is_file_exist(raster_file_or_files) image_tiles = [raster_file_or_files] elif isinstance(raster_file_or_files, list): image_tiles = raster_file_or_files else: raise ValueError('unsupport type for %s' % str(raster_file_or_files)) # check projection (assume we have the same projection), check them outside this function # get image box img_tile_boxes = [ raster_io.get_image_bound_box(tile) for tile in image_tiles ] img_tile_polygons = [ vector_gpd.convert_image_bound_to_shapely_polygon(box) for box in img_tile_boxes ] polygons = vector_gpd.read_polygons_gpd(in_shp) if len(polygons) < 1: basic.outputlogMessage('No polygons in %s' % in_shp) return False # polygons_json = [mapping(item) for item in polygons] # no need when use new verion of rasterio # process polygons one by one polygons and the corresponding image tiles (parallel and save memory) # also to avoid error: daemonic processes are not allowed to have children if process_num == 1: stats_res_list = [] for idx, polygon in enumerate(polygons): out_stats = zonal_stats_one_polygon(idx, polygon, image_tiles, img_tile_polygons, stats, nodata=nodata, range=range, band=band, all_touched=all_touched) stats_res_list.append(out_stats) elif process_num > 1: threadpool = Pool(process_num) para_list = [(idx, polygon, image_tiles, img_tile_polygons, stats, nodata, range, band, all_touched) for idx, polygon in enumerate(polygons)] stats_res_list = threadpool.starmap(zonal_stats_one_polygon, para_list) else: raise ValueError('Wrong process number: %s ' % str(process_num)) # save to shapefile add_attributes = {} new_key_list = [prefix + '_' + key for key in stats_res_list[0].keys()] for new_ley in new_key_list: add_attributes[new_ley] = [] for stats_result in stats_res_list: for key in stats_result.keys(): add_attributes[prefix + '_' + key].append(stats_result[key]) vector_gpd.add_attributes_to_shp(in_shp, add_attributes) pass
def yolo_results_to_shapefile(curr_dir, img_idx, area_save_dir, nms_overlap_thr, test_id): img_save_dir = os.path.join(area_save_dir, 'I%d' % img_idx) res_yolo_json = img_save_dir + '_result.json' res_json_files = [] if os.path.isfile(res_yolo_json): print('found %s in %s, will get shapefile from it' % (res_yolo_json, area_save_dir)) else: if os.path.isdir(img_save_dir): res_json_files = io_function.get_file_list_by_ext( '.json', img_save_dir, bsub_folder=False) if len(res_json_files) < 1: print('Warning, no YOLO results in %s, skip' % (img_save_dir)) return None print( 'found %d json files for patches in %s, will get shapefile from them' % (len(res_json_files), img_save_dir)) else: print('Warning, folder: %s doest not exist, skip' % img_save_dir) return None out_name = os.path.basename(area_save_dir) + '_' + test_id # to shapefile out_shp = 'I%d' % img_idx + '_' + out_name + '.shp' out_shp_path = os.path.join(img_save_dir, out_shp) if os.path.isfile(out_shp_path): print('%s already exist' % out_shp_path) else: class_id_list = [] name_list = [] box_bounds_list = [] confidence_list = [] source_image_list = [] if len(res_json_files) < 1: # use the result in *_result.json yolo_res_dict_list = io_function.read_dict_from_txt_json( res_yolo_json) total_frame = len(yolo_res_dict_list) image1 = yolo_res_dict_list[0]['filename'] for idx, res_dict in enumerate(yolo_res_dict_list): id_list, na_list, con_list, box_list, image1 = boxes_yoloXY_to_imageXY( idx, total_frame, res_dict, ref_image=None) class_id_list.extend(id_list) name_list.extend(na_list) confidence_list.extend(con_list) box_bounds_list.extend(box_list) source_image_list.extend([os.path.basename(image1)] * len(box_list)) else: # use the results in I0/*.json image1 = io_function.read_list_from_txt( os.path.join(area_save_dir, '%d.txt' % img_idx))[0] total_frame = len(res_json_files) # the patch numbers # only open image once with rasterio.open(image1) as src: for idx, f_json in enumerate(res_json_files): id_list, na_list, con_list, box_list = boxes_minXYmaxXY_to_imageXY( idx, total_frame, f_json, src) class_id_list.extend(id_list) name_list.extend(na_list) confidence_list.extend(con_list) box_bounds_list.extend(box_list) source_image_list.extend([os.path.basename(image1)] * len(box_bounds_list)) if len(box_bounds_list) < 1: print('Warning, no predicted boxes in %s' % img_save_dir) return None # apply non_max_suppression # print('box_bounds_list',box_bounds_list) # print('confidence_list',confidence_list) pick_index = non_max_suppression(np.array(box_bounds_list), probs=np.array(confidence_list), overlapThresh=nms_overlap_thr, b_geo=True) # print('pick_index', pick_index) class_id_list = [class_id_list[idx] for idx in pick_index] name_list = [name_list[idx] for idx in pick_index] confidence_list = [confidence_list[idx] for idx in pick_index] box_bounds_list = [box_bounds_list[idx] for idx in pick_index] source_image_list = [source_image_list[idx] for idx in pick_index] # to polygon box_poly_list = [ vector_gpd.convert_image_bound_to_shapely_polygon(item) for item in box_bounds_list ] # box_poly_list # save to shapefile detect_boxes_dict = { 'class_id': class_id_list, 'name': name_list, 'source_img': source_image_list, 'confidence': confidence_list, "Polygon": box_poly_list } save_pd = pd.DataFrame(detect_boxes_dict) ref_prj = map_projection.get_raster_or_vector_srs_info_proj4(image1) vector_gpd.save_polygons_to_files(save_pd, 'Polygon', ref_prj, out_shp_path) return out_shp_path
def zonal_stats_multiRasters(in_shp, raster_file_or_files, nodata=None, band = 1, stats = None, prefix='', range=None,buffer=None, all_touched=True, process_num=1): ''' zonal statistic based on vectors, along multiple rasters (image tiles) Args: in_shp: input vector file raster_file_or_files: a raster file or multiple rasters nodata: band: band stats: like [mean, std, max, min] range: interested values [min, max], None means infinity buffer: expand polygon with buffer (meter) before the statistic all_touched: process_num: process number for calculation Returns: ''' io_function.is_file_exist(in_shp) if stats is None: basic.outputlogMessage('warning, No input stats, set to ["mean"])') stats = ['mean'] stats_backup = stats.copy() if 'area' in stats: stats.remove('area') if 'count' not in stats: stats.append('count') if isinstance(raster_file_or_files,str): io_function.is_file_exist(raster_file_or_files) image_tiles = [raster_file_or_files] elif isinstance(raster_file_or_files,list): image_tiles = raster_file_or_files else: raise ValueError('unsupport type for %s'%str(raster_file_or_files)) # check projection (assume we have the same projection), check them outside this function # get image box img_tile_boxes = [raster_io.get_image_bound_box(tile) for tile in image_tiles] img_tile_polygons = [vector_gpd.convert_image_bound_to_shapely_polygon(box) for box in img_tile_boxes] polygons = vector_gpd.read_polygons_gpd(in_shp) if len(polygons) < 1: basic.outputlogMessage('No polygons in %s'%in_shp) return False # polygons_json = [mapping(item) for item in polygons] # no need when use new verion of rasterio if buffer is not None: polygons = [ poly.buffer(buffer) for poly in polygons] # process polygons one by one polygons and the corresponding image tiles (parallel and save memory) stats_res_list = [] for idx, polygon in enumerate(polygons): out_stats = zonal_stats_one_polygon(idx, polygon, image_tiles, img_tile_polygons, stats, nodata=nodata, range=range, band=band, all_touched=all_touched) stats_res_list.append(out_stats) # threadpool = Pool(process_num) # para_list = [ (idx, polygon, image_tiles, img_tile_polygons, stats, nodata, range,band, all_touched) # for idx, polygon in enumerate(polygons)] # stats_res_list = threadpool.starmap(zonal_stats_one_polygon,para_list) # save to shapefile add_attributes = {} new_key_list = [ prefix + '_' + key for key in stats_res_list[0].keys()] for new_ley in new_key_list: add_attributes[new_ley] = [] for stats_result in stats_res_list: for key in stats_result.keys(): add_attributes[prefix + '_' + key].append(stats_result[key]) if 'area' in stats_backup: dx, dy = raster_io.get_xres_yres_file(image_tiles[0]) add_attributes[prefix + '_' + 'area'] = [ count*dx*dy for count in add_attributes[prefix + '_' + 'count'] ] if 'count' not in stats_backup: del add_attributes[prefix + '_' + 'count'] vector_gpd.add_attributes_to_shp(in_shp,add_attributes) pass