def get_overlap_image_index(polygons,image_boxes,min_overlap_area=1):
'''
get the index of images that the polygons overlap
:param polygons: a list of polygons
:param image_boxes: the extent of the all the images
:param min_overlap_area: minumum areas for checking the overlap, ignore the image if it's too small
:return:
'''
# find the images which the polygons overlap (one or two images)
img_idx = []
# for a_poly in polygons:
# a_poly_json = mapping(a_poly)
# polygon_box = rasterio.features.bounds(a_poly_json)
polygon_box = get_bounds_of_polygons(polygons)
for idx, img_box in enumerate(image_boxes):
if rasterio.coords.disjoint_bounds(img_box, polygon_box) is False:
if idx not in img_idx:
img_idx.append(idx)
# check overlap
for idx in img_idx:
box_poly = convert_image_bound_to_shapely_polygon(image_boxes[idx])
poly_index = get_poly_index_within_extent(polygons,box_poly,min_overlap_area=min_overlap_area)
# if no overlap, remove index
if len(poly_index) < 1:
img_idx.remove(idx)
return img_idx
def read_polygons_from_small_patch(in_shp, in_raster):
'''read polygons and seperate to touch and not touch image edge groups'''
print(datetime.now(),
'reading polygons to touch and not touch image edge group')
polygons = vector_gpd.read_polygons_gpd(in_shp,
b_fix_invalid_polygon=False)
img_bound = raster_io.get_image_bound_box(in_raster)
img_resx, img_resy = raster_io.get_xres_yres_file(in_raster)
half_res = img_resx / 2.0
image_edge = vector_gpd.convert_image_bound_to_shapely_polygon(img_bound)
polygons_buff = [item.buffer(half_res)
for item in polygons] # buffer half pixel
# polygons_touch_img_edge_index = []
polygon_no_touch = []
polygons_touch = []
for idx, (polybuff, poly) in enumerate(zip(polygons_buff, polygons)):
if polybuff.within(image_edge):
polygon_no_touch.append(poly)
else:
# polygons_touch_img_edge_index.append(idx)
polygons_touch.append(poly)
# return polygons,polygons_touch_img_edge_index
return polygon_no_touch, polygons_touch
def latlon_2_polygons(one_nc_info):
print(one_nc_info)
# .nc4 filename, top (max) latitude, bottom (min) latitude, left (min) longitude, right (max) longitude
nc_filename = one_nc_info[0].strip()
top_lat = float(one_nc_info[1].strip())
bottom_lat = float(one_nc_info[2].strip())
left_lon = float(one_nc_info[3].strip())
right_lon = float(one_nc_info[4].strip())
bounding_box = (left_lon, bottom_lat, right_lon, top_lat
) # box: (left, bottom, right, top)
print(nc_filename, bounding_box)
polygon = vector_gpd.convert_image_bound_to_shapely_polygon(bounding_box)
return nc_filename, polygon
def main(options, args):
img_dir = args[0]
output = options.output
if os.path.isdir(img_dir):
img_pattern = options.image_pattern
if output is None:
output = os.path.basename(img_dir) + '_boxes.gpkg'
img_list = io_function.get_file_list_by_pattern(img_dir, img_pattern)
if len(img_list) < 1:
raise ValueError('No images in %s with pattern: %s' %
(img_dir, img_pattern))
else:
# if it's a file
img_list = [img_dir]
if output is None:
output = os.path.basename(img_dir) + '_bound.gpkg'
print('Find %d rasters in %s' % (len(img_list), img_dir))
# check projections?
prj4_1st = raster_io.get_projection(img_list[0], 'proj4')
for idx in range(1, len(img_list)):
prj4 = raster_io.get_projection(img_list[idx], 'proj4')
if prj4_1st != prj4:
raise ValueError('projection inconsistent between %s and %s ' %
(img_list[0], img_list[idx]))
img_boxes = [
raster_io.get_image_bound_box(img_path) for img_path in img_list
]
img_box_polygons = [
vector_gpd.convert_image_bound_to_shapely_polygon(box)
for box in img_boxes
]
# save to file
wkt = map_projection.get_raster_or_vector_srs_info_proj4(img_list[0])
save_pd = pd.DataFrame({'raster': img_list, 'Polygon': img_box_polygons})
vector_gpd.save_polygons_to_files(save_pd,
'Polygon',
wkt,
output,
format='GPKG')
print('save raster extents to %s' % output)
return
def get_dem_tif_ext_polygons(dem_list):
bounding_box_list = [ raster_io.get_image_bound_box(tif) for tif in dem_list ]
ext_polys = [ vector_gpd.convert_image_bound_to_shapely_polygon(box) for box in bounding_box_list]
return ext_polys
def zonal_stats_multiRasters(in_shp,
raster_file_or_files,
nodata=None,
band=1,
stats=None,
prefix='',
range=None,
all_touched=True,
process_num=1):
'''
zonal statistic based on vectors, along multiple rasters (image tiles)
Args:
in_shp: input vector file
raster_file_or_files: a raster file or multiple rasters
nodata:
band: band
stats: like [mean, std, max, min]
range: interested values [min, max], None means infinity
all_touched:
process_num: process number for calculation
Returns:
'''
io_function.is_file_exist(in_shp)
if stats is None:
basic.outputlogMessage('warning, No input stats, set to ["mean"])')
stats = ['mean']
if isinstance(raster_file_or_files, str):
io_function.is_file_exist(raster_file_or_files)
image_tiles = [raster_file_or_files]
elif isinstance(raster_file_or_files, list):
image_tiles = raster_file_or_files
else:
raise ValueError('unsupport type for %s' % str(raster_file_or_files))
# check projection (assume we have the same projection), check them outside this function
# get image box
img_tile_boxes = [
raster_io.get_image_bound_box(tile) for tile in image_tiles
]
img_tile_polygons = [
vector_gpd.convert_image_bound_to_shapely_polygon(box)
for box in img_tile_boxes
]
polygons = vector_gpd.read_polygons_gpd(in_shp)
if len(polygons) < 1:
basic.outputlogMessage('No polygons in %s' % in_shp)
return False
# polygons_json = [mapping(item) for item in polygons] # no need when use new verion of rasterio
# process polygons one by one polygons and the corresponding image tiles (parallel and save memory)
# also to avoid error: daemonic processes are not allowed to have children
if process_num == 1:
stats_res_list = []
for idx, polygon in enumerate(polygons):
out_stats = zonal_stats_one_polygon(idx,
polygon,
image_tiles,
img_tile_polygons,
stats,
nodata=nodata,
range=range,
band=band,
all_touched=all_touched)
stats_res_list.append(out_stats)
elif process_num > 1:
threadpool = Pool(process_num)
para_list = [(idx, polygon, image_tiles, img_tile_polygons, stats,
nodata, range, band, all_touched)
for idx, polygon in enumerate(polygons)]
stats_res_list = threadpool.starmap(zonal_stats_one_polygon, para_list)
else:
raise ValueError('Wrong process number: %s ' % str(process_num))
# save to shapefile
add_attributes = {}
new_key_list = [prefix + '_' + key for key in stats_res_list[0].keys()]
for new_ley in new_key_list:
add_attributes[new_ley] = []
for stats_result in stats_res_list:
for key in stats_result.keys():
add_attributes[prefix + '_' + key].append(stats_result[key])
vector_gpd.add_attributes_to_shp(in_shp, add_attributes)
pass
def yolo_results_to_shapefile(curr_dir, img_idx, area_save_dir,
nms_overlap_thr, test_id):
img_save_dir = os.path.join(area_save_dir, 'I%d' % img_idx)
res_yolo_json = img_save_dir + '_result.json'
res_json_files = []
if os.path.isfile(res_yolo_json):
print('found %s in %s, will get shapefile from it' %
(res_yolo_json, area_save_dir))
else:
if os.path.isdir(img_save_dir):
res_json_files = io_function.get_file_list_by_ext(
'.json', img_save_dir, bsub_folder=False)
if len(res_json_files) < 1:
print('Warning, no YOLO results in %s, skip' % (img_save_dir))
return None
print(
'found %d json files for patches in %s, will get shapefile from them'
% (len(res_json_files), img_save_dir))
else:
print('Warning, folder: %s doest not exist, skip' % img_save_dir)
return None
out_name = os.path.basename(area_save_dir) + '_' + test_id
# to shapefile
out_shp = 'I%d' % img_idx + '_' + out_name + '.shp'
out_shp_path = os.path.join(img_save_dir, out_shp)
if os.path.isfile(out_shp_path):
print('%s already exist' % out_shp_path)
else:
class_id_list = []
name_list = []
box_bounds_list = []
confidence_list = []
source_image_list = []
if len(res_json_files) < 1:
# use the result in *_result.json
yolo_res_dict_list = io_function.read_dict_from_txt_json(
res_yolo_json)
total_frame = len(yolo_res_dict_list)
image1 = yolo_res_dict_list[0]['filename']
for idx, res_dict in enumerate(yolo_res_dict_list):
id_list, na_list, con_list, box_list, image1 = boxes_yoloXY_to_imageXY(
idx, total_frame, res_dict, ref_image=None)
class_id_list.extend(id_list)
name_list.extend(na_list)
confidence_list.extend(con_list)
box_bounds_list.extend(box_list)
source_image_list.extend([os.path.basename(image1)] *
len(box_list))
else:
# use the results in I0/*.json
image1 = io_function.read_list_from_txt(
os.path.join(area_save_dir, '%d.txt' % img_idx))[0]
total_frame = len(res_json_files) # the patch numbers
# only open image once
with rasterio.open(image1) as src:
for idx, f_json in enumerate(res_json_files):
id_list, na_list, con_list, box_list = boxes_minXYmaxXY_to_imageXY(
idx, total_frame, f_json, src)
class_id_list.extend(id_list)
name_list.extend(na_list)
confidence_list.extend(con_list)
box_bounds_list.extend(box_list)
source_image_list.extend([os.path.basename(image1)] *
len(box_bounds_list))
if len(box_bounds_list) < 1:
print('Warning, no predicted boxes in %s' % img_save_dir)
return None
# apply non_max_suppression
# print('box_bounds_list',box_bounds_list)
# print('confidence_list',confidence_list)
pick_index = non_max_suppression(np.array(box_bounds_list),
probs=np.array(confidence_list),
overlapThresh=nms_overlap_thr,
b_geo=True)
# print('pick_index', pick_index)
class_id_list = [class_id_list[idx] for idx in pick_index]
name_list = [name_list[idx] for idx in pick_index]
confidence_list = [confidence_list[idx] for idx in pick_index]
box_bounds_list = [box_bounds_list[idx] for idx in pick_index]
source_image_list = [source_image_list[idx] for idx in pick_index]
# to polygon
box_poly_list = [
vector_gpd.convert_image_bound_to_shapely_polygon(item)
for item in box_bounds_list
]
# box_poly_list
# save to shapefile
detect_boxes_dict = {
'class_id': class_id_list,
'name': name_list,
'source_img': source_image_list,
'confidence': confidence_list,
"Polygon": box_poly_list
}
save_pd = pd.DataFrame(detect_boxes_dict)
ref_prj = map_projection.get_raster_or_vector_srs_info_proj4(image1)
vector_gpd.save_polygons_to_files(save_pd, 'Polygon', ref_prj,
out_shp_path)
return out_shp_path
def zonal_stats_multiRasters(in_shp, raster_file_or_files, nodata=None, band = 1, stats = None, prefix='',
range=None,buffer=None, all_touched=True, process_num=1):
'''
zonal statistic based on vectors, along multiple rasters (image tiles)
Args:
in_shp: input vector file
raster_file_or_files: a raster file or multiple rasters
nodata:
band: band
stats: like [mean, std, max, min]
range: interested values [min, max], None means infinity
buffer: expand polygon with buffer (meter) before the statistic
all_touched:
process_num: process number for calculation
Returns:
'''
io_function.is_file_exist(in_shp)
if stats is None:
basic.outputlogMessage('warning, No input stats, set to ["mean"])')
stats = ['mean']
stats_backup = stats.copy()
if 'area' in stats:
stats.remove('area')
if 'count' not in stats:
stats.append('count')
if isinstance(raster_file_or_files,str):
io_function.is_file_exist(raster_file_or_files)
image_tiles = [raster_file_or_files]
elif isinstance(raster_file_or_files,list):
image_tiles = raster_file_or_files
else:
raise ValueError('unsupport type for %s'%str(raster_file_or_files))
# check projection (assume we have the same projection), check them outside this function
# get image box
img_tile_boxes = [raster_io.get_image_bound_box(tile) for tile in image_tiles]
img_tile_polygons = [vector_gpd.convert_image_bound_to_shapely_polygon(box) for box in img_tile_boxes]
polygons = vector_gpd.read_polygons_gpd(in_shp)
if len(polygons) < 1:
basic.outputlogMessage('No polygons in %s'%in_shp)
return False
# polygons_json = [mapping(item) for item in polygons] # no need when use new verion of rasterio
if buffer is not None:
polygons = [ poly.buffer(buffer) for poly in polygons]
# process polygons one by one polygons and the corresponding image tiles (parallel and save memory)
stats_res_list = []
for idx, polygon in enumerate(polygons):
out_stats = zonal_stats_one_polygon(idx, polygon, image_tiles, img_tile_polygons, stats, nodata=nodata, range=range,
band=band, all_touched=all_touched)
stats_res_list.append(out_stats)
# threadpool = Pool(process_num)
# para_list = [ (idx, polygon, image_tiles, img_tile_polygons, stats, nodata, range,band, all_touched)
# for idx, polygon in enumerate(polygons)]
# stats_res_list = threadpool.starmap(zonal_stats_one_polygon,para_list)
# save to shapefile
add_attributes = {}
new_key_list = [ prefix + '_' + key for key in stats_res_list[0].keys()]
for new_ley in new_key_list:
add_attributes[new_ley] = []
for stats_result in stats_res_list:
for key in stats_result.keys():
add_attributes[prefix + '_' + key].append(stats_result[key])
if 'area' in stats_backup:
dx, dy = raster_io.get_xres_yres_file(image_tiles[0])
add_attributes[prefix + '_' + 'area'] = [ count*dx*dy for count in add_attributes[prefix + '_' + 'count'] ]
if 'count' not in stats_backup:
del add_attributes[prefix + '_' + 'count']
vector_gpd.add_attributes_to_shp(in_shp,add_attributes)
pass