def __init__(self, core_dataset_name='buildchange', src_version='v0', dst_version='v2', imageset='train_shanghai', subimage_size=1024, gap=512, num_processor=16): self.core_dataset_name = core_dataset_name self.src_version = src_version self.dst_version = dst_version self.imageset = imageset self.subimage_size = subimage_size self.gap = gap self.image_path = './data/{}/{}/{}/images'.format( core_dataset_name, src_version, imageset) self.mask_path = './data/{}/{}/{}/anno_v2'.format( core_dataset_name, src_version, imageset) self.image_save_path = './data/{}/{}/{}/images'.format( core_dataset_name, dst_version, "{}_{}".format(imageset, subimage_size)) wwtool.mkdir_or_exist(self.image_save_path) self.label_save_path = './data/{}/{}/{}/labels'.format( core_dataset_name, dst_version, "{}_{}".format(imageset, subimage_size)) wwtool.mkdir_or_exist(self.label_save_path) self.mask_parser = wwtool.MaskParse() self.pool = Pool(num_processor)
def __init__(self, rgb_file, foot_shp_file, geo_info, pred_segmentation_file, side_shp_file, pixel_anno): self.rgb_file = rgb_file self.foot_shp_file = foot_shp_file self.geo_info = geo_info self.pred_segmentation_file = pred_segmentation_file self.side_shp_file = side_shp_file self.pixel_anno = pixel_anno mask_parser = wwtool.MaskParse() objects = mask_parser(pixel_anno, category=255) self.ignore_polygons = [obj['polygon'] for obj in objects] self.rgb_image = cv2.imread(rgb_file) shp_parser = wwtool.ShpParse() objects = shp_parser(foot_shp_file, geo_info) self.foot_mask = cv2.imread(geo_info, 0) # annotated by floor heigh self.foot_polygons = [obj['converted_polygon'] for obj in objects] self.floors = [obj['converted_property']['Floor'] for obj in objects] self.pred_segmentation = cv2.imread(pred_segmentation_file, 0) try: side_shp = geopandas.read_file(side_shp_file, encoding='utf-8') self.side_annotations = [ side_coor for _, side_coor in side_shp.iterrows() ] except: print("Can't open this side shp file: {}".format(side_shp_file)) self.offset_and_floorheigh_list = []
def __init__(self, core_dataset_name='buildchange', src_version='v0', dst_version='v1', imageset='shanghai', subimage_size=1024, gap=512, multi_processing=False, num_processor=16, show=False): self.core_dataset_name = core_dataset_name self.src_version = src_version self.dst_version = dst_version self.imageset = imageset self.subimage_size = subimage_size self.gap = gap self.image_path = './data/{}/{}/{}/images'.format( core_dataset_name, src_version, imageset) self.merged_shp_path = './data/{}/{}/{}/merged_shp'.format( core_dataset_name, src_version, imageset) self.geo_path = './data/{}/{}/{}/geo_info'.format( core_dataset_name, src_version, imageset) self.pixel_anno_path = './data/{}/{}/{}/anno_v2'.format( core_dataset_name, src_version, imageset) self.image_save_path = './data/{}/{}/{}/images'.format( core_dataset_name, dst_version, "{}_{}".format(imageset, subimage_size)) if not os.path.exists(self.image_save_path): os.makedirs(self.image_save_path) else: shutil.rmtree(self.image_save_path) os.makedirs(self.image_save_path) self.label_save_path = './data/{}/{}/{}/labels'.format( core_dataset_name, dst_version, "{}_{}".format(imageset, subimage_size)) if not os.path.exists(self.label_save_path): os.makedirs(self.label_save_path) else: shutil.rmtree(self.label_save_path) os.makedirs(self.label_save_path) self.shp_parser = wwtool.ShpParse() self.mask_parser = wwtool.MaskParse() self.multi_processing = multi_processing self.pool = Pool(num_processor) self.show = show
def __setstate__(self, state): self.__dict__.update(state) if __name__ == '__main__': # cities = ['shanghai', 'beijing', 'jinan', 'haerbin', 'chengdu'] # sub_imageset_folds = {'beijing': ['arg', 'google', 'ms', 'tdt'], # 'chengdu': ['arg', 'google', 'ms', 'tdt'], # 'haerbin': ['arg', 'google', 'ms'], # 'jinan': ['arg', 'google', 'ms', 'tdt'], # 'shanghai': ['arg', 'google', 'ms', 'tdt', 'PHR2016', 'PHR2017']} cities = ['shanghai'] sub_imageset_folds = {'shanghai': ['arg']} core_dataset_name = 'buildchange' src_version = 'v0' dst_version = 'v2' sub_img_w, sub_img_h = 1024, 1024 mask_parser = wwtool.MaskParse() shp_parser = wwtool.ShpParse() for city in cities: convert = Simpletxt2Json(dst_version=dst_version, city=city, sub_imageset_folds=sub_imageset_folds, multi_processing=True, num_processor=8) convert.core() print(f"finish processing {city}")
def __call__(self, shp_fn, geom_img, ignore_file=None, return_ignore_index=False, show_ignored_polygons=False, coord='4326', merge_flag=False, merge_mode=2, connection_mode='floor', filter_small_size=0): """Parse shapefile of building change Arguments: shp_fn {str} -- file path of shapefile geom_img {rasterio class or string} -- png image contain coordinate information Keyword Arguments: coord {str} -- coordinate system (default: {'4326'}) merge_flag {bool} -- False: skip the polygon merge, True: output objects after merging (default: {False}) merge_mode {int} -- 1: merge by intersection, 2: merge by rule (default: {2}) connection_mode {str} -- "line": merge by line intersection, "floor": merge by floor (default: {'floor'}) Returns: [dict] -- dict which contains core information """ try: shp = gpd.read_file(shp_fn, encoding='utf-8') except: print("Can't open this shp file: {}".format(shp_fn)) return [] # open geometry information file when input is string (file path) if isinstance(geom_img, str): geom_img = rio.open(geom_img) # open the ignore file if ignore_file: mask_parser = wwtool.MaskParse() objects = mask_parser(ignore_file, category=255) if objects == []: return [] ignore_polygons = [obj['polygon'] for obj in objects] ori_polygon_list = [] ori_floor_list = [] ori_property_list = [] for idx, row_data in shp.iterrows(): polygon = row_data.geometry property_ = row_data[:-1] try: floor = row_data.Floor except: print("This file does not floor key: {}".format(shp_fn)) # for processing beijing shapefile try: floor = row_data.half_H except: print("This file does not half_H key: {}".format(shp_fn)) return [] if polygon == None: continue ori_polygon_list.append(polygon) ori_floor_list.append(floor) ori_property_list.append(property_) if merge_flag: # merge the splitted building when annotation merged_polygon_list, merged_property_list = self._merge_polygon( ori_polygon_list, ori_floor_list, ori_property_list, merge_mode=merge_mode, connection_mode=connection_mode) else: merged_polygon_list = ori_polygon_list merged_property_list = ori_property_list # converted coordinate converted_polygons = [] converted_properties = [] # original coordinate, add these to handle multipolygons after merging merged_ori_polygons = [] merged_ori_properties = [] for polygon, property_ in zip(merged_polygon_list, merged_property_list): if polygon.geom_type == 'Polygon': if coord == '4326': polygon_pixel = [(geom_img.index(c[0], c[1])[1], geom_img.index(c[0], c[1])[0]) for c in polygon.exterior.coords] polygon_pixel = Polygon(polygon_pixel) if polygon_pixel.area < filter_small_size: continue converted_polygons.append(polygon_pixel) converted_properties.append(property_) else: if polygon.area < filter_small_size: continue converted_polygons.append(polygon) converted_properties.append(property_) merged_ori_polygons.append(polygon) merged_ori_properties.append(property_) elif polygon.geom_type == 'MultiPolygon': for sub_polygon in polygon: if coord == '4326': polygon_pixel = [(geom_img.index(c[0], c[1])[1], geom_img.index(c[0], c[1])[0]) for c in sub_polygon.exterior.coords] polygon_pixel = Polygon(polygon_pixel) if polygon_pixel.area < filter_small_size: continue converted_polygons.append(polygon_pixel) converted_properties.append(property_) else: if sub_polygon.area < filter_small_size: continue converted_polygons.append(sub_polygon) converted_properties.append(property_) merged_ori_polygons.append(sub_polygon) merged_ori_properties.append(property_) else: raise(RuntimeError("type(polygon) = {}".format(type(polygon)))) if isinstance(converted_polygons, (list, shapely.geometry.multipolygon.MultiPolygon)): pass else: converted_polygons = [converted_polygons] masks = [] final_polygons = [] final_properties = converted_properties for polygon_pixel in converted_polygons: final_polygons.append(polygon_pixel) wkt = str(polygon_pixel) mask = self._wkt2coord(wkt) masks.append(mask) objects = [] if ignore_file: _, ignore_indexes = wwtool.cleaning_polygon_by_polygon(final_polygons, ignore_polygons, show=show_ignored_polygons) if return_ignore_index: # keep full polygons, and return ignored indexes ignore_list = [0] * len(final_polygons) for ignore_index in ignore_indexes[::-1]: ignore_list[ignore_index] = 1 else: # return the ignored polygons for ignore_index in ignore_indexes[::-1]: masks.pop(ignore_index) final_polygons.pop(ignore_index) final_properties.pop(ignore_index) merged_ori_polygons.pop(ignore_index) merged_ori_properties.pop(ignore_index) # ori -> original coordinate, converted -> converted coordinate for idx, (mask, converted_polygon, converted_property, ori_polygon, ori_property) in enumerate(zip(masks, final_polygons, final_properties, merged_ori_polygons, merged_ori_properties)): if mask == []: continue object_struct = dict() mask = [abs(_) for _ in mask] xmin, ymin, xmax, ymax = wwtool.pointobb2bbox(mask) bbox_w = xmax - xmin bbox_h = ymax - ymin object_struct['segmentation'] = mask object_struct['bbox'] = [xmin, ymin, bbox_w, bbox_h] object_struct['converted_polygon'] = converted_polygon object_struct['converted_property'] = converted_property object_struct['ori_polygon'] = ori_polygon object_struct['ori_property'] = ori_property object_struct['label'] = "1" if return_ignore_index: object_struct['ignore_index'] = ignore_list[idx] objects.append(object_struct) return objects