def __init__(self,
core_dataset_name='buildchange',
src_version='v0',
dst_version='v2',
imageset='train_shanghai',
subimage_size=1024,
gap=512,
num_processor=16):
self.core_dataset_name = core_dataset_name
self.src_version = src_version
self.dst_version = dst_version
self.imageset = imageset
self.subimage_size = subimage_size
self.gap = gap
self.image_path = './data/{}/{}/{}/images'.format(
core_dataset_name, src_version, imageset)
self.mask_path = './data/{}/{}/{}/anno_v2'.format(
core_dataset_name, src_version, imageset)
self.image_save_path = './data/{}/{}/{}/images'.format(
core_dataset_name, dst_version,
"{}_{}".format(imageset, subimage_size))
wwtool.mkdir_or_exist(self.image_save_path)
self.label_save_path = './data/{}/{}/{}/labels'.format(
core_dataset_name, dst_version,
"{}_{}".format(imageset, subimage_size))
wwtool.mkdir_or_exist(self.label_save_path)
self.mask_parser = wwtool.MaskParse()
self.pool = Pool(num_processor)
def __init__(self, rgb_file, foot_shp_file, geo_info,
pred_segmentation_file, side_shp_file, pixel_anno):
self.rgb_file = rgb_file
self.foot_shp_file = foot_shp_file
self.geo_info = geo_info
self.pred_segmentation_file = pred_segmentation_file
self.side_shp_file = side_shp_file
self.pixel_anno = pixel_anno
mask_parser = wwtool.MaskParse()
objects = mask_parser(pixel_anno, category=255)
self.ignore_polygons = [obj['polygon'] for obj in objects]
self.rgb_image = cv2.imread(rgb_file)
shp_parser = wwtool.ShpParse()
objects = shp_parser(foot_shp_file, geo_info)
self.foot_mask = cv2.imread(geo_info, 0) # annotated by floor heigh
self.foot_polygons = [obj['converted_polygon'] for obj in objects]
self.floors = [obj['converted_property']['Floor'] for obj in objects]
self.pred_segmentation = cv2.imread(pred_segmentation_file, 0)
try:
side_shp = geopandas.read_file(side_shp_file, encoding='utf-8')
self.side_annotations = [
side_coor for _, side_coor in side_shp.iterrows()
]
except:
print("Can't open this side shp file: {}".format(side_shp_file))
self.offset_and_floorheigh_list = []
def __init__(self,
core_dataset_name='buildchange',
src_version='v0',
dst_version='v1',
imageset='shanghai',
subimage_size=1024,
gap=512,
multi_processing=False,
num_processor=16,
show=False):
self.core_dataset_name = core_dataset_name
self.src_version = src_version
self.dst_version = dst_version
self.imageset = imageset
self.subimage_size = subimage_size
self.gap = gap
self.image_path = './data/{}/{}/{}/images'.format(
core_dataset_name, src_version, imageset)
self.merged_shp_path = './data/{}/{}/{}/merged_shp'.format(
core_dataset_name, src_version, imageset)
self.geo_path = './data/{}/{}/{}/geo_info'.format(
core_dataset_name, src_version, imageset)
self.pixel_anno_path = './data/{}/{}/{}/anno_v2'.format(
core_dataset_name, src_version, imageset)
self.image_save_path = './data/{}/{}/{}/images'.format(
core_dataset_name, dst_version,
"{}_{}".format(imageset, subimage_size))
if not os.path.exists(self.image_save_path):
os.makedirs(self.image_save_path)
else:
shutil.rmtree(self.image_save_path)
os.makedirs(self.image_save_path)
self.label_save_path = './data/{}/{}/{}/labels'.format(
core_dataset_name, dst_version,
"{}_{}".format(imageset, subimage_size))
if not os.path.exists(self.label_save_path):
os.makedirs(self.label_save_path)
else:
shutil.rmtree(self.label_save_path)
os.makedirs(self.label_save_path)
self.shp_parser = wwtool.ShpParse()
self.mask_parser = wwtool.MaskParse()
self.multi_processing = multi_processing
self.pool = Pool(num_processor)
self.show = show
def __setstate__(self, state):
self.__dict__.update(state)
if __name__ == '__main__':
# cities = ['shanghai', 'beijing', 'jinan', 'haerbin', 'chengdu']
# sub_imageset_folds = {'beijing': ['arg', 'google', 'ms', 'tdt'],
# 'chengdu': ['arg', 'google', 'ms', 'tdt'],
# 'haerbin': ['arg', 'google', 'ms'],
# 'jinan': ['arg', 'google', 'ms', 'tdt'],
# 'shanghai': ['arg', 'google', 'ms', 'tdt', 'PHR2016', 'PHR2017']}
cities = ['shanghai']
sub_imageset_folds = {'shanghai': ['arg']}
core_dataset_name = 'buildchange'
src_version = 'v0'
dst_version = 'v2'
sub_img_w, sub_img_h = 1024, 1024
mask_parser = wwtool.MaskParse()
shp_parser = wwtool.ShpParse()
for city in cities:
convert = Simpletxt2Json(dst_version=dst_version,
city=city,
sub_imageset_folds=sub_imageset_folds,
multi_processing=True,
num_processor=8)
convert.core()
print(f"finish processing {city}")
def __call__(self,
shp_fn,
geom_img,
ignore_file=None,
return_ignore_index=False,
show_ignored_polygons=False,
coord='4326',
merge_flag=False,
merge_mode=2,
connection_mode='floor',
filter_small_size=0):
"""Parse shapefile of building change
Arguments:
shp_fn {str} -- file path of shapefile
geom_img {rasterio class or string} -- png image contain coordinate information
Keyword Arguments:
coord {str} -- coordinate system (default: {'4326'})
merge_flag {bool} -- False: skip the polygon merge, True: output objects after merging (default: {False})
merge_mode {int} -- 1: merge by intersection, 2: merge by rule (default: {2})
connection_mode {str} -- "line": merge by line intersection, "floor": merge by floor (default: {'floor'})
Returns:
[dict] -- dict which contains core information
"""
try:
shp = gpd.read_file(shp_fn, encoding='utf-8')
except:
print("Can't open this shp file: {}".format(shp_fn))
return []
# open geometry information file when input is string (file path)
if isinstance(geom_img, str):
geom_img = rio.open(geom_img)
# open the ignore file
if ignore_file:
mask_parser = wwtool.MaskParse()
objects = mask_parser(ignore_file, category=255)
if objects == []:
return []
ignore_polygons = [obj['polygon'] for obj in objects]
ori_polygon_list = []
ori_floor_list = []
ori_property_list = []
for idx, row_data in shp.iterrows():
polygon = row_data.geometry
property_ = row_data[:-1]
try:
floor = row_data.Floor
except:
print("This file does not floor key: {}".format(shp_fn))
# for processing beijing shapefile
try:
floor = row_data.half_H
except:
print("This file does not half_H key: {}".format(shp_fn))
return []
if polygon == None:
continue
ori_polygon_list.append(polygon)
ori_floor_list.append(floor)
ori_property_list.append(property_)
if merge_flag:
# merge the splitted building when annotation
merged_polygon_list, merged_property_list = self._merge_polygon(
ori_polygon_list,
ori_floor_list,
ori_property_list,
merge_mode=merge_mode,
connection_mode=connection_mode)
else:
merged_polygon_list = ori_polygon_list
merged_property_list = ori_property_list
# converted coordinate
converted_polygons = []
converted_properties = []
# original coordinate, add these to handle multipolygons after merging
merged_ori_polygons = []
merged_ori_properties = []
for polygon, property_ in zip(merged_polygon_list, merged_property_list):
if polygon.geom_type == 'Polygon':
if coord == '4326':
polygon_pixel = [(geom_img.index(c[0], c[1])[1], geom_img.index(c[0], c[1])[0]) for c in polygon.exterior.coords]
polygon_pixel = Polygon(polygon_pixel)
if polygon_pixel.area < filter_small_size:
continue
converted_polygons.append(polygon_pixel)
converted_properties.append(property_)
else:
if polygon.area < filter_small_size:
continue
converted_polygons.append(polygon)
converted_properties.append(property_)
merged_ori_polygons.append(polygon)
merged_ori_properties.append(property_)
elif polygon.geom_type == 'MultiPolygon':
for sub_polygon in polygon:
if coord == '4326':
polygon_pixel = [(geom_img.index(c[0], c[1])[1], geom_img.index(c[0], c[1])[0]) for c in sub_polygon.exterior.coords]
polygon_pixel = Polygon(polygon_pixel)
if polygon_pixel.area < filter_small_size:
continue
converted_polygons.append(polygon_pixel)
converted_properties.append(property_)
else:
if sub_polygon.area < filter_small_size:
continue
converted_polygons.append(sub_polygon)
converted_properties.append(property_)
merged_ori_polygons.append(sub_polygon)
merged_ori_properties.append(property_)
else:
raise(RuntimeError("type(polygon) = {}".format(type(polygon))))
if isinstance(converted_polygons, (list, shapely.geometry.multipolygon.MultiPolygon)):
pass
else:
converted_polygons = [converted_polygons]
masks = []
final_polygons = []
final_properties = converted_properties
for polygon_pixel in converted_polygons:
final_polygons.append(polygon_pixel)
wkt = str(polygon_pixel)
mask = self._wkt2coord(wkt)
masks.append(mask)
objects = []
if ignore_file:
_, ignore_indexes = wwtool.cleaning_polygon_by_polygon(final_polygons, ignore_polygons, show=show_ignored_polygons)
if return_ignore_index:
# keep full polygons, and return ignored indexes
ignore_list = [0] * len(final_polygons)
for ignore_index in ignore_indexes[::-1]:
ignore_list[ignore_index] = 1
else:
# return the ignored polygons
for ignore_index in ignore_indexes[::-1]:
masks.pop(ignore_index)
final_polygons.pop(ignore_index)
final_properties.pop(ignore_index)
merged_ori_polygons.pop(ignore_index)
merged_ori_properties.pop(ignore_index)
# ori -> original coordinate, converted -> converted coordinate
for idx, (mask, converted_polygon, converted_property, ori_polygon, ori_property) in enumerate(zip(masks, final_polygons, final_properties, merged_ori_polygons, merged_ori_properties)):
if mask == []:
continue
object_struct = dict()
mask = [abs(_) for _ in mask]
xmin, ymin, xmax, ymax = wwtool.pointobb2bbox(mask)
bbox_w = xmax - xmin
bbox_h = ymax - ymin
object_struct['segmentation'] = mask
object_struct['bbox'] = [xmin, ymin, bbox_w, bbox_h]
object_struct['converted_polygon'] = converted_polygon
object_struct['converted_property'] = converted_property
object_struct['ori_polygon'] = ori_polygon
object_struct['ori_property'] = ori_property
object_struct['label'] = "1"
if return_ignore_index:
object_struct['ignore_index'] = ignore_list[idx]
objects.append(object_struct)
return objects