def __init__(self,
core_dataset_name='buildchange',
src_version='v0',
dst_version='v1',
imageset='shanghai',
subimage_size=1024,
gap=512,
num_processor=16):
self.core_dataset_name = core_dataset_name
self.src_version = src_version
self.dst_version = dst_version
self.imageset = imageset
self.subimage_size = subimage_size
self.gap = gap
self.image_path = './data/{}/{}/{}/images'.format(core_dataset_name, src_version, imageset)
self.merged_shp_path = './data/{}/{}/{}/merged_shp'.format(core_dataset_name, src_version, imageset)
self.geo_path = './data/{}/{}/{}/geo_info'.format(core_dataset_name, src_version, imageset)
self.image_save_path = './data/{}/{}/{}/images'.format(core_dataset_name, dst_version, "{}_{}".format(imageset, subimage_size))
wwtool.mkdir_or_exist(self.image_save_path)
self.label_save_path = './data/{}/{}/{}/labels'.format(core_dataset_name, dst_version, "{}_{}".format(imageset, subimage_size))
wwtool.mkdir_or_exist(self.label_save_path)
self.shp_parser = wwtool.ShpParse()
self.pool = Pool(num_processor)
def __init__(self,
dst_version,
city,
multi_processing=False,
num_processor=16):
self.splitted_image_dir = './data/buildchange/{}/{}/images'.format(
dst_version, city)
self.splitted_label_dir = './data/buildchange/{}/{}/labels_json'.format(
dst_version, city)
self.wireframe_dir = './data/buildchange/v2/{}/labels_wireframe'.format(
city)
wwtool.mkdir_or_exist(self.wireframe_dir)
self.city = city
self.multi_processing = multi_processing
self.pool = Pool(num_processor)
def __init__(self,
dst_version,
city,
sub_imageset_folds,
multi_processing=False,
num_processor=16):
self.splitted_image_dir = './data/buildchange/{}/{}/images'.format(
dst_version, city)
self.splitted_label_dir = './data/buildchange/{}/{}/labels'.format(
dst_version, city)
self.json_dir = './data/buildchange/v2/{}/labels_json'.format(city)
self.wrong_shp_file_dict = dict()
for sub_fold in sub_imageset_folds[city]:
wrong_file = './data/buildchange/v0/{}/{}/wrongShpFile.txt'.format(
city, sub_fold)
ori_filenames = self.read_wrong_file(wrong_file)
self.wrong_shp_file_dict[sub_fold] = ori_filenames
wwtool.mkdir_or_exist(self.json_dir)
self.city = city
self.multi_processing = multi_processing
self.pool = Pool(num_processor)
def __init__(self,
core_dataset_name='buildchange',
src_version='v0',
dst_version='v1',
imageset='shanghai',
sub_imageset_fold='arg',
subimage_size=1024,
gap=512,
multi_processing=False,
num_processor=16,
show=False):
self.core_dataset_name = core_dataset_name
self.src_version = src_version
self.dst_version = dst_version
self.imageset = imageset
self.sub_imageset_fold = sub_imageset_fold
self.subimage_size = subimage_size
self.gap = gap
self.image_path = './data/{}/{}/{}/{}/images'.format(
core_dataset_name, src_version, imageset, sub_imageset_fold)
self.roof_shp_path = './data/{}/{}/{}/{}/roof_shp_4326'.format(
core_dataset_name, src_version, imageset, sub_imageset_fold)
self.geo_path = './data/{}/{}/{}/{}/geo_info'.format(
core_dataset_name, src_version, imageset, sub_imageset_fold)
self.pixel_anno_v2_path = './data/{}/{}/{}/{}/pixel_anno_v2'.format(
core_dataset_name, src_version, imageset, sub_imageset_fold)
self.image_save_path = './data/{}/{}/{}/{}/images'.format(
core_dataset_name, dst_version, imageset, sub_imageset_fold)
wwtool.mkdir_or_exist(self.image_save_path)
self.label_save_path = './data/{}/{}/{}/{}/labels'.format(
core_dataset_name, dst_version, imageset, sub_imageset_fold)
wwtool.mkdir_or_exist(self.label_save_path)
self.shp_parser = wwtool.ShpParse()
self.multi_processing = multi_processing
self.pool = Pool(num_processor)
self.show = show
def shuffle_dataset(origin_dataset_dir, trainval_dir, test_dir, trainval_rate=0.8, image_format='.png', label_format='.txt', seed=0):
"""Generate trainval and test sets from origin set by copying files randomly.
Arguments:
origin_dataset_dir {str} -- path of origin dataset, contains `images` and `labels` folds
trainval_dir {str} -- path of trainval set, contains `images` and `labels` folds
test_dir {str} -- path of test set, contains `images` and `labels` folds
seed {int} -- seed of random function
Returns:
None
"""
np.random.seed(seed)
src_label_path = os.path.join(origin_dataset_dir, 'labels')
src_image_path = os.path.join(origin_dataset_dir, 'images')
trainval_dst_label_path = os.path.join(trainval_dir, 'labels')
wwtool.mkdir_or_exist(trainval_dst_label_path)
trainval_dst_image_path = os.path.join(trainval_dir, 'images')
wwtool.mkdir_or_exist(trainval_dst_image_path)
test_dst_label_path = os.path.join(test_dir, 'labels')
wwtool.mkdir_or_exist(test_dst_label_path)
test_dst_image_path = os.path.join(test_dir, 'images')
wwtool.mkdir_or_exist(test_dst_image_path)
file_names = [label_file.split('.txt')[0] for label_file in os.listdir(src_label_path)]
file_names = sorted(file_names)
np.random.shuffle(file_names)
trainval_file_names = file_names[0 : int(len(file_names) * trainval_rate)]
test_file_names = file_names[int(len(file_names) * trainval_rate):]
for trainval_file_name in trainval_file_names:
print("From {} to {}.".format(os.path.join(src_label_path, trainval_file_name), os.path.join(trainval_dst_label_path, trainval_file_name)))
shutil.copy(os.path.join(src_label_path, trainval_file_name + label_format), os.path.join(trainval_dst_label_path, trainval_file_name + label_format))
shutil.copy(os.path.join(src_image_path, trainval_file_name + image_format), os.path.join(trainval_dst_image_path, trainval_file_name + image_format))
for test_file_name in test_file_names:
print("From {} to {}.".format(os.path.join(src_label_path, test_file_name), os.path.join(test_dst_label_path, test_file_name)))
shutil.copy(os.path.join(src_label_path, test_file_name + label_format), os.path.join(test_dst_label_path, test_file_name + label_format))
shutil.copy(os.path.join(src_image_path, test_file_name + image_format), os.path.join(test_dst_image_path, test_file_name + image_format))
keypoint = False
just_keep_small = True
generate_small_dataset = True
small_size = 16 * 16
small_object_rate = 0.5
large_object_size = 64 * 64
anno_name = [core_dataset_name, release_version, rate]
if just_keep_small:
anno_name.append('small_object')
if generate_small_dataset:
dst_image_path = '/data/small/{}/images'.format(core_dataset_name)
dst_label_path = '/data/small/{}/labels'.format(core_dataset_name)
wwtool.mkdir_or_exist(dst_image_path)
wwtool.mkdir_or_exist(dst_label_path)
if keypoint:
for idx in range(len(converted_class)):
converted_class[idx]["keypoints"] = [
'top', 'right', 'bottom', 'left'
]
converted_class[idx]["skeleton"] = [[1, 2], [2, 3], [3, 4], [4, 1]]
anno_name.append('keypoint')
if groundtruth == False:
anno_name.append('no_ground_truth')
for imageset in imagesets:
imgpath = './data/{}/{}/{}/images'.format(core_dataset_name,
import cv2
from PIL import Image
from skimage.io import imread
import wwtool
Image.MAX_IMAGE_PIXELS = int(2048 * 2048 * 2048 // 4 // 3)
if __name__ == '__main__':
image_sets = ['trainval']
for image_set in image_sets:
image_path = '/data/gaofen/gaofen3/v0/{}/images'.format(image_set)
label_path = '/data/gaofen/gaofen3/v0/{}/labels'.format(image_set)
image_save_path = '/data/gaofen/gaofen3/v1/{}/images'.format(image_set)
wwtool.mkdir_or_exist(image_save_path)
label_save_path = '/data/gaofen/gaofen3/v1/{}/labels'.format(image_set)
wwtool.mkdir_or_exist(label_save_path)
# print(os.listdir(label_path))
for idx, label_file in enumerate(os.listdir(label_path)):
print(idx, label_file)
file_name = label_file.split('.xml')[0]
label_file = os.path.join(label_path, file_name + '.xml')
image_file = os.path.join(image_path, file_name + '.tif')
img = imread(image_file)
objects = wwtool.rovoc_parse(label_file)
bboxes = np.array([obj['bbox'] for obj in objects])
'roundabout': None,
'swimming-pool': 'swimming-pool',
'ground-track-field': None,
'basketball-court': None,
'container-crane': None}
if __name__ == "__main__":
image_format = '.png'
origin_image_path = './data/dota-v1.5/v1/trainval/images'
origin_label_path = './data/dota-v1.5/v1/trainval/labelTxt-v1.5'
filtered_image_path = './data/dota-v1.5/v1/trainval_iltered/images'
filtered_label_path = './data/dota-v1.5/v1/trainval_filtered/labels'
wwtool.mkdir_or_exist(filtered_image_path)
wwtool.mkdir_or_exist(filtered_label_path)
filter_count = 1
progress_bar = mmcv.ProgressBar(len(os.listdir(origin_label_path)))
for label_name in os.listdir(origin_label_path):
image_objects = wwtool.dota_parse(os.path.join(origin_label_path, label_name))
filtered_objects = []
for image_object in image_objects:
if convert_classes[image_object['label']] == None:
filter_count += 1
continue
else:
image_object['label'] = convert_classes[image_object['label']]
filtered_objects.append(image_object)
if __name__ == '__main__':
# cities = ['shanghai', 'beijing', 'jinan', 'haerbin', 'chengdu']
cities = ['shanghai']
label_list = []
for city in cities:
for label_fn in os.listdir(
'./data/buildchange/v2/{}/labels_json'.format(city)):
label_list.append([city, label_fn])
label_list = sorted(label_list)
np.random.shuffle(label_list)
dst_label_dir = './data/buildchange/v2/sampling/labels_json'
dst_image_dir = './data/buildchange/v2/sampling/images'
wwtool.mkdir_or_exist(dst_label_dir)
wwtool.mkdir_or_exist(dst_image_dir)
for label_fn in label_list[0:1000]:
basename = wwtool.get_basename(label_fn[1])
src_label_file = './data/buildchange/v2/{}/labels_json/{}'.format(
label_fn[0], basename + '.json')
src_image_file = './data/buildchange/v2/{}/images/{}'.format(
label_fn[0], basename + '.png')
dst_label_file = os.path.join(dst_label_dir, basename + '.json')
dst_image_file = os.path.join(dst_image_dir, basename + '.png')
shutil.copy(src_label_file, dst_label_file)
shutil.copy(src_image_file, dst_image_file)
rle = maskUtils.merge(rles)
elif isinstance(mask_ann['counts'], list):
# uncompressed RLE
rle = maskUtils.frPyObjects(mask_ann, img_h, img_w)
else:
# rle
rle = mask_ann
mask = maskUtils.decode(rle)
return mask
if __name__ == '__main__':
image_dir = '/data/buildchange/v2/shanghai/images'
label_dir = '/data/buildchange/v2/shanghai/labels_ignore'
save_dir = '/data/buildchange/v2/shanghai/vis'
wwtool.mkdir_or_exist(save_dir)
img_scale = 1024
for image_fn in os.listdir(image_dir):
# if 'L18_106968_219512' not in image_fn:
# continue
image_file = os.path.join(image_dir, image_fn)
label_file = os.path.join(label_dir, image_fn.replace('png', 'txt'))
save_file = os.path.join(save_dir, image_fn)
img = cv2.imread(image_file)
with open(label_file, 'r') as f:
lines = f.readlines()
gt_masks = []
for line in lines:
