def convert_annotations(data,
path_prefix,
num_sample,
nproc,
start_img_id=0,
start_ann_id=0):
modify_image_info_with_params = partial(modify_image_info,
path_prefix=path_prefix,
start_img_id=start_img_id)
modify_annotation_with_params = partial(modify_annotation,
num_sample=num_sample,
start_img_id=start_img_id,
start_ann_id=start_ann_id)
if nproc > 1:
data['annotations'] = mmcv.track_parallel_progress(
modify_annotation_with_params, data['annotations'], nproc=nproc)
data['images'] = mmcv.track_parallel_progress(
modify_image_info_with_params, data['images'], nproc=nproc)
else:
data['annotations'] = mmcv.track_progress(
modify_annotation_with_params, data['annotations'])
data['images'] = mmcv.track_progress(
modify_image_info_with_params,
data['images'],
)
data['categories'] = [{'id': 1, 'name': 'text'}]
return data
def main():
args = parse_args()
cityscapes_path = args.cityscapes_path
out_dir = args.out_dir if args.out_dir else cityscapes_path
mmcv.mkdir_or_exist(out_dir)
gt_dir = osp.join(cityscapes_path, args.gt_dir)
poly_files = []
for poly in mmcv.scandir(gt_dir, '_polygons.json', recursive=True):
poly_file = osp.join(gt_dir, poly)
poly_files.append(poly_file)
if args.nproc > 1:
mmcv.track_parallel_progress(convert_json_to_label, poly_files,
args.nproc)
else:
mmcv.track_progress(convert_json_to_label, poly_files)
split_names = ['train', 'val', 'test']
for split in split_names:
filenames = []
for poly in mmcv.scandir(
osp.join(gt_dir, split), '_polygons.json', recursive=True):
filenames.append(poly.replace('_gtFine_polygons.json', ''))
with open(osp.join(out_dir, f'{split}.txt'), 'w') as f:
f.writelines(f + '\n' for f in filenames)
def main():
# parse cfg and args
args = parse_args()
cfg = Config.fromfile(args.config)
if args.cfg_options is not None:
cfg.merge_from_dict(args.cfg_options)
# touch output file to save broken files list.
output_path = Path(args.out_path)
if not output_path.parent.exists():
raise Exception('log_file parent directory not found.')
if output_path.exists():
os.remove(output_path)
output_path.touch()
# do valid
validator = DatasetValidator(cfg, output_path, args.phase)
if args.num_process > 1:
# The default chunksize calcuation method of Pool.map
chunksize, extra = divmod(len(validator), args.num_process * 8)
if extra:
chunksize += 1
track_parallel_progress(validator.valid_idx,
list(range(len(validator))),
args.num_process,
chunksize=chunksize,
keep_order=False)
else:
track_progress(validator.valid_idx, list(range(len(validator))))
print_info(output_path)
def main():
args = parse_args()
coco_path = args.coco_path
nproc = args.nproc
out_dir = args.out_dir or coco_path
out_img_dir = osp.join(out_dir, 'images')
out_mask_dir = osp.join(out_dir, 'annotations')
mmcv.mkdir_or_exist(osp.join(out_img_dir, 'train2014'))
mmcv.mkdir_or_exist(osp.join(out_img_dir, 'test2014'))
mmcv.mkdir_or_exist(osp.join(out_mask_dir, 'train2014'))
mmcv.mkdir_or_exist(osp.join(out_mask_dir, 'test2014'))
train_list, test_list = generate_coco_list(coco_path)
assert (len(train_list) +
len(test_list)) == COCO_LEN, 'Wrong length of list {} & {}'.format(
len(train_list), len(test_list))
if args.nproc > 1:
mmcv.track_parallel_progress(
partial(
convert_to_trainID,
in_img_dir=osp.join(coco_path, 'images'),
in_ann_dir=osp.join(coco_path, 'annotations'),
out_img_dir=out_img_dir,
out_mask_dir=out_mask_dir,
is_train=True),
train_list,
nproc=nproc)
mmcv.track_parallel_progress(
partial(
convert_to_trainID,
in_img_dir=osp.join(coco_path, 'images'),
in_ann_dir=osp.join(coco_path, 'annotations'),
out_img_dir=out_img_dir,
out_mask_dir=out_mask_dir,
is_train=False),
test_list,
nproc=nproc)
else:
mmcv.track_progress(
partial(
convert_to_trainID,
in_img_dir=osp.join(coco_path, 'images'),
in_ann_dir=osp.join(coco_path, 'annotations'),
out_img_dir=out_img_dir,
out_mask_dir=out_mask_dir,
is_train=True), train_list)
mmcv.track_progress(
partial(
convert_to_trainID,
in_img_dir=osp.join(coco_path, 'images'),
in_ann_dir=osp.join(coco_path, 'annotations'),
out_img_dir=out_img_dir,
out_mask_dir=out_mask_dir,
is_train=False), test_list)
print('Done!')
def collect_annotations(files, dataset, nproc=1):
"""Collect the annotation information.
Args:
files(list): The list of tuples (image_file, groundtruth_file)
dataset(str): The dataset name, icdar2015 or icdar2017
nproc(int): The number of process to collect annotations
Returns:
images(list): The list of image information dicts
"""
assert isinstance(files, list)
assert isinstance(dataset, str)
assert dataset
assert isinstance(nproc, int)
load_img_info_with_dataset = partial(load_img_info, dataset=dataset)
if nproc > 1:
images = mmcv.track_parallel_progress(load_img_info_with_dataset,
files,
nproc=nproc)
else:
images = mmcv.track_progress(load_img_info_with_dataset, files)
return images
def cvt_annotations(devkit_path, years, split, out_file):
if not isinstance(years, list):
years = [years]
annotations = []
for year in years:
filelist = osp.join(devkit_path,
f'VOC{year}/ImageSets/Main/{split}.txt')
if not osp.isfile(filelist):
print(f'filelist does not exist: {filelist}, '
f'skip voc{year} {split}')
return
img_names = mmcv.list_from_file(filelist)
xml_paths = [
osp.join(devkit_path, f'VOC{year}/Annotations/{img_name}.xml')
for img_name in img_names
]
img_paths = [
f'VOC{year}/JPEGImages/{img_name}.jpg' for img_name in img_names
]
part_annotations = mmcv.track_progress(parse_xml,
list(zip(xml_paths, img_paths)))
annotations.extend(part_annotations)
if out_file.endswith('json'):
annotations = cvt_to_coco_json(annotations)
mmcv.dump(annotations, out_file)
return annotations
def cvt_annotations(devkit_path, years, split, out_file):
if not isinstance(years, list):
years = [years]
annotations = []
for year in years:
filelist = osp.join(devkit_path,
'VOC{}/ImageSets/Main/{}.txt'.format(year, split))
if not osp.isfile(filelist):
print('filelist does not exist: {}, skip voc{} {}'.format(
filelist, year, split))
return
img_names = mmcv.list_from_file(filelist)
xml_paths = [
osp.join(devkit_path,
'VOC{}/Annotations/{}.xml'.format(year, img_name))
for img_name in img_names
]
img_paths = [
'VOC{}/JPEGImages/{}.jpg'.format(year, img_name)
for img_name in img_names
]
part_annotations = mmcv.track_progress(parse_xml,
list(zip(xml_paths, img_paths)))
annotations.extend(part_annotations)
mmcv.dump(annotations, out_file)
return annotations
def collect_annotations(files, nproc=1):
print('Loading annotation images')
if nproc > 1:
images = mmcv.track_parallel_progress(img2coco, files, nproc=nproc)
else:
images = mmcv.track_progress(img2coco, files)
return images
def main():
args = parse_args()
coco_path = args.coco_path
nproc = args.nproc
out_dir = args.out_dir or coco_path
out_img_dir = osp.join(out_dir, 'images')
out_mask_dir = osp.join(out_dir, 'annotations')
mmcv.mkdir_or_exist(osp.join(out_mask_dir, 'train2017'))
mmcv.mkdir_or_exist(osp.join(out_mask_dir, 'val2017'))
if out_dir != coco_path:
shutil.copytree(osp.join(coco_path, 'images'), out_img_dir)
train_list = glob(osp.join(coco_path, 'annotations', 'train2017', '*.png'))
train_list = [file for file in train_list if '_labelTrainIds' not in file]
test_list = glob(osp.join(coco_path, 'annotations', 'val2017', '*.png'))
test_list = [file for file in test_list if '_labelTrainIds' not in file]
assert (len(train_list) +
len(test_list)) == COCO_LEN, 'Wrong length of list {} & {}'.format(
len(train_list), len(test_list))
if args.nproc > 1:
mmcv.track_parallel_progress(partial(convert_to_trainID,
out_mask_dir=out_mask_dir,
is_train=True),
train_list,
nproc=nproc)
mmcv.track_parallel_progress(partial(convert_to_trainID,
out_mask_dir=out_mask_dir,
is_train=False),
test_list,
nproc=nproc)
else:
mmcv.track_progress(
partial(convert_to_trainID,
out_mask_dir=out_mask_dir,
is_train=True), train_list)
mmcv.track_progress(
partial(convert_to_trainID,
out_mask_dir=out_mask_dir,
is_train=False), test_list)
print('Done!')
def test_track_progress_list():
out = StringIO()
ret = mmcv.track_progress(sleep_1s, [1, 2, 3], bar_width=3, file=out)
assert out.getvalue() == (
'[ ] 0/3, elapsed: 0s, ETA:'
'\r[> ] 1/3, 1.0 task/s, elapsed: 1s, ETA: 2s'
'\r[>> ] 2/3, 1.0 task/s, elapsed: 2s, ETA: 1s'
'\r[>>>] 3/3, 1.0 task/s, elapsed: 3s, ETA: 0s\n')
assert ret == [1, 2, 3]
def test_track_progress_list(capsys):
ret = mmcv.track_progress(sleep_1s, [1, 2, 3], bar_width=3)
out, _ = capsys.readouterr()
assert out == ('[ ] 0/3, elapsed: 0s, ETA:'
'\r[> ] 1/3, 1.0 task/s, elapsed: 1s, ETA: 2s'
'\r[>> ] 2/3, 1.0 task/s, elapsed: 2s, ETA: 1s'
'\r[>>>] 3/3, 1.0 task/s, elapsed: 3s, ETA: 0s\n')
assert ret == [1, 2, 3]
def collect_annotations(files, nproc=1):
print("Loading annotation images")
if nproc > 1:
images = mmcv.track_parallel_progress(load_img_info,
files,
nproc=nproc)
else:
images = mmcv.track_progress(load_img_info, files)
return images
def main():
args = parse_args()
devkit_path = args.devkit_path
if args.out_dir is None:
out_dir = osp.join(devkit_path, 'VOC2010', 'SegmentationClassContext')
else:
out_dir = args.out_dir
json_path = args.json_path
mmcv.mkdir_or_exist(out_dir)
img_dir = osp.join(devkit_path, 'VOC2010', 'JPEGImages')
train_detail = Detail(json_path, img_dir, 'train')
train_ids = train_detail.getImgs()
val_detail = Detail(json_path, img_dir, 'val')
val_ids = val_detail.getImgs()
mmcv.mkdir_or_exist(
osp.join(devkit_path, 'VOC2010/ImageSets/SegmentationContext'))
train_list = mmcv.track_progress(
partial(generate_labels, detail=train_detail, out_dir=out_dir),
train_ids)
with open(
osp.join(devkit_path, 'VOC2010/ImageSets/SegmentationContext',
'train.txt'), 'w') as f:
f.writelines(line + '\n' for line in sorted(train_list))
val_list = mmcv.track_progress(
partial(generate_labels, detail=val_detail, out_dir=out_dir), val_ids)
with open(
osp.join(devkit_path, 'VOC2010/ImageSets/SegmentationContext',
'val.txt'), 'w') as f:
f.writelines(line + '\n' for line in sorted(val_list))
print('Done!')
def process(json_dir,
img_dir,
out_dir,
tasks=['det'],
nproc=1,
recog_format='jsonl',
warp=False):
mmcv.mkdir_or_exist(out_dir)
json_file_list = glob.glob(osp.join(json_dir, '*.json'))
parse_labelme_json_func = partial(parse_labelme_json,
img_dir=img_dir,
out_dir=out_dir,
tasks=tasks,
recog_format=recog_format,
warp_flag=warp)
if nproc <= 1:
total_results = mmcv.track_progress(parse_labelme_json_func,
json_file_list)
else:
total_results = mmcv.track_parallel_progress(parse_labelme_json_func,
json_file_list,
keep_order=True,
nproc=nproc)
total_det_line_json_list = []
total_recog_crop_line_str = []
total_recog_warp_line_str = []
for res in total_results:
total_det_line_json_list.extend(res[0])
if 'recog' in tasks:
total_recog_crop_line_str.extend(res[1])
total_recog_warp_line_str.extend(res[2])
mmcv.mkdir_or_exist(out_dir)
det_out_file = osp.join(out_dir, 'instances_training.txt')
list_to_file(det_out_file, total_det_line_json_list)
if 'recog' in tasks:
recog_out_file_crop = osp.join(out_dir, f'train_label.{recog_format}')
list_to_file(recog_out_file_crop, total_recog_crop_line_str)
if warp:
recog_out_file_warp = osp.join(out_dir,
f'warp_train_label.{recog_format}')
list_to_file(recog_out_file_warp, total_recog_warp_line_str)
def collect_annotations(files, nproc=1):
"""Collect the annotation information.
Args:
files(list): The list of tuples (image_file, groundtruth_file)
nproc(int): The number of process to collect annotations
Returns:
images(list): The list of image information dicts
"""
assert isinstance(files, list)
assert isinstance(nproc, int)
if nproc > 1:
images = mmcv.track_parallel_progress(
load_img_info, files, nproc=nproc)
else:
images = mmcv.track_progress(load_img_info, files)
return images
def collect_annotations(files, split, nproc=1):
"""Collect the annotation information.
Args:
files(list): The list of tuples (image_file, groundtruth_file)
split(str): The split of dataset. Namely: training or test
nproc(int): The number of process to collect annotations
Returns:
images(list): The list of image information dicts
"""
assert isinstance(files, list)
assert isinstance(split, str)
assert isinstance(nproc, int)
load_img_info_with_split = partial(load_img_info, split=split)
if nproc > 1:
images = mmcv.track_parallel_progress(
load_img_info_with_split, files, nproc=nproc)
else:
images = mmcv.track_progress(load_img_info_with_split, files)
return images
def main():
"""Program for making Audi's A2D2 dataset compatible with mmsegmentation.
NOTE: The input argument path must be the ABSOLUTE PATH to the dataset
- NOT the symbolically linked one (i.e. data/a2d2)!
Segmentation label conversion:
The A2D2 labels are instance segmentations (i.e. car_1, car_2, ...),
while semantic segmentation requires categorical segmentations.
The function 'convert_TYPE_trainids()' converts all instance
segmentation to their corresponding categorical segmentation and saves
them as new label image files.
Conversion type options
A2D2: Generates segmentations using inherent categories.
Cityscapes: Generates segmentations according to the categories and
indexing (i.e. 'trainIds') as in Cityscapes.
Directory restructuring:
A2D2 files are not arranged in the required 'train/val/test' directory
structure.
The function 'restructure_a2d2_directory' creates a new compatible
directory structure in the root directory, and fills it with symbolic
links or file copies to the input and segmentation label images.
Example usage:
python tools/convert_datasets/a2d2.py path/to/camera_lidar_semantic
"""
args = parse_args()
a2d2_path = args.a2d2_path
out_dir = args.out_dir if args.out_dir else a2d2_path
mmcv.mkdir_or_exist(out_dir)
# Create a list of filepaths to all original labels
# NOTE: Original label files have a number before '.png'
label_filepaths = glob.glob(osp.join(a2d2_path, '*/label/*/*[0-9].png'))
# Convert segmentation images to the Cityscapes 'TrainIds' values
if args.convert:
seg_choice = args.choice
if seg_choice == 'cityscapes':
if args.nproc > 1:
mmcv.track_parallel_progress(convert_cityscapes_trainids,
label_filepaths, args.nproc)
else:
mmcv.track_progress(convert_cityscapes_trainids,
label_filepaths)
elif seg_choice == 'a2d2':
if args.nproc > 1:
mmcv.track_parallel_progress(convert_a2d2_trainids,
label_filepaths, args.nproc)
else:
mmcv.track_progress(convert_a2d2_trainids, label_filepaths)
else:
raise ValueError
# Restructure directory structure into 'img_dir' and 'ann_dir'
if args.restruct:
restructure_a2d2_directory(out_dir, args.val, args.test, args.symlink)
# -*- coding: utf-8 -* -
'''
mmcv创建进度条
参考：https://mmcv.readthedocs.io/en/latest/utils.html#progressbar
'''
import mmcv
import time
def do_task(i):
time.sleep(1)
return i + 1
'''
依次调用task，输出进度
'''
tasks = list(range(10))
results = mmcv.track_progress(do_task, tasks)
print(results)
'''
多进程执行task，8个进程
'''
results = mmcv.track_parallel_progress(do_task, tasks, 8)
print(results)
# test ping results.
ping_result = multithread_ping_ssrserver(ssr_config)
valid_index = [
i for i, v in enumerate(ping_result) if v[0] <= speedtest_ping_timeout
and '回国' not in ssr_config[i]['remarks'] and '打机' not in ssr_config[i]
['remarks'] and '游戏' not in ssr_config[i]['remarks']
]
ping_result_str = '%d/%d servers are ping accessible!!!\n%.2f%% ping pass rate!!!' % (
len(valid_index), len(ssr_config), 100.0 *
(float(len(valid_index)) / len(ssr_config)))
ssr_config = [ssr_config[i] for i in valid_index]
ping_result = [ping_result[i] for i in valid_index]
# test speed
results_list = track_progress(outwall_speed_test, ssr_config)
for i in range(len(results_list)):
results_list[i]['remarks'] = ssr_config[i]['remarks']
results_list[i]['ssr_config'] = ssr_config[i]
results_list[i]['ping'] = ping_result[i][0]
if len(results_list) == 0:
print('No available ssr server!!!')
else:
results_list = sorted(results_list,
key=lambda s:
(-s['www.google.com']['success_rate'], s[
'www.google.com']['ave_time']))
results_list = [
r for r in results_list if r['www.google.com']['success_rate'] != 0
]
