def do_evaluate_unlabeled(pred_config, output_file, reuse=True):
"""Evaluate unlabled data."""
for i, dataset in enumerate(cfg.DATA.VAL):
output = output_file + '-' + dataset
if not os.path.isfile(output) or not reuse:
if i == 0:
num_tower = max(cfg.TRAIN.NUM_GPUS, 1)
graph_funcs = MultiTowerOfflinePredictor(
pred_config, list(range(num_tower))).get_predictors()
logger.info('Evaluating {} ...'.format(dataset))
dataflows = [
get_eval_dataflow(dataset, shard=k, num_shards=num_tower)
for k in range(num_tower)
]
all_results = multithread_predict_dataflow(dataflows, graph_funcs)
eval_metrics = DatasetRegistry.get(
dataset).eval_inference_results2(
all_results, output, threshold=cfg.TRAIN.CONFIDENCE)
else:
all_results = json.load(open(output, 'r'))
eval_metrics = DatasetRegistry.get(
dataset).eval_inference_results2(
all_results,
output,
threshold=cfg.TRAIN.CONFIDENCE,
metric_only=True)
with open(output + '_cocometric.json', 'w') as f:
json.dump(eval_metrics, f)
def register_voc(basedir):
"""
Add COCO datasets like "coco_train201x" to the registry,
so you can refer to them with names in `cfg.DATA.TRAIN/VAL`.
Note that train2017==trainval35k==train2014+val2014-minival2014, and
val2017==minival2014.
"""
# 80 names for COCO
# For your own coco-format dataset, change this.
#class_names = [
# "boat", "cat", "dog", "person", "tvmonitor", "sofa", "car", "bird", "horse", "pottedplant", "bicycle", "chair", "bottle", "cow", "aeroplane", "train", "sheep", "diningtable", "bus", "motorbike"] # noqa
#class_names = [
# "person", "chair", "aeroplane", "bus", "cow", "bird", "motorbike", "boat", "car", "horse", "sofa", "pottedplant", "tvmonitor", "cat", "train", "bottle", "diningtable", "dog", "bicycle", "sheep"]
class_names = [
'motorbike', 'dog', 'person', 'horse', 'sofa', 'bicycle', 'cow',
'boat', 'train', 'car', 'bird', 'cat', 'chair', 'pottedplant', 'sheep',
'aeroplane', 'bottle', 'bus', 'diningtable', 'tvmonitor'
]
class_names = ['BG'] + class_names
for split in [
'VOC2007/instances_trainval', 'VOC2007/instances_test',
'VOC2012/instances_trainval'
]:
name = split
DatasetRegistry.register(name,
lambda x=split: VOCDetection(basedir, x))
DatasetRegistry.register_metadata(name, 'class_names', class_names)
logger.info('Register dataset {}'.format(
[a for a in DatasetRegistry._registry.keys()]))
def get_eval_unlabeled_dataflow(name,
shard=0,
num_shards=1,
return_size=False):
"""
Return a training dataflow. Each datapoint consists of the following:
An image: (h, w, 3),
1 or more pairs of (anchor_labels, anchor_boxes):
anchor_labels: (h', w', NA)
anchor_boxes: (h', w', NA, 4)
gt_boxes: (N, 4)
gt_labels: (N,)
If MODE_MASK, gt_masks: (N, h, w)
"""
if isinstance(name, (list, tuple)) and len(name) > 1:
if "VOC" not in name[0]:
assert "VOC" not in name[
1], "VOC has to be put before coco in cfg.DATA.TRAIN"
roidbs = []
for x in name:
_roidbs = DatasetRegistry.get(x).training_roidbs()
print_class_histogram(_roidbs)
roidbs.extend(_roidbs)
# roidbs = list(itertools.chain.from_iterable(DatasetRegistry.get(x).training_roidbs() for x in name))
logger.info("Merged roidbs from {}".format(name))
print_class_histogram(roidbs)
else:
if isinstance(name, (list, tuple)):
name = name[0]
roidbs = DatasetRegistry.get(name).training_roidbs()
print_class_histogram(roidbs)
num_imgs = len(roidbs)
img_per_shard = num_imgs // num_shards
img_range = (shard * img_per_shard, (shard + 1) *
img_per_shard if shard + 1 < num_shards else num_imgs)
logger.info("Found {} images for inference.".format(img_range[1] -
img_range[0] + 1))
# no filter for training
ds = DataFromListOfDict(roidbs[img_range[0]:img_range[1]],
["file_name", "image_id"])
def f(fname):
im = cv2.imread(fname, cv2.IMREAD_COLOR)
assert im is not None, fname
return im
ds = MapDataComponent(ds, f, 0)
# Evaluation itself may be multi-threaded, therefore don't add prefetch
# here.
if return_size:
return ds, num_imgs
return ds
def register_coco_for_voc(basedir):
class_names = [
"person", "chair", "aeroplane", "bus", "cow", "bird", "motorbike",
"boat", "car", "horse", "sofa", "pottedplant", "tvmonitor", "cat",
"train", "bottle", "diningtable", "dog", "bicycle", "sheep"
]
class_names = ["BG"] + class_names
for split in SEMI_SUPERVISED_SPLITS[-NUM_20CLASS:]:
name = "coco_" + split
DatasetRegistry.register(
name, lambda x=split: COCODetectionForVOC(basedir, x))
DatasetRegistry.register_metadata(name, "class_names", class_names)
logger.info("Register dataset {}".format(
[a for a in DatasetRegistry._registry.keys()]))
def register_coco(basedir):
"""Register COCO.
Add COCO datasets like "coco_train201x" to the registry,
so you can refer to them with names in `cfg.DATA.TRAIN/VAL`.
Note that train2017==trainval35k==train2014+val2014-minival2014, and
val2017==minival2014.
Args:
basedir: root dir that saves datasets.
"""
# 80 names for COCO
# For your own coco-format dataset, change this.
class_names = [
"person", "bicycle", "car", "motorcycle", "airplane", "bus", "train",
"truck", "boat", "traffic light", "fire hydrant", "stop sign",
"parking meter", "bench", "bird", "cat", "dog", "horse", "sheep",
"cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella",
"handbag", "tie", "suitcase", "frisbee", "skis", "snowboard",
"sports ball", "kite", "baseball bat", "baseball glove", "skateboard",
"surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork",
"knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange",
"broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair",
"couch", "potted plant", "bed", "dining table", "toilet", "tv",
"laptop", "mouse", "remote", "keyboard", "cell phone", "microwave",
"oven", "toaster", "sink", "refrigerator", "book", "clock", "vase",
"scissors", "teddy bear", "hair drier", "toothbrush"
] # noqa
class_names = ["BG"] + class_names
register_coco_supervised(basedir)
for split in SEMI_SUPERVISED_SPLITS[:-NUM_20CLASS]:
name = "coco_" + split
DatasetRegistry.register(name,
lambda x=split: COCODetection(basedir, x))
DatasetRegistry.register_metadata(name, "class_names", class_names)
logger.info("Register dataset {}".format(
[a for a in DatasetRegistry._registry.keys()])) # pylint: disable=protected-access
assert os.environ["COCODIR"], "COCODIR environ variable is not set".format(
os.environ["COCODIR"])
# also register coco train set 20 class for voc experiments
register_coco_for_voc(os.environ["COCODIR"])
def get_train_dataflow():
"""
Return a training dataflow. Each datapoint consists of the following:
An image: (h, w, 3),
1 or more pairs of (anchor_labels, anchor_boxes):
anchor_labels: (h', w', NA)
anchor_boxes: (h', w', NA, 4)
gt_boxes: (N, 4)
gt_labels: (N,)
If MODE_MASK, gt_masks: (N, h, w)
"""
roidbs = list(
itertools.chain.from_iterable(
DatasetRegistry.get(x).training_roidbs() for x in cfg.DATA.TRAIN))
print_class_histogram(roidbs)
# Filter out images that have no gt boxes, but this filter shall not be applied for testing.
# The model does support training with empty images, but it is not useful for COCO.
num = len(roidbs)
roidbs = list(
filter(lambda img: len(img["boxes"][img["is_crowd"] == 0]) > 0,
roidbs))
logger.info(
"Filtered {} images which contain no non-crowd groudtruth boxes. Total #images for training: {}"
.format(num - len(roidbs), len(roidbs)))
ds = DataFromList(roidbs, shuffle=True)
preprocess = TrainingDataPreprocessorAug(cfg)
if cfg.DATA.NUM_WORKERS > 0:
if cfg.TRAINER == "horovod":
buffer_size = cfg.DATA.NUM_WORKERS * 10 # one dataflow for each process, therefore don't need large buffer
ds = MultiThreadMapData(ds,
cfg.DATA.NUM_WORKERS,
preprocess,
buffer_size=buffer_size)
# MPI does not like fork()
else:
buffer_size = cfg.DATA.NUM_WORKERS * 20
ds = MultiProcessMapData(ds,
cfg.DATA.NUM_WORKERS,
preprocess,
buffer_size=buffer_size)
else:
ds = MapData(ds, preprocess)
return ds
def visualize_dataflow2(cfg, unlabled2017_used=True, VISPATH="./", maxvis=50):
"""Visualize the dataflow with labeled and unlabled strong augmentation."""
def prase_name(x):
if not unlabled2017_used:
return x + "-unlabeled"
else: # return coco2017 unlabeled data
return "coco_unlabeled2017"
def remove_no_box_data(_roidbs, filter_fn):
num = len(_roidbs)
_roidbs = filter_fn(_roidbs)
logger.info(
"Filtered {} images which contain no non-crowd groudtruth boxes. Total #images for training: {}"
.format(num - len(_roidbs), len(_roidbs)))
return _roidbs
pseudo_path = os.path.join(os.environ["PSEUDO_PATH"], "pseudo_data.npy")
pseudo_targets = dd.io.load(pseudo_path)
roidbs = list(
itertools.chain.from_iterable(
DatasetRegistry.get(x).training_roidbs() for x in cfg.DATA.TRAIN))
roidbs_u = list(
itertools.chain.from_iterable(
DatasetRegistry.get(prase_name(x)).training_roidbs()
for x in cfg.DATA.TRAIN))
roidbs = remove_no_box_data(
roidbs, lambda x: list(
filter(lambda img: len(img["boxes"][img["is_crowd"] == 0]) > 0, x))
)
roidbs_u = remove_no_box_data(
roidbs_u, lambda x: list(
filter(
lambda img: len(pseudo_targets[img["image_id"]]["boxes"]) > 0,
x)))
print_class_histogram(roidbs)
print_class_histogram(roidbs_u)
preprocess = TrainingDataPreprocessorSSlAug(
cfg, confidence=cfg.TRAIN.CONFIDENCE, pseudo_targets=pseudo_targets)
for jj, (rob, robu) in tqdm(enumerate(zip(roidbs, roidbs_u))):
data = preprocess((rob, robu))
# import pdb; pdb.set_trace()
nn = len(pseudo_targets[robu["image_id"]]["boxes"])
if data is None or len(data["gt_boxes_strong"]) == 0:
print("empty annotation, {} (original {})".format(jj, nn))
continue
ims = viz.draw_boxes(data["image"], data["gt_boxes"],
[str(a) for a in data["gt_labels"]])
ims_t = viz.draw_boxes(data["image_strong"], data["gt_boxes_strong"], [
str(a)
for a in data["gt_labels_strong"][:len(data["gt_boxes_strong"])]
])
ims = cv2.resize(ims, (ims_t.shape[1], ims_t.shape[0]))
vis = np.concatenate((ims, ims_t), axis=1)
if not os.path.exists(
os.path.dirname(
os.path.join(VISPATH, "result_{}.jpeg".format(jj)))):
os.makedirs(
os.path.dirname(
os.path.join(VISPATH, "result_{}.jpeg".format(jj))))
assert cv2.imwrite(os.path.join(VISPATH, "result_{}.jpeg".format(jj)),
vis)
if jj > maxvis:
break
def get_train_dataflow_w_unlabeled(load_path):
"""
Return a training dataflow. Each datapoint consists of the following:
An image: (h, w, 3),
1 or more pairs of (anchor_labels, anchor_boxes):
anchor_labels: (h', w', NA)
anchor_boxes: (h', w', NA, 4)
gt_boxes: (N, 4)
gt_labels: (N,)
If MODE_MASK, gt_masks: (N, h, w)
"""
assert os.path.isfile(load_path), "{} does not find".format(load_path)
roidbs = list(
itertools.chain.from_iterable(
DatasetRegistry.get(x).training_roidbs() for x in cfg.DATA.TRAIN))
print_class_histogram(roidbs)
if "VOC" in cfg.DATA.TRAIN[0]:
roidbs_u = list(
itertools.chain.from_iterable(
DatasetRegistry.get(x).training_roidbs()
for x in cfg.DATA.UNLABEL))
unlabled2017_used = False
else:
unlabled2017_used = np.any(["@" not in x for x in cfg.DATA.TRAIN])
def prase_name(x):
if not unlabled2017_used:
assert "@" in load_path, (
"{}: Did you use wrong pseudo_data.py for "
"this model?").format(load_path)
return x + "-unlabeled"
else:
# return coco2017 unlabeled data
return "coco_unlabeled2017"
roidbs_u = list(
itertools.chain.from_iterable(
DatasetRegistry.get(prase_name(x)).training_roidbs()
for x in cfg.DATA.TRAIN))
print_class_histogram(roidbs_u)
# Filter out images that have no gt boxes, but this filter shall not be applied for testing.
# The model does support training with empty images, but it is not useful for COCO.
def remove_no_box_data(_roidbs, filter_fn, dset):
num = len(_roidbs)
_roidbs = filter_fn(_roidbs)
logger.info(
"Filtered {} images which contain no non-crowd groudtruth boxes. Total {} #images for training: {}"
.format(num - len(_roidbs), dset, len(_roidbs)))
return _roidbs
roidbs = remove_no_box_data(
roidbs, lambda x: list(
filter(lambda img: len(img["boxes"][img["is_crowd"] == 0]) > 0, x)
), "labeled")
# load unlabeled
if unlabled2017_used:
assert "@" not in load_path, "Did you use the wrong pseudo path"
pseudo_targets = dd.io.load(load_path)
logger.info("Loaded {} pseudo targets from {}".format(
len(pseudo_targets), load_path))
roidbs_u = remove_no_box_data(
roidbs_u, lambda x: list(
filter(
lambda img: len(pseudo_targets[img["image_id"]]["boxes"]) > 0,
x)), "unlabeled")
preprocess = TrainingDataPreprocessorSSlAug(
cfg, confidence=cfg.TRAIN.CONFIDENCE, pseudo_targets=pseudo_targets)
ds = DataFrom2List(roidbs, roidbs_u, shuffle=True)
if cfg.DATA.NUM_WORKERS > 0:
if cfg.TRAINER == "horovod":
buffer_size = cfg.DATA.NUM_WORKERS * 10
ds = MultiThreadMapData(ds,
cfg.DATA.NUM_WORKERS,
preprocess,
buffer_size=buffer_size)
else:
buffer_size = cfg.DATA.NUM_WORKERS * 20
ds = MultiProcessMapData(ds,
cfg.DATA.NUM_WORKERS,
preprocess,
buffer_size=buffer_size)
else:
ds = MapData(ds, preprocess)
return ds