def do_test(cfg, model):
if "evaluator" in cfg.dataloader:
ret = inference_on_dataset(
model, instantiate(cfg.dataloader.test), instantiate(cfg.dataloader.evaluator)
)
print_csv_format(ret)
return ret
def do_test(cfg, model):
results = OrderedDict()
for dataset_name in cfg.DATASETS.TEST:
mapper = None if cfg.INPUT.TEST_INPUT_TYPE == 'default' else \
DatasetMapper(
cfg, False, augmentations=build_custom_augmentation(cfg, False))
data_loader = build_detection_test_loader(cfg,
dataset_name,
mapper=mapper)
output_folder = os.path.join(cfg.OUTPUT_DIR,
"inference_{}".format(dataset_name))
evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type
if evaluator_type == "lvis":
evaluator = LVISEvaluator(dataset_name, cfg, True, output_folder)
elif evaluator_type == 'coco':
evaluator = COCOEvaluator(dataset_name, cfg, True, output_folder)
else:
assert 0, evaluator_type
results[dataset_name] = inference_on_dataset(model, data_loader,
evaluator)
if comm.is_main_process():
logger.info("Evaluation results for {} in csv format:".format(
dataset_name))
print_csv_format(results[dataset_name])
if len(results) == 1:
results = list(results.values())[0]
return results
def test(cls, cfg, model, evaluators=None):
"""
Args:
cfg (CfgNode):
model (nn.Module):
evaluators (list[DatasetEvaluator] or None): if None, will call
:meth:`build_evaluator`. Otherwise, must have the same length as
``cfg.DATASETS.TEST``.
Returns:
dict: a dict of result metrics
"""
logger = logging.getLogger(__name__)
if isinstance(evaluators, DatasetEvaluator):
evaluators = [evaluators]
# 如果有多个测试集，那么就有多个DatasetEvaluator
if evaluators is not None:
assert len(
cfg.DATASETS.TEST) == len(evaluators), "{} != {}".format(
len(cfg.DATASETS.TEST), len(evaluators))
results = OrderedDict()
# 如果是多个数据集，就要枚举数据 coco_2017_val
for idx, dataset_name in enumerate(cfg.DATASETS.TEST):
data_loader = cls.build_test_loader(cfg, dataset_name)
# When evaluators are passed in as arguments,
# implicitly assume that evaluators can be created before data_loader.
if evaluators is not None:
evaluator = evaluators[idx]
else:
try:
# 在此会创建evaluator
evaluator = cls.build_evaluator(cfg, dataset_name)
except NotImplementedError:
logger.warn(
"No evaluator found. Use `DefaultTrainer.test(evaluators=)`, "
"or implement its `build_evaluator` method.")
results[dataset_name] = {}
continue
# 执行推理
"""
res = {
"AP", "AP50", "AP75", "APs", "APm", "APl", "AP-{#class_name}",...
}
"""
results_i = inference_on_dataset(model, data_loader, evaluator)
results[dataset_name] = results_i
if comm.is_main_process():
assert isinstance(
results_i, dict
), "Evaluator must return a dict on the main process. Got {} instead.".format(
results_i)
logger.info("Evaluation results for {} in csv format:".format(
dataset_name))
print_csv_format(results_i)
if len(results) == 1:
results = list(results.values())[0]
# 如果只有一个数据集，那只用返回单个，a dict
return results
def test(
cls,
cfg: CfgNode,
model: nn.Module,
evaluators: Optional[Union[DatasetEvaluator,
List[DatasetEvaluator]]] = None,
):
"""
Args:
cfg (CfgNode):
model (nn.Module):
evaluators (DatasetEvaluator, list[DatasetEvaluator] or None): if None, will call
:meth:`build_evaluator`. Otherwise, must have the same length as
``cfg.DATASETS.TEST``.
Returns:
dict: a dict of result metrics
"""
logger = logging.getLogger(__name__)
if isinstance(evaluators, DatasetEvaluator):
evaluators = [evaluators]
if evaluators is not None:
assert len(
cfg.DATASETS.TEST) == len(evaluators), "{} != {}".format(
len(cfg.DATASETS.TEST), len(evaluators))
results = OrderedDict()
for idx, dataset_name in enumerate(cfg.DATASETS.TEST):
data_loader = cls.build_test_loader(cfg, dataset_name)
# When evaluators are passed in as arguments,
# implicitly assume that evaluators can be created before data_loader.
if evaluators is not None:
evaluator = evaluators[idx]
else:
try:
embedder = cls.extract_embedder_from_model(model)
evaluator = cls.build_evaluator(cfg,
dataset_name,
embedder=embedder)
except NotImplementedError:
logger.warn(
"No evaluator found. Use `DefaultTrainer.test(evaluators=)`, "
"or implement its `build_evaluator` method.")
results[dataset_name] = {}
continue
results_i = inference_on_dataset(model, data_loader, evaluator)
results[dataset_name] = results_i
if comm.is_main_process():
assert isinstance(
results_i, dict
), "Evaluator must return a dict on the main process. Got {} instead.".format(
results_i)
logger.info("Evaluation results for {} in csv format:".format(
dataset_name))
print_csv_format(results_i)
if len(results) == 1:
results = list(results.values())[0]
return results
def test(cls, cfg, model, evaluators=None):
"""
Args:
cfg (CfgNode):
model (nn.Module):
evaluators (list[DatasetEvaluator] or None): if None, will call
:meth:`build_evaluator`. Otherwise, must have the same length as
`cfg.DATASETS.TEST`.
Returns:
dict: a dict of result metrics
"""
logger = logging.getLogger(__name__)
if isinstance(evaluators, DatasetEvaluator):
evaluators = [evaluators]
if evaluators is not None:
assert len(cfg.DATASETS.TEST) == len(evaluators), "{} != {}".format(
len(cfg.DATASETS.TEST), len(evaluators)
)
results = OrderedDict()
for idx, dataset_name in enumerate(cfg.DATASETS.TEST):
data_loader = cls.build_test_loader(cfg, dataset_name)
# When evaluators are passed in as arguments,
# implicitly assume that evaluators can be created before data_loader.
if evaluators is not None:
evaluator = evaluators[idx]
else:
try:
evaluator = cls.build_evaluator(cfg, dataset_name)
except NotImplementedError:
logger.warn(
"No evaluator found. Use `DefaultTrainer.test(evaluators=)`, "
"or implement its `build_evaluator` method."
)
results[dataset_name] = {}
continue
results_i = inference_on_dataset(model, data_loader, evaluator)
results[dataset_name] = results_i
if comm.is_main_process():
assert isinstance(
results_i, dict
), "Evaluator must return a dict on the main process. Got {} instead.".format(
results_i
)
logger.info("Evaluation results for {} in csv format:".format(dataset_name))
output_file = 'res_final.json'
with open(os.path.join(cfg.OUTPUT_DIR, 'inference',
output_file), 'w') as fp:
json.dump(results[dataset_name], fp, indent = 4)
print_csv_format(results_i)
if len(results) == 1:
results = list(results.values())[0]
return results
def _process_dataset_evaluation_results(self) -> OrderedDict:
results = OrderedDict()
for idx, dataset_name in enumerate(self.cfg.DATASETS.TEST):
results[dataset_name] = self._evaluators[idx].evaluate()
if comm.is_main_process():
print_csv_format(results[dataset_name])
if len(results) == 1:
results = list(results.values())[0]
return results
def do_test(cfg, model):
results = OrderedDict()
for dataset_name in cfg.DATASETS.TEST:
data_loader = build_detection_test_loader(cfg, dataset_name)
evaluator = get_evaluator(cfg, dataset_name)
results_i = inference_on_dataset(model, data_loader, evaluator)
results[dataset_name] = results_i
logger.info("Evaluation results for {} in csv format:".format(dataset_name))
print_csv_format(results_i)
if len(results) == 1:
results = list(results.values())[0]
return results
def do_test(cfg, model):
results = OrderedDict()
for dataset_name in cfg.DATASETS.TEST:
data_loader = build_detection_test_loader(cfg, dataset_name)
evaluator = get_evaluator(
cfg, dataset_name,
os.path.join(cfg.OUTPUT_DIR, "inference", dataset_name))
results_i = inference_on_dataset(model, data_loader, evaluator)
results[dataset_name] = results_i
print_csv_format(results_i)
if len(results) == 1:
results = list(results.values())[0]
return results
def test(cls, cfg, model, evaluators=None):
"""
Args:
cfg (CfgNode):
model (nn.Module):
evaluators (list[DatasetEvaluator] or None): if None, will call
:meth:`build_evaluator`. Otherwise, must have the same length as
`cfg.DATASETS.TEST`.
Returns:
dict: a dict of result metrics
"""
logger = logging.getLogger(__name__)
if isinstance(evaluators, DatasetEvaluator):
evaluators = [evaluators]
if evaluators is not None:
assert len(cfg.DATASETS.TEST) == len(evaluators), "{} != {}".format(
len(cfg.DATASETS.TEST), len(evaluators)
)
# print(evaluators)
results = OrderedDict()
relations =OrderedDict()
for idx, dataset_name in enumerate(cfg.DATASETS.TEST):
data_loader = cls.build_test_loader(cfg, dataset_name)
# When evaluators are passed in as arguments,
# implicitly assume that evaluators can be created before data_loader.
evaluator = (
evaluators[idx]
if evaluators is not None
else cls.build_evaluator(cfg, dataset_name)
)
results_i,relations_i = inference_on_dataset(model, data_loader, evaluator)
results[dataset_name] = results_i
relations[dataset_name] = relations_i
if comm.is_main_process():
assert isinstance(
results_i, dict
), "Evaluator must return a dict on the main process. Got {} instead.".format(
results_i
)
logger.info("Evaluation results for {} in csv format:".format(dataset_name))
print_csv_format(results_i)
# print_csv_format(relations_i)
if len(results) == 1 and len(relations):
results = list(results.values())[0]
relations = list(relations.values())[0]
return results,relations
def do_test(cfg, model):
results = OrderedDict()
for dataset_name in cfg.DATASETS.TEST:
data_loader = build_detection_test_loader(cfg, dataset_name, mapper=TaobaoMapper(cfg, False))
evaluator = get_evaluator(
cfg, dataset_name, pjoin(cfg.OUTPUT_DIR, "inference", dataset_name)
)
results_i = inference_on_dataset(model, data_loader, evaluator)
results[dataset_name] = results_i
if comm.is_main_process():
logger.info("Evaluation results for {} in csv format:".format(dataset_name))
print_csv_format(results_i)
if len(results) == 1:
results = list(results.values())[0]
return results
def do_evaluate(cfg, model):
"""
Evaluate on test set using coco evaluate
"""
results = OrderedDict()
dataset_name = cfg.DATASETS.TEST[1]
data_loader = build_detection_test_loader(cfg, dataset_name)
evaluator = COCOEvaluator(dataset_name, cfg, False, output_dir= cfg.OUTPUT_DIR)
results_i = inference_on_dataset(model, data_loader, evaluator)
results[dataset_name] = results_i
if comm.is_main_process():
logger.info("Evaluation results for {} in csv format:".format(dataset_name))
print_csv_format(results_i)
if len(results) == 1:
results = list(results.values())[0]
return results
def do_test(cfg, model):
results = OrderedDict()
for dataset_name in cfg.DATASETS.TEST:
data_loader = build_detection_test_loader(cfg, dataset_name)
evaluator = get_evaluator(
cfg, dataset_name, os.path.join(cfg.OUTPUT_DIR, "inference", dataset_name)
)
results_i = inference_on_dataset(model, data_loader, evaluator)
results[dataset_name] = results_i
wandb.log({'AP': results_i['bbox']['AP'], 'AP50': results_i['bbox']['AP50']})
if comm.is_main_process():
logger.info("Evaluation results for {} in csv format:".format(dataset_name))
print_csv_format(results_i)
if len(results) == 1:
results = list(results.values())[0]
return results
def main():
parser = default_argument_parser()
parser.add_argument("--prediction", help="predictions_file_path")
args = parser.parse_args()
cfg = setup(args)
with smart_path(args.prediction).open("rb") as fp:
buf = io.BytesIO(fp.read())
predictions = torch.load(buf)
pred_by_image = defaultdict(list)
for p in predictions:
pred_by_image[p["image_id"]].append(p)
dataset = cfg.DATASETS.TEST[0]
metadata = MetadataCatalog.get(dataset)
if hasattr(metadata, "thing_dataset_id_to_contiguous_id"):
def dataset_id_map(ds_id):
return metadata.thing_dataset_id_to_contiguous_id[ds_id]
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference")
evaluator = COCOEvaluator(dataset, cfg, False, output_folder)
evaluator.reset()
dicts = list(DatasetCatalog.get(dataset))
count = 0
for dic in tqdm.tqdm(dicts):
#assert len(pred_by_image[dic["image_id"]]) == 1, str(dic["image_id"])
if len(pred_by_image[dic["image_id"]]) == 0:
continue
prediction = pred_by_image[dic["image_id"]][0]
prediction = create_instances(prediction, (dic["height"], dic["width"]))
# Push an image
dic["annotations"] = reconstruct_ann(dic["annotations"])
evaluator.process([dic], [{"instances": prediction}])
count += 1
result = evaluator.evaluate()
prediction_path = smart_path(args.prediction)
save_path = prediction_path.parent.joinpath(prediction_path.stem + ".pkl")
with save_path.open("wb") as writer:
pickle.dump(result, writer)
print_csv_format(result)
def do_test(cfg, model):
results = OrderedDict()
for idx, dataset_name in enumerate(cfg.DATASETS.TEST):
data_loader = build_detection_test_loader(cfg, dataset_name)
evaluator = get_evaluator(
cfg, dataset_name,
os.path.join(cfg.OUTPUT_DIR, "inference", dataset_name))
results_i = inference_on_dataset(model, data_loader, evaluator)
results[dataset_name] = results_i
if comm.is_main_process():
logger.info("Evaluation results for {} in csv format:".format(
dataset_name))
print_csv_format(results_i)
if len(results) == 1:
results = list(results.values())[0]
# Store JSON of results somewhere, for visualization.
return results
def do_test(cfg1, model1, model2):
results = OrderedDict()
for dataset_name in cfg1.DATASETS.TEST:
data_loader = custom_test_loader(cfg1, dataset_name)
evaluator = get_evaluator(
cfg1, dataset_name,
os.path.join(cfg1.OUTPUT_DIR, "inference", dataset_name))
results_i = inference_first_stream(model1, model2, data_loader,
evaluator)
results[dataset_name] = results_i
if comm.is_main_process():
logger.info("Evaluation results for {} in csv format:".format(
dataset_name))
print_csv_format(results_i)
if len(results) == 1:
results = list(results.values())[0]
return results
def do_test(cfg, model, cat_heatmap_file):
"""
Run the model on the test sets and output the results
"""
results = OrderedDict()
for dataset_name in cfg.DATASETS.TEST:
data_loader = build_detection_test_loader(cfg, dataset_name)
evaluator = get_evaluator(
cfg, dataset_name, os.path.join(cfg.OUTPUT_DIR, "inference", dataset_name),
cat_heatmap_file
)
results_i = inference_on_dataset(model, data_loader, evaluator)
results[dataset_name] = results_i
if comm.is_main_process():
logger.info("Evaluation results for {} in csv format:".format(dataset_name))
print_csv_format(results_i)
if len(results) == 1:
results = list(results.values())[0]
return results
def do_test(cfg, model):
"""
# TODO: Write docstring
"""
# Initialise results dictionary
results = OrderedDict()
# Loop through the datasets in the config file
for dataset_name in cfg.DATASETS.TEST:
data_loader = build_detection_test_loader(cfg, dataset_name)
# Generate the evaluator
evaluator = get_evaluator(
cfg,
dataset_name,
os.path.join(cfg.OUTPUT_DIR, "inference", dataset_name)
)
# Run inference and add to results dictionary
results_i = inference_on_dataset(model, data_loader, evaluator)
results[dataset_name] = results_i
# Log the result set to weights and biases
result_log = {}
result_dict = results_i
for iou_type in result_dict:
for metric, result in result_dict[iou_type].items():
metric_log = f"{iou_type}_{metric}"
result_log[metric_log] = result
logger.debug('Log the eval results on Weights & Biases')
wandb.log(result_log)
# Print to terminal
if comm.is_main_process():
logger.info("Evaluation results for {} in csv format:".format(dataset_name))
print_csv_format(results_i)
if len(results) == 1:
results = list(results.values())[0]
return results
def do_test(cfg, model):
results = OrderedDict()
for dataset_name in cfg.DATASETS.TEST:
print('--', dataset_name)
try:
data_loader = build_detection_test_loader(cfg,
dataset_name,
mapper=TblDatasetMapper(
cfg, is_train=False))
except Exception as ex:
logging.exception("Something awful happened!")
evaluator = get_evaluator(
cfg, dataset_name,
os.path.join(cfg.OUTPUT_DIR, "inference", dataset_name))
results_i = inference_on_dataset(model, data_loader, evaluator)
results[dataset_name] = results_i
print_csv_format(results_i)
if len(results) == 1:
results = list(results.values())[0]
return results
def do_test(cfg, model):
img_folder = "../data/HICO_DET/images/test2015"
json_folder = "../data/HICO_DET/hico_det_json"
results = OrderedDict()
for dataset_name in cfg.DATASETS.TEST:
data_loader = custom_test_loader(cfg, dataset_name)
evaluator = get_evaluator(
cfg, dataset_name,
os.path.join(cfg.OUTPUT_DIR, "inference", dataset_name),
img_folder, json_folder)
results_i = inference_custom(model, data_loader, evaluator)
results[dataset_name] = results_i
if comm.is_main_process():
logger.info("Evaluation results for {} in csv format:".format(
dataset_name))
print_csv_format(results_i)
if len(results) == 1:
results = list(results.values())[0]
return results
def do_test(cfg, model):
import random
results = OrderedDict()
for dataset_name in cfg.DATASETS.TEST:
data_loader = build_detection_test_loader(cfg, dataset_name)
# evaluator = get_evaluator(
# cfg, dataset_name, os.path.join(cfg.OUTPUT_DIR, "inference", dataset_name)
# )
from detectron2.utils.visualizer import ColorMode
predictor = DefaultPredictor(cfg)
dataset_dicts = get_car_dicts("./data/dataset/car_val")
for d in random.sample(dataset_dicts, 10):
im = cv2.imread(d["file_name"])
outputs = predictor(
im
) # format is documented at https://detectron2.readthedocs.io/tutorials/models.html#model-output-format
v = Visualizer(
im[:, :, ::-1],
metadata=MetadataCatalog.get("car_val"),
scale=0.7,
instance_mode=ColorMode.
IMAGE_BW # remove the colors of unsegmented pixels. This option is only available for segmentation models
)
out = v.draw_instance_predictions(outputs["instances"].to("cpu"))
cv2.imshow('hello', out.get_image()[:, :, ::-1])
cv2.waitKey(0)
cv2.destroyAllWindows()
evaluator = COCOEvaluator("car_val",
None,
False,
output_dir="./output/2021_03_12")
results_i = inference_on_dataset(model, data_loader, evaluator)
results[dataset_name] = results_i
if comm.is_main_process():
logger.info("Evaluation results for {} in csv format:".format(
dataset_name))
print_csv_format(results_i)
if len(results) == 1:
results = list(results.values())[0]
return results
def do_test(cfg, model):
results = OrderedDict()
for dataset_name in cfg.DATASETS.TEST:
if cfg.MULTI_DATASET.ENABLED:
# TODO: refactor
try:
model.set_eval_dataset(dataset_name)
except:
try:
model.module.set_eval_dataset(dataset_name)
except:
try:
model.model.set_eval_dataset(dataset_name)
except:
print('set eval dataset failed.')
data_loader = build_detection_test_loader(cfg, dataset_name)
output_folder = os.path.join(
cfg.OUTPUT_DIR, "inference_{}".format(dataset_name))
evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type
if cfg.MULTI_DATASET.UNIFIED_EVAL:
evaluator = get_unified_evaluator(
evaluator_type, dataset_name, cfg, True, output_folder)
# print('evaluator', evaluator)
else:
if evaluator_type == "lvis":
evaluator = LVISEvaluator(dataset_name, cfg, True, output_folder)
elif evaluator_type == 'oid':
evaluator = OIDEvaluator(dataset_name, cfg, True, output_folder)
else:
evaluator = COCOEvaluator(dataset_name, cfg, True, output_folder)
results[dataset_name] = inference_on_dataset(
model, data_loader, evaluator)
if comm.is_main_process():
logger.info("Evaluation results for {} in csv format:".format(
dataset_name))
print_csv_format(results[dataset_name])
if len(results) == 1:
results = list(results.values())[0]
return results
def test_multi_models(cls, cfg, models, evaluators=None):
logger = logging.getLogger(__name__)
if isinstance(evaluators, DatasetEvaluator):
evaluators = [evaluators]
if evaluators is not None:
assert len(
cfg.DATASETS.TEST) == len(evaluators), "{} != {}".format(
len(cfg.DATASETS.TEST), len(evaluators))
results = OrderedDict()
for idx, dataset_name in enumerate(cfg.DATASETS.TEST):
data_loader = cls.build_test_loader(cfg, dataset_name)
# When evaluators are passed in as arguments,
# implicitly assume that evaluators can be created before data_loader.
if evaluators is not None:
evaluator = evaluators[idx]
else:
try:
evaluator = cls.build_evaluator(cfg, dataset_name)
except NotImplementedError:
logger.warn(
"No evaluator found. Use `DefaultTrainer.test(evaluators=)`, "
"or implement its `build_evaluator` method.")
results[dataset_name] = {}
continue
results_i = inference_ensemble_on_dataset(models, data_loader,
evaluator)
results[dataset_name] = results_i
if comm.is_main_process():
assert isinstance(
results_i, dict
), "Evaluator must return a dict on the main process. Got {} instead.".format(
results_i)
logger.info("Evaluation results for {} in csv format:".format(
dataset_name))
print_csv_format(results_i)
if len(results) == 1:
results = list(results.values())[0]
return results
def main(args):
cfg = setup(args)
assert cfg.TEST.TOPK_CAT.ENABLED
topk = cfg.TEST.TOPK_CAT.K
model = build_model(cfg)
logger.info("Model:\n{}".format(model))
DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load(
cfg.MODEL.WEIGHTS, resume=args.resume
)
results = OrderedDict()
for dataset_name in cfg.DATASETS.TEST:
data_loader = build_detection_test_loader(cfg, dataset_name)
evaluator = get_evaluator(
cfg, dataset_name, os.path.join(cfg.OUTPUT_DIR, "inference", dataset_name)
)
results_i = inference_on_dataset(
model,
data_loader,
evaluator,
num_classes=cfg.MODEL.ROI_HEADS.NUM_CLASSES,
topk=topk,
num_estimate=cfg.TEST.TOPK_CAT.NUM_ESTIMATE,
min_score=cfg.TEST.TOPK_CAT.MIN_SCORE,
)
results[dataset_name] = results_i
if comm.is_main_process():
for max_dets, max_dets_results in results_i.items():
logger.info(
f"Evaluation results for {dataset_name},max_dets={max_dets} in "
f"csv format:"
)
print_csv_format(max_dets_results)
if len(results) == 1:
results = list(results.values())[0]
with open(Path(cfg.OUTPUT_DIR) / "metrics_infer_topk.json", "w") as f:
json.dump(results, f)
return results
def do_test(cfg, model, iteration):
results = OrderedDict()
for dataset_name in cfg.DATASETS.TEST:
data_loader = build_detection_test_loader(cfg, dataset_name)
evaluator = get_evaluator(
cfg, dataset_name,
os.path.join(cfg.OUTPUT_DIR, "inference", dataset_name))
results_i = inference_on_dataset(model, data_loader, evaluator)
results[dataset_name] = results_i
if comm.is_main_process():
logger.info("Evaluation results for {} in csv format:".format(
dataset_name))
print_csv_format(results_i)
# log inference result
# write csv head if file not exists
if not os.path.exists('inference0.csv'):
with open('inference0.csv', 'w') as f:
for task, res in results_i.items():
important_res = [(k, v) for k, v in res.items()
if "-" not in k]
f.write("Time,Task,Iteration,")
f.write(",".join([k[0] for k in important_res]))
f.write("\n")
break
# write result
localtime = time.asctime(time.localtime(time.time()))
with open('inference0.csv', 'a') as f:
for task, res in results_i.items():
important_res = [(k, v) for k, v in res.items()
if "-" not in k]
f.write(localtime + ",")
f.write(task + ",")
f.write(str(iteration) + ",")
f.write(",".join(
["{0:.4f}".format(k[1]) for k in important_res]))
f.write("\n")
if len(results) == 1:
results = list(results.values())[0]
return results
def do_test(cfg, model):
results = OrderedDict()
for dataset_name in cfg.DATASETS.TEST:
data_loader = build_detection_test_loader(cfg=cfg,
dataset_name=dataset_name,
mapper=PathwayDatasetMapper(
cfg, False))
evaluator = RotatedCOCOEvaluator(dataset_name=dataset_name,
cfg=cfg,
distributed=False,
output_dir=os.path.join(
cfg.OUTPUT_DIR, "inference",
dataset_name))
results_i = inference_on_dataset(model, data_loader, evaluator)
results[dataset_name] = results_i
if comm.is_main_process():
logger.info("Evaluation results for {} in csv format:".format(
dataset_name))
print_csv_format(results_i)
if len(results) == 1:
results = list(results.values())[0]
return results
def do_test(cfg, model):
results = OrderedDict()
for dataset_name in cfg.DATASETS.TEST:
data_loader = build_detection_test_loader(cfg, dataset_name)
# Create the evaluator
evaluator = get_evaluator(cfg,
dataset_name,
output_folder=os.path.join(
cfg.OUTPUT_DIR, "inference",
dataset_name))
# Make inference on dataset
results_i = inference_on_dataset(model, data_loader, evaluator)
# Update results dictionary
results[dataset_name] = results_i
print("### Returning results_i...")
#print(results_i)
#print(f"### Average Precision: {results_i['AP']}")
# Let's get some communication happening
if comm.is_main_process():
logger.info("Evaluation results for {} in csv format:".format(
dataset_name))
## wandb.log()? TODO/NOTE: This may be something Weights & Biases can track
#print("### Calculating results...")
print_csv_format(results_i)
# Check to see length of results
if len(results) == 1:
results = list(results.values())[0]
#print("### Returning results...")
#print(results)
# TODO : log results_i dict with different parameters
print("### Saving results to Weights & Biases...")
wandb.log(results_i)
return results
torch_model = build_model(cfg)
DetectionCheckpointer(torch_model).resume_or_load(cfg.MODEL.WEIGHTS)
torch_model.eval()
# get a sample data
data_loader = build_detection_test_loader(cfg, cfg.DATASETS.TEST[0])
first_batch = next(iter(data_loader))
# convert and save model
if args.export_method == "caffe2_tracing":
exported_model = export_caffe2_tracing(cfg, torch_model, first_batch)
elif args.export_method == "scripting":
exported_model = export_scripting(torch_model)
elif args.export_method == "tracing":
exported_model = export_tracing(torch_model, first_batch)
# run evaluation with the converted model
if args.run_eval:
assert exported_model is not None, (
"Python inference is not yet implemented for "
f"export_method={args.export_method}, format={args.format}.")
logger.info(
"Running evaluation ... this takes a long time if you export to CPU."
)
dataset = cfg.DATASETS.TEST[0]
data_loader = build_detection_test_loader(cfg, dataset)
# NOTE: hard-coded evaluator. change to the evaluator for your dataset
evaluator = COCOEvaluator(dataset, output_dir=args.output)
metrics = inference_on_dataset(exported_model, data_loader, evaluator)
print_csv_format(metrics)
def _do_test(self, cfg, model, train_iter=None, model_tag="default"):
"""train_iter: Current iteration of the model, None means final iteration"""
assert len(cfg.DATASETS.TEST)
assert cfg.OUTPUT_DIR
is_final = (train_iter is None) or (train_iter
== cfg.SOLVER.MAX_ITER - 1)
logger.info(
f"Running evaluation for model tag {model_tag} at iter {train_iter}..."
)
def _get_inference_dir_name(base_dir, inference_type, dataset_name):
return os.path.join(
base_dir,
inference_type,
model_tag,
str(train_iter) if train_iter is not None else "final",
dataset_name,
)
add_print_flops_callback(cfg, model, disable_after_callback=True)
results = OrderedDict()
results[model_tag] = OrderedDict()
for dataset_name in cfg.DATASETS.TEST:
# Evaluator will create output folder, no need to create here
output_folder = _get_inference_dir_name(cfg.OUTPUT_DIR,
"inference", dataset_name)
# NOTE: creating evaluator after dataset is loaded as there might be dependency. # noqa
data_loader = self.build_detection_test_loader(cfg, dataset_name)
evaluator = self.get_evaluator(cfg,
dataset_name,
output_folder=output_folder)
if not isinstance(evaluator, DatasetEvaluators):
evaluator = DatasetEvaluators([evaluator])
if comm.is_main_process():
tbx_writer = _get_tbx_writer(
get_tensorboard_log_dir(cfg.OUTPUT_DIR))
logger.info("Adding visualization evaluator ...")
mapper = self.get_mapper(cfg, is_train=False)
evaluator._evaluators.append(
self.get_visualization_evaluator()(
cfg,
tbx_writer,
mapper,
dataset_name,
train_iter=train_iter,
tag_postfix=model_tag,
))
results_per_dataset = inference_on_dataset(model, data_loader,
evaluator)
if comm.is_main_process():
results[model_tag][dataset_name] = results_per_dataset
if is_final:
print_csv_format(results_per_dataset)
if is_final and cfg.TEST.AUG.ENABLED:
# In the end of training, run an evaluation with TTA
# Only support some R-CNN models.
output_folder = _get_inference_dir_name(
cfg.OUTPUT_DIR, "inference_TTA", dataset_name)
logger.info(
"Running inference with test-time augmentation ...")
data_loader = self.build_detection_test_loader(
cfg, dataset_name, mapper=lambda x: x)
evaluator = self.get_evaluator(cfg,
dataset_name,
output_folder=output_folder)
inference_on_dataset(GeneralizedRCNNWithTTA(cfg, model),
data_loader, evaluator)
if is_final and cfg.TEST.EXPECTED_RESULTS and comm.is_main_process():
assert len(
results
) == 1, "Results verification only supports one dataset!"
verify_results(cfg, results[model_tag][cfg.DATASETS.TEST[0]])
# write results to tensorboard
if comm.is_main_process() and results:
from detectron2.evaluation.testing import flatten_results_dict
flattened_results = flatten_results_dict(results)
for k, v in flattened_results.items():
tbx_writer = _get_tbx_writer(
get_tensorboard_log_dir(cfg.OUTPUT_DIR))
tbx_writer._writer.add_scalar("eval_{}".format(k), v,
train_iter)
if comm.is_main_process():
tbx_writer = _get_tbx_writer(
get_tensorboard_log_dir(cfg.OUTPUT_DIR))
tbx_writer._writer.flush()
return results
def test(cls, cfg, model, evaluators=None):
"""
Args:
cfg (CfgNode):
model (nn.Module):
evaluators (list[DatasetEvaluator] or None): if None, will call
:meth:`build_evaluator`. Otherwise, must have the same length as
`cfg.DATASETS.TEST`.
Returns:
dict: a dict of result metrics
"""
logger = logging.getLogger(__name__)
if isinstance(evaluators, DatasetEvaluator):
evaluators = [evaluators]
if evaluators is not None:
assert len(
cfg.DATASETS.TEST) == len(evaluators), "{} != {}".format(
len(cfg.DATASETS.TEST), len(evaluators))
logger.info("Version:{}".format(cfg.OUTPUT.TRAIN_VERSION))
results = OrderedDict()
dataset = cfg.DATASETS.TEST[0].split("_")[0]
if (not cfg.MODEL.ROI_MASK_HEAD.RECON_NET.LOAD_CODEBOOK or not os.path.exists("{}_codebook.npy".format(dataset))) \
and cfg.MODEL.ROI_MASK_HEAD.RECON_NET.NAME != "":
for idx, dataset_name in enumerate(cfg.DATASETS.TRAIN):
embedding_dataloader = cls.build_test_loader(cfg, dataset_name)
model = embedding_inference_on_train_dataset(
model, embedding_dataloader)
logger.info("Start KMEANS clustering")
model.roi_heads.recon_net.cluster()
logger.info("KMEANS clustering has finished")
torch.save(
model.roi_heads.recon_net.state_dict(),
"{}/{}_recon_net.pth".format(model._cfg.OUTPUT_DIR, dataset))
logger.info("Recon net saved")
np.save(
'{}/{}_codebook.npy'.format(model._cfg.OUTPUT_DIR, dataset),
model.roi_heads.recon_net.vector_dict)
# if cfg.MODEL.ROI_MASK_HEAD.RECON_NET.NAME == "General_Recon_Net":
# # torch.save(model.roi_heads.recon_net.state_dict(), "{}/recon_net.pth".format(model._cfg.OUTPUT_DIR))
# # logger.info("Recon net saved")
# dataset = "d2sa"
# if os.path.exists("{}_codebook.npy".format(dataset)) and :
# model.roi_heads.recon_net.load_state_dict(torch.load("{}_recon_net.pth".format(dataset)))
# model.roi_heads.recon_net.vector_dict = np.load("{}_codebook.npy".format(dataset))[()]
# logger.info("codebook loaded")
#
# else:
# for idx, dataset_name in enumerate(cfg.DATASETS.TRAIN):
# # if idx == 0:
# # continue
# embedding_dataloader = cls.build_test_loader(cfg, dataset_name)
# model = embedding_inference_on_train_dataset(model, embedding_dataloader)
# logger.info("Start KMEANS clustering")
# model.roi_heads.recon_net.cluster()
# logger.info("KMEANS clustering has finished")
# torch.save(model.roi_heads.recon_net.state_dict(), "{}/recon_net.pth".format(model._cfg.OUTPUT_DIR))
# logger.info("Recon net saved")
# np.save('{}/codebook.npy'.format(model._cfg.OUTPUT_DIR), model.roi_heads.recon_net.vector_dict)
# codebook = model.roi_heads.recon_net.vector_dict
# # space = []
# # label = []
# # name = []
# # for i in range(10):
# # name.append(i)
# # space.append(codebook[i + 1][: 256])
# # label.append(torch.ones(256) * i)
# # space = torch.cat(space, dim=0)
# # label = torch.cat(label, dim=0)
# for cls_id in range(1, 21):
# num_groups = 6
# space = codebook[cls_id]
# kmeans = KMeans(n_clusters=num_groups)
# kmeans.fit(space.cpu())
#
# coord = TSNE(random_state=20200605).fit_transform(np.array(space.cpu().data))
# vis.viz.scatter(coord, win="memory{}".format(cls_id +20), opts=dict(legend=[0,1,2,3,4,5], markersize=5), Y=kmeans.labels_ + 1, name="{}".format(cls_id))
#
# tensor = torch.FloatTensor(kmeans.cluster_centers_).cuda().view(num_groups, 8, 4, 4)
# mask = model.roi_heads.recon_net.decode(tensor)
# vis.images(mask, win_name='mask{}'.format(cls_id))
# ipdb.set_trace()
# ipdb.set_trace()
for idx, dataset_name in enumerate(cfg.DATASETS.TEST):
data_loader = cls.build_test_loader(cfg, dataset_name)
# When evaluators are passed in as arguments,
# implicitly assume that evaluators can be created before data_loader.
if evaluators is not None:
evaluator = evaluators[idx]
else:
try:
evaluator = cls.build_evaluator(cfg, dataset_name)
except NotImplementedError:
logger.warn(
"No evaluator found. Use `DefaultTrainer.test(evaluators=)`, "
"or implement its `build_evaluator` method.")
results[dataset_name] = {}
continue
results_i = inference_on_dataset(model, data_loader, evaluator)
results[dataset_name] = results_i
if comm.is_main_process():
assert isinstance(
results_i, dict
), "Evaluator must return a dict on the main process. Got {} instead.".format(
results_i)
logger.info("Evaluation results for {} in csv format:".format(
dataset_name))
print_csv_format(results_i)
if len(results) == 1:
results = list(results.values())[0]
return results
def test(cls, cfg, model, mapper_object, evaluators=None):
"""
Args:
cfg (CfgNode):
model (nn.Module):
evaluators (list[DatasetEvaluator] or None): if None, will call
:meth:`build_evaluator`. Otherwise, must have the same length as
`cfg.DATASETS.TEST`.
Returns:
dict: a dict of result metrics
"""
logger = logging.getLogger(__name__)
if isinstance(evaluators, DatasetEvaluator):
evaluators = [evaluators]
if evaluators is not None:
assert len(
cfg.DATASETS.TEST) == len(evaluators), "{} != {}".format(
len(cfg.DATASETS.TEST), len(evaluators))
results = OrderedDict()
for idx, dataset_name in enumerate(cfg.DATASETS.TEST):
# if(not isTrackAccuracy):
# break
data_loader = cls.build_test_loader(cfg, dataset_name,
mapper_object)
# When evaluators are passed in as arguments,
# implicitly assume that evaluators can be created before data_loader.
if evaluators is not None:
evaluator = evaluators[idx]
else:
try:
evaluator = cls.build_evaluator(cfg, dataset_name)
except NotImplementedError:
logger.warn(
"No evaluator found. Use `DefaultTrainer.test(evaluators=)`, "
"or implement its `build_evaluator` method.")
results[dataset_name] = {}
continue
# if(True): return results
# return results
results_i = evaluate.inference_on_dataset(model, data_loader,
evaluator)
accuracy_test = round((results_i["accuracy"] * 100), 2)
storage = get_event_storage()
storage.put_scalar("accuracy_" + dataset_name,
accuracy_test,
smoothing_hint=False)
results[dataset_name] = results_i
if comm.is_main_process():
assert isinstance(
results_i, dict
), "Evaluator must return a dict on the main process. Got {} instead.".format(
results_i)
logger.info("Evaluation results for {} in csv format:".format(
dataset_name))
print_csv_format(results_i)
if len(results) == 1:
results = list(results.values())[0]
return results
