def test(cls, cfg, model):
"""
Args:
cfg (CfgNode):
model (nn.Module):
Returns:
dict: a dict of result metrics
"""
logger = logging.getLogger(__name__)
results = OrderedDict()
for idx, dataset_name in enumerate(cfg.DATASETS.TESTS):
logger.info("Prepare testing set")
try:
data_loader, evaluator = cls.build_evaluator(cfg, dataset_name)
except NotImplementedError:
logger.warn(
"No evaluator found. implement its `build_evaluator` method."
)
results[dataset_name] = {}
continue
results_i = inference_on_dataset(model, data_loader, evaluator)
results[dataset_name] = results_i
if comm.is_main_process():
assert isinstance(
results, dict
), "Evaluator must return a dict on the main process. Got {} instead.".format(
results)
print_csv_format(results)
if len(results) == 1: results = list(results.values())[0]
return results
def do_test(cfg, model):
results = OrderedDict()
for idx, dataset_name in enumerate(cfg.DATASETS.TESTS):
logger.info("Prepare testing set")
try:
data_loader, evaluator = get_evaluator(cfg, dataset_name)
except NotImplementedError:
logger.warn(
"No evaluator found. implement its `build_evaluator` method.")
results[dataset_name] = {}
continue
results_i = inference_on_dataset(model,
data_loader,
evaluator,
flip_test=cfg.TEST.FLIP_ENABLED)
results[dataset_name] = results_i
if comm.is_main_process():
assert isinstance(
results, dict
), "Evaluator must return a dict on the main process. Got {} instead.".format(
results)
print_csv_format(results)
if len(results) == 1: results = list(results.values())[0]
return results
def main(args):
cfg = setup(args)
logger = logging.getLogger("fastreid.trainer")
cfg.defrost()
cfg.MODEL.BACKBONE.PRETRAIN = False
model = H_Trainer.build_model(cfg)
Checkpointer(model).load(cfg.MODEL.WEIGHTS) # load trained model
res = H_Trainer.test(cfg, model)
print_csv_format(res)
def test(cls, cfg, model, evaluators=None):
"""
Args:
cfg (CfgNode):
model (nn.Module):
evaluators (list[DatasetEvaluator] or None): if None, will call
:meth:`build_evaluator`. Otherwise, must have the same length as
`cfg.DATASETS.TEST`.
Returns:
dict: a dict of result metrics
"""
logger = logging.getLogger(__name__)
if isinstance(evaluators, DatasetEvaluator):
evaluators = [evaluators]
if evaluators is not None:
assert len(cfg.DATASETS.TEST) == len(evaluators), "{} != {}".format(
len(cfg.DATASETS.TEST), len(evaluators)
)
results = OrderedDict()
for idx, dataset_name in enumerate(cfg.DATASETS.TESTS):
logger.info("Prepare testing set")
data_loader, num_query = cls.build_test_loader(cfg, dataset_name)
# When evaluators are passed in as arguments,
# implicitly assume that evaluators can be created before data_loader.
if evaluators is not None:
evaluator = evaluators[idx]
else:
try:
evaluator = cls.build_evaluator(cfg, num_query)
except NotImplementedError:
logger.warn(
"No evaluator found. Use `DefaultTrainer.test(evaluators=)`, "
"or implement its `build_evaluator` method."
)
results[dataset_name] = {}
continue
results_i = inference_on_dataset(model, data_loader, evaluator)
results[dataset_name] = results_i
if comm.is_main_process():
assert isinstance(
results, dict
), "Evaluator must return a dict on the main process. Got {} instead.".format(
results
)
print_csv_format(results)
if len(results) == 1: results = list(results.values())[0]
return results
def test(self, cfg, model):
results = OrderedDict()
for idx, dataset_name in enumerate(cfg.DATASETS.TESTS):
results_i = inference_on_dataset(
model, self.test_data_loader[dataset_name],
self.evaluator[dataset_name])
results[dataset_name] = results_i
self.eval_results = results
if comm.is_main_process():
assert isinstance(
results, dict
), "Evaluator must return a dict on the main process. Got {} instead.".format(
results)
print_csv_format(results)
if len(results) == 1: results = list(results.values())[0]
return results
def test(cls, cfg, model):
"""
Args:
cfg (CfgNode):
model (nn.Module):
Returns:
dict: a dict of result metrics
"""
logger = logging.getLogger('fastreid')
results = OrderedDict()
dataset_name = cfg.DATASETS.TGT
logger.info("Prepare testing set")
try:
data_loader, evaluator = cls.build_evaluator(cfg, dataset_name)
except NotImplementedError:
logger.warn(
"No evaluator found. implement its `build_evaluator` method.")
results[dataset_name] = {}
results_i = inference_on_dataset(model,
data_loader,
evaluator,
flip_test=cfg.TEST.FLIP.ENABLED)
results[dataset_name] = results_i
if comm.is_main_process():
assert isinstance(
results, dict
), "Evaluator must return a dict on the main process. Got {} instead.".format(
results)
logger.info("Evaluation results for {} in csv format:".format(
dataset_name))
results_i['dataset'] = dataset_name
print_csv_format(results_i)
# if len(results) == 1:
# results = list(results.values())[0]
return results
for cfg in cfgs:
self.predictors.append(DefaultPredictor(cfg))
def run_on_loader(self, data_loader):
for batch in data_loader:
predictions = []
for predictor in self.predictors:
predictions.append(predictor(batch["images"]))
yield torch.cat(predictions, dim=-1), batch
if __name__ == "__main__":
args = get_parser().parse_args()
logger = setup_logger()
cfgs = []
for config_file in args.config_file:
cfg = setup_cfg(config_file, args.opts)
cfgs.append(cfg)
results = OrderedDict()
for dataset_name in cfgs[0].DATASETS.TESTS:
test_loader, num_query = build_reid_test_loader(cfgs[0], dataset_name)
evaluator = ReidEvaluator(cfgs[0], num_query)
feat_extract = FeatureExtraction(cfgs)
for (feat, batch) in tqdm.tqdm(feat_extract.run_on_loader(test_loader),
total=len(test_loader)):
evaluator.process(batch, feat)
result = evaluator.evaluate()
results[dataset_name] = result
print_csv_format(results)
