def test_model(model_file, multi_gpu_testing, opts=None):
"""Test a model."""
# Clear memory before inference
workspace.ResetWorkspace()
# Run inference
run_inference(
model_file, multi_gpu_testing=multi_gpu_testing,
check_expected_results=True,
)
def test_model(model_file, multi_gpu_testing, opts=None):
"""Test a model."""
# Clear memory before inference
workspace.ResetWorkspace()
# Run inference
run_inference(
model_file, multi_gpu_testing=multi_gpu_testing,
check_expected_results=True,
)
def test_model(self, model_file):
"""Test a model."""
# Clear memory before inference
workspace.ResetWorkspace()
# Run inference
run_inference(
model_file,
check_expected_results=True,
)
def train_model():
"""Model training loop."""
model, weights_file, start_iter, checkpoints, output_dir = create_model()
if 'final' in checkpoints:
# The final model was found in the output directory, so nothing to do
return checkpoints
setup_model_for_training(model, weights_file, output_dir)
training_stats = TrainingStats(model)
CHECKPOINT_PERIOD = int(cfg.TRAIN.SNAPSHOT_ITERS / cfg.NUM_GPUS)
TEST_PERIOD = int(cfg.TRAIN.TEST_ITERS)
for cur_iter in range(start_iter, cfg.SOLVER.MAX_ITER):
if model.roi_data_loader.has_stopped():
handle_critical_error(model, 'roi_data_loader failed')
training_stats.IterTic()
lr = model.UpdateWorkspaceLr(cur_iter,
lr_policy.get_lr_at_iter(cur_iter))
workspace.RunNet(model.net.Proto().name)
if cur_iter == start_iter:
nu.print_net(model)
training_stats.IterToc()
training_stats.UpdateIterStats()
training_stats.LogIterStats(cur_iter, lr)
if (cur_iter + 1) % CHECKPOINT_PERIOD == 0 and cur_iter > start_iter:
print("================SAVING MODEL=================")
checkpoints[cur_iter] = os.path.join(
output_dir, 'model_iter{}.pkl'.format(cur_iter))
nu.save_model_to_weights_file(checkpoints[cur_iter], model)
if (cur_iter + 1) % TEST_PERIOD == 0 and cur_iter > start_iter:
print("================RUN INFERENCE==================")
checkpoints[cur_iter] = os.path.join(
output_dir, 'model_iter{}.pkl'.format(cur_iter))
nu.save_model_to_weights_file(checkpoints[cur_iter], model)
run_inference(checkpoints[cur_iter],
multi_gpu_testing=False,
check_expected_results=True)
if cur_iter == start_iter + training_stats.LOG_PERIOD:
# Reset the iteration timer to remove outliers from the first few
# SGD iterations
training_stats.ResetIterTimer()
if np.isnan(training_stats.iter_total_loss):
handle_critical_error(model, 'Loss is NaN')
# Save the final model
print("=====================FINAL=======================")
checkpoints['final'] = os.path.join(output_dir, 'model_final.pkl')
nu.save_model_to_weights_file(checkpoints['final'], model)
# Shutdown data loading threads
model.roi_data_loader.shutdown()
return checkpoints
def eval(args, cfg, logger, files):
files = extract(files)
files.sort(key=sortMethod)
for f in files:
iter_string = re.findall(r'(?<=model_iter)\d+(?=\.pkl)', f)
checkpoint_iter = int(iter_string[0])
resume_weights_file = f
weights_file = os.path.join(cfg.TEST.WEIGHTS, resume_weights_file)
logger.info(
'========> Resuming from checkpoint {} at iter {}'.
format(weights_file, checkpoint_iter)
)
run_inference(
weights_file,
ind_range=args.range,
multi_gpu_testing=args.multi_gpu_testing,
check_expected_results=True,
)
if len(sys.argv) == 1:
parser.print_help()
sys.exit(1)
return parser.parse_args()
if __name__ == '__main__':
workspace.GlobalInit(['caffe2', '--caffe2_log_level=0'])
logger = setup_logging(__name__)
args = parse_args()
logger.info('Called with args:')
logger.info(args)
if args.cfg_file is not None:
merge_cfg_from_file(args.cfg_file)
if args.opts is not None:
merge_cfg_from_list(args.opts)
assert_and_infer_cfg()
logger.info('Testing with config:')
logger.info(pprint.pformat(cfg))
while not os.path.exists(cfg.TEST.WEIGHTS) and args.wait:
logger.info('Waiting for \'{}\' to exist...'.format(cfg.TEST.WEIGHTS))
time.sleep(10)
run_inference(
cfg.TEST.WEIGHTS,
ind_range=args.range,
multi_gpu_testing=args.multi_gpu_testing,
check_expected_results=True,
)
logger.info(args)
if args.cfg_file is not None:
merge_cfg_from_file(args.cfg_file)
if args.opts is not None:
merge_cfg_from_list(args.opts)
assert_and_infer_cfg()
logger.info('Testing with config:')
logger.info(pprint.pformat(cfg))
while not os.path.exists(cfg.TEST.WEIGHTS) and args.wait:
logger.info('Waiting for \'{}\' to exist...'.format(cfg.TEST.WEIGHTS))
time.sleep(10)
all_results = run_inference(cfg.TEST.WEIGHTS,
ind_range=args.range,
multi_gpu_testing=args.multi_gpu_testing,
check_expected_results=False,
evaluation=False)
all_boxes = all_results['all_boxes']
test_dataset = JsonDataset(cfg.TEST.DATASETS[0])
image_set = test_dataset.name.split('_')[-1]
root_path = DATASETS[test_dataset.name][ROOT_DIR]
image_set_path = os.path.join(root_path, 'ImageSets', 'Main',
image_set + '.txt')
with open(image_set_path, 'r') as f:
image_index = [x.strip() for x in f.readlines()]
test_roidb = test_dataset.get_roidb()
for i, entry in enumerate(test_roidb):
logger.info(args)
if args.cfg_file is not None:
merge_cfg_from_file(args.cfg_file)
if args.opts is not None:
merge_cfg_from_list(args.opts)
assert_and_infer_cfg()
logger.info('Testing with config:')
logger.info(pprint.pformat(cfg))
# output_dir = get_output_dir(cfg.TRAIN.DATASETS, training=True)
output_dir = '/mnt/fcav/self_training/object_detection/upperbound1/train/voc_GTA_caronly_train:cityscapes_caronly_train:voc_GTA_caronly_val/generalized_rcnn'
files = os.listdir(output_dir)
for f in files:
iter_string = re.findall(r'(?<=model_iter)\d+(?=\.pkl)', f)
if len(iter_string) > 0:
checkpoint_iter = int(iter_string[0])
start_iter = checkpoint_iter + 1
resume_weights_file = f
# Override the initialization weights with the found checkpoint
weights_file = os.path.join(output_dir, resume_weights_file)
logger.info(
'========> Resuming from checkpoint {} at start iter {}'.format(
weights_file, start_iter))
run_inference(
weights_file,
ind_range=args.range,
multi_gpu_testing=args.multi_gpu_testing,
check_expected_results=True,
)
models = []
afroc_scores = []
for i in res.keys():
afroc_scores.append(res[i][dataset]['box']['AFROC'])
models.append(i)
for model in model_list_keys:
if args.last_model >= model >= args.first_model and model > last_model_from_json:
count += 1
if count % args.samples != 0:
continue
logger.info('Model to test:')
logger.info(model)
workspace.ResetWorkspace()
results, auc_score, afroc_score = run_inference(
model_list[model],
ind_range=args.range,
multi_gpu_testing=args.multi_gpu_testing,
check_expected_results=True,
)
res[model] = results
with open(
os.path.join(
cfg.OUTPUT_DIR,
'RESULTS_{model}_{dataset}_{iter}.json'.format(
model=backbone,
dataset=cfg.TEST.DATASETS[0],
iter=cfg.SOLVER['MAX_ITER'])), 'w') as fp:
json.dump(res,
fp,
indent=1,
