def test(cfg):
logger = setup_logger(name="Test", level=cfg.LOGGER.LEVEL)
logger.info(cfg)
model = torch.load('ms_loss_net_car_5000.pkl')
model.eval()
logger.info("Validation")
val_loader = build_data(cfg, is_train=False)
labels = val_loader.dataset.label_list
labels = np.array([int(k) for k in labels])
feats = feat_extractor(model, val_loader, logger=logger)
ret_metric = RetMetric(feats=feats, labels=labels)
recall_curr = ret_metric.recall_k(1)
logger.info(f'Recall@1 : {recall_curr:.3f}')
def test(cfg):
logger = setup_logger(name="Train", level=cfg.LOGGER.LEVEL)
logger.info(cfg)
model = torch.load('net.pkl')
model.eval()
logger.info("Validation")
val_loader = build_data(cfg, is_train=False)
labels = val_loader[0].dataset.label_list
labels = np.array([int(k) for k in labels])
feats = feat_extractor(model, val_loader[0], logger=logger)
ret_metric = AccuracyCalculator(include=("precision_at_1", "mean_average_precision_at_r", "r_precision"), exclude=())
ret_metric = ret_metric.get_accuracy(feats, feats, labels, labels, True)
mapr_curr = ret_metric['mean_average_precision_at_r']
logger.info(f"ret_metric: {ret_metric}")
def extract(cfg, img_source, model_path=None):
logger = setup_logger(name="Feat", level=cfg.LOGGER.LEVEL)
model = build_model(cfg)
device = torch.device(cfg.MODEL.DEVICE)
model.to(device)
if model_path is not None:
print(f"load model {model_path} .......")
model_dict = torch.load(model_path)["model"]
model.load_state_dict(model_dict, strict=True)
print("model successfully loaded")
num_gpus = len(os.environ["CUDA_VISIBLE_DEVICES"].split(","))
if num_gpus > 1:
model = torch.nn.DataParallel(model)
transforms = build_transforms(cfg, is_train=False)
dataset = BaseDataSet(img_source,
False,
transforms=transforms,
mode=cfg.INPUT.MODE)
data_loader = DataLoader(
dataset,
collate_fn=collate_fn,
shuffle=False,
batch_size=cfg.DATA.TEST_BATCHSIZE * num_gpus,
num_workers=cfg.DATA.NUM_WORKERS,
pin_memory=False,
)
labels = dataset.label_list
feats = feat_extractor(model, data_loader, logger)
day_time = time.strftime("%Y-%m-%d", time.localtime())
npz_path = f"output/{day_time}_feat.npz"
np.savez(npz_path, feat=feats, upc=labels)
print(f"FEATS : \t {npz_path}")
def do_train(cfg, model, train_loader, val_loader, optimizer, scheduler,
criterion, checkpointer, device, checkpoint_period, arguments,
logger):
logger.info("Start training")
meters = MetricLogger(delimiter=" ")
max_iter = len(train_loader)
start_iter = arguments["iteration"]
best_iteration = -1
best_recall = 0
start_training_time = time.time()
end = time.time()
for iteration, (images, targets) in enumerate(train_loader, start_iter):
if iteration % cfg.VALIDATION.VERBOSE == 0 or iteration == max_iter:
model.eval()
logger.info('Validation')
labels = val_loader.dataset.label_list
labels = np.array([int(k) for k in labels])
feats = feat_extractor(model, val_loader, logger=logger)
ret_metric = RetMetric(feats=feats, labels=labels)
recall_curr = ret_metric.recall_k(1)
if recall_curr > best_recall:
best_recall = recall_curr
best_iteration = iteration
logger.info(
f'Best iteration {iteration}: recall@1: {best_recall:.3f}')
checkpointer.save(f"best_model")
else:
logger.info(
f'Recall@1 at iteration {iteration:06d}: {recall_curr:.3f}'
)
model.train()
model.apply(set_bn_eval)
data_time = time.time() - end
iteration = iteration + 1
arguments["iteration"] = iteration
scheduler.step()
images = images.to(device)
targets = torch.stack([target.to(device) for target in targets])
feats = model(images)
loss = criterion(feats, targets)
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time = time.time() - end
end = time.time()
meters.update(time=batch_time, data=data_time, loss=loss.item())
eta_seconds = meters.time.global_avg * (max_iter - iteration)
eta_string = str(datetime.timedelta(seconds=int(eta_seconds)))
if iteration % 20 == 0 or iteration == max_iter:
logger.info(
meters.delimiter.join([
"eta: {eta}",
"iter: {iter}",
"{meters}",
"lr: {lr:.6f}",
"max mem: {memory:.1f} GB",
]).format(
eta=eta_string,
iter=iteration,
meters=str(meters),
lr=optimizer.param_groups[0]["lr"],
memory=torch.cuda.max_memory_allocated() / 1024.0 /
1024.0 / 1024.0,
))
if iteration % checkpoint_period == 0:
checkpointer.save("model_{:06d}".format(iteration))
total_training_time = time.time() - start_training_time
total_time_str = str(datetime.timedelta(seconds=total_training_time))
logger.info("Total training time: {} ({:.4f} s / it)".format(
total_time_str, total_training_time / (max_iter)))
logger.info(
f"Best iteration: {best_iteration :06d} | best recall {best_recall} ")
def do_train(
cfg,
model,
train_loader,
val_loader,
optimizer,
scheduler,
criterion,
checkpointer,
writer,
device,
checkpoint_period,
arguments,
logger,
):
logger.info("Start training")
meters = MetricLogger(delimiter=" ")
max_iter = cfg.SOLVER.MAX_ITERS
best_iteration = -1
best_mapr = 0
start_training_time = time.time()
end = time.time()
if cfg.XBM.ENABLE:
logger.info(">>> use XBM")
xbm = XBM(cfg)
iteration = 0
_train_loader = iter(train_loader)
while iteration <= max_iter:
try:
images, targets, indices = _train_loader.next()
except StopIteration:
_train_loader = iter(train_loader)
images, targets, indices = _train_loader.next()
if (iteration % cfg.VALIDATION.VERBOSE == 0
or iteration == max_iter) and iteration > 0:
model.eval()
logger.info("Validation")
labels = val_loader[0].dataset.label_list
labels = np.array([int(k) for k in labels])
feats = feat_extractor(model, val_loader[0], logger=logger)
ret_metric = AccuracyCalculator(
include=("precision_at_1", "mean_average_precision_at_r",
"r_precision"),
exclude=())
ret_metric = ret_metric.get_accuracy(feats, feats, labels, labels,
True)
mapr_curr = ret_metric['mean_average_precision_at_r']
for k, v in ret_metric.items():
log_info[f"e_{k}"] = v
scheduler.step(log_info[f"e_precision_at_1"])
# scheduler.step(log_info[f"R@1"])
log_info["lr"] = optimizer.param_groups[0]["lr"]
if mapr_curr > best_mapr:
best_mapr = mapr_curr
best_iteration = iteration
logger.info(f"Best iteration {iteration}: {ret_metric}")
else:
logger.info(
f"Performance at iteration {iteration:06d}: {ret_metric}")
flush_log(writer, iteration)
np.save(f"xbm_pos_freqs_{iteration:06d}.npy",
criterion.total_pos_freqs,
allow_pickle=True)
np.save(f"xbm_neg_freqs_{iteration:06d}.npy",
criterion.total_neg_freqs,
allow_pickle=True)
model.train()
data_time = time.time() - end
iteration = iteration + 1
arguments["iteration"] = iteration
images = images.to(device)
targets = targets.to(device)
feats = model(images)
if cfg.XBM.ENABLE and iteration > cfg.XBM.START_ITERATION:
xbm.enqueue_dequeue(feats.detach(), targets.detach())
loss = criterion(feats, targets, feats, targets)
log_info["batch_loss"] = loss.item()
if cfg.XBM.ENABLE and iteration > cfg.XBM.START_ITERATION:
xbm_feats, xbm_targets = xbm.get()
xbm_loss = criterion(feats, targets, xbm_feats, xbm_targets)
log_info["xbm_loss"] = xbm_loss.item()
loss = loss + cfg.XBM.WEIGHT * xbm_loss
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time = time.time() - end
end = time.time()
meters.update(time=batch_time, data=data_time, loss=loss.item())
eta_seconds = meters.time.global_avg * (max_iter - iteration)
eta_string = str(datetime.timedelta(seconds=int(eta_seconds)))
if iteration % 20 == 0 or iteration == max_iter:
logger.info(
meters.delimiter.join([
"eta: {eta}",
"iter: {iter}",
"{meters}",
"lr: {lr:.6f}",
"max mem: {memory:.1f} GB",
]).format(
eta=eta_string,
iter=iteration,
meters=str(meters),
lr=optimizer.param_groups[0]["lr"],
memory=torch.cuda.max_memory_allocated() / 1024.0 /
1024.0 / 1024.0,
))
log_info["loss"] = loss.item()
flush_log(writer, iteration)
if iteration % checkpoint_period == 0 and cfg.SAVE:
checkpointer.save("model_{:06d}".format(iteration))
pass
del feats
del loss
total_training_time = time.time() - start_training_time
total_time_str = str(datetime.timedelta(seconds=total_training_time))
logger.info("Total training time: {} ({:.4f} s / it)".format(
total_time_str, total_training_time / (max_iter)))
logger.info(
f"Best iteration: {best_iteration :06d} | best MAP@R {best_mapr} ")
writer.close()
def do_train(
cfg,
model,
train_loader,
trainVal_loader,
val_loader,
optimizer,
optimizer_center,
scheduler,
scheduler_center,
criterion,
criterion_xbm,
checkpointer,
writer,
device,
arguments,
logger,
):
logger.info("Start training")
meters = MetricLogger(delimiter=" ")
max_iter = cfg.SOLVER.MAX_ITERS
best_iteration = -1
best_mapr = 0
start_training_time = time.time()
end = time.time()
USE_PRISM='PRISM' in cfg.LOSSES.NAME or 'PRISM' in cfg.LOSSES.NAME_XBM_LOSS
CHECK_NOISE=cfg.NOISE.CHECK_NOISE
if cfg.XBM.ENABLE:
logger.info(">>> use XBM")
xbm = XBM(cfg)
iteration = arguments["iteration"]+1
_train_loader = iter(train_loader)
while iteration <= max_iter:
try:
images, targets, indices = _train_loader.next()
except StopIteration:
_train_loader = iter(train_loader)
images, targets, indices = _train_loader.next()
if (iteration % cfg.VALIDATION.VERBOSE == 0 or iteration == max_iter) and iteration > 0:
model.eval()
logger.info("Validation")
if len(val_loader)==1:
labels = val_loader[0].dataset.label_list
labels = np.array([int(k) for k in labels])
feats = feat_extractor(model, val_loader[0], logger=logger)
feats_gallery=feats
feats_query=feats
labels_gallery=labels
labels_query=labels
else:
labels = val_loader[0].dataset.label_list
labels_gallery = np.array([int(k) for k in labels])
labels = val_loader[1].dataset.label_list
labels_query = np.array([int(k) for k in labels])
feats_gallery = feat_extractor(model, val_loader[0], logger=logger)
feats_query = feat_extractor(model, val_loader[1], logger=logger)
ret_metric = AccuracyCalculator(include=("precision_at_1", "mean_average_precision_at_r", "r_precision",'mean_average_precision_at_100'), exclude=())
ret_metric = ret_metric.get_accuracy(feats_query, feats_gallery, labels_query, labels_gallery, len(val_loader)==1)
mapr_curr = ret_metric['precision_at_1']
for k, v in ret_metric.items():
log_info[f"e_{k}"] = v
if cfg.SOLVER.LR_SCHEDULAR=='val':
scheduler.step(log_info['e_precision_at_1'])
log_info["lr"] = optimizer.param_groups[0]["lr"]
if mapr_curr > best_mapr:
best_mapr = mapr_curr
best_iteration = iteration
logger.info(f"Best iteration {iteration}: {ret_metric}")
checkpointer.save("model_best")
else:
logger.info(f"Performance at iteration {iteration:06d}: {ret_metric}")
flush_log(writer, iteration)
#======training set precision====
if iteration % cfg.VALIDATION.VERBOSE == 0 and iteration > 0:
logger.info("Validation on Training set")
labels = trainVal_loader.dataset.label_list
labels = np.array([int(k) for k in labels])
feats = feat_extractor(model, trainVal_loader, logger=logger)
feats_gallery=feats
feats_query=feats
labels_gallery=labels
labels_query=labels
ret_metric = AccuracyCalculator(include=("precision_at_1", "mean_average_precision_at_r", "r_precision",'mean_average_precision_at_100'), exclude=())
ret_metric = ret_metric.get_accuracy(feats_query, feats_gallery, labels_query, labels_gallery, len(trainVal_loader)==1)
mapr_curr = ret_metric['precision_at_1']
for k, v in ret_metric.items():
log_info[f"t_{k}"] = v
logger.info(f"Performance on Training set at iteration {iteration:06d}: {ret_metric}")
flush_log(writer, iteration)
#========
model.train()
data_time = time.time() - end
iteration = iteration + 1
arguments["iteration"] = iteration
images = images.to(device)
targets = targets.to(device)
feats = model(images)
if CHECK_NOISE:
is_noise=train_loader.dataset.is_noise[indices]
else:
is_noise=None
if cfg.XBM.ENABLE and iteration > cfg.XBM.START_ITERATION and (not USE_PRISM):
xbm.enqueue_dequeue(feats.detach(), targets.detach())
if (not USE_PRISM) or (iteration <= cfg.XBM.START_ITERATION):
loss = criterion(feats, targets, feats, targets)
log_info["batch_loss"] = loss.item()
if criterion_xbm is not None:
criterion_=criterion_xbm
else:
criterion_=criterion
if cfg.XBM.ENABLE and iteration > cfg.XBM.START_ITERATION:
xbm_feats, xbm_targets = xbm.get()
if USE_PRISM:
xbm_loss,p_in = criterion_(feats, targets, xbm_feats, xbm_targets,is_noise=is_noise)
if p_in is not None:
if p_in.dtype is torch.bool:
enqueue_feats=feats[p_in]
enqueue_targets=targets[p_in]
else:
enqueue_feats=feats
enqueue_targets=targets
if (p_in is not None) and (p_in.dtype is torch.bool):
selected_feats=feats[p_in]
selected_tar=targets[p_in]
loss = criterion(selected_feats, selected_tar, selected_feats, selected_tar)
else:
loss = criterion(feats, targets, feats, targets)
if not isinstance(loss,tuple):
log_info["batch_loss"] = loss.item()
else:
loss = 0
else:
xbm_loss = criterion(feats, targets, xbm_feats, xbm_targets)
loss = loss + cfg.XBM.WEIGHT * xbm_loss
optimizer.zero_grad()
if optimizer_center is not None:
optimizer_center.zero_grad()
if not isinstance(loss,(float,int)):
loss.backward()
optimizer.step()
if optimizer_center is not None:
optimizer_center.step()
if cfg.XBM.ENABLE and iteration > cfg.XBM.START_ITERATION and USE_PRISM:
xbm.enqueue_dequeue(enqueue_feats.detach(), enqueue_targets.detach())
if cfg.SOLVER.LR_SCHEDULAR!='val':
scheduler.step()
if scheduler_center is not None:
scheduler_center.step()
batch_time = time.time() - end
end = time.time()
meters.update(time=batch_time, data=data_time, loss=loss.item())
eta_seconds = meters.time.global_avg * (max_iter - iteration)
eta_string = str(datetime.timedelta(seconds=int(eta_seconds)))
if iteration % 20 == 0 or iteration == max_iter:
logger.info(
meters.delimiter.join(
[
"eta: {eta}",
"iter: {iter}",
"{meters}",
"lr: {lr:.6f}",
"max mem: {memory:.1f} GB",
]
).format(
eta=eta_string,
iter=iteration,
meters=str(meters),
lr=optimizer.param_groups[0]["lr"],
memory=torch.cuda.max_memory_allocated() / 1024.0 / 1024.0 / 1024.0,
)
)
log_info["loss"] = loss.item()
flush_log(writer, iteration)
del feats
del loss
total_training_time = time.time() - start_training_time
total_time_str = str(datetime.timedelta(seconds=total_training_time))
logger.info(
"Total training time: {} ({:.4f} s / it)".format(
total_time_str, total_training_time / (max_iter)
)
)
logger.info(f"Best iteration: {best_iteration :06d} | best MAP@R {best_mapr} ")
writer.close()
