def main(args):
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
enc_score_metrics = []
enc_target_metrics = []
dec_score_metrics = [[] for i in range(args.dec_steps)]
dec_target_metrics = [[] for i in range(args.dec_steps)]
if osp.isfile(args.checkpoint):
checkpoint = torch.load(args.checkpoint)
else:
raise(RuntimeError('Cannot find the checkpoint {}'.format(args.checkpoint)))
model = build_model(args).to(device)
model.load_state_dict(checkpoint['model_state_dict'])
model.train(False)
softmax = nn.Softmax(dim=1).to(device)
for session_idx, session in enumerate(args.test_session_set, start=1):
start = time.time()
with torch.set_grad_enabled(False):
camera_inputs = np.load(osp.join(args.data_root, args.camera_feature, session+'.npy'), mmap_mode='r')
sensor_inputs = np.load(osp.join(args.data_root, 'sensor', session+'.npy'), mmap_mode='r')
target = np.load(osp.join(args.data_root, 'target', session+'.npy'))
future_input = to_device(torch.zeros(model.future_size), device)
enc_hx = to_device(torch.zeros(model.hidden_size), device)
enc_cx = to_device(torch.zeros(model.hidden_size), device)
for l in range(target.shape[0]):
camera_input = to_device(
torch.as_tensor(camera_inputs[l].astype(np.float32)), device)
sensor_input = to_device(
torch.as_tensor(sensor_inputs[l].astype(np.float32)), device)
future_input, enc_hx, enc_cx, enc_score, dec_score_stack = \
model.step(camera_input, sensor_input, future_input, enc_hx, enc_cx)
enc_score_metrics.append(softmax(enc_score).cpu().numpy()[0])
enc_target_metrics.append(target[l])
for step in range(args.dec_steps):
dec_score_metrics[step].append(softmax(dec_score_stack[step]).cpu().numpy()[0])
dec_target_metrics[step].append(target[min(l + step, target.shape[0] - 1)])
end = time.time()
print('Processed session {}, {:2} of {}, running time {:.2f} sec'.format(
session, session_idx, len(args.test_session_set), end - start))
save_dir = osp.dirname(args.checkpoint)
result_file = osp.basename(args.checkpoint).replace('.pth', '.json')
# Compute result for encoder
utl.compute_result(args.class_index,
enc_score_metrics, enc_target_metrics,
save_dir, result_file, save=True, verbose=True)
# Compute result for decoder
for step in range(args.dec_steps):
utl.compute_result(args.class_index,
dec_score_metrics[step], dec_target_metrics[step],
save_dir, result_file, save=False, verbose=True)
def main(args):
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
enc_score_metrics = []
enc_target_metrics = []
dec_score_metrics = [[] for i in range(args.dec_steps)]
dec_target_metrics = [[] for i in range(args.dec_steps)]
if osp.isfile(args.checkpoint):
checkpoint = torch.load(args.checkpoint)
else:
raise (RuntimeError('Cannot find the checkpoint {}'.format(
args.checkpoint)))
model = build_model(args).to(device)
model.load_state_dict(checkpoint['model_state_dict'])
model.train(False)
softmax = nn.Softmax(dim=1).to(device)
localizations = {}
for session_idx, session in enumerate(args.test_session_set, start=1):
start = time.time()
with torch.set_grad_enabled(False):
print("ID {}: {}".format(session_idx, session))
camera_inputs = np.load(
osp.join(args.data_root, args.camera_feature,
session + '.npy'))
motion_inputs = np.load(
osp.join(args.data_root, args.motion_feature,
session + '.npy'))
# For Cricket
camera_inputs = np.squeeze(camera_inputs, axis=1)
motion_inputs = np.squeeze(motion_inputs, axis=1)
camera_inputs[camera_inputs == float("-Inf")] = 0
camera_inputs[camera_inputs == float("Inf")] = 0
motion_inputs[motion_inputs == float("-Inf")] = 0
motion_inputs[motion_inputs == float("Inf")] = 0
target = np.load(
osp.join(args.data_root, 'target', session + '.npy'))
future_input = to_device(torch.zeros(model.future_size), device)
enc_hx = to_device(torch.zeros(model.hidden_size), device)
enc_cx = to_device(torch.zeros(model.hidden_size), device)
niters = min([motion_inputs.shape[0], camera_inputs.shape[0]])
seq_diff = camera_inputs.shape[0] - motion_inputs.shape[0]
vid_enc_score_metrics = []
for l in range(niters):
camera_input = to_device(
torch.as_tensor(camera_inputs[l + seq_diff].astype(
np.float32)), device)
motion_input = to_device(
torch.as_tensor(motion_inputs[l].astype(np.float32)),
device)
future_input, enc_hx, enc_cx, enc_score, dec_score_stack = \
model.step(camera_input, motion_input, future_input, enc_hx, enc_cx)
vid_enc_score_metrics.append(
softmax(enc_score).cpu().numpy()[0])
enc_target_metrics.append(target[l + seq_diff, 1])
for step in range(args.dec_steps):
dec_score_metrics[step].append(
softmax(dec_score_stack[step]).cpu().numpy()[0])
dec_target_metrics[step].append(
target[min(l + step, target.shape[0] - 1), 1])
end = time.time()
enc_score_metrics.extend(vid_enc_score_metrics)
print(
'Processed session {}, {:2} of {}, running time {:.2f} sec'.format(
session, session_idx, len(args.test_session_set), end - start))
if osp.isfile(osp.join(VAL_DATASET, session + '.mp4')):
session = session + '.mp4'
else:
session = session + '.avi'
if session.startswith("v_"):
localizations["youtube/" + session] = getScoredLocalizations(
vid_enc_score_metrics, 1)
elif session.startswith("IPL2017"):
localizations["ipl2017/" + session] = getScoredLocalizations(
vid_enc_score_metrics, 1)
elif session.startswith("Game "):
localizations["cpl2015/" + session] = getScoredLocalizations(
vid_enc_score_metrics, 1)
else:
localizations["hotstar/" + session] = getScoredLocalizations(
vid_enc_score_metrics, 1)
save_dir = osp.dirname(args.checkpoint)
result_file = osp.basename(args.checkpoint).replace('.pth', '.json')
# Compute result for encoder
utl.compute_result(args.class_index, enc_score_metrics, enc_target_metrics, \
save_dir, result_file, ignore_class=[0], save=True, verbose=True)
# Compute result for decoder
for step in range(args.dec_steps):
utl.compute_result(args.class_index, dec_score_metrics[step], \
dec_target_metrics[step], save_dir, result_file, \
ignore_class=[0], save=False, verbose=True)
#print(localizations)
with open("prediction_localizations_TRN_mainVal.json", "w") as fp:
json.dump(localizations, fp)
def run(config, is_train, eval_name):
torch.manual_seed(config['training_parameters']['seed'])
args.gpu = config['training_parameters']['gpu']
output = config['logs']['dir_logs']
batch_size = config['training_parameters']['batch_size']
if args.gpu:
torch.cuda.manual_seed(config['training_parameters']['seed'])
torch.backends.cudnn.benchmark = True
if is_train:
'''
eval_name 为 test 时会同时加载test 数据集
'''
print("training . . .")
model, train_dset, eval_dset, embedding_weight, test_dset = load_model_data(
config, is_train=is_train, eval_name=eval_name)
else:
print("testing . . .")
model, eval_dset = load_model_data(config,
is_train=is_train,
eval_name=eval_name)
if args.gpu:
# model = model.cuda()
model = nn.DataParallel(model).cuda()
model_dir = os.path.join(output, "model_epoch16.pth")
eval_loader = DataLoader(
eval_dset,
batch_size,
shuffle=False,
num_workers=config['training_parameters']['num_workers'],
collate_fn=utils.trim_collate)
utils.compute_result(eval_name, model, model_dir, eval_loader, output)
return
logger = utils.logger(os.path.join(output, 'log.json'))
model_size = utils.params_count(model)
print("nParams:", model_size)
logger.add("model size(Params)", model_size)
logger.add("train set", len(train_dset))
logger.add("val set", len(eval_dset))
with open(output + "config.yaml", "w") as yaml_file:
yaml.dump(config, yaml_file)
# model.embedding.init_embedding(embedding_weight)
if args.gpu:
# model = model.cuda()
model = nn.DataParallel(model).cuda()
print("sucees to create model.")
# use_vg = config['data']['use_vg']
evaluation = True if eval_name == "val" else False #config['data']['evaluation']
if evaluation:
print("train with train dataset")
eval_loader = DataLoader(
eval_dset,
batch_size,
shuffle=False,
num_workers=config['training_parameters']['num_workers'],
collate_fn=utils.trim_collate)
train_loader = DataLoader(
train_dset,
batch_size,
shuffle=True,
num_workers=config['training_parameters']['num_workers'],
collate_fn=utils.trim_collate)
else:
print("train with train and val dataset")
eval_loader = None
train_dset = ConcatDataset([train_dset, eval_dset])
train_loader = DataLoader(
train_dset,
batch_size,
shuffle=True,
num_workers=config['training_parameters']['num_workers'],
collate_fn=utils.trim_collate)
# model_data = torch.load(output+'model_epoch8.pth')
# model.load_state_dict(model_data.get('model_state', model_data))
# print("success to load model!")
# 初始化优化器
# ignored_params = list(map(id, model.module.bert.parameters()))
# base_params = filter(lambda p: id(p) not in ignored_params, model.parameters())
# optim = torch.optim.Adamax([
# {'params': base_params},
# {'params': model.module.bert.parameters(), 'lr': 1e-6} #FC层使用较大的学习率
# ],
# lr = 0.0015
# )
optim = torch.optim.Adamax(filter(lambda p: p.requires_grad,
model.parameters()),
lr=0.0015)
# optim = torch.optim.Adam(
# filter(lambda p:p.requires_grad, model.parameters()),
# lr=0.00015,
# betas = (0.9, 0.98),
# eps = 1e-9
# # weight_decay=0.001
# )
train(model, train_loader, eval_loader, logger, optim, output,
**config['training_parameters'])
if eval_name == "val":
model_dir = os.path.join(output, "model_best.pth")
utils.compute_result(eval_name, model, model_dir, eval_loader, output)
else: # test
model_dir = os.path.join(output, "model_epoch5.pth")
test_loader = DataLoader(
test_dset,
batch_size,
shuffle=False,
num_workers=config['training_parameters']['num_workers'],
collate_fn=utils.trim_collate)
utils.compute_result(eval_name, model, model_dir, test_loader, output)
def main(args):
this_dir = osp.join(osp.dirname(__file__), '.')
save_dir = osp.join(this_dir, 'checkpoints')
if not osp.isdir(save_dir):
os.makedirs(save_dir)
command = 'python ' + ' '.join(sys.argv)
logger = utl.setup_logger(osp.join(this_dir, 'log.txt'), command=command)
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
utl.set_seed(int(args.seed))
model = build_model(args)
if osp.isfile(args.checkpoint):
checkpoint = torch.load(args.checkpoint,
map_location=torch.device('cpu'))
model.load_state_dict(checkpoint['model_state_dict'])
else:
model.apply(utl.weights_init)
if args.distributed:
model = nn.DataParallel(model)
model = model.to(device)
criterion = nn.CrossEntropyLoss().to(device)
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
if osp.isfile(args.checkpoint):
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
for param_group in optimizer.param_groups:
param_group['lr'] = args.lr
args.start_epoch += checkpoint['epoch']
softmax = nn.Softmax(dim=1).to(device)
for epoch in range(args.start_epoch, args.start_epoch + args.epochs):
data_loaders = {
phase: utl.build_data_loader(args, phase)
for phase in args.phases
}
enc_losses = {phase: 0.0 for phase in args.phases}
enc_score_metrics = []
enc_target_metrics = []
enc_mAP = 0.0
dec_losses = {phase: 0.0 for phase in args.phases}
dec_score_metrics = []
dec_target_metrics = []
dec_mAP = 0.0
start = time.time()
for phase in args.phases:
training = phase == 'train'
if training:
model.train(True)
elif not training and args.debug:
model.train(False)
else:
continue
with torch.set_grad_enabled(training):
for batch_idx, (camera_inputs, sensor_inputs, enc_target, dec_target) \
in enumerate(data_loaders[phase], start=1):
batch_size = camera_inputs.shape[0]
camera_inputs = camera_inputs.to(device)
sensor_inputs = sensor_inputs.to(device)
enc_target = enc_target.to(device).view(-1)
dec_target = dec_target.to(device).view(-1)
enc_score, dec_score = model(camera_inputs, sensor_inputs)
enc_loss = criterion(enc_score, enc_target)
dec_loss = criterion(dec_score, dec_target)
enc_losses[phase] += enc_loss.item() * batch_size
dec_losses[phase] += dec_loss.item() * batch_size
if args.verbose:
print(
'Epoch: {:2} | iteration: {:3} | enc_loss: {:.5f} dec_loss: {:.5f}'
.format(epoch, batch_idx, enc_loss.item(),
dec_loss.item()))
if training:
optimizer.zero_grad()
loss = enc_loss + dec_loss
loss.backward()
optimizer.step()
else:
# Prepare metrics for encoder
enc_score = softmax(enc_score).cpu().numpy()
enc_target = enc_target.cpu().numpy()
enc_score_metrics.extend(enc_score)
enc_target_metrics.extend(enc_target)
# Prepare metrics for decoder
dec_score = softmax(dec_score).cpu().numpy()
dec_target = dec_target.cpu().numpy()
dec_score_metrics.extend(dec_score)
dec_target_metrics.extend(dec_target)
end = time.time()
if args.debug:
result_file = 'inputs-{}-epoch-{}.json'.format(args.inputs, epoch)
# Compute result for encoder
enc_mAP = utl.compute_result(
args.class_index,
enc_score_metrics,
enc_target_metrics,
save_dir,
result_file,
save=True,
)
# Compute result for decoder
dec_mAP = utl.compute_result(
args.class_index,
dec_score_metrics,
dec_target_metrics,
save_dir,
result_file,
save=False,
)
# Output result
logger.output(epoch,
enc_losses,
dec_losses,
len(data_loaders['train'].dataset),
len(data_loaders['test'].dataset),
enc_mAP,
dec_mAP,
end - start,
debug=args.debug)
# Save model
checkpoint_file = 'inputs-{}-epoch-{}.pth'.format(args.inputs, epoch)
torch.save(
{
'epoch':
epoch,
'model_state_dict':
model.module.state_dict()
if args.distributed else model.state_dict(),
'optimizer_state_dict':
optimizer.state_dict(),
}, osp.join(save_dir, checkpoint_file))