def main():
with open('dont_commit.json') as f:
data = json.load(f)
#print(set_nickname(data))
print(format_dict(set_emoji(data)))
f = open("./../changes.txt", "w+")
f.write(format_dict(set_emoji(data)))
f.close()
def main():
with open('dont_commit.json') as f:
data = json.load(f)
dict = created_poll(data)
f = open("poll_data.json", "w+")
f.write(format_dict(dict))
f.close()
def main():
import argparse
parser = argparse.ArgumentParser(
description="imsitu VSRL. Training, evaluation and prediction.")
parser.add_argument("--gpuid",
default=-1,
help="put GPU id > -1 in GPU mode",
type=int)
#parser.add_argument("--command", choices = ["train", "eval", "resume", 'predict'], required = True)
parser.add_argument('--resume_training',
action='store_true',
help='Resume training from the model [resume_model]')
parser.add_argument('--resume_model',
type=str,
default='',
help='The model we resume')
parser.add_argument('--verb_module',
type=str,
default='',
help='pretrained verb module')
parser.add_argument('--train_role',
action='store_true',
help='cnn fix, verb fix, role train from the scratch')
parser.add_argument(
'--finetune_verb',
action='store_true',
help='cnn fix, verb finetune, role train from the scratch')
parser.add_argument(
'--finetune_cnn',
action='store_true',
help='cnn finetune, verb finetune, role train from the scratch')
parser.add_argument('--output_dir',
type=str,
default='./trained_models',
help='Location to output the model')
parser.add_argument('--evaluate',
action='store_true',
help='Only use the testing mode')
parser.add_argument('--test',
action='store_true',
help='Only use the testing mode')
parser.add_argument('--dataset_folder',
type=str,
default='./imSitu',
help='Location of annotations')
parser.add_argument('--imgset_dir',
type=str,
default='./resized_256',
help='Location of original images')
parser.add_argument('--frcnn_feat_dir',
type=str,
help='Location of output from detectron')
#todo: train role module separately with gt verbs
args = parser.parse_args()
batch_size = 640
#lr = 5e-6
lr = 0.0001
lr_max = 5e-4
lr_gamma = 0.1
lr_step = 25
clip_norm = 50
weight_decay = 1e-4
n_epoch = 500
n_worker = 3
#dataset_folder = 'imSitu'
#imgset_folder = 'resized_256'
dataset_folder = args.dataset_folder
imgset_folder = args.imgset_dir
print('model spec :, top down att with role q ')
train_set = json.load(open(dataset_folder + "/train.json"))
imsitu_roleq = json.load(open("imsitu_data/imsitu_questions.json"))
encoder = imsitu_encoder(train_set, imsitu_roleq)
model = model_role_noun.BaseModel(encoder, args.gpuid)
# To group up the features
cnn_features, role_features = utils.group_features_noun(model)
train_set = imsitu_loader_roleq(imgset_folder, train_set, encoder,
model.train_preprocess())
train_loader = torch.utils.data.DataLoader(train_set,
batch_size=64,
shuffle=True,
num_workers=n_worker)
dev_set = json.load(open(dataset_folder + "/dev.json"))
dev_set = imsitu_loader_roleq(imgset_folder, dev_set, encoder,
model.dev_preprocess())
dev_loader = torch.utils.data.DataLoader(dev_set,
batch_size=64,
shuffle=True,
num_workers=n_worker)
test_set = json.load(open(dataset_folder + "/test.json"))
test_set = imsitu_loader_roleq(imgset_folder, test_set, encoder,
model.dev_preprocess())
test_loader = torch.utils.data.DataLoader(test_set,
batch_size=64,
shuffle=True,
num_workers=n_worker)
traindev_set = json.load(open(dataset_folder + "/dev.json"))
traindev_set = imsitu_loader_roleq(imgset_folder, traindev_set, encoder,
model.dev_preprocess())
traindev_loader = torch.utils.data.DataLoader(traindev_set,
batch_size=8,
shuffle=True,
num_workers=n_worker)
utils.set_trainable(model, False)
if args.train_role:
print('CNN fix, Verb fix, train role from the scratch from: {}'.format(
args.verb_module))
args.train_all = False
if len(args.verb_module) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.verb_module, [model.conv, model.verb],
['conv', 'verb'])
optimizer_select = 1
model_name = 'cfx_vfx_rtrain'
elif args.finetune_verb:
print('CNN fix, Verb finetune, train role from the scratch from: {}'.
format(args.verb_module))
args.train_all = True
if len(args.verb_module) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.verb_module, [model.conv, model.verb],
['conv', 'verb'])
optimizer_select = 2
model_name = 'cfx_vft_rtrain'
elif args.finetune_cnn:
print(
'CNN finetune, Verb finetune, train role from the scratch from: {}'
.format(args.verb_module))
args.train_all = True
if len(args.verb_module) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.verb_module, [model.conv, model.verb],
['conv', 'verb'])
optimizer_select = 3
model_name = 'cft_vft_rtrain'
elif args.resume_training:
print('Resume training from: {}'.format(args.resume_model))
args.train_all = True
if len(args.resume_model) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.resume_model, [model])
optimizer_select = 0
model_name = 'resume_all'
else:
print('Training from the scratch.')
optimizer_select = 0
args.train_all = True
model_name = 'train_full'
optimizer = utils.get_optimizer_noun(lr, weight_decay, optimizer_select,
cnn_features, role_features)
if not os.path.exists(args.output_dir):
os.mkdir(args.output_dir)
torch.manual_seed(1234)
if args.gpuid >= 0:
#print('GPU enabled')
model.cuda()
torch.cuda.manual_seed(1234)
torch.backends.cudnn.deterministic = True
optimizer = torch.optim.Adamax([{
'params': cnn_features,
'lr': 5e-5
}, {
'params': role_features
}],
lr=1e-3)
#optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay)
#scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=lr_step, gamma=lr_gamma)
#gradient clipping, grad check
scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.9)
if args.evaluate:
top1, top5, val_loss = eval(model,
dev_loader,
encoder,
args.gpuid,
write_to_file=True)
top1_avg = top1.get_average_results_nouns()
top5_avg = top5.get_average_results_nouns()
avg_score = top1_avg["verb"] + top1_avg["value"] + top1_avg["value-all"] + top5_avg["verb"] + \
top5_avg["value"] + top5_avg["value-all"] + top5_avg["value*"] + top5_avg["value-all*"]
avg_score /= 8
print('Dev average :{:.2f} {} {}'.format(
avg_score * 100, utils.format_dict(top1_avg, '{:.2f}', '1-'),
utils.format_dict(top5_avg, '{:.2f}', '5-')))
#write results to csv file
role_dict = top1.role_dict
fail_val_all = top1.value_all_dict
pass_val_dict = top1.vall_all_correct
fail_role = top1.fail_verb_role
all_role = top1.all_verb_role
with open('role_pred_data.json', 'w') as fp:
json.dump(role_dict, fp, indent=4)
with open('fail_val_all.json', 'w') as fp:
json.dump(fail_val_all, fp, indent=4)
with open('pass_val_all.json', 'w') as fp:
json.dump(pass_val_dict, fp, indent=4)
with open('fail_role.json', 'w') as fp:
json.dump(fail_role, fp, indent=4)
with open('all_role.json', 'w') as fp:
json.dump(all_role, fp, indent=4)
print('Writing predictions to file completed !')
elif args.test:
top1, top5, val_loss = eval(model,
test_loader,
encoder,
args.gpuid,
write_to_file=True)
top1_avg = top1.get_average_results_nouns()
top5_avg = top5.get_average_results_nouns()
avg_score = top1_avg["verb"] + top1_avg["value"] + top1_avg["value-all"] + top5_avg["verb"] + \
top5_avg["value"] + top5_avg["value-all"] + top5_avg["value*"] + top5_avg["value-all*"]
avg_score /= 8
print('Test average :{:.2f} {} {}'.format(
avg_score * 100, utils.format_dict(top1_avg, '{:.2f}', '1-'),
utils.format_dict(top5_avg, '{:.2f}', '5-')))
else:
print('Model training started!')
train(model, train_loader, dev_loader, traindev_loader, optimizer,
scheduler, n_epoch, args.output_dir, encoder, args.gpuid,
clip_norm, lr_max, model_name, args)
def train(model,
train_loader,
dev_loader,
traindev_loader,
optimizer,
scheduler,
max_epoch,
model_dir,
encoder,
gpu_mode,
clip_norm,
lr_max,
model_name,
args,
eval_frequency=4000):
model.train()
train_loss = 0
total_steps = 0
print_freq = 400
dev_score_list = []
time_all = time.time()
'''if model.gpu_mode >= 0 :
ngpus = 2
device_array = [i for i in range(0,ngpus)]
pmodel = torch.nn.DataParallel(model, device_ids=device_array)
else:
pmodel = model'''
pmodel = model
'''if scheduler.get_lr()[0] < lr_max:
scheduler.step()'''
top1 = imsitu_scorer(encoder, 1, 3)
top5 = imsitu_scorer(encoder, 5, 3)
'''print('init param data check :')
for f in model.parameters():
if f.requires_grad:
print(f.data.size())'''
for epoch in range(max_epoch):
#print('current sample : ', i, img.size(), verb.size(), roles.size(), labels.size())
#sizes batch_size*3*height*width, batch*504*1, batch*6*190*1, batch*3*6*lebale_count*1
mx = len(train_loader)
for i, (_, img, verb, roles, role_q, q_len,
labels) in enumerate(train_loader):
#print("epoch{}-{}/{} batches\r".format(epoch,i+1,mx)) ,
t0 = time.time()
t1 = time.time()
total_steps += 1
if gpu_mode >= 0:
img = torch.autograd.Variable(img.cuda())
roles = torch.autograd.Variable(roles.cuda())
role_q = torch.autograd.Variable(role_q.cuda())
q_len = torch.autograd.Variable(q_len.cuda())
verb = torch.autograd.Variable(verb.cuda())
labels = torch.autograd.Variable(labels.cuda())
else:
img = torch.autograd.Variable(img)
verb = torch.autograd.Variable(verb)
role_q = torch.autograd.Variable(role_q)
q_len = torch.autograd.Variable(q_len)
roles = torch.autograd.Variable(roles)
labels = torch.autograd.Variable(labels)
'''print('all inputs')
print(img)
print('=========================================================================')
print(verb)
print('=========================================================================')
print(roles)
print('=========================================================================')
print(labels)'''
noun_predict, role_predict = pmodel(img, verb)
#verb_predict, rol1pred, role_predict = pmodel.forward_eval5(img)
#print ("forward time = {}".format(time.time() - t1))
t1 = time.time()
'''g = make_dot(verb_predict, model.state_dict())
g.view()'''
noun_loss = model.calculate_noun_loss(verb, noun_predict, labels,
args)
role_loss = model.calculate_role_loss(verb, role_predict, roles,
args)
alpha = 0.5
loss = noun_loss + (role_loss * 3)
#loss = model.calculate_eval_loss_new(verb_predict, verb, rol1pred, labels, args)
#loss = loss_ * random.random() #try random loss
#print ("loss time = {}".format(time.time() - t1))
t1 = time.time()
#print('current loss = ', loss)
loss.backward()
#print ("backward time = {}".format(time.time() - t1))
#torch.nn.utils.clip_grad_norm_(model.parameters(), clip_norm)
'''for param in filter(lambda p: p.requires_grad,model.parameters()):
print(param.grad.data.sum())'''
#start debugger
#import pdb; pdb.set_trace()
optimizer.step()
optimizer.zero_grad()
'''print('grad check :')
for f in model.parameters():
print('data is')
print(f.data)
print('grad is')
print(f.grad)'''
train_loss += loss.item()
#top1.add_point_eval5(verb_predict, verb, role_predict, labels)
#top5.add_point_eval5(verb_predict, verb, role_predict, labels)
top1.add_point_noun(verb, noun_predict, labels)
top5.add_point_noun(verb, noun_predict, labels)
if total_steps % print_freq == 0:
top1_a = top1.get_average_results_nouns()
top5_a = top5.get_average_results_nouns()
print("{},{},{}, {} , {}, loss = {:.2f}, avg loss = {:.2f}".
format(total_steps - 1, epoch, i,
utils.format_dict(top1_a, "{:.2f}", "1-"),
utils.format_dict(top5_a, "{:.2f}", "5-"),
loss.item(), train_loss /
((total_steps - 1) % eval_frequency)))
if total_steps % eval_frequency == 0:
top1, top5, val_loss = eval(model, dev_loader, encoder,
gpu_mode)
model.train()
top1_avg = top1.get_average_results_nouns()
top5_avg = top5.get_average_results_nouns()
avg_score = top1_avg["verb"] + top1_avg["value"] + top1_avg["value-all"] + top5_avg["verb"] + \
top5_avg["value"] + top5_avg["value-all"] + top5_avg["value*"] + top5_avg["value-all*"]
avg_score /= 8
print('Dev {} average :{:.2f} {} {}'.format(
total_steps - 1, avg_score * 100,
utils.format_dict(top1_avg, '{:.2f}', '1-'),
utils.format_dict(top5_avg, '{:.2f}', '5-')))
#print('Dev loss :', val_loss)
dev_score_list.append(avg_score)
max_score = max(dev_score_list)
if max_score == dev_score_list[-1]:
torch.save(
model.state_dict(), model_dir +
"/{}_td_noun_role1.model".format(model_name))
print('New best model saved! {0}'.format(max_score))
#eval on the trainset
'''top1, top5, val_loss = eval(model, traindev_loader, encoder, gpu_mode)
model.train()
top1_avg = top1.get_average_results()
top5_avg = top5.get_average_results()
avg_score = top1_avg["verb"] + top1_avg["value"] + top1_avg["value-all"] + top5_avg["verb"] + \
top5_avg["value"] + top5_avg["value-all"] + top5_avg["value*"] + top5_avg["value-all*"]
avg_score /= 8
print ('TRAINDEV {} average :{:.2f} {} {}'.format(total_steps-1, avg_score*100,
utils.format_dict(top1_avg,'{:.2f}', '1-'),
utils.format_dict(top5_avg, '{:.2f}', '5-')))'''
print('current train loss', train_loss)
train_loss = 0
top1 = imsitu_scorer(encoder, 1, 3)
top5 = imsitu_scorer(encoder, 5, 3)
del noun_predict, role_predict, loss, img, verb, roles, labels
#break
print('Epoch ', epoch, ' completed!')
scheduler.step()
def train(model,
train_loader,
dev_loader,
traindev_loader,
optimizer,
scheduler,
max_epoch,
model_dir,
encoder,
gpu_mode,
clip_norm,
lr_max,
model_name,
args,
eval_frequency=2000):
model.train()
train_loss = 0
total_steps = 0
print_freq = 400
dev_score_list = []
'''if model.gpu_mode >= 0 :
ngpus = 2
device_array = [i for i in range(0,ngpus)]
pmodel = torch.nn.DataParallel(model, device_ids=device_array)
else:
pmodel = model'''
pmodel = model
top1 = imsitu_scorer(encoder, 1, 3)
top5 = imsitu_scorer(encoder, 5, 3)
'''print('init param data check :')
for f in model.parameters():
if f.requires_grad:
print(f.data.size())'''
for epoch in range(max_epoch):
#print('current sample : ', i, img.size(), verb.size(), roles.size(), labels.size())
#sizes batch_size*3*height*width, batch*504*1, batch*6*190*1, batch*3*6*lebale_count*1
mx = len(train_loader)
for i, (_id, img, verb, verbq) in enumerate(train_loader):
#print("epoch{}-{}/{} batches\r".format(epoch,i+1,mx)) ,
total_steps += 1
if gpu_mode >= 0:
img = torch.autograd.Variable(img.cuda())
verb = torch.autograd.Variable(verb.cuda())
verbq = torch.autograd.Variable(verbq.cuda())
else:
img = torch.autograd.Variable(img)
verb = torch.autograd.Variable(verb)
verbq = torch.autograd.Variable(verbq)
#optimizer.zero_grad()
'''print('all inputs')
print(img)
print('=========================================================================')
print(verb)
print('=========================================================================')
print(roles)
print('=========================================================================')
print(labels)'''
verb_predict = pmodel(img, verbq)
'''g = make_dot(verb_predict, model.state_dict())
g.view()'''
loss = model.calculate_loss(verb_predict, verb)
#print('current loss = ', loss)
loss.backward()
# torch.nn.utils.clip_grad_norm_(model.parameters(), clip_norm)
optimizer.step()
optimizer.zero_grad()
train_loss += loss.item()
top1.add_point_verb_only(verb_predict, verb)
top5.add_point_verb_only(verb_predict, verb)
if total_steps % print_freq == 0:
top1_a = top1.get_average_results()
top5_a = top5.get_average_results()
print("{},{},{}, {} , {}, loss = {:.2f}, avg loss = {:.2f}".
format(total_steps - 1, epoch, i,
utils.format_dict(top1_a, "{:.2f}", "1-"),
utils.format_dict(top5_a, "{:.2f}", "5-"),
loss.item(), train_loss /
((total_steps - 1) % eval_frequency)))
if total_steps % eval_frequency == 0:
top1, top5, val_loss = eval(model, dev_loader, encoder,
gpu_mode)
model.train()
top1_avg = top1.get_average_results()
top5_avg = top5.get_average_results()
#todo : top 5
avg_score = top1_avg["verb"] + top1_avg["value"] + top1_avg["value-all"] + top5_avg["verb"] + \
top5_avg["value"] + top5_avg["value-all"] + top5_avg["value*"] + top5_avg["value-all*"]
#avg_score = top1_avg["value*"]
avg_score /= 2
print('Dev {} average :{:.2f} {} {}'.format(
total_steps - 1, avg_score * 100,
utils.format_dict(top1_avg, '{:.2f}', '1-'),
utils.format_dict(top5_avg, '{:.2f}', '5-')))
#print('Dev loss :', val_loss)
dev_score_list.append(avg_score)
max_score = max(dev_score_list)
if max_score == dev_score_list[-1]:
torch.save(
model.state_dict(), model_dir +
"/{}_agent2verbq_td_predagent_agentobj.model".format(
model_name))
print('New best model saved! {0}'.format(max_score))
#eval on the trainset
'''top1, top5, val_loss = eval(model, traindev_loader, encoder, gpu_mode)
model.train()
top1_avg = top1.get_average_results()
top5_avg = top5.get_average_results()
avg_score = top1_avg["verb"] + top1_avg["value"] + top1_avg["value-all"] + top5_avg["verb"] + \
top5_avg["value"] + top5_avg["value-all"] + top5_avg["value*"] + top5_avg["value-all*"]
avg_score /= 8
print ('TRAINDEV {} average :{:.2f} {} {}'.format(total_steps-1, avg_score*100,
utils.format_dict(top1_avg,'{:.2f}', '1-'),
utils.format_dict(top5_avg, '{:.2f}', '5-')))'''
print('current train loss', train_loss)
train_loss = 0
top1 = imsitu_scorer(encoder, 1, 3)
top5 = imsitu_scorer(encoder, 5, 3)
del loss, img
#break
print('Epoch ', epoch, ' completed!')
scheduler.step()
def train(model,
train_loader,
dev_loader,
traindev_loader,
optimizer,
scheduler,
max_epoch,
model_dir,
encoder,
gpu_mode,
clip_norm,
lr_max,
checkpoint_at,
eval_frequency=4000):
model.train()
train_loss = 0
total_steps = 0
print_freq = 5
dev_score_list = []
checkp_list = []
'''if model.gpu_mode >= 0 :
ngpus = 2
device_array = [i for i in range(0,ngpus)]
pmodel = torch.nn.DataParallel(model, device_ids=device_array)
else:
pmodel = model'''
pmodel = model
'''if scheduler.get_lr()[0] < lr_max:
scheduler.step()'''
top1 = imsitu_scorer(encoder, 1, 3)
top5 = imsitu_scorer(encoder, 5, 3)
'''print('init param data check :')
for f in model.parameters():
if f.requires_grad:
print(f.data.size())'''
for epoch in range(max_epoch):
#print('current sample : ', i, img.size(), verb.size(), roles.size(), labels.size())
#sizes batch_size*3*height*width, batch*504*1, batch*6*190*1, batch*3*6*lebale_count*1
mx = len(train_loader)
for i, (img, verb, roles, labels) in enumerate(train_loader):
#print("epoch{}-{}/{} batches\r".format(epoch,i+1,mx)) ,
total_steps += 1
if gpu_mode >= 0:
img = torch.autograd.Variable(img.cuda())
roles = torch.autograd.Variable(roles.cuda())
verb = torch.autograd.Variable(verb.cuda())
labels = torch.autograd.Variable(labels.cuda())
else:
img = torch.autograd.Variable(img)
verb = torch.autograd.Variable(verb)
roles = torch.autograd.Variable(roles)
labels = torch.autograd.Variable(labels)
#optimizer.zero_grad()
'''print('all inputs')
print(img)
print('=========================================================================')
print(verb)
print('=========================================================================')
print(roles)
print('=========================================================================')
print(labels)'''
verb_predict = pmodel(img)
'''g = make_dot(verb_predict, model.state_dict())
g.view()'''
loss = model.calculate_loss(verb_predict, verb)
#print('current loss = ', loss)
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), clip_norm)
'''for param in filter(lambda p: p.requires_grad,model.parameters()):
print(param.grad.data.sum())'''
#start debugger
#import pdb; pdb.set_trace()
optimizer.step()
optimizer.optimizer.zero_grad()
#optimizer.step()
#optimizer.zero_grad()
'''print('grad check :')
for f in model.parameters():
print('data is')
print(f.data)
print('grad is')
print(f.grad)'''
train_loss += loss.item()
top1.add_point_verb_only(verb_predict, verb)
top5.add_point_verb_only(verb_predict, verb)
if total_steps % print_freq == 0:
top1_a = top1.get_average_results()
top5_a = top5.get_average_results()
print("{},{},{}, {} , {}, loss = {:.2f}, avg loss = {:.2f}".
format(total_steps - 1, epoch, i,
utils.format_dict(top1_a, "{:.2f}", "1-"),
utils.format_dict(top5_a, "{:.2f}", "5-"),
loss.item(), train_loss /
((total_steps - 1) % eval_frequency)))
if total_steps % checkpoint_at == 0:
#assuming this is the best model at each cycle
checkpoint_number = math.ceil(total_steps / checkpoint_at)
torch.save(
model.state_dict(), model_dir +
"/verb_only256_cosanwr_p1_chkp{0}.model".format(
checkpoint_number))
print('New model saved at cycle {0}'.format(checkpoint_number))
checkp_list.append(checkpoint_number)
top1, top5, val_loss = eval(
model, checkp_list, model_dir,
"/verb_only256_cosanwr_p1_chkp{0}.model", dev_loader,
encoder, gpu_mode)
model.train()
top1_avg = top1.get_average_results()
top5_avg = top5.get_average_results()
avg_score = top1_avg["verb"] + top5_avg["verb"]
avg_score /= 2
print('Dev {} average :{:.2f} {} {}'.format(
total_steps - 1, avg_score * 100,
utils.format_dict(top1_avg, '{:.2f}', '1-'),
utils.format_dict(top5_avg, '{:.2f}', '5-')))
print('current train loss', train_loss)
train_loss = 0
top1 = imsitu_scorer(encoder, 1, 3)
top5 = imsitu_scorer(encoder, 5, 3)
del verb_predict, loss, img, verb, roles, labels
#break
print('Epoch ', epoch, ' completed!')
def main():
import argparse
parser = argparse.ArgumentParser(
description="imsitu VSRL. Training, evaluation and prediction.")
parser.add_argument("--gpuid",
default=-1,
help="put GPU id > -1 in GPU mode",
type=int)
#parser.add_argument("--command", choices = ["train", "eval", "resume", 'predict'], required = True)
parser.add_argument('--resume_training',
action='store_true',
help='Resume training from the model [resume_model]')
parser.add_argument('--resume_model',
type=str,
default='',
help='The model we resume')
parser.add_argument('--verb_module',
type=str,
default='',
help='pretrained verb module')
parser.add_argument('--role_module',
type=str,
default='',
help='pretrained role module')
parser.add_argument('--train_role',
action='store_true',
help='cnn fix, verb fix, role train from the scratch')
parser.add_argument(
'--finetune_verb',
action='store_true',
help='cnn fix, verb finetune, role train from the scratch')
parser.add_argument(
'--finetune_cnn',
action='store_true',
help='cnn finetune, verb finetune, role train from the scratch')
parser.add_argument('--output_dir',
type=str,
default='./trained_models',
help='Location to output the model')
parser.add_argument('--evaluate',
action='store_true',
help='Only use the testing mode')
parser.add_argument('--test',
action='store_true',
help='Only use the testing mode')
parser.add_argument('--dataset_folder',
type=str,
default='./imSitu',
help='Location of annotations')
parser.add_argument('--imgset_dir',
type=str,
default='./resized_256',
help='Location of original images')
parser.add_argument('--frcnn_feat_dir',
type=str,
help='Location of output from detectron')
#todo: train role module separately with gt verbs
args = parser.parse_args()
batch_size = 640
#lr = 5e-6
lr = 0.0001
lr_max = 5e-4
lr_gamma = 0.1
lr_step = 25
clip_norm = 50
weight_decay = 1e-4
n_epoch = 500
n_worker = 3
#dataset_folder = 'imSitu'
#imgset_folder = 'resized_256'
dataset_folder = args.dataset_folder
imgset_folder = args.imgset_dir
print('model spec :, verb role with context ')
train_set = json.load(open(dataset_folder + "/train.json"))
imsitu_roleq = json.load(open("imsitu_data/imsitu_questions_prev.json"))
encoder = imsitu_encoder(train_set, imsitu_roleq)
model = model_verbmlp_roletd_new.BaseModel(encoder, args.gpuid)
# To group up the features
#all verb and role feat are under role as it's a single unit
train_set = imsitu_loader_roleq(imgset_folder, train_set, encoder,
model.train_preprocess())
train_loader = torch.utils.data.DataLoader(train_set,
batch_size=64,
shuffle=True,
num_workers=n_worker)
dev_set = json.load(open(dataset_folder + "/dev.json"))
dev_set = imsitu_loader_roleq(imgset_folder, dev_set, encoder,
model.dev_preprocess())
dev_loader = torch.utils.data.DataLoader(dev_set,
batch_size=64,
shuffle=True,
num_workers=n_worker)
test_set = json.load(open(dataset_folder + "/test.json"))
test_set = imsitu_loader_roleq(imgset_folder, test_set, encoder,
model.dev_preprocess())
test_loader = torch.utils.data.DataLoader(test_set,
batch_size=64,
shuffle=True,
num_workers=n_worker)
traindev_set = json.load(open(dataset_folder + "/dev.json"))
traindev_set = imsitu_loader_roleq(imgset_folder, traindev_set, encoder,
model.dev_preprocess())
traindev_loader = torch.utils.data.DataLoader(traindev_set,
batch_size=8,
shuffle=True,
num_workers=n_worker)
utils.set_trainable(model, False)
utils.load_net(args.verb_module, [model.verb])
utils.load_net(args.role_module, [model.roles])
if not os.path.exists(args.output_dir):
os.mkdir(args.output_dir)
torch.manual_seed(1234)
if args.gpuid >= 0:
#print('GPU enabled')
model.cuda()
torch.cuda.manual_seed(1234)
torch.backends.cudnn.deterministic = True
top1, top5, val_loss = eval(model,
dev_loader,
encoder,
args.gpuid,
write_to_file=True)
top1_avg = top1.get_average_results()
avg_score = top1_avg["verb"] + top1_avg["value"] + top1_avg["value-all"]
avg_score /= 3
print('Dev average :{:.2f} {} '.format(
avg_score * 100, utils.format_dict(top1_avg, '{:.2f}', '1-')))
def show_config(args=[]):
print utils.format_dict(configs.sla_configs)
def train(model,
train_loader,
dev_loader,
traindev_loader,
optimizer,
scheduler,
max_epoch,
model_dir,
encoder,
gpu_mode,
clip_norm,
lr_max,
eval_frequency=4000):
model.train()
train_loss = 0
total_steps = 0
print_freq = 400
dev_score_list = []
'''if model.gpu_mode >= 0 :
ngpus = 2
device_array = [i for i in range(0,ngpus)]
pmodel = torch.nn.DataParallel(model, device_ids=device_array)
else:
pmodel = model'''
pmodel = model
top1 = imsitu_scorer(encoder, 1, 3)
top5 = imsitu_scorer(encoder, 5, 3)
'''print('init param data check :')
for f in model.parameters():
if f.requires_grad:
print(f.data.size())'''
for epoch in range(max_epoch):
#print('current sample : ', i, img.size(), verb.size(), roles.size(), labels.size())
#sizes batch_size*3*height*width, batch*504*1, batch*6*190*1, batch*3*6*lebale_count*1
mx = len(train_loader)
for i, (p_img, p_verb, n_img, n_verb) in enumerate(train_loader):
#print("epoch{}-{}/{} batches\r".format(epoch,i+1,mx)) ,
total_steps += 1
if gpu_mode >= 0:
p_img = torch.autograd.Variable(p_img.cuda())
p_verb = torch.autograd.Variable(p_verb.cuda())
n_img = torch.autograd.Variable(n_img.cuda())
n_verb = torch.autograd.Variable(n_verb.cuda())
else:
p_img = torch.autograd.Variable(p_img)
p_verb = torch.autograd.Variable(p_verb)
n_img = torch.autograd.Variable(n_img)
n_verb = torch.autograd.Variable(n_verb)
#optimizer.zero_grad()
'''print('all inputs')
print(img)
print('=========================================================================')
print(verb)
print('=========================================================================')
print(roles)
print('=========================================================================')
print(labels)'''
p_img_rep, p_verb_rep = pmodel(p_img, p_verb)
n_img_rep, n_verb_rep = pmodel(n_img, n_verb)
p_img_pred = pmodel.classifier_forward(p_img)
n_img_pred = pmodel.classifier_forward(n_img)
#print('came here')
'''g = make_dot(verb_predict, model.state_dict())
g.view()'''
predicted_labels = torch.cat([p_img_pred, n_img_pred], 0)
gt_labels = torch.cat([p_verb, n_verb], 0)
p_loss = model.calculate_loss(predicted_labels, gt_labels)
#n_loss = model.calculate_loss(n_img_pred, n_verb)
triplet_loss = model.triplet_loss(p_img_rep, p_verb_rep, n_img_rep,
n_verb_rep)
loss = triplet_loss + p_loss
#loss = p_loss
#loss = triplet_loss
#print('current loss = ', loss)
loss.backward()
#print('done')
# torch.nn.utils.clip_grad_norm_(model.parameters(), clip_norm)
optimizer.step()
optimizer.zero_grad()
train_loss += loss.item()
top1.add_point_verb_only(p_img_pred, p_verb)
top5.add_point_verb_only(p_img_pred, p_verb)
if total_steps % print_freq == 0:
top1_a = top1.get_average_results()
top5_a = top5.get_average_results()
print("{},{},{}, {} , {}, loss = {:.2f}, avg loss = {:.2f}".
format(total_steps - 1, epoch, i,
utils.format_dict(top1_a, "{:.2f}", "1-"),
utils.format_dict(top5_a, "{:.2f}", "5-"),
loss.item(), train_loss /
((total_steps - 1) % eval_frequency)))
#del verb_predict, loss, img, verb, roles, labels
#break
print('Epoch ', epoch, ' completed!')
top1, top5, val_loss = eval(model, dev_loader, encoder, gpu_mode)
model.train()
top1_avg = top1.get_average_results()
top5_avg = top5.get_average_results()
avg_score = top1_avg["verb"] + top5_avg["verb"]
avg_score /= 8
print('Dev {} average :{:.2f} {} {}'.format(
total_steps - 1, avg_score * 100,
utils.format_dict(top1_avg, '{:.2f}', '1-'),
utils.format_dict(top5_avg, '{:.2f}', '5-')))
dev_score_list.append(avg_score)
max_score = max(dev_score_list)
if max_score == dev_score_list[-1]:
torch.save(model.state_dict(),
model_dir + "/verbonly_vgg16_rank.model")
print('New best model saved! {0}'.format(max_score))
print('current train loss', train_loss)
train_loss = 0
top1 = imsitu_scorer(encoder, 1, 3)
top5 = imsitu_scorer(encoder, 5, 3)
scheduler.step()
def main():
import argparse
parser = argparse.ArgumentParser(
description="imsitu VSRL. Training, evaluation and prediction.")
parser.add_argument("--gpuid",
default=-1,
help="put GPU id > -1 in GPU mode",
type=int)
#parser.add_argument("--command", choices = ["train", "eval", "resume", 'predict'], required = True)
parser.add_argument('--resume_training',
action='store_true',
help='Resume training from the model [resume_model]')
parser.add_argument('--resume_model',
type=str,
default='',
help='The model we resume')
parser.add_argument('--verb_module',
type=str,
default='',
help='pretrained verb module')
parser.add_argument('--train_role',
action='store_true',
help='cnn fix, verb fix, role train from the scratch')
parser.add_argument(
'--finetune_verb',
action='store_true',
help='cnn fix, verb finetune, role train from the scratch')
parser.add_argument(
'--finetune_cnn',
action='store_true',
help='cnn finetune, verb finetune, role train from the scratch')
parser.add_argument('--output_dir',
type=str,
default='./trained_models',
help='Location to output the model')
parser.add_argument('--evaluate',
action='store_true',
help='Only use the testing mode')
#todo: train role module separately with gt verbs
args = parser.parse_args()
batch_size = 640
#lr = 5e-6
lr = 0.0001
lr_max = 5e-4
lr_gamma = 0.1
lr_step = 25
clip_norm = 50
weight_decay = 1e-4
n_epoch = 500
n_worker = 3
dataset_folder = 'imSitu'
imgset_folder = 'resized_256'
train_set = json.load(open(dataset_folder + "/train.json"))
encoder = imsitu_encoder(train_set)
model = model_vsrl_small_finetune.RelationNetworks(encoder, args.gpuid)
# To group up the features
cnn_features, verb_features, role_features = utils.group_features(model)
train_set = imsitu_loader(imgset_folder, train_set, encoder,
model.train_preprocess())
train_loader = torch.utils.data.DataLoader(train_set,
batch_size=32,
shuffle=True,
num_workers=n_worker)
dev_set = json.load(open(dataset_folder + "/dev.json"))
dev_set = imsitu_loader(imgset_folder, dev_set, encoder,
model.train_preprocess())
dev_loader = torch.utils.data.DataLoader(dev_set,
batch_size=32,
shuffle=True,
num_workers=n_worker)
traindev_set = json.load(open(dataset_folder + "/dev.json"))
traindev_set = imsitu_loader(imgset_folder, traindev_set, encoder,
model.train_preprocess())
traindev_loader = torch.utils.data.DataLoader(traindev_set,
batch_size=8,
shuffle=True,
num_workers=n_worker)
utils.set_trainable(model, False)
if args.train_role:
print('CNN fix, Verb fix, train role from the scratch from: {}'.format(
args.verb_module))
args.train_all = False
if len(args.verb_module) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.verb_module, [model.conv, model.verb],
['conv', 'verb'])
optimizer_select = 1
model_name = 'cfx_vfx_rtrain'
elif args.finetune_verb:
print('CNN fix, Verb finetune, train role from the scratch from: {}'.
format(args.verb_module))
args.train_all = True
if len(args.verb_module) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.verb_module, [model.conv, model.verb],
['conv', 'verb'])
optimizer_select = 2
model_name = 'cfx_vft_rtrain'
elif args.finetune_cnn:
print(
'CNN finetune, Verb finetune, train role from the scratch from: {}'
.format(args.verb_module))
args.train_all = True
if len(args.verb_module) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.verb_module, [model.conv, model.verb],
['conv', 'verb'])
optimizer_select = 3
model_name = 'cft_vft_rtrain'
elif args.resume_training:
print('Resume training from: {}'.format(args.resume_model))
args.train_all = True
if len(args.resume_model) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.resume_model, [model])
optimizer_select = 0
model_name = 'resume_all'
else:
if not args.evaluate:
print('Training from the scratch.')
optimizer_select = 0
args.train_all = True
model_name = 'train_full'
optimizer = utils.get_optimizer(lr, weight_decay, optimizer_select,
cnn_features, verb_features, role_features)
if not os.path.exists(args.output_dir):
os.mkdir(args.output_dir)
if args.gpuid >= 0:
#print('GPU enabled')
model.cuda()
#optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=lr_step,
gamma=lr_gamma)
#gradient clipping, grad check
if args.evaluate:
top1, top5, val_loss = eval(model,
dev_loader,
encoder,
args.gpuid,
write_to_file=True)
top1_avg = top1.get_average_results()
top5_avg = top5.get_average_results()
avg_score = top1_avg["verb"] + top1_avg["value"] + top1_avg["value-all"] + top5_avg["verb"] + \
top5_avg["value"] + top5_avg["value-all"]
avg_score /= 8
print('Dev average :{:.2f} {} {}'.format(
avg_score * 100, utils.format_dict(top1_avg, '{:.2f}', '1-'),
utils.format_dict(top5_avg, '{:.2f}', '5-')))
#write results to csv file
gt_labels = top1.gt_situation
pred_labels = top1.predicted_situation
verb_pred = top1.verb_pred
with open("gt_rn_only.csv", "w") as f:
writer = csv.writer(f)
writer.writerows(gt_labels)
with open("pred_rn_only.csv", "w") as f:
writer = csv.writer(f)
writer.writerows(pred_labels)
with open("verbpred_rn_only.csv", "w") as f:
writer = csv.writer(f)
writer.writerow(['verb', 'total', 'predicted'])
for key, value in verb_pred.items():
writer.writerow([key, value[0], value[1]])
print('Writing predictions to file completed !')
else:
print('Model training started!')
train(model, train_loader, dev_loader, traindev_loader, optimizer,
scheduler, n_epoch, args.output_dir, encoder, args.gpuid,
clip_norm, lr_max, model_name, args)
def main():
import argparse
parser = argparse.ArgumentParser(
description="VQA using MAC. Training, evaluation and prediction.")
parser.add_argument("--gpuid",
default=-1,
help="put GPU id > -1 in GPU mode",
type=int)
#parser.add_argument("--command", choices = ["train", "eval", "resume", 'predict'], required = True)
parser.add_argument('--resume_training',
action='store_true',
help='Resume training from the model [resume_model]')
parser.add_argument('--resume_model',
type=str,
default='',
help='The model we resume')
parser.add_argument('--verb_module',
type=str,
default='',
help='pretrained verb module')
parser.add_argument('--train_role',
action='store_true',
help='cnn fix, verb fix, role train from the scratch')
parser.add_argument(
'--finetune_verb',
action='store_true',
help='cnn fix, verb finetune, role train from the scratch')
parser.add_argument(
'--finetune_cnn',
action='store_true',
help='cnn finetune, verb finetune, role train from the scratch')
parser.add_argument('--output_dir',
type=str,
default='./trained_models',
help='Location to output the model')
parser.add_argument('--evaluate',
action='store_true',
help='Only use the testing mode')
parser.add_argument('--test',
action='store_true',
help='Only use the testing mode')
parser.add_argument('--imgset_folder',
type=str,
default='',
help='path to image set')
#todo: train role module separately with gt verbs
args = parser.parse_args()
batch_size = 640
#lr = 5e-6
lr = 0.0001
lr_max = 5e-4
lr_gamma = 0.1
lr_step = 25
clip_norm = 50
weight_decay = 1e-4
n_epoch = 500
n_worker = 3
dataset_folder = 'vqa_data'
imgset_folder = args.imgset_folder
print('model spec :, mac net for vqa 2.0')
train_set = json.load(open(dataset_folder + "/vqa_openended_train.json"))
encoder = vqa_encoder(train_set)
model = model_mac_vqa.E2ENetwork(encoder, args.gpuid)
# To group up the features
cnn_features, mac_features = utils.group_features_vqa(model)
train_set = vqa_loader(imgset_folder + '/train2014', train_set, encoder,
model.train_preprocess())
train_loader = torch.utils.data.DataLoader(train_set,
batch_size=64,
shuffle=True,
num_workers=n_worker,
collate_fn=collate_data)
dev_set = json.load(open(dataset_folder + "/vqa_openended_val.json"))
dev_set = vqa_loader(imgset_folder + '/val2014', dev_set, encoder,
model.dev_preprocess())
dev_loader = torch.utils.data.DataLoader(dev_set,
batch_size=64,
shuffle=True,
num_workers=n_worker,
collate_fn=collate_data)
'''test_set = json.load(open(dataset_folder +"/test.json"))
test_set = imsitu_loader_roleq(imgset_folder, test_set, encoder, model.dev_preprocess())
test_loader = torch.utils.data.DataLoader(test_set, batch_size=64, shuffle=True, num_workers=n_worker)
traindev_set = json.load(open(dataset_folder +"/dev.json"))
traindev_set = imsitu_loader_roleq(imgset_folder, traindev_set, encoder, model.dev_preprocess())
traindev_loader = torch.utils.data.DataLoader(traindev_set, batch_size=8, shuffle=True, num_workers=n_worker)'''
utils.set_trainable(model, False)
if args.train_role:
print('CNN fix, Verb fix, train role from the scratch from: {}'.format(
args.verb_module))
args.train_all = False
if len(args.verb_module) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.verb_module, [model.conv, model.verb],
['conv', 'verb'])
optimizer_select = 1
model_name = 'cfx_vfx_rtrain'
elif args.finetune_verb:
print('CNN fix, Verb finetune, train role from the scratch from: {}'.
format(args.verb_module))
args.train_all = True
if len(args.verb_module) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.verb_module, [model.conv, model.verb],
['conv', 'verb'])
optimizer_select = 2
model_name = 'cfx_vft_rtrain'
elif args.finetune_cnn:
print(
'CNN finetune, Verb finetune, train role from the scratch from: {}'
.format(args.verb_module))
args.train_all = True
if len(args.verb_module) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.verb_module, [model.conv, model.verb],
['conv', 'verb'])
optimizer_select = 3
model_name = 'cft_vft_rtrain'
elif args.resume_training:
print('Resume training from: {}'.format(args.resume_model))
args.train_all = True
if len(args.resume_model) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.resume_model, [model])
optimizer_select = 0
model_name = 'resume_all'
else:
print('Training from the scratch.')
optimizer_select = 0
args.train_all = True
model_name = 'train_full'
optimizer = utils.get_optimizer_vqa(optimizer_select, cnn_features,
mac_features)
if not os.path.exists(args.output_dir):
os.mkdir(args.output_dir)
if args.gpuid >= 0:
#print('GPU enabled')
model.cuda()
#optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay)
#scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=lr_step, gamma=lr_gamma)
#gradient clipping, grad check
scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.9)
if args.evaluate:
top1, top5, val_loss = eval(model,
dev_loader,
encoder,
args.gpuid,
write_to_file=True)
top1_avg = top1.get_average_results()
top5_avg = top5.get_average_results()
avg_score = top1_avg["verb"] + top1_avg["value"] + top1_avg["value-all"] + top5_avg["verb"] + \
top5_avg["value"] + top5_avg["value-all"]
avg_score /= 8
print('Dev average :{:.2f} {} {}'.format(
avg_score * 100, utils.format_dict(top1_avg, '{:.2f}', '1-'),
utils.format_dict(top5_avg, '{:.2f}', '5-')))
#write results to csv file
role_dict = top1.role_dict
fail_val_all = top1.value_all_dict
with open('role_pred_data.json', 'w') as fp:
json.dump(role_dict, fp, indent=4)
with open('fail_val_all.json', 'w') as fp:
json.dump(fail_val_all, fp, indent=4)
print('Writing predictions to file completed !')
elif args.test:
top1, top5, val_loss = eval(model,
test_loader,
encoder,
args.gpuid,
write_to_file=True)
top1_avg = top1.get_average_results()
top5_avg = top5.get_average_results()
avg_score = top1_avg["verb"] + top1_avg["value"] + top1_avg["value-all"] + top5_avg["verb"] + \
top5_avg["value"] + top5_avg["value-all"]
avg_score /= 8
print('Test average :{:.2f} {} {}'.format(
avg_score * 100, utils.format_dict(top1_avg, '{:.2f}', '1-'),
utils.format_dict(top5_avg, '{:.2f}', '5-')))
else:
print('Model training started!')
train(model, train_loader, dev_loader, None, optimizer, scheduler,
n_epoch, args.output_dir, encoder, args.gpuid, clip_norm, lr_max,
model_name, args)
