def main():
from vocab import Vocabulary
import evaluation
#evaluation.evalrank("runs/coco_vse++/model_best.pth.tar", data_path="data", split="test")
evaluation.evalrank("runs/coco_vse++_vse/model_best.pth.tar",
data_path="data",
split="test")
def main_test():
global args
args = parser.parse_args()
if args.transfer_test != True:
evaluation.evalrank(model_path=args.model_path, data_path=args.data_path, data_name=args.data_name,
data_name_vocab=args.data_name_vocab, split=args.split, VSE_model=CVSE)
else:
evaluation.evalrank(model_path=args.model_path, data_path=args.data_path, data_name=args.data_name,
data_name_vocab=args.data_name_vocab, split="test", VSE_model=CVSE,
concept_path=args.concept_path, transfer_test=True)
def main():
# Hyper Parameters
parser = argparse.ArgumentParser()
parser.add_argument('--data_path',
default='./data/',
help='path to datasets')
parser.add_argument('--model_path',
default='./data/',
help='path to model')
parser.add_argument('--split', default='test', help='val/test')
parser.add_argument('--gpuid', default=0., type=str, help='gpuid')
parser.add_argument('--fold5', action='store_true', help='fold5')
opts = parser.parse_args()
device_id = opts.gpuid
print("use GPU:", device_id)
os.environ['CUDA_VISIBLE_DEVICES'] = str(device_id)
device_id = 0
torch.cuda.set_device(0)
# load model and options
checkpoint = torch.load(opts.model_path)
opt = checkpoint['opt']
opt.loss_verbose = False
opt.split = opts.split
opt.data_path = opts.data_path
opt.fold5 = opts.fold5
# load vocabulary used by the model
vocab = deserialize_vocab(
os.path.join(opt.vocab_path, '%s_vocab.json' % opt.data_name))
opt.vocab_size = len(vocab)
# construct model
model = SCAN(opt)
model.cuda()
model = nn.DataParallel(model)
# load model state
model.load_state_dict(checkpoint['model'])
print('Loading dataset')
data_loader = data.get_test_loader(opt.split, opt.data_name, vocab,
opt.batch_size, opt.workers, opt)
print(opt)
print('Computing results...')
evaluation.evalrank(model.module,
data_loader,
opt,
split=opt.split,
fold5=opt.fold5)
def test(run=run):
with open('%s_params_%s.pkl' % (saveto, run), 'rb') as f:
model_options = pkl.load(f)
with open('%s.dictionary_%s.pkl' % (saveto, run), 'rb') as f:
worddict = pkl.load(f)
model = LIUMCVC_Encoder(model_options)
model.load_state_dict(torch.load('%s_model_%s.pkl' % (saveto, run)))
model = model.cuda()
best_model = {}
best_model['options'] = model_options
best_model['en_cn_model'] = model
best_model['worddict'] = worddict
evalrank(best_model, data, split='dev')
evalrank(best_model, data, split='test')
def execute(input_string, n=1000):
final_top_n = eval.evalrank(
input_string,
img_feature,
n,
"/scan/SCAN/runs/coco_scan/log/model_best.pth.tar",
data_path="/scan/SCAN/data",
split="test",
fold5=False)
return final_top_n
def execute(input_string, n=1000):
final_top_n = eval.evalrank(input_string,
n,
"runs/coco_scan/log/model_best.pth.tar",
data_path="./data",
split="test",
fold5=False)
ids = dict(result=None)
ids['result'] = final_top_n
jsonFile = json.dumps(ids)
requests.post("http://localhost:5000/getScanResult",
data={'Results': jsonFile})
def main(args):
run = args.run
checkpoint = args.checkpoint
data_path = args.data_path
folders = glob.glob("{}{}/seed*/".format(args.model_path, run))
nr_runs = len(folders)
r1 = 0
r5 = 0
r10 = 0
medr = 0
meanr = 0
ri1 = 0
ri5 = 0
ri10 = 0
medri = 0
meanri = 0
for i in range(nr_runs):
print("Evaluating seed{}".format(str(i+1)))
model_path = "{}{}/seed{}/checkpoint/{}".format(args.model_path, run, i+1, checkpoint )
# plot_path = "{}{}/seed{}checkpoint/".format(args.plot_path, run)
rt, rti, r, ri = evaluation.evalrank(model_path, run, data_path=args.data_path, split="test", vocab_path=args.vocab_path)
r1 += r[0]
r5 += r[1]
r10 += r[2]
medr += r[3]
meanr += r[4]
ri1 += ri[0]
ri5 += ri[1]
ri10 += ri[2]
medri += ri[3]
meanri += ri[4]
r1 = r1 / nr_runs
r5 = r5 / nr_runs
r10 = r10 / nr_runs
medr = medr / nr_runs
meanr = meanr / nr_runs
ri1 = ri1 / nr_runs
ri5 = ri5 / nr_runs
ri10 = ri10 / nr_runs
medri = medri / nr_runs
meanri = meanri / nr_runs
print("AVERAGE Image to text: {:.1f} {:.1f} {:.1f} {:.1f} {:.1f}".format( r1, r5, r10, medr, meanr))
print("AVERAGE Text to image: {:.1f} {:.1f} {:.1f} {:.1f} {:.1f}".format(ri1, ri5, ri10, medri, meanri))
def main(opt, current_config):
model_checkpoint = opt.checkpoint
checkpoint = torch.load(model_checkpoint)
print('Checkpoint loaded from {}'.format(model_checkpoint))
loaded_config = checkpoint['config']
if opt.size == "1k":
fold5 = True
elif opt.size == "5k":
fold5 = False
else:
raise ValueError('Test split size not recognized!')
# Override some mandatory things in the configuration (paths)
loaded_config['dataset']['images-path'] = current_config['dataset'][
'images-path']
loaded_config['dataset']['data'] = current_config['dataset']['data']
loaded_config['image-model'][
'pre-extracted-features-root'] = current_config['image-model'][
'pre-extracted-features-root']
evaluation.evalrank(loaded_config, checkpoint, split="test", fold5=fold5)
def main(opt, current_config):
model_checkpoint = opt.checkpoint
if opt.gpu:
checkpoint = torch.load(
model_checkpoint) # , map_location=torch.device("cpu"))
else:
checkpoint = torch.load(model_checkpoint,
map_location=torch.device("cpu"))
print('Checkpoint loaded from {}'.format(model_checkpoint))
loaded_config = checkpoint['config']
if opt.size == "1k":
fold5 = True
elif opt.size == "5k":
fold5 = False
else:
raise ValueError('Test split size not recognized!')
# Override some mandatory things in the configuration (paths)
if current_config is not None:
loaded_config['dataset']['images-path'] = current_config['dataset'][
'images-path']
loaded_config['dataset']['data'] = current_config['dataset']['data']
loaded_config['image-model'][
'pre-extracted-features-root'] = current_config['image-model'][
'pre-extracted-features-root']
loaded_config['training']['bs'] = current_config['training']['bs']
evaluation.evalrank(loaded_config,
checkpoint,
split="test",
fold5=False,
eval_t2i=opt.t2i,
eval_i2t=opt.i2t)
from vocab import Vocabulary
import evaluation
import numpy
DATA_PATH = '/hdd2/niluthpol/VTT/vsepp_data/'
RUN_PATH = '/home/niluthpol/VTT/models/'
shared_space = 'both' # help='both'|'object_text'|'activity_text' ; default = 'both'
evaluation.evalrank(RUN_PATH + "msrvtt_object_text/model_best.pth.tar",
RUN_PATH + "msrvtt_activity_text/model_best.pth.tar",
data_path=DATA_PATH,
split="test",
shared_space="both")
def main():
# Hyper Parameters
opt = opts.parse_opt()
device_id = opt.gpuid
device_count = len(str(device_id).split(","))
#assert device_count == 1 or device_count == 2
print("use GPU:", device_id, "GPUs_count", device_count, flush=True)
os.environ['CUDA_VISIBLE_DEVICES']=str(device_id)
device_id = 0
torch.cuda.set_device(0)
# Load Vocabulary Wrapper
vocab = deserialize_vocab(os.path.join(opt.vocab_path, '%s_vocab.json' % opt.data_name))
opt.vocab_size = len(vocab)
# Load data loaders
train_loader, val_loader = data.get_loaders(
opt.data_name, vocab, opt.batch_size, opt.workers, opt)
# Construct the model
model = SCAN(opt)
model.cuda()
model = nn.DataParallel(model)
# Loss and Optimizer
criterion = ContrastiveLoss(opt=opt, margin=opt.margin, max_violation=opt.max_violation)
mse_criterion = nn.MSELoss(reduction="batchmean")
optimizer = torch.optim.Adam(model.parameters(), lr=opt.learning_rate)
# optionally resume from a checkpoint
if not os.path.exists(opt.model_name):
os.makedirs(opt.model_name)
start_epoch = 0
best_rsum = 0
if opt.resume:
if os.path.isfile(opt.resume):
print("=> loading checkpoint '{}'".format(opt.resume))
checkpoint = torch.load(opt.resume)
start_epoch = checkpoint['epoch']
best_rsum = checkpoint['best_rsum']
model.load_state_dict(checkpoint['model'])
print("=> loaded checkpoint '{}' (epoch {}, best_rsum {})"
.format(opt.resume, start_epoch, best_rsum))
else:
print("=> no checkpoint found at '{}'".format(opt.resume))
evalrank(model.module, val_loader, opt)
print(opt, flush=True)
# Train the Model
for epoch in range(start_epoch, opt.num_epochs):
message = "epoch: %d, model name: %s\n" % (epoch, opt.model_name)
log_file = os.path.join(opt.logger_name, "performance.log")
logging_func(log_file, message)
print("model name: ", opt.model_name, flush=True)
adjust_learning_rate(opt, optimizer, epoch)
run_time = 0
for i, (images, captions, lengths, masks, ids, _) in enumerate(train_loader):
start_time = time.time()
model.train()
optimizer.zero_grad()
if device_count != 1:
images = images.repeat(device_count,1,1)
score = model(images, captions, lengths, masks, ids)
loss = criterion(score)
loss.backward()
if opt.grad_clip > 0:
clip_grad_norm_(model.parameters(), opt.grad_clip)
optimizer.step()
run_time += time.time() - start_time
# validate at every val_step
if i % 100 == 0:
log = "epoch: %d; batch: %d/%d; loss: %.4f; time: %.4f" % (epoch,
i, len(train_loader), loss.data.item(), run_time / 100)
print(log, flush=True)
run_time = 0
if (i + 1) % opt.val_step == 0:
evalrank(model.module, val_loader, opt)
print("-------- performance at epoch: %d --------" % (epoch))
# evaluate on validation set
rsum = evalrank(model.module, val_loader, opt)
#rsum = -100
filename = 'model_' + str(epoch) + '.pth.tar'
# remember best R@ sum and save checkpoint
is_best = rsum > best_rsum
best_rsum = max(rsum, best_rsum)
save_checkpoint({
'epoch': epoch + 1,
'model': model.state_dict(),
'best_rsum': best_rsum,
'opt': opt,
}, is_best, filename=filename, prefix=opt.model_name + '/')
from vocab import Vocabulary
import evaluation
evaluation.evalrank(
"data/runs/coco_vse++_resnet_restval_finetune/model_best.pth.tar",
data_path="data/data",
split="test")
from vocab import Vocabulary
import evaluation
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--model_path', default='$RUN_PATH/coco_vse/model_best.pth.tar', help='path to model')
parser.add_argument('--data_path', default='data/data', help='path to datasets')
parser.add_argument('--fold5', action='store_true',
help='Use fold5')
parser.add_argument('--save_embeddings', action='store_true',
help='save_embeddings')
parser.add_argument('--save_csv', default='')
opt_eval = parser.parse_args()
evaluation.evalrank(opt_eval, split='test')
import tools, evaluation, os
# Hey Kipster! For this to work, use a python virtualenv
# and pip install -r requirements.txt in IF-root
# you might also need to install numpy or gfortran with your os pkg manager
# First lets make sure the model kinda works
__dirname = os.path.dirname(os.path.realpath(__file__))
model = tools.load_model(__dirname + '/data/coco.npz')
evaluation.evalrank(model, data='coco', split='test')
# Now lets compute sentence vecs for something specific
example_sentences = [
'black tie women', 'warm winter coat',
'long dressi gown tuxedo cocktail black_ti'
]
sentence_vectors = tools.encode_sentences(model,
example_sentences,
verbose=True)
print sentence_vectors.shape
print sentence_vectors[0].shape
from vocab import Vocabulary
import evaluation
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
evaluation.evalrank(
"/home/wangzheng/neurltalk/new/SCAN_2loss500_3_0.8_10x/runs/f30k_scan/log/model_best.pth.tar",
data_path="/data6/wangzheng/data",
split="test")
from vocab import Vocabulary
import evaluation
evaluation.evalrank("models/model_best.pth.tar",
data_path="../data/",
split="test")
from vocab import Vocabulary import evaluation import os os.environ["CUDA_VISIBLE_DEVICES"] = "0" RUN_PATH = "models1/model_best.pth.tar" DATA_PATH = "/media/ubuntu/data/chunxiao/" evaluation.evalrank(RUN_PATH, data_path=DATA_PATH, split="test")
from vocab import Vocabulary
import evaluation
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--model', '-m',)
parser.add_argument('--data_path', '-d', default=None)
parser.add_argument('--data_name', default=None)
parser.add_argument('--split', '-s', default='test')
parser.add_argument('--no_cv', action='store_false')
parser.add_argument('--batch_size', type=int, default=64)
args = parser.parse_args()
evaluation.evalrank(
model_path=args.model,
data_path=args.data_path,
data_name=args.data_name,
split=args.split,
fold5=args.no_cv,
batch_size=args.batch_size,
)
# ------------------------------------------------------------------------------------- # A Bidirectional Focal Atention Network implementation based on # https://arxiv.org/abs/1909.11416. # "Focus Your Atention: A Bidirectional Focal Atention Network for Image-Text Matching" # Chunxiao Liu, Zhendong Mao, An-An Liu, Tianzhu Zhang, Bin Wang, Yongdong Zhang # # Writen by Chunxiao Liu, 2019 # ------------------------------------------------------------------------------------- """testall on MSCOCO""" from vocab import Vocabulary import evaluation import os os.environ["CUDA_VISIBLE_DEVICES"] = "0" RUN_PATH = "/userhome/BFAN/models/model_best.pth.tar" DATA_PATH = "/userhome/" evaluation.evalrank(RUN_PATH, data_path=DATA_PATH, split="testall",fold5=True)
from vocab import Vocabulary
import evaluation
import pickle
evaluation.evalrank("",
data_path="./data", split="test",
fold5=True)
"""print (rt,rti)
print(len(rt),len(rti))
dic_now = {}
dic_now["rt_ranks"]=rt[0]
dic_now["rt_top1"]=rt[1]
dic_now["rti_ranks"] =rti[0]
dic_now["rti_top1"]=rti[1]
with open('vsepp' + '.results.pickle', 'wb') as handle:
pickle.dump(dic_now, handle, protocol=pickle.HIGHEST_PROTOCOL)"""
# import evaluation_vsepp
# evaluation_vsepp.evalrank("runs/save_ori_vse/coco_vse++_combine/model_best.pth.tar", data_path='data/', split="test", fold5=False)
# evaluation.evalrank("runs/coco_vse++_combine_P_S/model_best.pth.tar", data_path='data/', split="test", fold5=False)
# for i in range(1,17):
# print('model:' + str(i) + '\n')
# evaluation.evalrank("runs/save_fc_attn/coco_combine_double_GCN_attn_" + str(i) +"/model_best.pth.tar", data_path='../vsepp-master/data_SCAN/', split="test", fold5=False)
# print('\n')
# # print('\n')
# # evaluation.evalrank("runs/coco_combine_double_GCN_attn/checkpoint.pth.tar", data_path='../vsepp-master/data_SCAN/', split="test", fold5=False)
# for i in range(1,10):
# print('model:' + str(i) + '\n')
# evaluation.evalrank("runs/camera/GCN_Attn_" + str(i) +"/model_best.pth.tar", data_path='../vsepp-master/data_SCAN/', split="test", fold5=False)
# print('\n')
# # print('\n')
# # evaluation.evalrank("runs/coco_combine_double_GCN_attn/checkpoint.pth.tar", data_path='../vsepp-master/data_SCAN/', split="test", fold5=False)
for i in range(1,5):
print('model:' + str(i) + '\n')
evaluation.evalrank("runs/camera/Only_Visual_GCN_Attn_" + str(i) +"/model_best.pth.tar", data_path='../vsepp-master/data_SCAN/', split="test", fold5=False)
print('\n')
# print('\n')
# evaluation.evalrank("runs/coco_combine_double_GCN_attn/checkpoint.pth.tar", data_path='../vsepp-master/data_SCAN/', split="test", fold5=False)
import torch
from vocab import Vocabulary
import evaluation_models
import evaluation
# for flickr
print('\nEvaluation on Flickr30K:')
evaluation.evalrank(
"/SSD/VSRN_CTC/runs/full_model3_newfeats/model_best.pth.tar",
data_path='/SSD/Datasets/Flickr30K/data/',
split="test",
fold5=False)
# for TextCaps Validation set
print('\nEvaluation on TextCaps Validation set:')
evaluation.evalrank("/SSD/VSRN/runs/full_model3_newfeats/model_best.pth.tar",
data_path='/SSD/Datasets/TextCaps/Flickr_Format/',
split="dev",
fold5=False)
# for NewSplit STACMR set
print('\nEvaluation on STACMR:')
evaluation.evalrank(
"/SSD/VSRN_CTC/runs/full_model3_newfeats/model_best.pth.tar",
data_path=
'/SSD/Datasets/Coco-Text/ST_CMR_testdataset/New_Split/Flickr_Format/',
split="dev",
fold5=False)
evaluation.evalrank(
"/SSD/VSRN_CTC/runs/full_model3_newfeats/model_best.pth.tar",
data_path=
