parser.add_argument('--seed', type=int, default=1204, help='random seed')
args = parser.parse_args()
return args
if __name__ == '__main__':
args = parse_args()
utils.create_dir(args.output)
logger = utils.Logger(os.path.join(args.output, 'log.txt'))
logger.write(args.__repr__())
torch.backends.cudnn.benchmark = True
dictionary = Dictionary.load_from_file('data/dictionary.pkl')
train_dset = VQAFeatureDataset('train', dictionary, adaptive=True)
val_dset = VQAFeatureDataset('val', dictionary, adaptive=True)
batch_size = args.batch_size
constructor = 'build_%s' % args.model
model = getattr(base_model, constructor)(train_dset, args.num_hid, args.op,
args.gamma).cuda()
tfidf = None
weights = None
if args.tfidf:
dict = Dictionary.load_from_file('data/dictionary.pkl')
tfidf, weights = tfidf_from_questions(['train', 'val', 'test2015'],
dict)
return args
if __name__ == '__main__':
args = parse_args()
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
torch.backends.cudnn.benchmark = True
if args.task == 'vqa':
from train import train
dict_path = 'data/dictionary.pkl'
dictionary = Dictionary.load_from_file(dict_path)
train_dset = VQAFeatureDataset('train',
dictionary,
adaptive=args.unadaptive,
use_counter=args.use_counter)
val_dset = VQAFeatureDataset('val',
dictionary,
adaptive=args.unadaptive,
use_counter=args.use_counter)
w_emb_path = 'data/glove6b_init_300d.npy'
elif args.task == 'flickr':
from train_flickr import train
dict_path = 'data/flickr30k/dictionary.pkl'
dictionary = Dictionary.load_from_file(dict_path)
train_dset = Flickr30kFeatureDataset('train', dictionary)
val_dset = Flickr30kFeatureDataset('val', dictionary)
w_emb_path = 'data/flickr30k/glove6b_init_300d.npy'
args.op = ''
def main():
args = parse_args()
dataset = args.dataset
args.output = os.path.join('logs', args.output)
if not os.path.isdir(args.output):
utils.create_dir(args.output)
else:
if click.confirm('Exp directory already exists in {}. Erase?'.format(
args.output, default=False)):
os.system('rm -r ' + args.output)
utils.create_dir(args.output)
else:
os._exit(1)
if dataset == 'cpv1':
dictionary = Dictionary.load_from_file('data/dictionary_v1.pkl')
elif dataset == 'cpv2' or dataset == 'v2':
dictionary = Dictionary.load_from_file('data/dictionary.pkl')
print("Building train dataset...")
train_dset = VQAFeatureDataset('train',
dictionary,
dataset=dataset,
cache_image_features=args.cache_features)
print("Building test dataset...")
eval_dset = VQAFeatureDataset('val',
dictionary,
dataset=dataset,
cache_image_features=args.cache_features)
get_bias(train_dset, eval_dset)
# Build the model using the original constructor 使用原始构造函数构建模型
constructor = 'build_%s' % args.model
model = getattr(base_model, constructor)(train_dset, args.num_hid).cuda()
if dataset == 'cpv1':
model.w_emb.init_embedding('data/glove6b_init_300d_v1.npy')
elif dataset == 'cpv2' or dataset == 'v2':
model.w_emb.init_embedding('data/glove6b_init_300d.npy')
# Add the loss_fn based our arguments 根据我们的参数添加loss_fn
if args.debias == "bias_product":
model.debias_loss_fn = BiasProduct()
elif args.debias == "none":
model.debias_loss_fn = Plain()
elif args.debias == "reweight":
model.debias_loss_fn = ReweightByInvBias()
elif args.debias == "learned_mixin":
model.debias_loss_fn = LearnedMixin(args.entropy_penalty)
elif args.debias == 'focal':
model.debias_loss_fn = Focal()
else:
raise RuntimeError(args.mode)
with open('util/qid2type_%s.json' % args.dataset, 'r') as f:
qid2type = json.load(f)
model = model.cuda()
batch_size = args.batch_size
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
torch.backends.cudnn.benchmark = True
train_loader = DataLoader(train_dset,
batch_size,
shuffle=True,
num_workers=0)
eval_loader = DataLoader(eval_dset,
batch_size,
shuffle=False,
num_workers=0)
print("Starting training...")
train(model, train_loader, eval_loader, args, qid2type)
results = []
for i in range(logits.size(0)):
result = {}
result['question_id'] = qIds[i].item()
result['answer'] = get_answer(logits[i], dataloader)
results.append(result)
return results
if __name__ == '__main__':
args = parse_args()
torch.backends.cudnn.benchmark = True
dictionary = Dictionary.load_from_file('data/dictionary.pkl')
eval_dset = VQAFeatureDataset(args.split, dictionary, adaptive=True)
n_device = torch.cuda.device_count()
batch_size = args.batch_size * n_device
constructor = 'build_%s' % args.model
model = getattr(base_model, constructor)(eval_dset, args.num_hid, args.op,
args.gamma).cuda()
eval_loader = DataLoader(eval_dset,
batch_size,
shuffle=False,
num_workers=1,
collate_fn=utils.trim_collate)
def process(args, model, eval_loader):
model_path = args.input+'/model%s.pth' % \
print(args)
if args.task == 'train':
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
torch.backends.cudnn.benchmark = True
start = time.time()
batch_size = args.batch_size
train_batch = batch_size
test_batch = batch_size
dictionary = Dictionary.load_from_file('data/dictionary.pkl')
if args.train_dataset == 'all':
train_dset = VQAFeatureDataset('train', dictionary, filter_pair=False)
elif args.train_dataset == 'filter':
train_dset = VQAFeatureDataset('train', dictionary, filter_pair=True)
elif args.train_dataset == 'pairwise':
train_batch = batch_size / 2
train_dset = VQAFeatureDatasetWithPair('train', dictionary)
elif args.train_dataset == 'end2end':
train_dset = VQAFeatureDatasetEnd2End('train', dictionary, filter_pair=False)
elif args.train_dataset == 'all_pair':
train_dset_all = VQAFeatureDataset('train', dictionary, filter_pair=False)
train_dset_pair = VQAFeatureDatasetWithPair('train', dictionary, preloaded=train_dset_all.pre_loaded())
train_dset = train_dset_all # for model building: dset.vdim and co.
elif args.train_dataset == 'allpair':
train_batch = batch_size / 2
train_dset = VQAFeatureDatasetAllPair('train', dictionary)
elif args.train_dataset == 'trainval':
parser.add_argument('--num_hid', type=int, default=1024)
parser.add_argument('--model', type=str, default='baseline0_newatt')
parser.add_argument('--output', type=str, default='saved_models/exp0')
parser.add_argument('--batch_size', type=int, default=512)
parser.add_argument('--seed', type=int, default=1111, help='random seed')
args = parser.parse_args()
return args
if __name__ == '__main__':
args = parse_args()
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
torch.backends.cudnn.benchmark = True
dictionary = Dictionary.load_from_file('../../bottom-up-attention-vqa-tf/data/dictionary.pkl')
train_dset = VQAFeatureDataset('train', dictionary)
eval_dset = VQAFeatureDataset('val', dictionary)
batch_size = args.batch_size
constructor = 'build_%s' % args.model
model = getattr(base_model, constructor)(train_dset, args.num_hid).cuda()
model.w_emb.init_embedding('../../bottom-up-attention-vqa-tf/data/glove6b_init_300d.npy')
model = nn.DataParallel(model).cuda()
train_loader = DataLoader(train_dset, batch_size, shuffle=True, num_workers=1)
eval_loader = DataLoader(eval_dset, batch_size, shuffle=True, num_workers=1)
train(model, train_loader, eval_loader, args.epochs, args.output)
def imageAdv(args, imageAdvF):
# Fetch data.
questionIds = cPickle.load(open("data/goodQuestions.pkl", "rb"))
targetDict = cPickle.load(open("data/q2t.pkl", "rb"))
dictionary = Dictionary.load_from_file('data/dictionary.pkl')
print "Fetching eval data"
imageLoader = imageModel.ImageLoader("data/val2014img", "val")
dataset = VQAFeatureDataset('valSample',
args.evalset_name,
dictionary,
imageLoader=imageLoader,
questionIds=questionIds)
# Fetch model.
model = imageModel.getCombinedModel(args, dataset)
model = nn.DataParallel(model).cuda()
# Train and save.
label2ans = dataset.label2ans
# imageSaverOld = imageModel.ImageSaver("data/adv1Old")
imageSaverNew = imageModel.ImageSaver("data/adv6New", True)
numSuccess = 0
successList = []
iterList = []
targetList = []
predictedList = []
for i, vqaInfo in enumerate(dataset):
entry = dataset.entries[i]
if entry["question_id"] not in targetDict:
continue
targets = targetDict[entry["question_id"]]
target = None
while target is None or target in entry["answer"]["labels"]:
target = targets[np.random.randint(len(targets))]
print ""
print "questionId: {}, imgId: {}".format(entry["question_id"],
entry["image_id"])
print "Answers: {}".format(
[label2ans[label] for label in entry["answer"]["labels"]])
print "Target: {}".format(label2ans[target])
startTime = time.time()
success, imgOld, imgNew, predicted, iters = imageAdvF(
model, vqaInfo, entry, target, dataset)
print "Num: {0}, Success: {1}, Predicted: {2}, Iters: {3}, Taken: {4:.2f}".format(
i, success, label2ans[predicted], iters,
time.time() - startTime)
if success:
numSuccess += 1
successList.append(True)
else:
successList.append(False)
iterList.append(iters)
targetList.append(target)
predictedList.append(int(predicted.numpy()))
qidStr = str(entry["question_id"])
# imageSaverOld.saveImage(imgOld, prefix + imgIdStr + ".jpg")
imageSaverNew.saveImage(imgNew, qidStr + ".png")
print ""
print "Success: {}".format(numSuccess)
print "SuccessList: {}".format(successList)
print "IterList: {}".format(iterList)
print "TargetList: {}".format(targetList)
print "PredictedList: {}".format(predictedList)
np.savez("data/adv6Out.npz",
successList=successList,
iterList=iterList,
targetList=targetList,
predictedList=predictedList)
score += batch_score
upper_bound += (a.max(1)[0]).sum()
num_data += pred.size(0)
score = score / len(dataloader.dataset)
V_loss /= len(dataloader.dataset)
upper_bound = upper_bound / len(dataloader.dataset)
return score, upper_bound, V_loss
args = parse_args()
dictionary = Dictionary.load_from_file('./data/dictionary.pkl')
# test_dset = VQAFeatureDataset('test', dictionary)
eval_dset = VQAFeatureDataset('val', dictionary)
model = build_model_A3x2(eval_dset, num_hid=args.num_hid, dropout= args.dropout, norm=args.norm,\
activation=args.activation, dropL=args.dropout_L, dropG=args.dropout_G,\
dropW=args.dropout_W, dropC=args.dropout_C)
model = model.cuda()
model.w_emb.init_embedding('data/glove6b_init_300d.npy')
model = nn.DataParallel(model).cuda()
# model.apply(weights_init_ku)
ckpt = torch.load(MODEL_PATH)
model.load_state_dict(ckpt)
eval_loader = DataLoader(eval_dset, BATCH_SIZE, shuffle=True, num_workers=4)
# test_loader = DataLoader(test_dset, BATCH_SIZE, shuffle=True, num_workers=4)
parser.add_argument('--lambda', type=float, default=0.01)
args = parser.parse_args()
return args
if __name__ == '__main__':
args = parse_args()
params = vars(args)
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
torch.backends.cudnn.benchmark = True
# dictionary = Dictionary.load_from_file('./dictionary.pkl')
dictionary = load_dictionary(args.sentense_file_path, args.task)
if not args.test_phase:
train_dset = VQAFeatureDataset(args.task, dictionary, args.sentense_file_path,args.feat_category,args.feat_path, mode='Train')
eval_dset = VQAFeatureDataset(args.task, dictionary, args.sentense_file_path,args.feat_category,args.feat_path, mode='Test')
batch_size = args.batch_size
model_name = args.task+'_model'
model = getattr(locals()[model_name], 'build_%s' % args.model)(args.task, train_dset, params).cuda()
# model.w_emb.init_embedding(dictionary,args.glove_file_path,args.task)
print('========start train========')
model = model.cuda()
train_loader = DataLoader(train_dset, batch_size, shuffle=True, num_workers=1)
eval_loader = DataLoader(eval_dset, batch_size, shuffle=True, num_workers=1)
train(model, train_loader, eval_loader, params)
else:
test_dset = VQAFeatureDataset(args.task, dictionary, args.sentense_file_path,args.feat_category,args.feat_path, mode='Valid')
osf_object,
args.relation_type,
adaptive=args.adaptive,
pos_emb_dim=args.imp_pos_emb_dim,
dataroot=args.data_folder)
train_dset = GQAFeatureDataset('train_balanced',
dictionary,
osf_object,
args.relation_type,
adaptive=args.adaptive,
pos_emb_dim=args.imp_pos_emb_dim,
dataroot=args.data_folder)
else:
val_dset = VQAFeatureDataset('val',
dictionary,
args.relation_type,
adaptive=args.adaptive,
pos_emb_dim=args.imp_pos_emb_dim,
dataroot=args.data_folder)
train_dset = VQAFeatureDataset('train',
dictionary,
args.relation_type,
adaptive=args.adaptive,
pos_emb_dim=args.imp_pos_emb_dim,
dataroot=args.data_folder)
model = build_regat(val_dset, args).to(device)
tfidf = None
weights = None
if args.dataset == "gqa" and args.tfidf:
tfidf, weights = tfidf_from_questions_gqa(
if __name__ == '__main__':
args = parse_args()
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
# torch.backends.cudnn.benchmark = True
device = torch.device('cpu')
if torch.cuda.is_available():
device = torch.device('cuda:' + args.gpu)
dictionary = Dictionary.load_from_file('data/dictionary.pkl')
train_dset = VQAFeatureDataset('train', dictionary, device)
finetune_dset = VQAFeatureDataset('finetune', dictionary, device)
dev_dset = VQAFeatureDataset('dev', dictionary, device)
eval_dset = VQAFeatureDataset('test', dictionary, device)
batch_size = args.batch_size
constructor = 'build_%s' % args.model
model = getattr(base_model, constructor)(train_dset, args.num_hid)
model.w_emb.init_embedding('data/glove6b_init_300d.npy')
model = model.to(device)
if args.model_ckpt:
model.load_state_dict(torch.load(args.model_ckpt))
print('Loaded checkpoint')
def main(args):
softmax = torch.nn.Softmax(dim=-1)
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
dictionary = Dictionary.load_from_file(args.dictionary_path)
ques_net = Question_Classifier(args.bert_mode,
args.bert_pretrain,
num_classes=args.ques_num_classes)
img_net = Network(backbone_type=args.backbone_type,
num_classes=args.img_num_classes)
cls_net = models.resnet34(pretrained=False, num_classes=2)
cls_net = cls_net.cuda()
ques_net = ques_net.cuda()
img_net = img_net.cuda()
cls_net.load_state_dict(torch.load(args.cls2_model_path))
ques_net.load_state_dict(
torch.load(args.ques_model_path,
map_location=lambda storage, loc: storage))
img_net.load_model(args.img_model_path)
fd = json.load(open(args.feature_dict_path, 'r'))
eval_dset = VQAFeatureDataset(args.split,
dictionary,
args.data_root,
question_len=12,
clip=True)
eval_loader = DataLoader(eval_dset,
args.batch_size,
shuffle=False,
num_workers=2)
label2ans = eval_dset.label2ans # ans2label = {'no': 0, 'yes': 1, diseases...}
cls_net.eval()
ques_net.eval()
img_net.eval()
score = 0
closed_ques_num = 28
closed_score = 0
cnt = 0
out_dir = os.path.join(args.out_dir, args.split, args.model_name)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
pred_file = open(out_dir + '/prediction.txt', 'w')
ques_pred_file = open(out_dir + '/question_type_prediction.txt', 'w')
all_list = [0 for i in range(332)]
true_list = [0 for i in range(332)]
csc = 0
csctotal = 0
with torch.no_grad():
for v, q, a, ans_type, q_types, image_name in tqdm(iter(eval_loader)):
v, q, a = v.cuda(), q.cuda(), a.cuda()
v = v.reshape(v.shape[0], 3, 224, 224)
q_prob = ques_net(q) # 1 x ques_num_classes
q_prob = q_prob[
0] # [0: closed-ended-normal, 1: closed-ended-abnormal 2: open-ended]
q_type = torch.argmax(q_prob)
v_prob, feature = img_net(v) # 1 x img_num_classes
if q_type == 0:
normal_prob2 = softmax(cls_net(v)[0])[0].item()
abnormal_prob = 1 - normal_prob2
pred = 0 if abnormal_prob > normal_prob2 else 1
elif q_type == 1:
normal_prob2 = softmax(cls_net(v)[0])[0].item()
abnormal_prob = 1 - normal_prob2
pred = 1 if abnormal_prob > normal_prob2 else 0
else:
disease_prob = softmax(v_prob)
prob1, pred_idx = torch.topk(v_prob, 5, dim=-1)
prob1 = softmax(prob1[0])
if prob1[0] > 0.6:
csctotal += 1
pred = torch.argmax(disease_prob) + 2
else:
pred_idx = pred_idx.cpu().numpy().tolist()[0]
p_idx = []
for i in pred_idx:
p_list = []
for fdict in fd[str(i + 2)]:
fdict = np.asarray(fdict)
fdict = torch.from_numpy(fdict)
cs = torch.cosine_similarity(feature.cpu(),
fdict,
dim=1)
p_list.append(cs.item())
p_idx.append(max(p_list))
pred = pred_idx[p_idx.index(max(p_idx))] + 2
if args.split != 'test':
gt = torch.argmax(a[0])
all_list[gt.item()] += 1
if pred == gt:
if pred > 1:
csc += 1
true_list[gt.item()] += 1
score += 1
if cnt < closed_ques_num:
closed_score += 1
img_name = image_name[0]
pred_ans = label2ans[pred]
pred_file.write(img_name[:-4] + '|' + pred_ans + '\n')
ques_pred_file.write(img_name[:-4] + '|' +
str(q_type.cpu().numpy()) + '\n')
cnt += 1
if args.split != 'test':
open_score = score - closed_score
score = (score * 100.0) / cnt
open_score = (open_score * 100.0) / (cnt - closed_ques_num)
closed_score = (closed_score * 100.0) / closed_ques_num
file = open(out_dir + '/score.txt', 'w')
file.write('score: %.4f\n' % (score))
file.write('closed score: %.4f\n' % (closed_score))
file.write('open score: %.4f\n' % (open_score))
print(csc / csctotal)
print('score: %.4f' % (score))
