def eval(args):
args = handle_args(args)
if os.path.isdir(os.getcwd() + '/results/images/' + args.run_name) is False:
os.mkdir(os.getcwd() + '/results/images/' + args.run_name)
if os.path.isdir(os.getcwd() + '/results/history/' + args.run_name) is False:
os.mkdir(os.getcwd() + '/results/history/' + args.run_name)
if os.path.isdir(os.getcwd() + '/results/files/' + args.run_name) is False:
os.mkdir(os.getcwd() + '/results/files/' + args.run_name)
datapath = args.datadir
dataset, data_loader = utils.get_dataset(datapath, args.img_size, \
args.batch_size)
classes, class_to_idx, idx_to_class = utils.get_classes(dataset)
word_dim = 300
label_dim = len(classes)
# todo! change the number of classes to intersect with quickdraws classes!
#label_criterion = nn.L1Loss()
label_criterion = nn.CrossEntropyLoss()
#label_criterion = nn.MultiLabelMarginLoss()
reconstr_criterion = nn.MSELoss()
model = DistributedWordLabeller(width=args.img_size, height=args.img_size, \
word_dim=word_dim, label_dim=label_dim)
optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate,
weight_decay=1e-5)
print('\nNum classes: %r, num images: %r' % (len(classes), len(dataset)))
#word_vecs = {i: [0, 0, 0] for i in range(300)}
word_vecs = utils.get_word_vectors('/data/nlp/glove/glove_300d.json', classes, word_dim)
matrix = np.zeros((len(classes), len(classes)))
if args.eval == 'True':
datapath = args.test_datadir
dataset, data_loader = utils.get_dataset(datapath, args.img_size, \
args.batch_size)
checkpoint = torch.load(os.getcwd() + '/model_states/' + args.run_name)
model.load_state_dict(checkpoint['state_dict'])
with torch.no_grad():
for batch_idx, (img, target_tensor) in enumerate(data_loader):
target_idxs = target_tensor.data.numpy().tolist()
target_names = [idx_to_class[idx] for idx in target_idxs]
print(target_names)
target_labels = torch.tensor([[1 if i == idx else 0 for i in \
range(len(classes))] for idx in target_idxs], \
dtype=torch.float32)
reconstr, word_dist, label_dist = model.forward(img)
labels = model.pred_labels(label_dist)
matrix = update_label_matrix(matrix, labels, target_idxs)
avg_acc, metric_dict = matrix_to_metrics(train_matrix, idx_to_class)
print(avg_acc); print(metric_dict)
return
f = open(os.getcwd() + '/results/files/' + args.run_name + \
'/matrix.json', 'w+')
temp = {'matrix': matrix.tolist(), 'metrics': metric_dict, 'avg_acc': avg_acc}
f.write(json.dumps(temp))
def model_1(args):
if os.path.isdir(os.getcwd() + '/results/images/' +
args.run_name) is False:
os.mkdir(os.getcwd() + '/results/images/' + args.run_name)
if os.path.isdir(os.getcwd() + '/results/history/' +
args.run_name) is False:
os.mkdir(os.getcwd() + '/results/history/' + args.run_name)
if os.path.isdir(os.getcwd() + '/results/files/' + args.run_name) is False:
os.mkdir(os.getcwd() + '/results/files/' + args.run_name)
datapath = args.datadir
#args.batch_size = 2
args.img_size = 224
dataset, data_loader = utils.get_dataset(datapath, args.img_size, \
args.batch_size)
classes, class_to_idx, idx_to_class = utils.get_classes(dataset)
word_dim = 300
label_criterion = nn.CrossEntropyLoss()
reconstr_criterion = nn.L1Loss()
#reconstr_criterion = nn.MSELoss()
model = BimodalDAEImage(300, 2048, n_classes=len(classes))
cnn = resnet101(pretrained=True)
optimizer = torch.optim.Adam(model.parameters(),
lr=args.learning_rate,
weight_decay=1e-5)
print('\nNum classes: %r, num images: %r' % (len(classes), len(dataset)))
#### change temp
#word_vecs = utils.get_wvecs_json(os.getcwd() + '/data/files/wvecs.json', classes, word_dim)
word_vecs = utils.get_word_vectors(os.getcwd() + '/data/files/wvecs.json',
classes, word_dim)
loss_hist, metric_hist = {}, {}
softmax = nn.Softmax(dim=1)
for epoch in range(args.epochs):
print('Epoch %r' % epoch)
log.info('Epoch %r' % epoch)
loss_hist[epoch], metric_hist[epoch] = {}, {}
for batch_idx, (img, target_tensor) in enumerate(data_loader):
batch_acc, batch_loss = [], {'reconstr': [], 'classification': []}
target_idxs = target_tensor.data.numpy().tolist()
target_names = [idx_to_class[idx] for idx in target_idxs]
target_labels = torch.tensor([[1 if i == idx else 0 for i in \
range(len(classes))] for idx in target_idxs], \
dtype=torch.long)
# previously target dist reps
target_textual = torch.tensor([word_vecs[name] for name in target_names], \
dtype=torch.float32)
#print('Text', target_textual.size())
#img_rep = img[0].reshape(1, 3, args.img_size, args.img_size)
#print(img_rep.size())
#rep = vgg.forward(img_rep)
#print(rep.size())
target_visual = torch.tensor([
cnn.forward(img[idx].reshape(1, 3, args.img_size,
args.img_size)).data.numpy()
for idx in range(len(target_idxs))
],
dtype=torch.float32)
#print('Visual', target_visual.size())
n_samples = len(target_idxs)
optimizer.zero_grad()
img_reconstr, text_reconstr, hidden = model.forward(target_visual, \
target_textual)
textual_loss = reconstr_criterion(text_reconstr, target_textual)
textual_loss.backward(retain_graph=True)
visual_loss = reconstr_criterion(img_reconstr, target_visual)
visual_loss.backward(retain_graph=True)
#print('Textual reconstr', text_reconstr.size())
#print('Visual reconstr', img_reconstr.size())
#print('Hidden', hidden.size())
preds = softmax(hidden)
pred_loss = label_criterion(preds, target_tensor)
pred_loss.backward()
optimizer.step()
if epoch % 10 == 0:
state = {'epoch': epoch + 1, 'state_dict': \
model.state_dict(), 'optimizer': optimizer.state_dict()}
torch.save(state,
os.getcwd() + "/model_states/" + args.run_name)
return
def model_1(args):
if os.path.isdir(os.getcwd() + '/results/images/' + args.run_name) is False:
os.mkdir(os.getcwd() + '/results/images/' + args.run_name)
if os.path.isdir(os.getcwd() + '/results/history/' + args.run_name) is False:
os.mkdir(os.getcwd() + '/results/history/' + args.run_name)
if os.path.isdir(os.getcwd() + '/results/files/' + args.run_name) is False:
os.mkdir(os.getcwd() + '/results/files/' + args.run_name)
datapath = args.datadir
args.img_size = 224
dataset, data_loader = utils.get_dataset(datapath, args.img_size, \
args.batch_size)
classes, class_to_idx, idx_to_class = utils.get_classes(dataset)
word_dim = 300
label_dim = len(classes)
model = BimodalDAEImage(300, 2048, n_classes=len(classes))
cnn = resnet101(pretrained=True)
optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate,
weight_decay=1e-5)
print('\nNum classes: %r, num images: %r' % (len(classes), len(dataset)))
word_vecs = utils.get_wvecs_json(os.getcwd() + '/data/files/wvecs.json', classes, word_dim)
#word_vecs = utils.get_word_vectors(os.getcwd() + '/data/files/wvecs.json', classes, word_dim)
encoding_dict = {}
with torch.no_grad():
for batch_idx, (img, target_tensor) in enumerate(data_loader):
target_idxs = target_tensor.data.numpy().tolist()
target_names = [idx_to_class[idx] for idx in target_idxs]
target_labels = torch.tensor([[1 if i == idx else 0 for i in \
range(len(classes))] for idx in target_idxs], \
dtype=torch.long)
# previously target dist reps
target_textual = torch.tensor([word_vecs[name] for name in target_names], \
dtype=torch.float32)
target_visual = torch.tensor(
[cnn.forward(
img[idx].reshape(1, 3, args.img_size, args.img_size)).data.numpy() for idx in range(len(target_idxs))], dtype=torch.float32
)
n_samples = len(target_idxs)
img_reconstr, text_reconstr, hidden = model.forward(target_visual, \
target_textual)
print('Hidden', hidden.size())
#preds = softmax(hidden)
reps = hidden.data.numpy()
for idx in range(len(reps)):
target = target_names[idx]
print(target)
if target not in encoding_dict.keys():
encoding_dict[target] = []
#val = reps[idx].view(1, -1)
encoding_dict[target].append(list(reps[idx].tolist()))
f = open(os.getcwd() + '/results/files/' + args.run_name + '/encoding_dict.json', 'w+')
f.write(json.dumps(encoding_dict))
print('Eval done!')