def main():
# Load vocabulary wrapper.
with open(vocab_path) as f:
vocab = pickle.load(f)
encoder = EncoderCNN(4096, embed_dim)
encoder.load_state_dict(torch.load('searchimage.pkl'))
for p in encoder.parameters():
p.requires_grad = False
word_encoder = EncoderRNN(embed_dim, embed_dim, len(vocab), num_layers_rnn)
word_encoder.load_state_dict(torch.load('searchword.pkl'))
if torch.cuda.is_available():
encoder.cuda()
word_encoder.cuda()
# Loss and Optimizer
criterion = nn.MSELoss()
params = list(
word_encoder.parameters()) # + list(encoder.linear.parameters())
optimizer = torch.optim.Adam(params, lr=2e-6, weight_decay=0.001)
#load data
with open(image_data_file) as f:
image_data = pickle.load(f)
image_features = si.loadmat(image_feature_file)
img_features = image_features['fc7'][0]
img_features = np.concatenate(img_features)
print 'here'
iteration = 0
for i in range(10): # epoch
use_caption = i % 5
print 'Epoch', i
losses = []
for x, y in make_mini_batch(img_features,
image_data,
use_caption=use_caption):
encoder.zero_grad()
word_encoder.zero_grad()
word_padding, lengths = make_word_padding(y, vocab)
x = Variable(torch.from_numpy(x).cuda())
word_index = Variable(torch.from_numpy(word_padding).cuda())
features = encoder(x)
outputs = word_encoder(word_index, lengths)
loss = torch.mean((features - outputs).pow(2))
loss.backward()
optimizer.step()
losses.append(loss.data[0])
if iteration % 100 == 0:
print 'loss', sum(losses) / float(len(losses))
losses = []
iteration += 1
torch.save(word_encoder.state_dict(), 'searchword.pkl')
torch.save(encoder.state_dict(), 'searchimage.pkl')
def main(args):
with open('data/vocab.pkl', 'rb') as f:
vocab = pickle.load(f)
encoder = EncoderCNN(256)
encoder.eval() # evaluation mode (BN uses moving mean/variance)
decoder = DecoderRNN(256, 512, len(vocab), 1)
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
# Load the trained model parameters
encoder.load_state_dict(torch.load(args.encoder))
decoder.load_state_dict(torch.load(args.decoder))
measurement_score = test(encoder, decoder, vocab, args.num_samples,
args.num_hints, args.debug, args.c_step,
args.no_avg)
if args.msm == "co":
scores = cocoEval()
scores_u = cocoEval(res='data/captions_val2014_results_u.json')
print(scores)
print(scores_u)
with open(args.filepath, 'w+') as f:
pickle.dump((scores, scores_u), f)
def main():
# Load vocabulary wrapper.
with open(vocab_path) as f:
vocab = pickle.load(f)
encoder = EncoderCNN(4096, embed_dim)
decoder = DecoderRNN(embed_dim, hidden_size, len(vocab), num_layers_rnn)
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
# Loss and Optimizer
criterion = nn.CrossEntropyLoss()
params = list(decoder.parameters()) + list(encoder.linear.parameters())
optimizer = torch.optim.Adam(params, lr=0.001)
#load data
with open(image_data_file) as f:
image_data = pickle.load(f)
image_features = si.loadmat(image_feature_file)
img_features = image_features['fc7'][0]
img_features = np.concatenate(img_features)
print 'here'
iteration = 0
save_loss = []
for i in range(10): # epoch
use_caption = i % 5
print 'Epoch', i
for x, y in make_mini_batch(img_features,
image_data,
use_caption=use_caption):
word_padding, lengths = make_word_padding(y, vocab)
x = Variable(torch.from_numpy(x).cuda())
word_index = Variable(torch.from_numpy(word_padding).cuda())
encoder.zero_grad()
decoder.zero_grad()
features = encoder(x)
targets = pack_padded_sequence(word_index,
lengths,
batch_first=True)[0]
outputs = decoder(features, word_index, lengths)
loss = criterion(outputs, targets)
loss.backward()
optimizer.step()
if iteration % 100 == 0:
print 'loss', loss.data[0]
save_loss.append(loss.data[0])
iteration += 1
torch.save(decoder.state_dict(), 'decoder.pkl')
torch.save(encoder.state_dict(), 'encoder.pkl')
with open('losses.txt', 'w') as f:
print >> f, losses
def main(image):
# Configuration for hyper-parameters
config = Config()
# Image Preprocessing
transform = config.test_transform
# Load vocabulary
with open(os.path.join(config.vocab_path, 'vocab.pkl'), 'rb') as f:
vocab = pickle.load(f)
# Build Models
encoder = EncoderCNN(config.embed_size)
encoder.eval() # evaluation mode (BN uses moving mean/variance)
decoder = DecoderRNN(config.embed_size, config.hidden_size, len(vocab),
config.num_layers)
# Load the trained model parameters
encoder.load_state_dict(
torch.load(
os.path.join(config.teacher_cnn_path, config.trained_encoder)))
decoder.load_state_dict(
torch.load(
os.path.join(config.teacher_lstm_path, config.trained_decoder)))
# Prepare Image
image = Image.open(image)
image_tensor = Variable(transform(image).unsqueeze(0))
# Set initial states
state = (Variable(torch.zeros(config.num_layers, 1, config.hidden_size)),
Variable(torch.zeros(config.num_layers, 1, config.hidden_size)))
# If use gpu
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
state = [s.cuda() for s in state]
image_tensor = image_tensor.cuda()
# Generate caption from image
feature = encoder(image_tensor)
sampled_ids = decoder.sample(feature, state)
sampled_ids = sampled_ids.cpu().data.numpy()
# Decode word_ids to words
sampled_caption = []
for word_id in sampled_ids:
word = vocab.idx2word[word_id]
sampled_caption.append(word)
if word_id == 96:
sampled_caption.append('<end>')
break
if word == '<end>':
break
sentence = ' '.join(sampled_caption)
# Print out image and generated caption.
print(sentence)
return sentence
def main(args):
# Val images folder
filepath = '/scratch/ys2542/pytorch-tutorial/tutorials/03-advanced/image_captioning/data/resizedval2014'
onlyfiles = [fl for fl in listdir(filepath) if isfile(join(filepath, fl))]
# image preprocessing
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
])
# load vocabulary wrapper pickle file
with open(args.vocab_path, 'rb') as f:
vocab = pickle.load(f)
encoder = EncoderCNN(args.embed_size) # build encoder
encoder.eval() # evaluation mode by moving mean and variance
decoder = DecoderRNN(args.embed_size, args.hidden_size, len(vocab),
args.num_layers) # build decoder
# load the trained CNN and RNN parameters
encoder.load_state_dict(torch.load(args.encoder_path))
decoder.load_state_dict(torch.load(args.decoder_path))
# Load all images in val folder
for i in onlyfiles:
badsize = 0 # count the unload images
args_image = filepath + '/' # val folder path with image names
args_image = args_image + i
# transform image and wrap it to tensor
image = load_image(args_image, transform)
image_tensor = to_var(image, volatile=True)
if torch.cuda.is_available(): # load GPU
encoder.cuda()
decoder.cuda()
# generate caption from image
try:
feature = encoder(image_tensor)
sampled_ids = decoder.sample(feature)
sampled_ids = sampled_ids.cpu().data.numpy()
# decode word_ids to words
sampled_caption = []
for word_id in sampled_ids:
word = vocab.idx2word[word_id]
sampled_caption.append(word)
if word == '<end>':
break
sentence = ' '.join(sampled_caption)
# print out image and generated caption without start and end
print('beam_size_1' + '\t' + i + '\t' + sentence[8:-8])
except:
badsize = badsize + 1 # count some wrong images
def main(args):
# Image preprocessing
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
])
# Load vocabulary wrapper
with open(args.vocab_path, 'rb') as f:
vocab = pickle.load(f)
# Build Models
encoder = EncoderCNN(args.embed_size)
encoder.eval() # evaluation mode (BN uses moving mean/variance)
decoder = DecoderRNN(args.embed_size, args.hidden_size, len(vocab),
args.num_layers)
# Load the trained model parameters
encoder.load_state_dict(torch.load(args.encoder_path))
decoder.load_state_dict(torch.load(args.decoder_path))
# Prepare Image
image = load_image(args.image, transform)
image_tensor = to_var(image, volatile=True)
# If use gpu
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
# Generate caption from image
feature = encoder(image_tensor)
sampled_ids = decoder.sample(feature)
sampled_ids = sampled_ids.cpu().data.numpy()
# Decode word_ids to words
sampled_caption = []
for word_id in sampled_ids:
word = vocab.idx2word[word_id]
sampled_caption.append(word)
if word == '<end>':
break
sentence = ' '.join(sampled_caption)
# Print out image and generated caption.
sentence = sentence.replace('<start> ',
'').replace(' <end>', '').replace('.',
'').strip()
translator = Translator()
sentence_indo = translator.translate(sentence, dest='id').text
print('This is an image of: ' + sentence_indo)
tts = gTTS(sentence_indo, 'id')
tts.save('result.mp3')
playsound('result.mp3')
image = Image.open(args.image)
plt.imshow(np.asarray(image))
plt.show()
def main(args):
# Image preprocessing
transform = transforms.Compose([
transforms.Scale(args.crop_size),
transforms.CenterCrop(args.crop_size),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])
# Load vocabulary wrapper
with open(args.vocab_path, 'rb') as f:
vocab = pickle.load(f)
alexnet = models.alexnet(pretrained=True)
alexnet2 = AlexNet2(alexnet)
# Build Models
encoder = EncoderCNN(4096, args.embed_size)
encoder.eval() # evaluation mode (BN uses moving mean/variance)
decoder = DecoderRNN(args.embed_size, args.hidden_size, len(vocab),
args.num_layers)
# Load the trained model parameters
encoder.load_state_dict(torch.load(args.encoder_path))
decoder.load_state_dict(torch.load(args.decoder_path))
# Prepare Image
image = Image.open(args.image)
image_tensor = Variable(transform(image).unsqueeze(0))
# Set initial states
state = (Variable(torch.zeros(args.num_layers, 1, args.hidden_size)),
Variable(torch.zeros(args.num_layers, 1, args.hidden_size)))
# If use gpu
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
alexnet2.cuda()
state = [s.cuda() for s in state]
image_tensor = image_tensor.cuda()
# Generate caption from image
alexnet2(image_tensor)
feature = encoder(alexnet2.fc7_value)
sampled_ids = decoder.sample(feature, state)
sampled_ids = sampled_ids.cpu().data.numpy()
# Decode word_ids to words
sampled_caption = []
for word_id in sampled_ids:
word = vocab.idx2word[word_id]
sampled_caption.append(word)
if word == '<end>':
break
sentence = ' '.join(sampled_caption)
# Print out image and generated caption.
print(sentence)
def main(args):
# Image preprocessing
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
])
# Load vocabulary wrapper
with open(args.vocab_path, 'rb') as f:
vocab = pickle.load(f)
# Build Models
encoder = EncoderCNN(args.embed_size)
encoder.eval() # evaluation mode (BN uses moving mean/variance)
decoder = DecoderRNN(args.embed_size, args.hidden_size, len(vocab),
args.num_layers)
# Load the trained model parameters
encoder.load_state_dict(torch.load(args.encoder_path))
decoder.load_state_dict(torch.load(args.decoder_path))
# Prepare Image
#image = load_image(args.image, transform)
#image_tensor = to_var(image, volatile=True)
# If use gpu
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
data = []
try:
img_path = args.image
# Prepare Image
image = load_image(img_path, transform)
image_tensor = to_var(image, volatile=True)
# Generate caption from image
feature = encoder(image_tensor)
#pdb.set_trace()
sampled_ids = decoder.sample(feature)
sampled_ids = sampled_ids.cpu().data.numpy()
# Decode word_ids to words
sampled_caption = []
for word_id in sampled_ids:
word = vocab.idx2word[word_id]
if word == '<start>':
continue
if word == '<end>':
break
sampled_caption.append(word)
sentence = ' '.join(sampled_caption)
# Print out image and generated caption.
print(sentence)
data.append({'key': img_path.split('/')[-1], 'sentence': sentence})
except:
print(img_path)
def main(args):
# Image preprocessing
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406),
(0.229, 0.224, 0.225))])
# Load vocabulary wrapper
with open(args.vocab_path, 'rb') as f:
vocab = pickle.load(f)
# Build Models
encoder = EncoderCNN(args.embed_size)
encoder.eval() # evaluation mode (BN uses moving mean/variance)
decoder = DecoderRNN(args.embed_size, args.hidden_size,
len(vocab), args.num_layers)
# Load the trained model parameters
encoder.load_state_dict(torch.load(args.encoder_path))
decoder.load_state_dict(torch.load(args.decoder_path))
# If use gpu
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
# Prepare Image
image_dir = args.image
images = os.listdir(image_dir)
for image_id in images:
if not image_id.endswith('.jpg'):
continue
image = os.path.join(image_dir, image_id)
image = load_image(image, transform)
image_tensor = to_var(image, volatile=True)
# Generate caption from image
try:
feature, cnn_features = encoder(image_tensor)
sampled_ids = decoder.sample(feature, cnn_features)
sampled_ids = sampled_ids.cpu().data.numpy()
except:
continue
# Decode word_ids to words
sampled_caption = []
for word_id in sampled_ids:
word = vocab.idx2word[word_id]
sampled_caption.append(word)
if word == '<end>':
break
sentence = ' '.join(sampled_caption)
# Print out image and generated caption.
print (image_id + '\t' + sentence)
def main(args):
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406),
(0.229, 0.224, 0.225))])
with open(args.vocab_path, 'rb') as f:
vocab = pickle.load(f)
# Build Models
encoder = EncoderCNN(args.embed_size)
encoder.eval() # evaluation mode (BN uses moving mean/variance)
decoder = DecoderRNN(args.embed_size, args.hidden_size,
len(vocab), args.num_layers)
# Load the trained model parameters
encoder.load_state_dict(torch.load(args.encoder_path))
decoder.load_state_dict(torch.load(args.decoder_path))
# Prepare Image
image = load_image(args.image, transform)
image_tensor = to_var(image, volatile=True)
# If use gpu
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
# Generate caption from image
feature = encoder(image_tensor)
sentence = decode(feature,[],decoder,vocab)
print (sentence)
user_input = raw_input("Does it make sense to you?(y/n)\n")
if str(user_input) == "n":
f = open('data/step_1/caption_1.txt','r')
ground_true = f.read()
teach_wordid = []
teach_wordid.append(vocab.word2idx["<start>"])
while(True):
print "This is the ground true:\n"+ground_true+"\n"+\
"###################################################\n"
reference = ground_true.split()
hypothesis = sentence.split()
BLEUscore = nltk.translate.bleu_score.sentence_bleu([reference], hypothesis)
print "Current BLEU score is "+str(BLEUscore)
word = raw_input("next word:\n")
word_idx = vocab.word2idx[word]
teach_wordid.append(word_idx)
sentence = decode(feature,teach_wordid,decoder,vocab)
print "###################################################\n"
print "Current Translated sentence is: \n"+sentence+"\n"
def main(args):
# Image preprocessing
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
])
# Load vocabulary wrapper
with open(args.vocab_path, 'rb') as f:
vocab = pickle.load(f)
# Build Models
encoder = EncoderCNN(args.embed_size)
encoder.eval() # evaluation mode (BN uses moving mean/variance)
decoder = DecoderRNN(args.embed_size, args.hidden_size, len(vocab),
args.num_layers)
# Load the trained model parameters
encoder.load_state_dict(torch.load(args.encoder_path))
decoder.load_state_dict(torch.load(args.decoder_path))
# Prepare Image
image = load_image(args.image, transform)
image_tensor = to_var(image, volatile=True)
# If use gpu
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
bar = Bar('Processing', max=100)
for i in range(100):
bar.next()
# Generate caption from image
feature = encoder(image_tensor)
sampled_ids = decoder.sample(feature)
sampled_ids = sampled_ids.cpu().data.numpy()
# Decode word_ids to words
sampled_caption = []
for word_id in sampled_ids:
word = vocab.idx2word[word_id]
sampled_caption.append(word)
if word == '<end>':
break
sentence = ' '.join(sampled_caption)
bar.finish()
# Print out image and generated caption.
print("\n")
print(sentence)
image = Image.open(args.image)
imgplot = plt.imshow(np.asarray(image))
plt.show()
def main():
# TODO parse arguments
parser = argparse.ArgumentParser()
# DATA related
parser.add_argument('--data_path', type=str,
help='Path of training data')
parser.add_argument('--max_length', type=int, default=250,
help='Pad all data to this size')
# Model Specs
parser.add_argument('--embed_size', type=int, default=300,
help='Dimension of word embedding')
parser.add_argument('--filter_size', type=int, default=5,
help='size of filter')
parser.add_argument('--stride', type=int, default=2,
help='stride size for each layer')
parser.add_argument('--filter_nums', type=str, default='300,600',
help='filer number for each convolution layer')
parser.add_argument('--hidden_size', type=int, default=500,
help='size of hidden state in the middle')
# Optimization Specs
parser.add_argument('--batch_size', type=int, default=128,
help='Minibatch during training')
parser.add_argument('--optim', type=str, default='SGD',
help='Optimization method')
parser.add_argument('--learning_rate', type=float, default=0.0005,
help='Learning rate of model')
parser.add_argument('--decay', type=float, default=0.5,
help='Decay rate of learning rate')
parser.add_argument('--start_decay', type=int, default=5,
help='Start Epoch of decay learning rate')
args = parser.parse_args()
args.filter_nums = [int(i) for i in args.filter_nums.split(',')]
args.filter_nums.append(args.hidden_size)
final_filter_size = args.max_length
for num in args.filter_nums[:-1]:
final_filter_size = math.floor(
(final_filter_size - args.filter_size)/args.stride + 1
)
# TODO build model
encoder = EncoderCNN(args.embed_size, args.filter_size,
args.stride, args.filter_nums, final_filter_size)
decoder = DecoderCNN(args.embed_size, args.filter_size,
args.stride, args.filter_nums, final_filter_size)
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
creterion =
# TODO Load training data
with open(args.data_path, 'r') as f:
data = f.readlines()
def main(args):
# Image preprocessing
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406),
(0.229, 0.224, 0.225))])
# Load vocabulary wrapper
with open(args.vocab_path, 'rb') as f:
vocab = pickle.load(f)
# Build Models
encoder = EncoderCNN(args.embed_size)
encoder.eval() # evaluation mode (BN uses moving mean/variance)
decoder = DecoderRNN(args.embed_size, args.hidden_size,
len(vocab), args.num_layers)
# Load the trained model parameters
encoder.load_state_dict(torch.load(args.encoder_path))
decoder.load_state_dict(torch.load(args.decoder_path))
# Prepare Image
image = load_image(args.image, transform)
image_tensor = to_var(image, volatile=True)
# If use gpu
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
# Generate caption from image
feature = encoder(image_tensor)
sampled_ids = decoder.sample(feature)
sampled_ids = sampled_ids.cpu().data.numpy()
# Decode word_ids to words
sampled_caption = []
for word_id in sampled_ids:
word = vocab.idx2word[word_id]
sampled_caption.append(word)
if word == '<end>':
break
sentence = ' '.join(sampled_caption)
# Print out image and generated caption.
print (sentence)
image = Image.open(args.image)
plt.imshow(np.asarray(image))
def main(args):
# Image preprocessing
transform = transforms.Compose([
transforms.Scale(args.crop_size),
#transforms.CenterCrop(args.crop_size),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])
# Load vocabulary wrapper
with open(args.vocab_path, 'rb') as f:
vocab = pickle.load(f)
# Build Models
encoder = EncoderCNN(args.embed_size)
encoder.eval() # evaluation mode (BN uses moving mean/variance)
decoder = DecoderRNN(args.embed_size, args.hidden_size, len(vocab),
args.num_layers)
# Load the trained model parameters
encoder.load_state_dict(torch.load(args.encoder_path))
decoder.load_state_dict(torch.load(args.decoder_path))
# Prepare Image
image = load_image(args.image, transform)
image_tensor = to_var(image, volatile=True)
# If use gpu
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
image_tensor = image_tensor.cuda()
# Generate caption from image
feature = encoder(image_tensor)
sampled_ids = decoder.sample(feature, args.length)
sampled_ids = sampled_ids.cpu().data.numpy()
# Decode word_ids to words
sampled_caption = []
for word_id in sampled_ids:
word = vocab.idx2word[word_id]
if word != '<start>' and word != '<end>':
sampled_caption.append(word)
if word == '<end>':
break
sentence = ''.join(sampled_caption)
# Print out image and generated caption.
print(sentence)
def encode(img,vocab):
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406),
(0.229, 0.224, 0.225))])
encoder = EncoderCNN(256)
encoder.eval() # evaluation mode (BN uses moving mean/variance)
encoder.load_state_dict(torch.load('../models/encoder-4-3000.pkl'))
image = load_image(img, transform)
image_tensor = to_var(image, volatile=True)
# If use gpu
if torch.cuda.is_available():
encoder.cuda()
feature = encoder(image_tensor)
return feature
def main(args):
if not os.path.exists(
args.model_path
): # # create model folder to keep model setting pickle files
os.makedirs(args.model_path)
# image preprocessing and normailzation
transform = transforms.Compose([
transforms.RandomCrop(args.crop_size),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
])
with open(args.vocab_path, 'rb') as f:
vocab = pickle.load(f) # load vocabulary wrapper file
# get data loader
data_loader = get_loader(args.image_dir,
args.caption_path,
vocab,
transform,
args.batch_size,
shuffle=True,
num_workers=args.num_workers)
encoder = EncoderCNN(args.embed_size) # build encoder
decoder = DecoderRNN(args.embed_size, args.hidden_size, len(vocab),
args.num_layers) # build decoder
if torch.cuda.is_available(): # load GPU
encoder.cuda()
decoder.cuda()
criterion = nn.CrossEntropyLoss() # get loss
params = list(decoder.parameters()) + list(
encoder.linear.parameters()) + list(encoder.bn.parameters())
optimizer = torch.optim.Adam(params,
lr=args.learning_rate) # get optimization
# train the Models
total_step = len(data_loader)
for epoch in range(args.num_epochs):
for i, (images, captions, lengths) in enumerate(data_loader):
# set mini batch dataset
images = to_var(images, volatile=True)
captions = to_var(captions)
targets = pack_padded_sequence(captions, lengths,
batch_first=True)[0]
# forward and backward
decoder.zero_grad()
encoder.zero_grad()
features = encoder(images)
outputs = decoder(features, captions, lengths)
loss = criterion(outputs, targets)
loss.backward()
optimizer.step() # optimization
# Print loss and perplexity
if i % args.log_step == 0:
print(
'Epoch [%d/%d], Step [%d/%d], Loss: %.4f, Perplexity: %5.4f'
% (epoch, args.num_epochs, i, total_step, loss.data[0],
np.exp(loss.data[0])))
# save the models pickle file settings
if (i + 1) % args.save_step == 0:
torch.save(
decoder.state_dict(),
os.path.join(args.model_path,
'decoder-%d-%d.pkl' % (epoch + 1, i + 1)))
torch.save(
encoder.state_dict(),
os.path.join(args.model_path,
'encoder-%d-%d.pkl' % (epoch + 1, i + 1)))
def main():
# Configuration for hyper-parameters
config = Config()
# Image preprocessing
transform = config.train_transform
# Load vocabulary wrapper
with open(os.path.join(config.vocab_path, 'vocab.pkl'), 'rb') as f:
vocab = pickle.load(f)
# Build data loader
image_path = os.path.join(config.image_path, 'train2014')
json_path = os.path.join(config.caption_path, 'captions_train2014.json')
train_loader = get_data_loader(image_path, json_path, vocab,
transform, config.batch_size,
shuffle=True, num_workers=config.num_threads)
total_step = len(train_loader)
# Build Models
teachercnn = EncoderCNN(config.embed_size)
teachercnn.eval()
studentcnn = StudentCNN_Model1(config.embed_size)
#Load the best teacher model
teachercnn.load_state_dict(torch.load(os.path.join('../TrainedModels/TeacherCNN', config.trained_encoder)))
studentlstm = DecoderRNN(config.embed_size, config.hidden_size/2,
len(vocab), config.num_layers/2)
if torch.cuda.is_available():
teachercnn.cuda()
studentcnn.cuda()
studentlstm.cuda()
# Loss and Optimizer
criterion_lstm = nn.CrossEntropyLoss()
criterion_cnn = nn.MSELoss()
params = list(studentlstm.parameters()) + list(studentcnn.parameters())
optimizer_lstm = torch.optim.Adam(params, lr=config.learning_rate)
optimizer_cnn = torch.optim.Adam(studentcnn.parameters(), lr=config.cnn_learningrate)
print('entering in to training loop')
# Train the Models
for epoch in range(config.num_epochs):
for i, (images, captions, lengths, img_ids) in enumerate(train_loader):
images = Variable(images)
captions = Variable(captions)
if torch.cuda.is_available():
images = images.cuda()
captions = captions.cuda()
targets = pack_padded_sequence(captions, lengths, batch_first=True)[0]
# Forward, Backward and Optimize
optimizer_lstm.zero_grad()
optimizer_cnn.zero_grad()
features_tr = teachercnn(images)
features_st = studentcnn(images)
outputs = studentlstm(features_st, captions, lengths)
loss = criterion(features_st, features_tr.detach()) + criterion_lstm(outputs, targets)
loss.backward()
optimizer_cnn.step()
optimizer_lstm.step()
# Print log info
if i % config.log_step == 0:
print('Epoch [%d/%d], Step [%d/%d], Loss: %.4f, Perplexity: %5.4f'
%(epoch, config.num_epochs, i, total_step,
loss.data[0], np.exp(loss.data[0])))
# Save the Model
if (i+1) % config.save_step == 0:
torch.save(studentlstm.state_dict(),
os.path.join(config.student_lstm_path,
'decoder-%d-%d.pkl' %(epoch+1, i+1)))
torch.save(studentcnn.state_dict(),
os.path.join(config.student_cnn_path,
'encoder-%d-%d.pkl' %(epoch+1, i+1)))
def main(args):
# Create model directory
if not os.path.exists(args.model_path):
os.makedirs(args.model_path)
# Load vocabulary wrapper.
with open(args.vocab_path, 'rb') as f:
vocab = pickle.load(f)
# Image preprocessing
# For normalization, see https://github.com/pytorch/vision#models
transform = transforms.Compose([
transforms.RandomCrop(args.crop_size),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
])
val_loader = get_loader('./data/val_resized2014/',
'./data/annotations/captions_val2014.json', vocab,
transform, 1, False, 1)
start_epoch = 0
encoder_state = args.encoder
decoder_state = args.decoder
# Build the models
encoder = EncoderCNN(args.embed_size)
if not args.train_encoder:
encoder.eval()
decoder = DecoderRNN(args.embed_size, args.hidden_size, len(vocab),
args.num_layers)
if args.restart:
encoder_state, decoder_state = 'new', 'new'
if encoder_state == '': encoder_state = 'new'
if decoder_state == '': decoder_state = 'new'
if decoder_state != 'new':
start_epoch = int(decoder_state.split('-')[1])
print("Using encoder: {}".format(encoder_state))
print("Using decoder: {}".format(decoder_state))
# Build data loader
data_loader = get_loader(args.image_dir,
args.caption_path,
vocab,
transform,
args.batch_size,
shuffle=True,
num_workers=args.num_workers)
""" Make logfile and log output """
with open(args.model_path + args.logfile, 'a+') as f:
f.write("Training on vanilla loss (using new model). Started {} .\n".
format(str(datetime.now())))
f.write("Using encoder: new\nUsing decoder: new\n\n")
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
# Loss and Optimizer
criterion = nn.CrossEntropyLoss()
params = list(decoder.parameters()) + list(
encoder.linear.parameters()) + list(encoder.bn.parameters())
optimizer = torch.optim.Adam(params, lr=args.learning_rate)
batch_loss = []
batch_acc = []
# Train the Models
total_step = len(data_loader)
for epoch in range(start_epoch, args.num_epochs):
for i, (images, captions, lengths, _, _) in enumerate(data_loader):
# Set mini-batch dataset
images = to_var(images, volatile=True)
captions = to_var(captions)
targets = pack_padded_sequence(captions, lengths,
batch_first=True)[0]
# Forward, Backward and Optimize
decoder.zero_grad()
encoder.zero_grad()
features = encoder(images)
out = decoder(features, captions, lengths)
loss = criterion(out, targets)
batch_loss.append(loss.data[0])
loss.backward()
optimizer.step()
# # Print log info
# if i % args.log_step == 0:
# print('Epoch [%d/%d], Step [%d/%d], Loss: %.4f, Perplexity: %5.4f, Val: %.5f, %.5f'
# %(epoch, args.num_epochs, i, total_step,
# loss.data[0], np.exp(loss.data[0]), acc, gt_acc))
# with open(args.model_path + args.logfile, 'a') as f:
# f.write('Epoch [%d/%d], Step [%d/%d], Loss: %.4f, Perplexity: %5.4f, Val: %.5f, %.5f\n'
# %(epoch, args.num_epochs, i, total_step,
# loss.data[0], np.exp(loss.data[0]), acc, gt_acc))
# Save the models
if (i + 1) % args.save_step == 0:
torch.save(
decoder.state_dict(),
os.path.join(args.model_path,
'decoder-%d-%d.pkl' % (epoch + 1, i + 1)))
torch.save(
encoder.state_dict(),
os.path.join(args.model_path,
'encoder-%d-%d.pkl' % (epoch + 1, i + 1)))
with open(args.model_path + 'training_loss.pkl', 'w+') as f:
pickle.dump(batch_loss, f)
with open(args.model_path + 'training_val.pkl', 'w+') as f:
pickle.dump(batch_acc, f)
with open(args.model_path + args.logfile, 'a') as f:
f.write("Training finished at {} .\n\n".format(str(datetime.now())))
# Specify values for embed_size and hidden_size - we use the same values as in training step
embed_size = 256
hidden_size = 512
# Get the vocabulary and its size
vocab = data_loader.dataset.vocab
vocab_size = len(vocab)
# Initialize the encoder and decoder, and set each to inference mode
encoder = EncoderCNN(embed_size)
encoder.eval()
decoder = DecoderRNN(embed_size, hidden_size, vocab_size)
decoder.eval()
# Load the pre-trained weights
encoder.load_state_dict(checkpoint['encoder'])
decoder.load_state_dict(checkpoint['decoder'])
# Move models to GPU if CUDA is available.
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
# In[5]:
x = get_prediction(data_loader, encoder, decoder, vocab)
# In[6]:
print(x)
def main():
#write predicted caption
if not os.path.exists(args['generate_caption_path']):
os.makedirs(args['generate_caption_path'])
caption_string = os.path.join(args['generate_caption_path'], "caption_ncrt_class5.txt")
#mode = "a" if os.path.exists(caption_string) else "w"
fp =open(caption_string, "w+")
# Image preprocessing
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.9638, 0.9638, 0.9638),
(0.1861, 0.1861, 0.1861))])
# Load vocabulary wrapper
with open(args['vocab_path'], 'rb') as f:
vocab = pickle.load(f)
# Build Models
encoder = EncoderCNN(args['embed_size'])
encoder.eval() # evaluation mode (BN uses moving mean/variance)
decoder = DecoderRNN(args['embed_size'], args['hidden_size'],
len(vocab), args['num_layers'], max_seq_length=50)
decoder.eval()
# Load the trained model parameters
encoder.load_state_dict(torch.load(args['encoder_path']))
decoder.load_state_dict(torch.load(args['decoder_path']))
# If use gpu
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
# Prepare Image
image_dir = args['image_path']
images = os.listdir(image_dir)
i = 1
for image_id in images:
#print('i->',i)
#i = i+1
if not image_id.endswith('.jpg'):
continue
image = os.path.join(image_dir, image_id)
image = load_image(image, transform)
image_tensor = image.cuda()
# Generate caption from image
try:
feature, cnn_features = encoder(image_tensor)
sampled_ids = decoder.sample(feature, cnn_features)
sampled_ids = sampled_ids.cpu().data.numpy()
except:
continue
#print('image_ids->',image_id)
# Decode word_ids to words
sampled_caption = []
for word_id in sampled_ids:
word = vocab.idx2word[word_id]
sampled_caption.append(word)
if word == '<end>':
break
sentence = ' '.join(sampled_caption)
print ('i->', i, image_id + '\t' + sentence)
fp.write(image_id)
fp.write('\t')
fp.write(sentence)
if i<398:
fp.write("\n")
i = i+1
fp.close()
def main(args):
torch.manual_seed(args.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(args.seed)
# Create model directory
if not os.path.exists(args.model_path):
os.makedirs(args.model_path)
# Image preprocessing
# For normalization, see https://github.com/pytorch/vision#models
transform = transforms.Compose([
# transforms.RandomCrop(args.crop_size),
# transforms.RandomHorizontalFlip(),
transforms.Scale(args.crop_size),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
])
# Load vocabulary wrapper.
with open(args.vocab_path, 'rb') as f:
vocab = pickle.load(f)
# Build data loader
data_loader = get_loader(args.image_dir,
args.caption_path,
vocab,
args.MSCOCO_result,
args.coco_detection_result,
transform,
args.batch_size,
shuffle=True,
num_workers=args.num_workers,
dummy_object=99,
yolo=False)
# Build the models
encoder = EncoderCNN(args.embed_size)
# the layout encoder hidden state size must be the same with decoder input size
layout_encoder = LayoutEncoder(args.layout_embed_size, args.embed_size,
100, args.num_layers)
decoder = DecoderRNN(args.embed_size, args.hidden_size, len(vocab),
args.num_layers)
if torch.cuda.is_available():
encoder.cuda()
layout_encoder.cuda()
decoder.cuda()
# Loss and Optimizer
criterion = nn.CrossEntropyLoss()
params = list(layout_encoder.parameters()) + list(decoder.parameters()) + \
list(encoder.linear.parameters()) + list(encoder.bn.parameters())
optimizer = torch.optim.Adam(params, lr=args.learning_rate)
# Train the Models
total_step = len(data_loader)
for epoch in range(args.num_epochs):
for i, (images, captions, lengths, label_seqs, location_seqs,
visual_seqs, layout_lengths) in enumerate(data_loader):
# Set mini-batch dataset
images = to_var(images)
captions = to_var(captions)
targets = pack_padded_sequence(captions, lengths,
batch_first=True)[0]
# Forward, Backward and Optimize
# decoder.zero_grad()
# layout_encoder.zero_grad()
# encoder.zero_grad()
# Modify This part for using visual features or not
# features = encoder(images)
layout_encoding = layout_encoder(label_seqs, location_seqs,
layout_lengths)
# comb_features = features + layout_encoding
comb_features = layout_encoding
outputs = decoder(comb_features, captions, lengths)
loss = criterion(outputs, targets)
optimizer.zero_grad()
loss.backward()
optimizer.step()
# Print log info
if i % args.log_step == 0:
print(
'Epoch [%d/%d], Step [%d/%d], Loss: %.4f, Perplexity: %5.4f'
% (epoch, args.num_epochs, i, total_step, loss.data[0],
np.exp(loss.data[0])))
# Save the models
if (i + 1) % args.save_step == 0:
torch.save(
decoder.state_dict(),
os.path.join(args.model_path,
'decoder-%d-%d.pkl' % (epoch + 1, i + 1)))
torch.save(
layout_encoder.state_dict(),
os.path.join(
args.model_path,
'layout_encoding-%d-%d.pkl' % (epoch + 1, i + 1)))
def train(batch_size=32,
vocab_threshold=5,
vocab_from_file=True,
embed_size=256,
hidden_size=512,
num_epochs=10,
latest_model=None,
cocoapi_dir="./Coco/"):
# Keep track of train and validation losses and validation Bleu-4 scores by epoch
train_losses = []
# Define a transform to pre-process the training images
transform_train = transforms.Compose([
transforms.Resize(256),
transforms.RandomCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
])
# Build data loader, applying the transforms
train_loader = get_loader(transform=transform_train,
mode='train',
batch_size=batch_size,
vocab_threshold=vocab_threshold,
vocab_from_file=vocab_from_file,
cocoapi_loc=cocoapi_dir)
# The size of the vocabulary
vocab_size = len(train_loader.dataset.vocab)
# Initialize the encoder and decoder
checkpoint = None
if latest_model:
checkpoint = torch.load(latest_model)
start_epoch = 1
if checkpoint:
train_losses = checkpoint['train_losses']
val_losses = checkpoint['val_losses']
start_epoch = checkpoint['epoch']
encoder = EncoderCNN(embed_size)
decoder = DecoderRNN(embed_size, hidden_size, vocab_size)
if checkpoint:
encoder.load_state_dict(checkpoint['encoder'])
decoder.load_state_dict(checkpoint['decoder'])
# Move models to GPU if CUDA is available
if torch.cuda.is_available():
torch.cuda.set_device(1)
encoder.cuda()
decoder.cuda()
# Define the loss function
loss = nn.CrossEntropyLoss().cuda() if torch.cuda.is_available(
) else nn.CrossEntropyLoss()
# Specify the learnable parameters of the model
params = list(decoder.parameters()) + list(
encoder.embed.parameters()) + list(encoder.bn.parameters())
# Define the optimizer
optimizer = torch.optim.Adam(params=params, lr=0.001)
if checkpoint:
optimizer.load_state_dict(checkpoint['optimizer'])
# Set the total number of training and validation steps per epoch
total_train_step = math.ceil(
len(train_loader.dataset.caption_lengths) /
train_loader.batch_sampler.batch_size)
start_time = time.time()
for epoch in range(start_epoch, num_epochs + 1):
train_loss = train_one(train_loader, encoder, decoder, loss, optimizer,
vocab_size, epoch, total_train_step)
train_losses.append(train_loss)
# Save the entire model anyway, regardless of being the best model so far or not
filename = os.path.join("./models", "model-{}.pkl".format(epoch))
save_epoch(filename, encoder, decoder, optimizer, train_losses, epoch)
print("Epoch [%d/%d] took %ds" %
(epoch, num_epochs, time.time() - start_time))
start_time = time.time()
def main():
# Configuration for hyper-parameters
torch.cuda.set_device(0)
config = Config()
# Image preprocessing
transform = config.train_transform
# Load vocabulary wrapper
with open(os.path.join(config.vocab_path, 'vocab.pkl'), 'rb') as f:
vocab = pickle.load(f)
# Build data loader
train_image_path = os.path.join(config.image_path, 'train2017')
json_path = os.path.join(config.caption_path, 'captions_train2017.json')
train_loader = get_data_loader(train_image_path,
json_path,
vocab,
transform,
config.batch_size,
shuffle=False,
num_workers=config.num_threads)
val_image_path = os.path.join(config.image_path, 'val2017')
json_path = os.path.join(config.caption_path, 'captions_val2017.json')
val_loader = get_data_loader(val_image_path,
json_path,
vocab,
transform,
config.batch_size,
shuffle=False,
num_workers=config.num_threads)
total_step = len(train_loader)
# Build Models
encoder = EncoderCNN(config.embed_size)
encoder.eval()
decoder = DecoderRNN(config.embed_size, config.hidden_size, len(vocab),
config.num_layers)
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
# Loss and Optimizer
criterion = nn.CrossEntropyLoss()
params = list(decoder.parameters()) + list(encoder.resnet.fc.parameters())
optimizer = torch.optim.Adam(params, lr=config.learning_rate)
print('entering in to training loop')
# Train the Models
with open('train1_log.txt', 'w') as logfile:
logfile.write('Validation Error,Training Error')
for epoch in range(0, 25):
for i, (images, captions, lengths,
img_ids) in enumerate(train_loader):
images = Variable(images)
captions = Variable(captions)
if torch.cuda.is_available():
images = images.cuda()
captions = captions.cuda()
targets = pack_padded_sequence(captions,
lengths,
batch_first=True)[0]
# Forward, Backward and Optimize
optimizer.zero_grad()
features = encoder(images)
outputs = decoder(features, captions, lengths)
loss = criterion(outputs, targets)
loss.backward()
optimizer.step()
# Print log info
if i % config.log_step == 0:
print(
'Epoch [%d/%d], Step [%d/%d], Loss: %.4f, Perplexity: %5.4f'
% (epoch, config.num_epochs, i, total_step,
loss.data[0], np.exp(loss.data[0])))
# Save the Model
if (i + 1) % config.save_step == 0:
torch.save(
encoder.state_dict(),
os.path.join(config.teacher_cnn_path,
'encoder-%d-%d.pkl' % (epoch + 1, i + 1)))
torch.save(
decoder.state_dict(),
os.path.join(config.teacher_lstm_path,
'decoder-%d-%d.pkl' % (epoch + 1, i + 1)))
print('Just Completed an Epoch, Initite Validation Error Test')
avgvalloss = 0
for j, (images, captions, lengths,
img_ids) in enumerate(val_loader):
images = Variable(images)
captions = Variable(captions)
if torch.cuda.is_available():
images = images.cuda()
captions = captions.cuda()
targets = pack_padded_sequence(captions,
lengths,
batch_first=True)[0]
optimizer.zero_grad()
features = encoder(images)
outputs = decoder(features, captions, lengths)
valloss = criterion(outputs, targets)
if j == 0:
avgvalloss = valloss.data[0]
avgvalloss = (avgvalloss + valloss.data[0]) / 2
if ((j + 1) % 1000 == 0):
print('Average Validation Loss: %.4f' % (avgvalloss))
logfile.write(
str(avgvalloss) + ',' + str(loss.data[0]) + str('\n'))
break
def main(args):
# Image preprocessing
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406),
(0.229, 0.224, 0.225))])
# Load vocabulary wrapper
with open(args.vocab_path, 'rb') as f:
vocab = pickle.load(f)
# Build Models
encoder = EncoderCNN(args.embed_size)
encoder.eval() # evaluation mode (BN uses moving mean/variance)
decoder = DecoderRNN(args.embed_size, args.hidden_size,
len(vocab), args.num_layers)
# Load the trained model parameters
#print args.encoder_path
encoder.load_state_dict(torch.load(args.encoder_path))
decoder.load_state_dict(torch.load(args.decoder_path))
data_loader = get_loader(args.image_dir, args.caption_path, vocab,
transform, args.batch_size,
shuffle=True, num_workers=args.num_workers)
total_num = len(data_loader)*args.batch_size
print total_num
num_correct=0
tested=0
hypotheses=[]
references=[]
for i, (images, captions, lengths) in enumerate(data_loader):
tested += args.batch_size
if i==1:
break;
# If use gpu
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
# Prepare Image
images = to_var(images, volatile=True)
captions = to_var(captions)
#targets = pack_padded_sequence(captions, lengths, batch_first=True)[0]
max_sent_length=captions[-1].size(0)
print max_sent_length,'length'
print captions.size(),'caption_size'
#print captions[0].size()
#print captions[0]
#print targets.size()
# Generate caption from image
features=encoder(images)
sampled_captions = decoder.sample(features,max_sent_length)
targets=torch.transpose(sampled_captions.view(max_sent_length,-1),0,1);
print targets.size(),'ans'
#print targets
#print captions
ref_sents=translate(captions,vocab)
hypo_sents=translate(targets,vocab)
references.extend(ref_sents)
hypotheses.extend(hypo_sents)
num_correct_t = targets.data.eq(captions.data).sum()
print num_correct_t,'num correct'
num_correct += num_correct_t
#feature = encoder(image_tensor)
#sampled_ids = decoder.sample(feature)
#sampled_ids = sampled_ids.cpu().data.numpy()
# Decode word_ids to words
#sampled_caption = []
#for word_id in sampled_ids:
# word = vocab.idx2word[word_id]
# sampled_caption.append(word)
# if word == '<end>':
# break
#sentence = ' '.join(sampled_caption)
# Print out image and generated caption.
#print (sentence)
hypo_ref_out=(hypotheses,references)
with open('hypo_out.txt', 'wb') as handle:
pickle.dump(hypo_ref_out,handle)
print len(hypotheses)
print hypotheses[0:10]
print references[0:10]
bleu_score=bleu.BLEU(hypotheses,[references])
print bleu_score
print 'num_correct',num_correct,'total',tested,total_num
score = BLEU.corpus_bleu(references,hypotheses)
score1 = BLEU.corpus_bleu(references,hypotheses,weights=[1,0,0,0])
score2 = BLEU.corpus_bleu(references, hypotheses,weights=[0.5,0.5,0,0])
score3 = BLEU.corpus_bleu(references, hypotheses,weights=[0.3,0.3,0.3,0])
score4 = BLEU.corpus_bleu(references, hypotheses, weights=[0.25,0.25,0.25,0.25])
print score,score1,score2,score3,score4
def main(args):
with open('./data/vocab.pkl', 'rb') as f:
vocab = pickle.load(f)
encoder = EncoderCNN(256)
encoder.eval() # evaluation mode (BN uses moving mean/variance)
decoder = DecoderRNN(256, 512, len(vocab), 1)
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
# Load the trained model parameters
encoder.load_state_dict(torch.load(args.encoder))
decoder.load_state_dict(torch.load(args.decoder))
if args.test_prop0:
decoder.test_h_from_c()
return
if args.test_c_step:
data_points = test(encoder, decoder, vocab, args.num_samples,
args.num_hints)
with open(args.filepath, 'w+') as f:
pickle.dump(data_points, f)
print("Done sampling for c_step evaluation. Data saved to {}".format(
args.filepath))
return
measurement_score = test(encoder, decoder, vocab, args.num_samples,
args.num_hints, args.debug, args.c_step,
args.no_avg)
if args.msm == "ps":
if not args.no_avg:
print "ground truth prediction score without update\n" + str(
measurement_score[0])
print "ground truth prediction score with update\n" + str(
measurement_score[1])
print "Difference\n" + str(measurement_score[1] -
measurement_score[0])
else:
with open(args.filepath, 'w+') as f:
pickle.dump(measurement_score, f)
print "Done. Data saved to {}".format(args.filepath)
elif args.msm == "ce":
if not args.no_avg:
print "Cross Entropy Loss without update\n" + str(
measurement_score[0])
print "Cross Entropy Loss with update\n" + str(
measurement_score[1])
print "Difference\n" + str(measurement_score[1] -
measurement_score[0])
else:
with open(args.filepath, 'w+') as f:
pickle.dump(measurement_score, f)
print "Done. Data saved to {}".format(args.filepath)
elif args.msm == "co":
scores = cocoEval()
scores_u = cocoEval(res='data/captions_val2014_results_u.json')
print(scores)
print(scores_u)
def main(args):
#setup tensorboard
if args.tensorboard:
cc = CrayonClient(hostname="localhost")
print(cc.get_experiment_names())
#if args.name in cc.get_experiment_names():
try:
cc.remove_experiment(args.name)
except:
print("experiment didnt exist")
cc_server = cc.create_experiment(args.name)
# Create model directory
full_model_path = args.model_path + "/" + args.name
if not os.path.exists(full_model_path):
os.makedirs(full_model_path)
with open(full_model_path + "/parameters.json", 'w') as f:
f.write((json.dumps(vars(args))))
# Image preprocessing
transform = transforms.Compose([
transforms.Scale(args.crop_size),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])
mini_transform = transforms.Compose(
[transforms.ToPILImage(),
transforms.Scale(20),
transforms.ToTensor()])
# Load vocabulary wrapper.
if args.vocab_path is not None:
with open(args.vocab_path, 'rb') as f:
vocab = pickle.load(f)
else:
print("building new vocab")
vocab = build_vocab(args.image_dir, 1, None)
with open((full_model_path + "/vocab.pkl"), 'wb') as f:
pickle.dump(vocab, f)
# Build data loader
data_loader = get_loader(args.image_dir,
vocab,
transform,
args.batch_size,
shuffle=True,
num_workers=args.num_workers)
code_data_set = ProcessingDataset(root=args.image_dir,
vocab=vocab,
transform=transform)
train_ds, val_ds = validation_split(code_data_set)
train_loader = torch.utils.data.DataLoader(train_ds, collate_fn=collate_fn)
test_loader = torch.utils.data.DataLoader(val_ds, collate_fn=collate_fn)
train_size = len(train_loader)
test_size = len(test_loader)
# Build the models
encoder = EncoderCNN(args.embed_size, args.train_cnn)
print(encoder)
decoder = DecoderRNN(args.embed_size, args.hidden_size, len(vocab),
args.num_layers)
print(decoder)
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
# Loss and Optimizer
criterion = nn.CrossEntropyLoss()
params = list(decoder.parameters()) + list(
encoder.linear.parameters()) + list(encoder.bn.parameters())
#params = list(decoder.parameters()) #+ list(encoder.linear.parameters()) + list(encoder.bn.parameters())
optimizer = torch.optim.Adam(params, lr=args.learning_rate)
start_time = time.time()
add_log_entry(args.name, start_time, vars(args))
# Train the Models
total_step = len(data_loader)
for epoch in range(args.num_epochs):
for i, (images, captions, lengths) in enumerate(data_loader):
decoder.train()
encoder.train()
# Set mini-batch dataset
image_ts = to_var(images, volatile=True)
captions = to_var(captions)
targets = pack_padded_sequence(captions, lengths,
batch_first=True)[0]
count = images.size()[0]
# Forward, Backward and Optimize
decoder.zero_grad()
encoder.zero_grad()
features = encoder(image_ts)
outputs = decoder(features, captions, lengths)
loss = criterion(outputs, targets)
loss.backward()
optimizer.step()
total = targets.size(0)
max_index = outputs.max(dim=1)[1]
#correct = (max_index == targets).sum()
_, predicted = torch.max(outputs.data, 1)
correct = predicted.eq(targets.data).cpu().sum()
accuracy = 100. * correct / total
if args.tensorboard:
cc_server.add_scalar_value("train_loss", loss.data[0])
cc_server.add_scalar_value("perplexity", np.exp(loss.data[0]))
cc_server.add_scalar_value("accuracy", accuracy)
# Print log info
if i % args.log_step == 0:
print(
'Epoch [%d/%d], Step [%d/%d], Loss: %.4f, accuracy: %2.2f Perplexity: %5.4f'
% (epoch, args.num_epochs, i, total_step, loss.data[0],
accuracy, np.exp(loss.data[0])))
# Save the models
if (i + 1) % args.save_step == 0:
torch.save(
decoder.state_dict(),
os.path.join(full_model_path,
'decoder-%d-%d.pkl' % (epoch + 1, i + 1)))
torch.save(
encoder.state_dict(),
os.path.join(full_model_path,
'encoder-%d-%d.pkl' % (epoch + 1, i + 1)))
if 1 == 2 and i % int(train_size / 10) == 0:
encoder.eval()
#decoder.eval()
correct = 0
for ti, (timages, tcaptions,
tlengths) in enumerate(test_loader):
timage_ts = to_var(timages, volatile=True)
tcaptions = to_var(tcaptions)
ttargets = pack_padded_sequence(tcaptions,
tlengths,
batch_first=True)[0]
tfeatures = encoder(timage_ts)
toutputs = decoder(tfeatures, tcaptions, tlengths)
print(ttargets)
print(toutputs)
print(ttargets.size())
print(toutputs.size())
#correct = (ttargets.eq(toutputs[0].long())).sum()
accuracy = 100 * correct / test_size
print('accuracy: %.4f' % (accuracy))
if args.tensorboard:
cc_server.add_scalar_value("accuracy", accuracy)
torch.save(
decoder.state_dict(),
os.path.join(full_model_path,
'decoder-%d-%d.pkl' % (epoch + 1, i + 1)))
torch.save(
encoder.state_dict(),
os.path.join(full_model_path,
'encoder-%d-%d.pkl' % (epoch + 1, i + 1)))
end_time = time.time()
print("finished training, runtime: %d", [(end_time - start_time)])
def main(args):
# Create model directory
if not os.path.exists(args.model_path):
os.makedirs(args.model_path)
# Image preprocessing
# For normalization, see https://github.com/pytorch/vision#models
# transform = transforms.Compose([
# transforms.RandomCrop(args.crop_size),
# transforms.RandomHorizontalFlip(),
# transforms.ToTensor(),
# transforms.Normalize((0.485, 0.456, 0.406),
# (0.229, 0.224, 0.225))])
transform = transforms.Compose([
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
])
# Load vocabulary wrapper.
with open(args.vocab_path, 'rb') as f:
vocab = pickle.load(f)
# data_loader = get_loader(args.image_dir, args.caption_path, vocab,
# transform, args.batch_size,
# shuffle=True, num_workers=args.num_workers)
sasr_data_loader = SASR_Data_Loader(vocab, transform)
sasr_data_loader.load_data(args.data_file, args.init_flag)
frogger_data_loader = sasr_data_loader.data_loader(
args.batch_size, transform, shuffle=True, num_workers=args.num_workers)
# Build the models
encoder = EncoderCNN(args.embed_size)
decoder = DecoderRNN(args.embed_size, args.hidden_size, len(vocab),
args.num_layers)
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
# Loss and Optimizer
criterion = nn.CrossEntropyLoss()
params = list(decoder.parameters()) + list(
encoder.linear.parameters()) + list(encoder.bn.parameters())
optimizer = torch.optim.Adam(params, lr=args.learning_rate)
total_step = len(frogger_data_loader)
for epoch in range(args.num_epochs):
for i, (images, captions, lengths) in enumerate(frogger_data_loader):
images = to_var(images, volatile=True)
if (list(images.size())[0] != 1):
captions = to_var(captions)
# print(list(images.size())[0])
# print(captions)
# exit(0)
targets = pack_padded_sequence(captions,
lengths,
batch_first=True)[0]
decoder.zero_grad()
encoder.zero_grad()
features = encoder(images)
outputs = decoder(features, captions, lengths)
loss = criterion(outputs, targets)
loss.backward()
optimizer.step()
# Print log info
if i % args.log_step == 0:
print(
'Epoch [%d/%d], Step [%d/%d], Loss: %.4f, Perplexity: %5.4f'
% (epoch, args.num_epochs, i, total_step, loss.data[0],
np.exp(loss.data[0])))
# Save the models
if (i + 1) % args.save_step == 0:
torch.save(
decoder.state_dict(),
os.path.join(args.model_path,
'decoder-%d-%d.pkl' % (epoch + 1, i + 1)))
torch.save(
encoder.state_dict(),
os.path.join(args.model_path,
'encoder-%d-%d.pkl' % (epoch + 1, i + 1)))
# Initializing the Encoder and Decoder Network with arguments passed
encoder2 = EncoderCNN(embed_size=args.embed_dim).eval()
decoder2 = DecoderRNN(embed_size=args.embed_dim,
hidden_size=args.hidden_dim,
vocab_size=vocab_size,
num_layers=1,
max_seq_length=20)
else:
raise Exception('Unknown model {}'.format(args.model))
# Path where the input saved module is present
decoder_saved_module = '/content/drive/My Drive/Colab Notebooks/576_project/version5/lr0.01_2decoder.pt'
encoder_saved_module = '/content/drive/My Drive/Colab Notebooks/576_project/version5/lr0.01_2encoder.pt'
## Activate CUDA if specified and available.
if args.cuda:
encoder2.cuda()
decoder2.cuda()
# Reading the pretrained weights for Encoder and Decoder
encoder2.load_state_dict(torch.load(encoder_saved_module))
decoder2.load_state_dict(torch.load(decoder_saved_module))
for batch_idx, batch in enumerate(test_loader):
image, caption, urls = batch[0], batch[1], batch[2]
if args.cuda:
image, caption = image.cuda(), caption.cuda()
# Read images and their target labels in the current batch.
image, caption = Variable(image), caption
def main(args):
# Create model directory
if not os.path.exists(args.model_path):
os.makedirs(args.model_path)
# Image preprocessing
# Composea all processing together, to a tensor with (C,H,W) and value in range (0 - 1)
transform = transforms.Compose([
transforms.RandomCrop(args.crop_size),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])
# Load vocabulary wrapper.
with open(args.vocab_path, 'rb') as f:
vocab = pickle.load(f)
# Build data loader
data_loader = get_loader(args.image_dir,
args.caption_path,
vocab,
transform,
args.batch_size,
shuffle=True,
num_workers=args.num_workers)
# Build the models
encoder = EncoderCNN(args.embed_size)
decoder = DecoderRNN(args.embed_size, args.hidden_size, len(vocab),
args.num_layers)
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
# Loss and Optimizer
criterion = nn.CrossEntropyLoss()
params = list(decoder.parameters()) + list(encoder.resnet.fc.parameters())
optimizer = torch.optim.Adam(params, lr=args.learning_rate)
# Train the Models
total_step = len(data_loader)
for epoch in range(args.num_epochs):
for i, (images, captions, lengths) in enumerate(data_loader):
# Set mini-batch dataset
images = Variable(images)
captions = Variable(captions)
print("cap size %s" % str(captions.size()))
if torch.cuda.is_available():
images = images.cuda()
captions = captions.cuda()
targets = pack_padded_sequence(captions, lengths,
batch_first=True)[0]
print(targets)
# Forward, Backward and Optimize
decoder.zero_grad()
encoder.zero_grad()
features = encoder(images)
print("cnn feats %s" % str(features.size()))
outputs = decoder(features, captions, lengths)
loss = criterion(outputs, targets)
loss.backward()
optimizer.step()
# Print log info
if i % args.log_step == 0:
print(
'Epoch [%d/%d], Step [%d/%d], Loss: %.4f, Perplexity: %5.4f'
% (epoch, args.num_epochs, i, total_step, loss.data[0],
np.exp(loss.data[0])))
# Save the models
if (i + 1) % args.save_step == 0:
torch.save(
decoder.state_dict(),
os.path.join(args.model_path,
'decoder-%d-%d.pkl' % (epoch + 1, i + 1)))
torch.save(
encoder.state_dict(),
os.path.join(args.model_path,
'encoder-%d-%d.pkl' % (epoch + 1, i + 1)))
def main(args):
# Image preprocessing
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
])
# Load vocabulary wrapper
with open(args.vocab_path, 'rb') as f:
vocab = pickle.load(f)
# Build Models
encoder = EncoderCNN(args.embed_size)
encoder.eval() # evaluation mode (BN uses moving mean/variance)
decoder = DecoderRNN(args.embed_size, args.hidden_size, len(vocab),
args.num_layers)
# Load the trained model parameters
encoder.load_state_dict(torch.load(args.encoder_path))
decoder.load_state_dict(torch.load(args.decoder_path))
df = pd.read_csv(args.folder + "../" + "srumtextimageurl.csv",
usecols=[0, 1, 2])
df["caption"] = len(df) * [""]
caption = []
images = glob.glob(args.folder + "*")
print(images)
for image_name in images:
try:
#print(image_name)
# Prepare Image
image = load_image(image_name, transform)
image_tensor = to_var(image, volatile=True)
# If use gpu
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
# Generate caption from image
feature = encoder(image_tensor)
sampled_ids = decoder.sample(feature)
sampled_ids = sampled_ids.cpu().data.numpy()
# Decode word_ids to words
sampled_caption = []
for word_id in sampled_ids:
word = vocab.idx2word[word_id]
sampled_caption.append(word)
if word == '<end>':
break
sentence = ' '.join(sampled_caption[1:-1])
# Add generated caption to the df.
image_url = image_name.split("/")[-1]
for index, row in df.iterrows():
if type(row["URL"]) is str:
url = row["URL"].split("/")[-1]
print(url, image_url)
if row["URL"].startswith("http") and url == image_url:
df.loc[index, "caption"] = sentence
#caption.append(sentence)
print(sentence)
break
except RuntimeError:
continue
print(len(caption))
print(df)
df.to_csv(args.folder + "../" + "srumtextimageurl_results.csv")
'''image = Image.open(args.image)
def main(args):
# Create model directory
if not os.path.exists(args.model_path):
os.makedirs(args.model_path)
worker_thread_count = 1
retry_for_failed = 2
# Image preprocessing
# For normalization, see https://github.com/pytorch/vision#models
transform = transforms.Compose([
# transforms.RandomCrop(args.crop_size),
# transforms.RandomHorizontalFlip(),
transforms.Scale(args.crop_size),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])
#transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))])
# Load vocabulary wrapper.
with open(args.vocab_path, 'rb') as f:
vocab = pickle.load(f)
# Build data loader
data_loader = get_loader(args.image_dir,
vocab,
transform,
args.batch_size,
shuffle=True,
num_workers=args.num_workers)
# Build the models
encoder = EncoderCNN(args.embed_size)
decoder = DecoderRNN(args.embed_size, args.hidden_size, len(vocab),
args.num_layers)
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
# Loss and Optimizer
criterion = nn.L1Loss()
params = list(decoder.parameters()) + list(
encoder.linear.parameters()) + list(encoder.bn.parameters())
optimizer = torch.optim.Adam(params, lr=args.learning_rate)
# Train the Models
total_step = len(data_loader)
for epoch in range(args.num_epochs):
for i, (images, captions, lengths) in enumerate(data_loader):
processed_items = []
threads = []
has_data_to_process = True
def do_request(item):
position = item['position']
#print(position)
#print(item)
retry = retry_for_failed
while retry:
r = requests.post('http://localhost:4567/', data=item)
if r.status_code == 200:
pil = Image.open(io.BytesIO(r.content)).convert('RGB')
processed_items[position] = transform(pil)
#print(position, processed_items[position])
break
else:
print("shouldb be here")
time.sleep(2)
retry -= 1
# Set mini-batch dataset
image_tensors = to_var(images, volatile=True)
captions = to_var(captions)
targets = pack_padded_sequence(captions, lengths,
batch_first=True)[0]
#print(images.size())
#print(torch.equal(images[0] ,images[1]))
# Forward, Backward and Optimize
decoder.zero_grad()
encoder.zero_grad()
features = encoder(image_tensors)
outputs = decoder(features, captions, lengths)
codes = []
def worker():
while items_to_process.qsize() > 0 or has_data_to_process:
item = items_to_process.get()
if item is None:
break
do_request(item)
items_to_process.task_done()
print("ended thread processing")
for j in range(worker_thread_count):
t = threading.Thread(target=worker)
t.daemon = True # thread dies when main thread (only non-daemon thread) exits.
t.start()
threads.append(t)
for ii, image in enumerate(images):
image_tensor = to_var(image.unsqueeze(0), volatile=True)
feature = encoder(image_tensor)
sampled_ids = decoder.sample(feature)
sampled_ids = sampled_ids.cpu().data.numpy()
sampled_caption = []
for word_id in sampled_ids:
word = vocab.idx2word[word_id]
sampled_caption.append(word)
if word == '<end>':
break
sentence = ' '.join(sampled_caption)
payload = {'code': sentence}
data = {'position': ii, 'code': sentence}
items_to_process.put(data)
processed_items.append('failed')
codes.append(sentence)
has_data_to_process = False
print(codes)
print(items_to_process.qsize())
print(image.size())
print("waiting for threads")
for t in threads:
t.join()
print("done reassembling images")
for t in threads:
t.shutdown = True
t.join()
bad_value = False
for pi in processed_items:
if isinstance(pi, str) and pi == "failed":
bad_value = True
if bad_value == True:
print("failed conversion,skipping batch")
continue
output_tensor = torch.FloatTensor(len(processed_items), 3,
images.size()[2],
images.size()[3])
for ii, image_tensor in enumerate(processed_items):
output_tensor[ii] = processed_items[ii]
output_var = to_var(output_tensor, False)
target_var = to_var(images, False)
#loss = criterion(output_var,target_var)
print("loss")
print(loss)
loss.backward()
optimizer.step()
# Print log info
if i % args.log_step == 0:
print(
'Epoch [%d/%d], Step [%d/%d], Loss: %.4f, Perplexity: %5.4f'
% (epoch, args.num_epochs, i, total_step, loss.data[0],
np.exp(loss.data[0])))
# Save the models
if (i + 1) % args.save_step == 0:
torch.save(
decoder.state_dict(),
os.path.join(args.model_path,
'decoder-%d-%d.pkl' % (epoch + 1, i + 1)))
torch.save(
encoder.state_dict(),
os.path.join(args.model_path,
'encoder-%d-%d.pkl' % (epoch + 1, i + 1)))
def main(args):
# Create model directory
if not os.path.exists(args.model_path):
os.makedirs(args.model_path)
# Image preprocessing
# For normalization, see https://github.com/pytorch/vision#models
transform = transforms.Compose([
transforms.RandomCrop(args.crop_size),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406),
(0.229, 0.224, 0.225))])
# Load vocabulary wrapper.
with open(args.vocab_path, 'rb') as f:
vocab = pickle.load(f)
# Build data loader
data_loader = get_loader(args.image_dir, args.caption_path, vocab,
transform, args.batch_size,
shuffle=True, num_workers=args.num_workers)
# Build the models
encoder = EncoderCNN(args.embed_size)
decoder = DecoderRNN(args.embed_size, args.hidden_size,
len(vocab), args.num_layers)
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
# Loss and Optimizer
criterion = nn.CrossEntropyLoss()
params = list(decoder.parameters()) + list(encoder.linear.parameters()) + list(encoder.bn.parameters())
optimizer = torch.optim.Adam(params, lr=args.learning_rate)
# Train the Models
total_step = len(data_loader)
for epoch in range(args.num_epochs):
for i, (images, captions, lengths) in enumerate(data_loader):
# Set mini-batch dataset
images = to_var(images, volatile=True)
captions = to_var(captions)
targets = pack_padded_sequence(captions, lengths, batch_first=True)[0]
# Forward, Backward and Optimize
decoder.zero_grad()
encoder.zero_grad()
features = encoder(images)
outputs = decoder(features, captions, lengths)
loss = criterion(outputs, targets)
loss.backward()
optimizer.step()
# Print log info
if i % args.log_step == 0:
print('Epoch [%d/%d], Step [%d/%d], Loss: %.4f, Perplexity: %5.4f'
%(epoch, args.num_epochs, i, total_step,
loss.data[0], np.exp(loss.data[0])))
# Save the models
if (i+1) % args.save_step == 0:
torch.save(decoder.state_dict(),
os.path.join(args.model_path,
'decoder-%d-%d.pkl' %(epoch+1, i+1)))
torch.save(encoder.state_dict(),
os.path.join(args.model_path,
'encoder-%d-%d.pkl' %(epoch+1, i+1)))
