def main():
data = load_coco_data(data_path='./data', split='val')
with open('./data/train/word_to_idx.pkl', 'rb') as f:
word_to_idx = pickle.load(f)
model = CaptionGenerator(word_to_idx,
dim_feature=[196, 512],
dim_embed=512,
dim_hidden=1024,
n_time_step=16,
prev2out=True,
ctx2out=True,
alpha_c=1.0,
selector=True,
dropout=True)
solver = CaptioningSolver(model,
data,
data,
n_epochs=20,
batch_size=128,
update_rule='adam',
learning_rate=0.001,
print_every=1000,
save_every=1,
image_path='./image/val2014_resized',
pretrained_model=None,
model_path='model/lstmval/',
test_model='model/lstm/model-10',
print_bleu=True,
log_path='log/')
solver.test(data, split='val')
def main():
# load train dataset
with open("/nfs/private/cas/dataset_0_N/week/train_features.pkl",
"rb") as f:
train_features = pickle.load(f)
with open("/nfs/private/cas/dataset_0_N/week/train_labels.pkl", "rb") as f:
train_labels = pickle.load(f)
with open("/nfs/private/cas/dataset_0_N/week/test_features.pkl",
"rb") as f:
test_features = pickle.load(f)
with open("/nfs/private/cas/dataset_0_N/week/test_labels.pkl", "rb") as f:
test_labels = pickle.load(f)
word_to_idx = {"<START>": -3, "<END>": -2, "<NULL>": -1}
dim_feature = train_features.shape[1]
n_time_step = train_labels.shape[1] - 1
print "n_time_step:%d" % n_time_step
model = CaptionGenerator(word_to_idx,
V=int(np.max(train_features) + 1),
dim_feature=dim_feature,
dim_embed=128,
dim_hidden=128,
n_time_step=n_time_step,
prev2out=True,
ctx2out=True,
alpha_c=1.0,
selector=True,
dropout=True)
data = {"features": train_features, "labels": train_labels}
val_data = {"features": test_features, "labels": test_labels}
solver = CaptioningSolver(
model,
data,
val_data,
n_epochs=50000,
batch_size=100,
update_rule='adam',
learning_rate=1e-4,
print_every=100,
save_every=10,
image_path='./image/',
pretrained_model=None,
model_path='./model/0_N/cnn/week/',
test_model=
'/ais/gobi5/linghuan/basic-attention/model/lstm/lstm/model-19',
print_bleu=True,
log_path='./log/')
solver.train()
def main():
# load train dataset
data = load_coco_data(data_path='./data', split='train')
word_to_idx = data['word_to_idx']
model = CaptionGenerator(word_to_idx,
dim_feature=[196, 512],
dim_embed=512,
dim_hidden=1024,
n_time_step=16,
prev2out=True,
ctx2out=True,
alpha_c=1.0,
selector=True,
dropout=True)
discrim = Discriminator(word_to_idx,
dim_feature=[196, 512],
dim_embed=512,
dim_hidden=1024,
n_time_step=16,
prev2out=True,
ctx2out=True,
alpha_c=0.0,
selector=True,
dropout=True,
learning_rate=0.01)
solver = CaptioningSolver(model,
discrim,
data,
data,
n_epochs=20,
batch_size=128,
gpu_list="1,2,3",
update_rule='adam',
learning_rate=0.001,
print_every=1000,
save_every=1,
image_path='./image/',
pretrained_model=None,
model_path='model/lstm/',
train_new='./model/lstm/model-20',
test_model='model/lstm/model-21',
print_bleu=True,
log_path='log/')
solver.train()
def __init__(self, sess, model_path, use_inception):
path_prefix = os.path.dirname(os.path.realpath(__file__))
# word to index mapping
with open(os.path.join(path_prefix, 'data/train/word_to_idx.pkl'),
"rb") as f:
self.word_to_idx = pickle.load(f)
if use_inception:
L = 64
D = 2048
cnn_model_path = os.path.join(path_prefix,
'data/inception_v3.ckpt')
else:
L = 196
D = 512
cnn_model_path = os.path.join(
path_prefix, './data/imagenet-vgg-verydeep-19.mat')
self.batch_size = 128
self.sess = sess
self.use_inception = use_inception
print("Creating model...")
self.model = CaptionGenerator(
self.word_to_idx,
dim_feature=[L, D],
dim_embed=512,
dim_hidden=1800,
n_time_step=16,
prev2out=True,
ctx2out=True,
alpha_c=5.0,
selector=True,
dropout=True,
use_cnn="inception" if use_inception else "vgg",
cnn_model_path=cnn_model_path)
print("Loading CNN weights...")
self.model.cnn.load_weights(sess)
print("Building sampler...")
self.alphas, self.betas, self.generated_captions = self.model.build_sampler(
max_len=20)
# initialize model and load weights
print("Loading LSTM weights...")
# tf.global_variables_initializer().run()
saver = tf.train.Saver(self.model.sampler_vars)
saver.restore(sess, model_path)
def main():
# load train dataset
# data = load_coco_data(data_path='./our_data', split='train')
# word_to_idx = data['word_to_idx']
# # load val dataset to print out bleu scores every epoch
# test_data = load_coco_data(data_path='./our_data', split='test')
#our train:
data = load_coco_data(
data_path='.\image_data_to_be_labeled\Object_feature\our_data',
split='train')
our_test = load_coco_data(
data_path='.\image_data_to_be_labeled\Object_feature\our_data',
split='train')
word_to_idx = data['word_to_idx']
model = CaptionGenerator(word_to_idx,
dim_feature=[216, 512],
dim_embed=512,
dim_hidden=1024,
n_time_step=26,
prev2out=True,
ctx2out=True,
alpha_c=1.0,
selector=True,
dropout=False)
solver = CaptioningSolver(model,
data,
our_test,
n_epochs=5000,
batch_size=64,
update_rule='adam',
learning_rate=1e-4,
print_every=1000,
save_every=100,
image_path='./image/',
pretrained_model=None,
model_path='model/our_train0414/',
test_model='model/our_train0414/model-2000',
print_bleu=False,
log_path='log/')
# solver.train()
solver.test(our_test)
def main():
# load train dataset
print '@@@@@@@@@@@@@'
data = load_data(data_path='./data', split='train')
label_to_idx = data['label_to_idx']
print '@@@@@'
# load val dataset to print out bleu scores every epoch
#val_data = load_data(data_path='./data', split='val')
val_data = []
print label_to_idx
print data['labels']
model = CaptionGenerator(label_to_idx,
dim_feature=[196, 512],
dim_embed=512,
dim_hidden=1024,
n_time_step=30,
prev2out=True,
ctx2out=True,
alpha_c=1.0,
selector=True,
dropout=True)
solver = CaptioningSolver(model,
data,
val_data,
n_epochs=1,
batch_size=1,
update_rule='adam',
learning_rate=0.001,
print_every=3,
save_every=1,
image_path='./image/',
pretrained_model=None,
model_path='model/lstm/',
test_model='model/lstm/model-10',
print_bleu=True,
log_path='log/')
print 'model + solver'
solver.train()
def main():
# check if there are new images in pred_folder
if not os.path.exists(pred_folder):
os.makedirs(pred_folder)
new_images = []
for img in os.listdir(pred_folder):
if img[0] == '.':
continue
new_images.append(img)
if new_images:
shutil.rmtree(resized_folder)
os.makedirs(resized_folder)
for img in new_images:
with open(os.path.join(pred_folder, img), 'r+b') as f:
with Image.open(f) as image:
image = resize_image(image)
image.save(os.path.join(resized_folder, img))
shutil.rmtree(pred_folder)
os.makedirs(pred_folder)
prep = PreProcessor(batch_size=5,
max_length=15,
word_count_threshold=1,
cnn_model_path='data/imagenet-vgg-verydeep-19.mat')
prep.run('pred')
with open('./data/train/word_to_idx.pkl', 'rb') as f:
word_to_idx = pickle.load(f)
data = load_data(data_path='./data', split='pred')
model = CaptionGenerator(word_to_idx,
feature_dim=[196, 512],
embed_dim=512,
hidden_dim=1024,
len_sent=16,
lamda=1.0)
solver = CaptioningSolver(model, data, None, n_epochs=5, batch_size=5, learning_rate=0.001, print_every=1,\
save_every=1, image_path='./image/pred_resized', model_path='./model/lstm', test_model='./model/lstm/model-5')
return solver.test(
data, display=True) # display True to print results in terminal
def main():
# train_data = load_data(current_path + 'data_set/', 'test')
# length = len(train_data['video_ids'])
# train_data['features'] = train_data['features'][:int(0.7 * length)]
# train_data['labels'] = train_data['labels'][:int(0.7 * length)]
# train_data['video_ids'] = train_data['video_ids'][:int(0.7 * length)]
# train_data['video_filenames'] = train_data['video_filenames'][:int(0.7 * length)]
with open('train_data_vgg.pkl', 'rb') as handle:
train_data = pickle.load(handle)
# length = len(train_data['new_filename'])
train_data['features'] = train_data['features']
train_data['labels'] = train_data['labels']
train_data['video_ids'] = train_data['new_filename']
# train_data['video_filenames'] = train_data['video_filenames'][:int(0.7 * length)]
if train_for_test == 1:
with open('val_data_vgg.pkl', 'rb') as handle:
val_data = pickle.load(handle)
# length = len(train_data['new_filename'])
train_data['features'] = np.concatenate(
(train_data['features'], val_data['features']), axis=0)
train_data['labels'] = np.concatenate(
(train_data['labels'], val_data['labels']), axis=0)
train_data['video_ids'] = np.concatenate(
(train_data['new_filename'], val_data['new_filename']), axis=0)
# train_data = {}
data = {'train_data': train_data}
# label_to_idx = load_pickle(current_path + 'data_set/label_to_idx.pkl')
label_to_idx = load_pickle('labels_to_idx.pkl')
num_images_per_video = 17
model = CaptionGenerator(
label_to_idx=label_to_idx,
# dim_feature=[49, 1280],
dim_feature=[196, 512],
dim_hidden=1024,
n_time_step=num_images_per_video,
ctx2out=True,
alpha_c=1.0,
selector=True,
dropout=False)
solver = CaptioningSolver(model,
data,
n_epochs=300,
batch_size=15,
update_rule='adam',
learning_rate=0.0006,
print_every=3,
save_every=10,
pretrained_model=None,
model_path=current_path + 'model/lstm/',
test_model=current_path + 'model/lstm/model-310',
log_path=current_path + 'log/',
data_path=current_path + '/data_set/',
test_result_save_path=current_path +
'data_set/test/model_test_result/',
models_val_disp=current_path +
'model/lstm/models_accuracy_val.txt')
if train == 1:
solver.train()
if validate == 1:
solver.all_model_val()
if test == 1:
solver.test()
if train_for_test == 1:
solver.train()
def main(params):
loader = LoadData(input_json=params.input_labels,
input_h5=params.input_caps,
feature_path=params.input_feats,
batch_img=params.batch_img,
seq_per_img=params.seq_per_img,
train_only=params.train_only)
if params.mode == "EVAL":
model = CaptionGenerator(loader.word_to_idx,
num_features=params.num_objects,
dim_feature=params.dim_features,
dim_embed=params.dim_word_emb,
dim_hidden=params.rnn_hid_size,
dim_attention=params.att_hid_size,
n_time_step=loader.seq_length - 1)
solver = CaptioningSolver(data_loader=loader,
model=model,
ngram_file=params.input_ngram,
n_epochs=params.epoch,
update_rule=params.optimizer,
learning_rate=params.lr,
print_every=params.print_every,
start_epoch=params.start_epoch,
log_path=params.log_path,
model_path=params.model_path,
pretrained_model=params.pretrained,
test_model=params.test_model)
solver.output_result_for_eval()
elif params.mode == "TEST":
model = CaptionGenerator(loader.word_to_idx,
num_features=params.num_objects,
dim_feature=params.dim_features,
dim_embed=params.dim_word_emb,
dim_hidden=params.rnn_hid_size,
dim_attention=params.att_hid_size,
n_time_step=loader.seq_length - 1)
solver = CaptioningSolver(data_loader=loader,
model=model,
ngram_file=params.input_ngram,
n_epochs=params.epoch,
update_rule=params.optimizer,
learning_rate=params.lr,
print_every=params.print_every,
start_epoch=params.start_epoch,
log_path=params.log_path,
model_path=params.model_path,
pretrained_model=params.pretrained,
test_model=params.test_model)
solver.test()
elif params.mode == "TRAIN":
with tf.Graph().as_default():
model = CaptionGenerator(loader.word_to_idx,
num_features=params.num_objects,
dim_feature=params.dim_features,
dim_embed=params.dim_word_emb,
dim_hidden=params.rnn_hid_size,
dim_attention=params.att_hid_size,
n_time_step=loader.seq_length - 1)
solver = CaptioningSolver(data_loader=loader,
model=model,
ngram_file=params.input_ngram,
n_epochs=params.epoch,
update_rule=params.optimizer,
learning_rate=params.lr,
print_every=params.print_every,
start_epoch=params.start_epoch,
log_path=params.log_path,
model_path=params.model_path,
pretrained_model=params.pretrained,
test_model=params.test_model)
solver.train()
with tf.Graph().as_default():
model = CaptionGenerator(loader.word_to_idx,
num_features=params.num_objects,
dim_feature=params.dim_features,
dim_embed=params.dim_word_emb,
dim_hidden=params.rnn_hid_size,
dim_attention=params.att_hid_size,
n_time_step=loader.seq_length - 1)
solver = CaptioningSolver(data_loader=loader,
model=model,
ngram_file=params.input_ngram,
n_epochs=params.epoch,
update_rule=params.optimizer,
learning_rate=params.lr,
print_every=params.print_every,
start_epoch=params.start_epoch,
log_path=params.log_path,
model_path=params.model_path,
pretrained_model=params.pretrained,
test_model=params.test_model)
solver.train_reinforce()