def build_set2seq(len_sequence, batch_size, max_value, message_dim,
process_steps):
tag = 'set2seq_bs%d_ls%d_md%d_ps%d' % (batch_size, len_sequence,
message_dim, process_steps),
train_net = '%s_train.prototxt' % tag
solver_net = '%s_solver.prototxt' % tag
bash = 'run_%s.sh' % tag
param_str = {
'len_sequence': len_sequence,
'batch_size': batch_size,
'max_value': max_value,
'top_names': ['rand_data', 'label_data', 'train_label_data'],
'message_dim': message_dim,
'process_steps': process_steps
}
net = set2seq.set2sequence_net(param_str)
net.build_set2seq(param_str, train_net)
solver_params = {
'max_iter': 10000,
'stepsize': 5000,
'snapshot': 10000,
'base_lr': 0.01
}
cn.make_solver(solver_net, train_net, [], **solver_params)
#cn.make_bash_script(bash, solver_net)
return tag
def build_sentence_generation_model(model_id, class_conditional, image_conditional, solver_args={'max_iter': 15000, 'stepsize': 2000, 'snapshot': 1000}):
layer = 'extractGVEFeatures'
save_file_name_base = 'prototxt/%s' %(model_id)
save_file_train = '%s_%s.prototxt' %(save_file_name_base, 'train')
save_file_test_on_train = '%s_%s.prototxt' %(save_file_name_base, 'test_on_train')
save_file_test_on_test = '%s_%s.prototxt' %(save_file_name_base, 'test_on_test')
save_file_deploy = '%s_%s.prototxt' %(save_file_name_base,'deploy')
save_file_solver = '%s_%s.prototxt' %(save_file_name_base,'solver')
save_bash = '%s_%s.sh' %('train', model_id)
data_inputs_train = {}
data_inputs_train['module'] = 'data_layers'
data_inputs_train['layer'] = layer
data_inputs_test = {}
data_inputs_test['module'] = 'data_layers'
data_inputs_test['layer'] = layer
data_inputs_train = data_inputs_train
data_inputs_test = data_inputs_test
label_options = {'vector_file': 'data/description_sentence_features.p'}
param_str_train, param_str_test = make_lrcn_param_str(label_options=label_options)
data_inputs_train['param_str'] = param_str_train
data_inputs_test['param_str'] = param_str_test
model_train = lrcn.lrcn(data_inputs_train, lstm_dim=1000, embed_dim=1000, class_conditional=class_conditional, image_conditional=image_conditional, class_size=200, image_dim=8192)
model_train.make_sentence_generation_net(save_file_train, accuracy=False, loss=True)
cn.make_solver(save_file_solver, [save_file_train], [], **solver_args)
cn.make_bash_script(save_bash, save_file_solver)
def caption_classifier(embed_dim, lstm_dim, embed_drop, lstm_drop):
save_file_name_base = 'caption_classifier_embedDrop_%s_lstmDrop_%s_embedHidden_%s_lstmHidden_%s' %(int(embed_drop*100), int(lstm_drop*100), int(embed_dim), int(lstm_dim))
save_file_train = 'prototxt/%s_%s.prototxt' %(save_file_name_base, 'train')
save_file_test_on_train = 'prototxt/%s_%s.prototxt' %(save_file_name_base, 'test_on_train')
save_file_test_on_test = 'prototxt/%s_%s.prototxt' %(save_file_name_base, 'test_on_test')
save_file_deploy = 'prototxt/%s_deploy.prototxt' %save_file_name_base
save_file_solver = 'prototxt/%s_%s.prototxt' %(save_file_name_base, 'solver')
save_bash = '%s_%s.sh' %('train', save_file_name_base)
data_inputs_train = {}
data_inputs_train['module'] = 'data_layers'
data_inputs_train['layer'] = 'CaptionToLabel'
data_inputs_test = {}
data_inputs_test['module'] = 'data_layers'
data_inputs_test['layer'] = 'CaptionToLabel'
label_options = {'sentence_supervision': 'last', 'label_stream_size': 20}
param_str_train, param_str_test = make_lrcn_class_param_str(label_options=label_options)
data_inputs_train['param_str'] = param_str_train
data_inputs_test['param_str'] = param_str_test
model_train = lrcn.lrcn(data_inputs_train, lstm_dim=lstm_dim, embed_dim=embed_dim)
model_train.caption_classifier(save_file_train, accuracy=False, loss=True, embed_drop=embed_drop, lstm_drop=lstm_drop)
model_train.caption_classifier(save_file_deploy, accuracy=False, loss=False, deploy=True, embed_drop=embed_drop, lstm_drop=lstm_drop)
model_test_on_train = lrcn.lrcn(data_inputs_train, lstm_dim=lstm_dim, embed_dim=embed_dim)
model_test_on_train.caption_classifier(save_file_test_on_train, accuracy=True, loss=False)
model_test_on_test = lrcn.lrcn(data_inputs_test, lstm_dim=lstm_dim, embed_dim=embed_dim)
model_test_on_test.caption_classifier(save_file_test_on_test, accuracy=True, loss=False)
cn.make_solver(save_file_solver, [save_file_train], [save_file_test_on_train, save_file_test_on_test], **{'base_lr': 0.1, 'stepsize': 2000, 'max_iter': 6000})
cn.make_bash_script(save_bash, save_file_solver)
def sentence_generation_reinforce(save_file_name, weights=None, orig_proto=None, classify_model=None, classify_weights=None, RL_loss='lstm_classification', class_conditional=True, lw=20):
save_file_name_base = 'prototxt/%s' %save_file_name
save_file_train = '%s_%s.prototxt' %(save_file_name_base, 'train')
save_file_test_on_train = '%s_%s.prototxt' %(save_file_name_base, 'test_on_train')
save_file_test_on_test = '%s_%s.prototxt' %(save_file_name_base, 'test_on_test')
save_file_deploy = '%s_%%s.prototxt' %save_file_name_base
save_file_solver = '%s_%s.prototxt' %(save_file_name_base, 'solver')
save_bash = '%s_%s.sh' %('train', save_file_name)
label_options = {'vector_file': 'data/description_sentence_features.p'}
data_layer = 'extractGVEFeatures'
data_inputs_train = {}
data_inputs_train['module'] = 'data_layers'
data_inputs_train['layer'] = data_layer
data_inputs_test = {}
data_inputs_test['module'] = 'data_layers'
data_inputs_test['layer'] = data_layer
param_str_train, param_str_test = make_lrcn_param_str(label_options=label_options, data_layer=data_layer)
data_inputs_train['param_str'] = param_str_train
data_inputs_test['param_str'] = param_str_test
model_train = reinforce.reinforce(data_inputs_train, cc=class_conditional, baseline=False, separate_sents=True)
model_train.lrcn_reinforce(save_name=save_file_train, RL_loss=RL_loss, lw=lw)
model_lm_deploy = reinforce.reinforce(data_inputs_test, cc=class_conditional, T=1)
model_lm_deploy.lrcn_reinforce_wtd_deploy(save_name=save_file_deploy %'wtd')
cn.make_solver(save_file_solver, [save_file_train], [],
**{'base_lr': 0.001, 'stepsize': 2000, 'max_iter': 10000, 'snapshot': 1000})
if weights:
ind_model_weights = transfer_net_weights(orig_proto, weights, save_file_train)
save_file_train = transfer_combine_weights(save_file_train, classify_model, ind_model_weights, classify_weights)
if weights:
cn.make_bash_script(save_bash, save_file_solver, weights=save_file_train)
else:
cn.make_bash_script(save_bash, save_file_solver)