def load_data(self):
if self.signature == 'youtube2text':
print 'loading youtube2text %s features'%self.video_feature
dataset_path = common.get_rab_dataset_base_path()+'msvd_data/'
self.train = common.load_pkl(dataset_path + 'train.pkl')
self.valid = common.load_pkl(dataset_path + 'valid.pkl')
self.test = common.load_pkl(dataset_path + 'test.pkl')
self.CAP = common.load_pkl(dataset_path + 'CAP.pkl')
self.train_ids = ['vid%s'%i for i in range(1,1201)]
self.valid_ids = ['vid%s'%i for i in range(1201,1301)]
self.test_ids = ['vid%s'%i for i in range(1301,1971)]
else:
raise NotImplementedError()
self.worddict = common.load_pkl(dataset_path + 'worddict.pkl')
self.word_idict = dict()
# wordict start with index 2
for kk, vv in self.worddict.iteritems():
self.word_idict[vv] = kk
self.word_idict[0] = '<eos>'
self.word_idict[1] = 'UNK'
if self.video_feature == 'googlenet':
self.ctxg_dim = 1024#global_feature dimension
self.ctxl_dim = 4096#local_feature dimension
self.ctxglm_dim = 1024#fused dimension
self.ctxm_dim = 4096#motion_feature dimension
else:
raise NotImplementedError()
self.kf_train = common.generate_minibatch_idx(
len(self.train), self.mb_size_train)
self.kf_valid = common.generate_minibatch_idx(
len(self.valid), self.mb_size_test)
self.kf_test = common.generate_minibatch_idx(
len(self.test), self.mb_size_test)
def train_from_scratch(config, state, channel):
# Model options
save_model_dir = config[config.model].save_model_dir
np.random.seed(int(config.random_seed))
if save_model_dir == 'current':
config[config.model].save_model_dir = './'
save_model_dir = './'
# to facilitate the use of cluster for multiple jobs
save_path = './model_config.pkl'
else:
# run locally, save locally
save_path = os.path.join(save_model_dir, 'model_config.pkl')
print 'current save dir ', save_model_dir
common.create_dir_if_not_exist(save_model_dir)
reload_ = config[config.model].reload_
if reload_:
print 'preparing reload'
save_dir_backup = config[config.model].save_model_dir
from_dir_backup = config[config.model].from_dir
# never start retrain in the same folder
assert save_dir_backup != from_dir_backup
print 'save dir ', save_dir_backup
print 'from_dir ', from_dir_backup
print 'setting current model config with the old one'
if config[config.model].mode == 'train':
model_config_old = common.load_pkl(from_dir_backup +
'/model_config.pkl')
set_config(config, model_config_old)
config[config.model].save_model_dir = save_dir_backup
config[config.model].from_dir = from_dir_backup
config[config.model].reload_ = True
if config.erase_history:
print 'erasing everything in ', save_model_dir
os.system('rm %s/*' % save_model_dir)
# for stdout file logging
#sys.stdout = Unbuffered(sys.stdout, state.save_model_path + 'stdout.log')
print 'saving model config into %s' % save_path
common.dump_pkl(config, save_path)
# Also copy back from config into state.
for key in config:
setattr(state, key, config[key])
model_type = config.model
print 'Model Type: %s' % model_type
print 'Host: %s' % socket.gethostname()
print 'Command: %s' % ' '.join(sys.argv)
if config.model == 'attention':
model_attention.train_from_scratch(state, channel)
elif config.model == 'lstmdd':
model_lstmdd.train_from_scratch(state, channel)
elif config.model == 'mtle':
model_mtle.train_from_scratch(state, channel)
else:
raise NotImplementedError()
def save_test_samples_lsmdc(samples_test, engine): # for lsmdc16 challenge
out_dir = 'predictions/' + engine.signature + '_' + engine.video_feature + '_' + engine.model_type + '/'
if not os.path.exists('predictions/'):
os.mkdir('predictions/')
if not os.path.exists(out_dir):
os.mkdir(out_dir)
f = open(out_dir + 'samplestest.csv', 'wr')
gts_test = OrderedDict()
results = OrderedDict()
results['version'] = "1"
dict_path = os.path.join(
'/p/work2/projects/ryat/datasets/vid-desc/lsmdc16/pkls16',
'dict_vids_mapping.pkl')
vids_names = common.load_pkl(dict_path)
# D= None
# if engine.signature=='youtube2text':
# import cPickle
# d= open('data/youtube2text_iccv15/original/dict_youtube_mapping.pkl','rb')
# D = cPickle.load(d)
# D = dict((y,x) for x,y in D.iteritems())
samples = []
# for vidID in engine.test_ids:
for vidID in samples_test.keys():
gts_test[vidID] = engine.CAP[vidID]
# print samples_test[vidID]
sample = OrderedDict()
sample['video_id'] = vids_names[vidID]
# sample['ovid_id']=vidID
sample['caption'] = samples_test[vidID][0]['caption']
# sample['ocaption']=gts_test[vidID][0]['caption']
samples.append(sample)
# if engine.signature=='youtube2text':
# f.write(D[vidID]+','+ samples_test[vidID][0]['caption']+','+gts_test[vidID][0]['caption']+'\n')
# else:
f.write(vidID + ',' + samples_test[vidID][0]['caption'] + ',' +
gts_test[vidID][0]['caption'] + '\n')
f.close()
# results['result']= samples
# results['external_data']={'used': 'true','details':'First fully connected of C3D pretrained on Sports1M'}
samples = sorted(samples, key=lambda x: x['video_id'])
import json
with open(out_dir + 'publictest_burka_results.json', 'w') as outfile:
json.dump(samples, outfile, indent=4)
def load_data(self):
if self.signature == 'youtube2text':
print 'loading youtube2text %s features'%self.video_feature
dataset_path = common.get_rab_dataset_base_path()+'youtube2text_iccv15/'
self.train = common.load_pkl(dataset_path + 'train.pkl')
self.valid = common.load_pkl(dataset_path + 'valid.pkl')
self.test = common.load_pkl(dataset_path + 'test.pkl')
self.CAP = common.load_pkl(dataset_path + 'CAP.pkl')
self.FEAT = common.load_pkl(dataset_path + 'FEAT_key_vidID_value_features.pkl')
self.train_ids = ['vid%s'%i for i in range(1,1201)]
self.valid_ids = ['vid%s'%i for i in range(1201,1301)]
self.test_ids = ['vid%s'%i for i in range(1301,1971)]
else:
raise NotImplementedError()
self.worddict = common.load_pkl(dataset_path + 'worddict.pkl')
self.word_idict = dict()
# wordict start with index 2
for kk, vv in self.worddict.iteritems():
self.word_idict[vv] = kk
self.word_idict[0] = '<eos>'
self.word_idict[1] = 'UNK'
if self.video_feature == 'googlenet':
self.ctx_dim = 1024
else:
raise NotImplementedError()
self.kf_train = common.generate_minibatch_idx(
len(self.train), self.mb_size_train)
self.kf_valid = common.generate_minibatch_idx(
len(self.valid), self.mb_size_test)
self.kf_test = common.generate_minibatch_idx(
len(self.test), self.mb_size_test)
def train_from_scratch(config, state, channel):
# Model options
save_model_dir = config[config.model].save_model_dir
if save_model_dir == 'current':
config[config.model].save_model_dir = './'
save_model_dir = './'
# to facilitate the use of cluster for multiple jobs
save_path = './model_config.pkl'
else:
# run locally, save locally
save_path = save_model_dir + 'model_config.pkl'
print 'current save dir ',save_model_dir
common.create_dir_if_not_exist(save_model_dir)
reload_ = config[config.model].reload_
if reload_:
print 'preparing reload'
save_dir_backup = config[config.model].save_model_dir
from_dir_backup = config[config.model].from_dir
# never start retrain in the same folder
assert save_dir_backup != from_dir_backup
print 'save dir ',save_dir_backup
print 'from_dir ',from_dir_backup
print 'setting current model config with the old one'
model_config_old = common.load_pkl(from_dir_backup+'/model_config.pkl')
set_config(config, model_config_old)
config[config.model].save_model_dir = save_dir_backup
config[config.model].from_dir = from_dir_backup
config[config.model].reload_ = True
if config.erase_history:
print 'erasing everything in ',save_model_dir
os.system('rm %s/*'%save_model_dir)
# for stdout file logging
#sys.stdout = Unbuffered(sys.stdout, state.save_model_path + 'stdout.log')
print 'saving model config into %s'%save_path
common.dump_pkl(config, save_path)
# Also copy back from config into state.
for key in config:
setattr(state, key, config[key])
model_type = config.model
print 'Model Type: %s'%model_type
print 'Host: %s' % socket.gethostname()
print 'Command: %s' % ' '.join(sys.argv)
if config.model == 'attention':
model_attention.train_from_scratch(state, channel)
else:
raise NotImplementedError()
def save_blind_test_samples(samples_test, engine): # for lsmdc16 challenge
out_dir = 'submissions/' + engine.signature + '_' + engine.video_feature + '_' + engine.model_type + '/'
if not os.path.exists('submissions/'):
os.mkdir('submissions/')
if not os.path.exists(out_dir):
os.mkdir(out_dir)
# f=open(out_dir+'samplesbtest.csv','wr')
gts_test = OrderedDict()
results = OrderedDict()
results['version'] = "1"
dict_path = os.path.join('data/lsmdc16/', 'dict_bvids_mapping.pkl')
vids_names = common.load_pkl(dict_path)
samples = []
# for vidID in engine.test_ids:
for vidID in samples_test.keys():
# gts_test[vidID] = engine.CAP[vidID]
sample = OrderedDict()
sample['video_id'] = vids_names[vidID]
sample['caption'] = samples_test[vidID][0]['caption']
samples.append(sample)
# f.write(vidID+','+ samples_test[vidID][0]['caption']+','+gts_test[vidID][0]['caption']+'\n')
# f.close()
samples = sorted(samples, key=lambda x: x['video_id'])
import json
with open(out_dir + 'blindtest_burka_results.json', 'w') as outfile:
json.dump(samples, outfile, indent=4)
def load_data(self):
if self.signature == 'youtube2text':
print 'loading youtube2text %s features' % self.video_feature
dataset_path = common.get_rab_dataset_base_path()
self.train = common.load_pkl(dataset_path + 'train.pkl')
self.valid = common.load_pkl(dataset_path + 'valid.pkl')
self.test = common.load_pkl(dataset_path + 'test.pkl')
self.CAP = common.load_pkl(dataset_path + 'CAP.pkl')
self.FEAT = common.load_pkl(dataset_path +
'FEAT_key_vidID_value_features.pkl')
self.VALUES = common.load_pkl(
'../preprocessing/video_values_conv.pkl'
) #dataset_path + 'FEAT_key_vidID_value_features.pkl')
self.train_ids = ['vid%s' % i for i in range(1, 1201)]
self.valid_ids = ['vid%s' % i for i in range(1201, 1301)]
self.test_ids = ['vid%s' % i for i in range(1301, 1971)]
else:
raise NotImplementedError()
self.worddict = common.load_pkl(dataset_path + 'worddict.pkl')
self.word_idict = dict()
# wordict start with index 2
for kk, vv in self.worddict.iteritems():
self.word_idict[vv] = kk
self.word_idict[0] = '<eos>'
self.word_idict[1] = 'UNK'
if self.video_feature == 'googlenet':
self.ctx_dim = 1024
else:
raise NotImplementedError()
self.kf_train = common.generate_minibatch_idx(len(self.train),
self.mb_size_train)
self.kf_valid = common.generate_minibatch_idx(len(self.valid),
self.mb_size_test)
self.kf_test = common.generate_minibatch_idx(len(self.test),
self.mb_size_test)
def load_data(self):
if self.signature == 'youtube2text':
print 'loading youtube2text %s features' % self.video_feature
dataset_path = common.get_rab_dataset_base_path() + 'data/'
self.train = common.load_pkl(dataset_path + 'train.pkl')
self.valid = common.load_pkl(dataset_path + 'valid.pkl')
self.test = common.load_pkl(dataset_path + 'test.pkl')
self.CAP = common.load_pkl(dataset_path + 'CAP.pkl')
self.s_o = common.load_pkl(
dataset_path + 'sem_obj_new.pkl') # load extracted visual tags
self.train_ids = ['vid%s' % i for i in range(1, 1201)]
self.valid_ids = ['vid%s' % i for i in range(1201, 1301)]
self.test_ids = ['vid%s' % i for i in range(1301, 1971)]
else:
raise NotImplementedError()
self.worddict = common.load_pkl(dataset_path + 'worddict.pkl')
# adding the 3000 words in visual genome to our vocabulary
with open(dataset_path + 'vg_list', 'r') as f:
line = f.readline()
line = line[:-1]
semantic_object = []
while line:
semantic_object.append(line)
line = f.readline()
line = line[:-1]
new_s_o = []
new_s_os = []
for o in semantic_object:
new_s_o = o.split('_')
for i in new_s_o:
if i not in new_s_os:
new_s_os.append(i)
new_s_os_l = len(new_s_os)
for oo in new_s_os:
word_l = len(self.worddict)
# if oo not in self.word_idict.values():
if self.worddict.has_key(oo):
continue
else:
self.worddict[oo] = word_l
self.word_idict = dict()
# wordict start with index 2
for kk, vv in self.worddict.iteritems():
self.word_idict[vv] = kk
self.word_idict[0] = '<eos>'
self.word_idict[1] = 'UNK'
if self.video_feature == 'resnet152':
self.ctx_dim = 4096
else:
raise NotImplementedError()
self.kf_train = common.generate_minibatch_idx(len(self.train),
self.mb_size_train)
self.kf_valid = common.generate_minibatch_idx(len(self.valid),
self.mb_size_test)
self.kf_test = common.generate_minibatch_idx(len(self.test),
self.mb_size_test)
def load_data(self):
if self.signature == 'youtube2text' or self.signature == 'trecvid':
print 'loading {} {} features'.format(self.signature, self.video_feature)
if self.data_dir=='':
dataset_path = common.get_rab_dataset_base_path()+'youtube2text/'+self.video_feature
else:
dataset_path = self.data_dir
# dataset_path = common.get_rab_dataset_base_path()
self.train = common.load_pkl(os.path.join(dataset_path ,'train.pkl'))
self.valid = common.load_pkl(os.path.join(dataset_path ,'valid.pkl'))
self.test = common.load_pkl(os.path.join(dataset_path ,'test.pkl'))
self.CAP = common.load_pkl(os.path.join(dataset_path , 'CAP.pkl'))
# self.FEAT = common.load_pkl(os.path.join(dataset_path , 'FEAT_key_vidID_value_features_'+self.proc+'.pkl'))
self.load_feats(dataset_path)
self.train_ids = list(set(self.train[i].split('_')[0] for i in range(len(self.train))))
self.valid_ids = list(set(self.valid[i].split('_')[0] for i in range(len(self.valid))))
self.test_ids = list(set(self.test[i].split('_')[0] for i in range(len(self.test))))
elif self.signature == 'lsmdc' or self.signature == 'lsmdc16' or self.signature == 'mvad' or self.signature == 'mpii' or self.signature == 'tacos':
print 'loading {} {} features'.format(self.signature, self.video_feature)
dataset_path = self.data_dir
self.train = common.load_pkl(os.path.join(dataset_path, 'train.pkl'))
self.valid = common.load_pkl(os.path.join(dataset_path, 'valid.pkl'))
self.test = common.load_pkl(os.path.join(dataset_path, 'test.pkl'))
self.CAP = common.load_pkl(os.path.join(dataset_path, 'CAP.pkl'))
self.train_ids = self.train
self.valid_ids = self.valid
self.test_ids = self.test
if self.signature == 'lsmdc16':
self.btest = common.load_pkl(os.path.join(dataset_path, 'blindtest.pkl'))
self.btest_ids = self.btest
elif self.signature == 'ysvd':
print 'loading ysvd %s features'%self.video_feature
dataset_path = common.get_rab_dataset_base_path()+'ysvd/'
self.all = common.load_pkl(os.path.join(dataset_path, 'all_vids.pkl'))
self.CAP = common.load_pkl(os.path.join(dataset_path, 'CAP.pkl'))
self.FEAT = common.load_pkl(os.path.join(dataset_path, 'FEAT_key_vidID_value_features.pkl'))
self.train = self.all[0:500]
self.valid = self.all[501:750]
self.test = self.all[751:1000]
self.train_ids = self.train
self.valid_ids = self.valid
self.test_ids = self.test
elif self.signature == 'vtt16' or self.signature == 'vtt17':
print 'loading {} {} features'.format(self.signature, self.video_feature)
if self.data_dir=='':
dataset_path = common.get_rab_dataset_base_path()+'vtt/'+self.video_feature
else:
dataset_path = self.data_dir
self.train = common.load_pkl(os.path.join(dataset_path, 'train.pkl'))
self.valid = common.load_pkl(os.path.join(dataset_path, 'valid.pkl'))
self.test = common.load_pkl(os.path.join(dataset_path, 'test.pkl'))
self.CAP = common.load_pkl(os.path.join(dataset_path, 'CAP.pkl'))
self.load_feats(dataset_path)
# Get list of just the videoID, instead of videoID_CapID. Use set to ignore duplicates, then recast to list
self.train_ids = list(set(self.train[i].split('_')[0] for i in range(len(self.train))))
self.valid_ids = list(set(self.valid[i].split('_')[0] for i in range(len(self.valid))))
self.test_ids = list(set(self.test[i].split('_')[0] for i in range(len(self.test))))
self.test_ids = self.test_ids #only for testing
else:
raise NotImplementedError()
self.worddict = common.load_pkl(os.path.join(dataset_path ,'worddict.pkl'))
self.word_idict = dict()
# wordict start with index 2
for kk, vv in self.worddict.iteritems():
self.word_idict[vv] = kk
self.word_idict[0] = '<eos>'
self.word_idict[1] = 'UNK'
if self.video_feature == 'googlenet':
self.ctx_dim = 1024
elif self.video_feature == 'resnet' or self.video_feature == 'resnet152':
if self.proc=='nostd':
self.ctx_dim = 2048
elif self.proc=='pca':
self.ctx_dim=1024
elif self.video_feature == 'nasnetalarge':
self.ctx_dim = 4032
elif self.video_feature == 'pnasnet5large':
self.ctx_dim = 4320
elif self.video_feature == 'polynet':
self.ctx_dim = 2048
elif self.video_feature == 'senet154':
self.ctx_dim = 2048
elif self.video_feature == 'densenet121':
raise NotImplementedError()
elif self.video_feature == 'c3d':
if self.proc=='nostd':
self.ctx_dim = 4101
elif self.proc=='pca':
self.ctx_dim=1024
elif self.video_feature == 'c3d_resnet':
if self.proc=='nostd':
self.ctx_dim = 6149
elif self.proc=='pca':
self.ctx_dim=2048
elif self.proc=='pca512':
self.ctx_dim=1024
elif self.proc=='pca_c3d':
self.ctx_dim=3072
else:
raise NotImplementedError()
print "ctx_dim: "+str(self.ctx_dim)
self.kf_train = common.generate_minibatch_idx(
len(self.train), self.mb_size_train)
self.kf_valid = common.generate_minibatch_idx(
len(self.valid), self.mb_size_test)
self.kf_test = common.generate_minibatch_idx(
len(self.test), self.mb_size_test)
if self.dec == 'multi-stdist':
self.skip_vectors = common.load_pkl(os.path.join(dataset_path,'skip_vectors.pkl'))
def load_feats(self,dataset_path):
if self.video_feature=='c3d':
if self.proc=='pca':
self.FEAT = common.load_pkl(os.path.join(dataset_path , 'FEATS_c3d_'+self.proc+'.pkl'))
elif self.proc=='pca512':
self.FEAT = common.load_pkl(os.path.join(dataset_path , 'FEATS_c3d_'+self.proc+'.pkl'))
elif self.proc=='pca_c3d':
self.FEAT = common.load_pkl(os.path.join(dataset_path , 'FEATS_c3d_pca.pkl'))
else:
self.FEAT = common.load_pkl(os.path.join(dataset_path , 'FEATS_c3d.pkl'))
elif self.video_feature=='c3d_resnet':
if self.proc=='pca':
self.FEAT = common.load_pkl(os.path.join(dataset_path , 'FEATS_c3d_'+self.proc+'.pkl'))
self.FEAT2 = common.load_pkl(os.path.join(dataset_path , 'FEATS_resnet_'+self.proc+'.pkl'))
elif self.proc=='pca512':
self.FEAT = common.load_pkl(os.path.join(dataset_path , 'FEATS_c3d_'+self.proc+'.pkl'))
self.FEAT2 = common.load_pkl(os.path.join(dataset_path ,'FEATS_resnet_'+self.proc+'.pkl'))
elif self.proc=='pca_c3d':
self.FEAT = common.load_pkl(os.path.join(dataset_path , 'FEATS_c3d_pca.pkl'))
self.FEAT2 = common.load_pkl(os.path.join(dataset_path ,'FEATS_resnet_nostd.pkl'))
else:
self.FEAT = common.load_pkl(os.path.join(dataset_path , 'FEATS_c3d.pkl'))
self.FEAT2 = common.load_pkl(os.path.join(dataset_path ,'FEATS_resnet.pkl'))
elif self.video_feature == 'googlenet':
self.FEAT = common.load_pkl(os.path.join(dataset_path, 'FEATS_googlenet.pkl'))
elif self.video_feature == 'resnet':
if self.proc=='pca':
self.FEAT = common.load_pkl(os.path.join(dataset_path, 'FEATS_resnet_'+self.proc+'.pkl'))
else:
self.FEAT = common.load_pkl(os.path.join(dataset_path, 'FEATS_resnet.pkl'))
elif self.video_feature == 'nasnetalarge':
self.FEAT = common.load_pkl(os.path.join(dataset_path, 'FEATS_nasnetalarge.pkl'))
elif self.video_feature == 'resnet152':
self.FEAT = common.load_pkl(os.path.join(dataset_path, 'FEATS_resnet152.pkl'))
elif self.video_feature == 'pnasnet5large':
self.FEAT = common.load_pkl(os.path.join(dataset_path, 'FEATS_pnasnet5large.pkl'))
elif self.video_feature == 'polynet':
self.FEAT = common.load_pkl(os.path.join(dataset_path, 'FEATS_polynet.pkl'))
elif self.video_feature == 'senet154':
self.FEAT = common.load_pkl(os.path.join(dataset_path, 'FEATS_senet154.pkl'))
else:
self.FEAT = common.load_pkl(os.path.join(dataset_path , 'FEATS_'+self.proc+'.pkl'))
return self