def load_data(self):
print('loading {}-{} features'.format(self.dataset_name,self.cnn_name))
self.train_data_ids = utils.read_file_to_list(self.train_data_ids_path)
self.val_data_ids = utils.read_file_to_list(self.val_data_ids_path)
self.test_data_ids = utils.read_file_to_list(self.test_data_ids_path)
utils.shuffle_array(self.train_data_ids)
utils.shuffle_array(self.val_data_ids)
utils.shuffle_array(self.test_data_ids)
self.train_data_ids = self.train_data_ids[:1] # ONLY FOR DEBUG - REMOVE
self.val_data_ids = self.val_data_ids[:1]
self.test_data_ids = self.test_data_ids[:1]
self.train_caps = utils.read_from_json(self.train_caps_path)
self.val_caps = utils.read_from_json(self.val_caps_path)
self.test_caps = utils.read_from_json(self.test_caps_path)
self.vocab = utils.read_from_json(self.vocab_path)
self.reverse_vocab = utils.read_from_pickle(self.reverse_vocab_path)
self.vocab_size = len(self.vocab)
if self.cnn_name in ['ResNet50', 'ResNet152', 'InceptionV3']:
self.ctx_dim = 2048
elif self.cnn_name in ['MURALI']:
self.ctx_dim = 1024
elif self.cnn_name in ['VGG19']:
self.ctx_dim = 512
else:
raise NotImplementedError()
self.train_ids = self.get_vid_ids(self.train_data_ids)
self.val_ids = self.get_vid_ids(self.val_data_ids)
self.test_ids = self.get_vid_ids(self.test_data_ids)
self.kf_train = utils.generate_minibatch_idx(len(self.train_data_ids), self.mb_size_train)
self.kf_val = utils.generate_minibatch_idx(len(self.val_data_ids), self.mb_size_test) #TODO - verify test or val
self.kf_test = utils.generate_minibatch_idx(len(self.test_data_ids), self.mb_size_test)
def load_data(self):
print 'loading youtube2text %s features' % self.video_feature
dataset_path = config.RAB_DATASET_BASE_PATH
feature_path = config.RAB_FEATURE_BASE_PATH
self.train = utils.load_pkl(dataset_path + 'train.pkl')
# print len(self.train)
# self.train = self.train[:5000]
self.valid = utils.load_pkl(dataset_path + 'valid.pkl')
# print len(self.valid)
# self.valid = self.valid[:500]
self.test = utils.load_pkl(dataset_path + 'test.pkl')
# print len(self.test)
# self.test = self.test[:500]
self.CAP = utils.load_pkl(dataset_path + 'CAP.pkl')
self.FEAT_ROOT = feature_path
if self.signature == 'youtube2text':
# self.train_ids = ['vid%s' % i for i in range(1, 1201)]
# self.valid_ids = ['vid%s' % i for i in range(1201, 1301)]
# self.test_ids = ['vid%s' % i for i in range(1301, 1971)]
self.train_ids = ['vid%s' % i for i in range(1, 301)]
self.valid_ids = ['vid%s' % i for i in range(301, 351)]
self.test_ids = ['vid%s' % i for i in range(351, 401)]
self.worddict = utils.load_pkl(dataset_path + 'worddict.pkl')
elif self.signature == 'msr-vtt':
self.train_ids = ['video%s' % i for i in range(0, 6513)]
self.valid_ids = ['video%s' % i for i in range(6513, 7010)]
self.test_ids = ['video%s' % i for i in range(7010, 10000)]
self.worddict = utils.load_pkl(dataset_path + 'worddict_large.pkl')
else:
raise NotImplementedError()
self.word_idict = dict()
# wordict start with index 2
for kk, vv in self.worddict.iteritems():
self.word_idict[vv] = kk
self.word_idict[0] = '<eos>'
self.word_idict[1] = 'UNK'
# if len(self.word_idict) < self.n_words:
self.n_words = len(self.word_idict)
if self.video_feature == 'googlenet':
self.ctx_dim = 2048
else:
raise NotImplementedError()
self.kf_train = utils.generate_minibatch_idx(len(self.train),
self.mb_size_train)
self.kf_valid = utils.generate_minibatch_idx(len(self.valid),
self.mb_size_test)
self.kf_test = utils.generate_minibatch_idx(len(self.test),
self.mb_size_test)
def __init__(self):
self.x = numpy.random.uniform(0,
1,
size=(minibatch_size * 10, 3, 224,
224)).astype('float32')
self.y = numpy.random.randint(0, 1000,
size=(len(self.x, ))).astype('int32')
self.train_minibatch_idx = utils.generate_minibatch_idx(
len(self.x), minibatch_size)
def simple_train_sgd(self, trainset, epoch, epoch_end):
# train with SGD
print 'Train %s with SGD' % self.__class__
idx = range(trainset.shape[0])
minibatch_idx_overall = utils.generate_minibatch_idx(
trainset.shape[0], self.minibatch_size)
while (epoch < epoch_end):
costs_epoch = []
costs_by_step_epoch = []
for k, use_idx in enumerate(minibatch_idx_overall):
if self.verbose:
sys.stdout.write('\rTraining minibatches %d/%d' %
(k, len(minibatch_idx_overall)))
sys.stdout.flush()
minibatch_data = trainset[use_idx, :]
minibatch_mask = utils.generate_masks_deep_orderless_nade(
minibatch_data.shape, self.rng_numpy)
if 0:
# this is deep nade
cost = self.train_fn(minibatch_data, minibatch_mask)
else:
# len(results)==2
results = self.train_fn(minibatch_data, minibatch_mask)
cost = results[0]
# results[1]: (1,k)
costs_by_step = results[1].flatten()
costs_epoch.append(cost)
costs_by_step_epoch.append(costs_by_step)
# now linearly decrease the learning rate
current_lr = self.learning_rate.get_value()
new_lr = current_lr - numpy.float32(self.lr_decrease)
self.learning_rate.set_value(new_lr)
cost_epoch_avg = numpy.mean(costs_epoch)
cost_by_step_avg = numpy.asarray(costs_by_step_epoch).mean(axis=0)
self.costs_steps.append(cost_by_step_avg)
self.costs.append(cost_epoch_avg)
print '\rTraining %d/%d epochs, cost %.2f, costs by step %s lr %.5f' % (
epoch, epoch_end, cost_epoch_avg,
numpy.round(cost_by_step_avg, 2), current_lr)
if epoch != 0 and (epoch + 1) % self.valid_freq == 0:
numpy.savetxt(self.save_model_path + 'epoch_costs_by_step.txt',
self.costs_steps)
numpy.savetxt(self.save_model_path + 'epoch_costs.txt',
self.costs)
if self.channel:
self.channel.save()
self.sample_nade_v0(epoch)
self.make_plots(self.costs)
self.visualize_filters(epoch)
self.LL(epoch, save_nothing=False)
self.inpainting(epoch, self.k)
self.save_model(epoch)
epoch += 1
# end of training
print
def load_data(self):
print 'loading %s %s features' % (self.signature, self.video_feature)
dataset_path = config.RAB_DATASET_BASE_PATH
self.train = utils.load_pkl(dataset_path + 'train.pkl')
self.valid = utils.load_pkl(dataset_path + 'valid.pkl')
self.test = utils.load_pkl(dataset_path + 'test.pkl')
self.CAP = utils.load_pkl(dataset_path + 'CAP.pkl')
if self.signature == 'youtube2text':
self.FEAT_ROOT = config.RAB_FEATURE_BASE_PATH
self.train_ids = ['vid%s' % i for i in range(1, 1201)]
self.valid_ids = ['vid%s' % i for i in range(1201, 1301)]
self.test_ids = ['vid%s' % i for i in range(1301, 1971)]
elif self.signature == 'msr-vtt':
self.FEAT = utils.load_pkl(dataset_path + 'INCEPTION_FEAT.pkl')
self.train_ids = ['video%s' % i for i in range(0, 6513)]
self.valid_ids = ['video%s' % i for i in range(6513, 7010)]
self.test_ids = ['video%s' % i for i in range(7010, 10000)]
else:
raise NotImplementedError()
self.word_ix = utils.load_pkl(dataset_path + 'worddict.pkl')
self.ix_word = dict()
# word_ix start with index 2
for kk, vv in self.word_ix.iteritems():
self.ix_word[vv] = kk
self.ix_word[0] = '<eos>'
self.ix_word[1] = 'UNK'
if len(self.ix_word) < self.n_words:
self.n_words = len(self.ix_word)
if self.video_feature == 'googlenet':
self.ctx_dim = 2048
else:
raise NotImplementedError()
self.kf_train = utils.generate_minibatch_idx(len(self.train),
self.mb_size_train)
self.kf_valid = utils.generate_minibatch_idx(len(self.valid),
self.mb_size_test)
self.kf_test = utils.generate_minibatch_idx(len(self.test),
self.mb_size_test)
def __init__(self):
# load dataset
base_dir = '/dl1/home/lyao/Data/mnist/'
self.train_x = numpy.load(base_dir + 'train_x.npy')
self.train_y = numpy.load(base_dir + 'train_y.npy')
self.valid_x = numpy.load(base_dir + 'valid_x.npy')
self.valid_y = numpy.load(base_dir + 'valid_y.npy')
self.test_x = numpy.load(base_dir + 'test_x.npy')
self.test_y = numpy.load(base_dir + 'test_y.npy')
self.train_minibatch_idx = utils.generate_minibatch_idx(
len(self.train_x), minibatch_size)
def classify():
IMAGE = '/dl1/data/projects/imagenet/valid/'
LABELS = '/dl1/data/projects/imagenet/val.txt'
MEAN = '/dl1/data/projects/imagenet/ilsvrc_2012_mean.npy'
EXT = 'JPEG'
preprocessor = utils.VGGImageFuncs()
'''build theano fn'''
x = T.ftensor4('images')
model = build_model(x)
y = lasagne.layers.get_output(model['prob'], deterministic=True)
params = lasagne.layers.get_all_params(model['prob'], trainable=True)
classify_fn = theano.function([x], y)
'''perform classification'''
files = glob.glob(IMAGE + '/*.' + EXT)
files = utils.sort_by_numbers_in_file_name(files)
labels = utils.load_txt_file(LABELS)
labels = [int((label.split(' ')[-1]).strip()) for label in labels]
# go through minibatches
idx = utils.generate_minibatch_idx(len(files), 64)
TOP1s = []
TOP5s = []
for i, index in enumerate(idx):
t0 = time.time()
current = [files[j] for j in index]
gts = np.asarray([labels[j] for j in index])
#inputs =[load_image(im_f) for im_f in current]
inputs = preprocessor.preprocess(current)
import ipdb
ipdb.set_trace()
probs = classify_fn(inputs) # (m, 1000, 1, 1)
probs = np.squeeze(probs)
predictions = probs.argsort()[:, ::-1][:, :5]
for pred, gt in zip(predictions, gts):
TOP1 = pred[0] == gt
TOP5 = gt in pred
TOP1s.append(TOP1)
TOP5s.append(TOP5)
print '%d / %d minibatches, acu TOP1 %.4f, TOP5 %.4f, used %.2f' % (
i, len(idx), np.mean(TOP1s) * 100, np.mean(TOP5s) * 100,
time.time() - t0)
