def __init__(self, d, lr, lambda_z_wu, do_classify, use_kl=True):
""" model architecture """
self.MLP_SIZES = [512, 256, 256, 128, 128]
self.Z_SIZES = [64, 32, 32, 32, 32]
self.L = L = len(self.MLP_SIZES)
self.do_classify = do_classify
""" flags for regularizers """
self.use_kl = use_kl
""" data and external toolkits """
self.d = d # dataset manager
self.ls = Layers()
self.lf = LossFunctions(self.ls, d, self.encoder)
""" placeholders defined outside"""
self.lr = lr
self.lambda_z_wu = lambda_z_wu
""" cache for mu and sigma """
self.e_mus, self.e_logsigmas = [0] * L, [
0
] * L # q(z_i+1 | z_i), bottom-up inference as Eq.7-9
self.p_mus, self.p_logsigmas = [0] * L, [
0
] * L # p(z_i | z_i+1), top-down prior as Eq.1-3
self.d_mus, self.d_logsigmas = [0] * L, [
0
] * L # q(z_i | .), bidirectional inference as Eq.17-19
def __init__(self, d, lr, lambda_z_wu, read_attn, write_attn, do_classify,
do_reconst):
self.do_classify = do_classify
""" flags for each regularizor """
self.do_reconst = do_reconst
self.read_attn = read_attn
self.write_attn = write_attn
""" dataset information """
self.set_datainfo(d)
""" external toolkits """
self.ls = Layers()
self.lf = LossFunctions(self.ls, self.d, self.encoder)
self.ii = ImageInterface(_is_3d, self.read_attn, self.write_attn,
GLIMPSE_SIZE_READ, GLIMPSE_SIZE_WRITE, _h, _w,
_c)
# for refference from get_loss_kl_draw()
self.T = T
self.L = L
self.Z_SIZES = Z_SIZES
""" placeholders defined outside"""
self.lr = lr
self.lambda_z_wu = lambda_z_wu
"""sequence of canvases """
self.cs = [0] * T
""" initialization """
self.init_lstms()
self.init_time_zero()
""" workaround for variable_scope(reuse=True) """
self.DO_SHARE = None
def __init__(self,
num_inputs,
num_hidden,
num_outputs,
learn_rate,
h_w=None,
h_b=None,
o_w=None,
o_b=None):
self.num_inputs = num_inputs
self.hidden_layer = Layers(num_hidden, h_b)
self.output_layer = Layers(num_outputs, o_b)
self.init_weights_i2h(h_w)
self.init_weights_h2o(o_w)
self.learn_rate = learn_rate
def __init__(self, resource):
""" data and external toolkits """
self.d = resource.dh # dataset manager
self.ls = Layers()
self.lf = LossFunctions(self.ls, self.d, self.encoder)
""" placeholders defined outside"""
if c.DO_TRAIN:
self.lr = resource.ph['lr']
def __init__(self, d, lr, lambda_pi_usl, use_pi):
""" flags for each regularizor """
self.use_pi = use_pi
""" data and external toolkits """
self.d = d # dataset manager
self.ls = Layers()
self.lf = LossFunctions(self.ls, d, self.encoder)
""" placeholders defined outside"""
self.lr = lr
self.lambda_pi_usl = lambda_pi_usl
def __init__(self, variable_list, fully_index, sparse_index, sparse_num,
input_shape):
self._variable_list = copy.deepcopy(variable_list)
self.layers = Layers(
weight_init=tf.contrib.layers.xavier_initializer(),
regularizer=tf.contrib.layers.l2_regularizer(
FLAGS.regularizer_factor),
bias_init=tf.constant_initializer(0.0))
self.fully_index = fully_index
self.sparse_index = sparse_index
self.sparse_num = sparse_num
self.index_fix = [0, self.fully_index, self.sparse_index]
self._input_shape = input_shape
def __init__(self, is_3d, is_read_attention, is_write_attention, read_n, write_n, h, w, c):
""" to manage do_share flag inside Layers object, ImageInterface has Layers as its own property """
self.do_share = False
self.ls = Layers()
self.is_3d = is_3d
self.read_n = read_n
self.write_n = write_n
self.h = h
self.w = w
self.c = c
if is_read_attention:
self.read = self._read_attention
else:
self.read = self._read_no_attention
if is_write_attention:
self.write = self._write_attention
else:
self.write = self._write_no_attention
def __init__(self):
self.layers = Layers()
def __init__(self, channel=None):
self.rng_numpy, self.rng_theano = get_two_rngs()
self.layers = Layers()
self.predict = Predict()
self.channel = channel
MAXLEN = 10
HIDDEN_SIZE = 100
EPOCHS = 500
MIN_COUNT = 1
lr = 0.5
pretrained = False
activation = "linear"
helper = DataHelper()
emb_matrix, inputs, targets = helper.trainset_preparation(
path_data, EMB_DIM, MAXLEN, BATCH_SIZE, MIN_COUNT, pretrained)
VOCAB_SIZE = len(helper.stoi)
helper.save("./binaries/vocab.pkl")
itos = {i: t for t, i in helper.stoi.items()}
lm = Layers(VOCAB_SIZE, EMB_DIM, HIDDEN_SIZE, emb_matrix, activation)
def generate(sent_input, len_sent):
sent_input = helper.preprocessing(sent_input)
token = sent_input.split()
sequence = helper.transform(token)
for i in range(len_sent):
inputs = np.array([sequence])
prob = lm.forward(inputs)[-1][-1]
index = np.argmax(prob)
sequence.append(index)
print(" ".join([itos[idx] for idx in sequence]))
for e in range(EPOCHS):
def train(
random_seed=1234,
dim_word=256, # word vector dimensionality
ctx_dim=-1, # context vector dimensionality, auto set
dim=1000, # the number of LSTM units
n_layers_out=1,
n_layers_init=1,
encoder='none',
encoder_dim=100,
prev2out=False,
ctx2out=False,
patience=10,
max_epochs=5000,
dispFreq=100,
decay_c=0.,
alpha_c=0.,
alpha_entropy_r=0.,
lrate=0.01,
selector=False,
n_words=100000,
maxlen=100, # maximum length of the description
optimizer='adadelta',
clip_c=2.,
batch_size=64,
valid_batch_size=64,
save_model_dir='/data/lisatmp3/yaoli/exp/capgen_vid/attention/test/',
validFreq=10,
saveFreq=10, # save the parameters after every saveFreq updates
sampleFreq=10, # generate some samples after every sampleFreq updates
metric='blue',
dataset='youtube2text',
video_feature='googlenet',
use_dropout=False,
reload_=False,
from_dir=None,
K=10,
OutOf=240,
verbose=True,
debug=True):
rng_numpy, rng_theano = utils.get_two_rngs()
model_options = locals().copy()
if 'self' in model_options:
del model_options['self']
with open('%smodel_options.pkl' % save_model_dir, 'wb') as f:
pkl.dump(model_options, f)
# instance model
layers = Layers()
model = Model()
print 'Loading data'
engine = data_engine.Movie2Caption('attention', dataset, video_feature,
batch_size, valid_batch_size, maxlen,
n_words, K, OutOf)
model_options['ctx_dim'] = engine.ctx_dim
model_options['n_words'] = engine.n_words
print 'n_words:', model_options['n_words']
# set test values, for debugging
idx = engine.kf_train[0]
[x_tv, mask_tv, ctx_tv, ctx_mask_tv
] = data_engine.prepare_data(engine,
[engine.train[index] for index in idx])
print 'init params'
t0 = time.time()
params = model.init_params(model_options)
# reloading
if reload_:
model_saved = from_dir + '/model_best_so_far.npz'
assert os.path.isfile(model_saved)
print "Reloading model params..."
params = utils.load_params(model_saved, params)
tparams = utils.init_tparams(params)
trng, use_noise, \
x, mask, ctx, mask_ctx, \
cost, extra = \
model.build_model(tparams, model_options)
alphas = extra[1]
betas = extra[2]
print 'buliding sampler'
f_init, f_next = model.build_sampler(tparams, model_options, use_noise,
trng)
# before any regularizer
print 'building f_log_probs'
f_log_probs = theano.function([x, mask, ctx, mask_ctx],
-cost,
profile=False,
on_unused_input='ignore')
cost = cost.mean()
if decay_c > 0.:
decay_c = theano.shared(numpy.float32(decay_c), name='decay_c')
weight_decay = 0.
for kk, vv in tparams.iteritems():
weight_decay += (vv**2).sum()
weight_decay *= decay_c
cost += weight_decay
if alpha_c > 0.:
alpha_c = theano.shared(numpy.float32(alpha_c), name='alpha_c')
alpha_reg = alpha_c * ((1. - alphas.sum(0))**2).sum(-1).mean()
cost += alpha_reg
if alpha_entropy_r > 0:
alpha_entropy_r = theano.shared(numpy.float32(alpha_entropy_r),
name='alpha_entropy_r')
alpha_reg_2 = alpha_entropy_r * (-tensor.sum(
alphas * tensor.log(alphas + 1e-8), axis=-1)).sum(-1).mean()
cost += alpha_reg_2
else:
alpha_reg_2 = tensor.zeros_like(cost)
print 'building f_alpha'
f_alpha = theano.function([x, mask, ctx, mask_ctx], [alphas, betas],
name='f_alpha',
on_unused_input='ignore')
print 'compute grad'
grads = tensor.grad(cost, wrt=utils.itemlist(tparams))
if clip_c > 0.:
g2 = 0.
for g in grads:
g2 += (g**2).sum()
new_grads = []
for g in grads:
new_grads.append(
tensor.switch(g2 > (clip_c**2), g / tensor.sqrt(g2) * clip_c,
g))
grads = new_grads
lr = tensor.scalar(name='lr')
print 'build train fns'
f_grad_shared, f_update = eval(optimizer)(lr, tparams, grads,
[x, mask, ctx, mask_ctx], cost,
extra + grads)
print 'compilation took %.4f sec' % (time.time() - t0)
print 'Optimization'
history_errs = []
# reload history
if reload_:
print 'loading history error...'
history_errs = numpy.load(
from_dir + 'model_best_so_far.npz')['history_errs'].tolist()
bad_counter = 0
processes = None
queue = None
rqueue = None
shared_params = None
uidx = 0
uidx_best_blue = 0
uidx_best_valid_err = 0
estop = False
best_p = utils.unzip(tparams)
best_blue_valid = 0
best_valid_err = 999
alphas_ratio = []
for eidx in xrange(max_epochs):
n_samples = 0
train_costs = []
grads_record = []
print 'Epoch ', eidx
for idx in engine.kf_train:
tags = [engine.train[index] for index in idx]
n_samples += len(tags)
uidx += 1
use_noise.set_value(1.)
pd_start = time.time()
x, mask, ctx, ctx_mask = data_engine.prepare_data(engine, tags)
pd_duration = time.time() - pd_start
if x is None:
print 'Minibatch with zero sample under length ', maxlen
continue
ud_start = time.time()
rvals = f_grad_shared(x, mask, ctx, ctx_mask)
cost = rvals[0]
probs = rvals[1]
alphas = rvals[2]
betas = rvals[3]
grads = rvals[4:]
grads, NaN_keys = utils.grad_nan_report(grads, tparams)
if len(grads_record) >= 5:
del grads_record[0]
grads_record.append(grads)
if NaN_keys != []:
print 'grads contain NaN'
import pdb
pdb.set_trace()
if numpy.isnan(cost) or numpy.isinf(cost):
print 'NaN detected in cost'
import pdb
pdb.set_trace()
# update params
f_update(lrate)
ud_duration = time.time() - ud_start
if eidx == 0:
train_error = cost
else:
train_error = train_error * 0.95 + cost * 0.05
train_costs.append(cost)
if numpy.mod(uidx, dispFreq) == 0:
print 'Epoch ', eidx, 'Update ', uidx, 'Train cost mean so far', \
train_error, 'fetching data time spent (sec)', pd_duration, \
'update time spent (sec)', ud_duration, 'save_dir', save_model_dir
alphas, betas = f_alpha(x, mask, ctx, ctx_mask)
counts = mask.sum(0)
betas_mean = (betas * mask).sum(0) / counts
betas_mean = betas_mean.mean()
print 'alpha ratio %.3f, betas mean %.3f' % (
alphas.min(-1).mean() /
(alphas.max(-1)).mean(), betas_mean)
l = 0
for vv in x[:, 0]:
if vv == 0:
break
if vv in engine.word_idict:
print '(', numpy.round(betas[l, 0],
3), ')', engine.word_idict[vv],
else:
print '(', numpy.round(betas[l, 0], 3), ')', 'UNK',
l += 1
print '(', numpy.round(betas[l, 0], 3), ')'
if numpy.mod(uidx, saveFreq) == 0:
pass
if numpy.mod(uidx, sampleFreq) == 0:
use_noise.set_value(0.)
print '------------- sampling from train ----------'
x_s = x
mask_s = mask
ctx_s = ctx
ctx_mask_s = ctx_mask
model.sample_execute(engine, model_options, tparams, f_init,
f_next, x_s, ctx_s, ctx_mask_s, trng)
print '------------- sampling from valid ----------'
idx = engine.kf_valid[numpy.random.randint(
1,
len(engine.kf_valid) - 1)]
tags = [engine.valid[index] for index in idx]
x_s, mask_s, ctx_s, mask_ctx_s = data_engine.prepare_data(
engine, tags)
model.sample_execute(engine, model_options, tparams, f_init,
f_next, x_s, ctx_s, mask_ctx_s, trng)
if validFreq != -1 and numpy.mod(uidx, validFreq) == 0:
t0_valid = time.time()
alphas, _ = f_alpha(x, mask, ctx, ctx_mask)
ratio = alphas.min(-1).mean() / (alphas.max(-1)).mean()
alphas_ratio.append(ratio)
numpy.savetxt(save_model_dir + 'alpha_ratio.txt', alphas_ratio)
current_params = utils.unzip(tparams)
numpy.savez(save_model_dir + 'model_current.npz',
history_errs=history_errs,
**current_params)
use_noise.set_value(0.)
train_err = -1
train_perp = -1
valid_err = -1
valid_perp = -1
test_err = -1
test_perp = -1
if not debug:
# first compute train cost
if 0:
print 'computing cost on trainset'
train_err, train_perp = model.pred_probs(
engine,
'train',
f_log_probs,
verbose=model_options['verbose'])
else:
train_err = 0.
train_perp = 0.
if 1:
print 'validating...'
valid_err, valid_perp = model.pred_probs(
engine,
'valid',
f_log_probs,
verbose=model_options['verbose'],
)
else:
valid_err = 0.
valid_perp = 0.
if 1:
print 'testing...'
test_err, test_perp = model.pred_probs(
engine,
'test',
f_log_probs,
verbose=model_options['verbose'])
else:
test_err = 0.
test_perp = 0.
mean_ranking = 0
blue_t0 = time.time()
scores, processes, queue, rqueue, shared_params = \
metrics.compute_score(
model_type='attention',
model_archive=current_params,
options=model_options,
engine=engine,
save_dir=save_model_dir,
beam=5, n_process=5,
whichset='both',
on_cpu=False,
processes=processes, queue=queue, rqueue=rqueue,
shared_params=shared_params, metric=metric,
one_time=False,
f_init=f_init, f_next=f_next, model=model
)
'''
{'blue': {'test': [-1], 'valid': [77.7, 60.5, 48.7, 38.5, 38.3]},
'alternative_valid': {'Bleu_3': 0.40702270203174923,
'Bleu_4': 0.29276570520368456,
'CIDEr': 0.25247168210607884,
'Bleu_2': 0.529069629270047,
'Bleu_1': 0.6804308797115253,
'ROUGE_L': 0.51083584331688392},
'meteor': {'test': [-1], 'valid': [0.282787550236724]}}
'''
valid_B1 = scores['valid']['Bleu_1']
valid_B2 = scores['valid']['Bleu_2']
valid_B3 = scores['valid']['Bleu_3']
valid_B4 = scores['valid']['Bleu_4']
valid_Rouge = scores['valid']['ROUGE_L']
valid_Cider = scores['valid']['CIDEr']
valid_meteor = scores['valid']['METEOR']
test_B1 = scores['test']['Bleu_1']
test_B2 = scores['test']['Bleu_2']
test_B3 = scores['test']['Bleu_3']
test_B4 = scores['test']['Bleu_4']
test_Rouge = scores['test']['ROUGE_L']
test_Cider = scores['test']['CIDEr']
test_meteor = scores['test']['METEOR']
print 'computing meteor/blue score used %.4f sec, '\
'blue score: %.1f, meteor score: %.1f'%(
time.time()-blue_t0, valid_B4, valid_meteor)
history_errs.append([
eidx, uidx, train_err, train_perp, valid_perp, test_perp,
valid_err, test_err, valid_B1, valid_B2, valid_B3,
valid_B4, valid_meteor, valid_Rouge, valid_Cider, test_B1,
test_B2, test_B3, test_B4, test_meteor, test_Rouge,
test_Cider
])
numpy.savetxt(save_model_dir + 'train_valid_test.txt',
history_errs,
fmt='%.3f')
print 'save validation results to %s' % save_model_dir
# save best model according to the best blue or meteor
if len(history_errs) > 1 and \
valid_B4 > numpy.array(history_errs)[:-1,11].max():
print 'Saving to %s...' % save_model_dir,
numpy.savez(save_model_dir +
'model_best_blue_or_meteor.npz',
history_errs=history_errs,
**best_p)
if len(history_errs) > 1 and \
valid_err < numpy.array(history_errs)[:-1,6].min():
best_p = utils.unzip(tparams)
bad_counter = 0
best_valid_err = valid_err
uidx_best_valid_err = uidx
print 'Saving to %s...' % save_model_dir,
numpy.savez(save_model_dir + 'model_best_so_far.npz',
history_errs=history_errs,
**best_p)
with open('%smodel_options.pkl' % save_model_dir,
'wb') as f:
pkl.dump(model_options, f)
print 'Done'
elif len(history_errs) > 1 and \
valid_err >= numpy.array(history_errs)[:-1,6].min():
bad_counter += 1
print 'history best ', numpy.array(history_errs)[:,
6].min()
print 'bad_counter ', bad_counter
print 'patience ', patience
if bad_counter > patience:
print 'Early Stop!'
estop = True
break
if test_B4 > 0.52 and test_meteor > 0.32:
print 'Saving to %s...' % save_model_dir,
numpy.savez(save_model_dir + 'model_' + str(uidx) + '.npz',
history_errs=history_errs,
**current_params)
print 'Train ', train_err, 'Valid ', valid_err, 'Test ', test_err, \
'best valid err so far',best_valid_err
print 'valid took %.2f sec' % (time.time() - t0_valid)
# end of validatioin
if debug:
break
if estop:
break
if debug:
break
# end for loop over minibatches
print 'This epoch has seen %d samples, train cost %.2f' % (
n_samples, numpy.mean(train_costs))
# end for loop over epochs
print 'Optimization ended.'
if best_p is not None:
utils.zipp(best_p, tparams)
use_noise.set_value(0.)
valid_err = 0
test_err = 0
if not debug:
#if valid:
valid_err, valid_perp = model.pred_probs(
engine, 'valid', f_log_probs, verbose=model_options['verbose'])
#if test:
#test_err, test_perp = self.pred_probs(
# 'test', f_log_probs,
# verbose=model_options['verbose'])
print 'stopped at epoch %d, minibatch %d, '\
'curent Train %.2f, current Valid %.2f, current Test %.2f '%(
eidx,uidx,numpy.mean(train_err),numpy.mean(valid_err),numpy.mean(test_err))
params = copy.copy(best_p)
numpy.savez(save_model_dir + 'model_best.npz',
train_err=train_err,
valid_err=valid_err,
test_err=test_err,
history_errs=history_errs,
**params)
if history_errs != []:
history = numpy.asarray(history_errs)
best_valid_idx = history[:, 6].argmin()
numpy.savetxt(save_model_dir + 'train_valid_test.txt',
history,
fmt='%.4f')
print 'final best exp ', history[best_valid_idx]
return train_err, valid_err, test_err
def __init__(self,
batch: int = None,
hours: float = None,
width: int = None,
height: int = None,
level: Levels = None,
reset_chance: float = None,
failed_actions_chance: float = None,
**kwargs) -> None:
super().__init__()
self.batch: int = batch
self.hours: float = hours
self.level: Levels = level
self.uses = {
Levels.Causal1: {
LayerType.Blocks, LayerType.Goal, LayerType.Gold,
LayerType.Keys, LayerType.Door
},
Levels.Causal2: {
LayerType.Blocks, LayerType.Goal, LayerType.Diamond1,
LayerType.Diamond2, LayerType.Diamond3, LayerType.Diamond4
},
Levels.Causal3: {
LayerType.Blocks, LayerType.Goal, LayerType.Gold,
LayerType.Bluedoor, LayerType.Bluekeys, LayerType.Reddoor,
LayerType.Redkeys
},
Levels.Causal4: {
LayerType.Blocks, LayerType.Goal, LayerType.Gold,
LayerType.Bluedoor, LayerType.Bluekeys, LayerType.Reddoor,
LayerType.Redkeys, LayerType.Rock, LayerType.Dirt
},
Levels.Rocks:
{LayerType.Blocks, LayerType.Goal, LayerType.Rock, LayerType.Dirt},
Levels.Maze: {
LayerType.Blocks, LayerType.Goal, LayerType.Gold,
LayerType.Door, LayerType.Keys, LayerType.Holder,
LayerType.Putter
},
Levels.Causal5: {
LayerType.Blocks, LayerType.Goal, LayerType.Brown1,
LayerType.Brown2, LayerType.Brown3, LayerType.Pink1,
LayerType.Pink2, LayerType.Pink3
},
Levels.Coconuts: {
LayerType.Blocks, LayerType.Goal, LayerType.Rock,
LayerType.Dirt, LayerType.Gold, LayerType.Coconut
},
Levels.Causal6: {
LayerType.Blocks, LayerType.Goal, LayerType.Greendown,
LayerType.Greenup, LayerType.Greenstar, LayerType.Yellowstar,
LayerType.Bluestar
},
Levels.SuperLevel: {
LayerType.Blocks, LayerType.Goal, LayerType.Gold,
LayerType.Bluedoor, LayerType.Bluekeys, LayerType.Reddoor,
LayerType.Redkeys, LayerType.Rock, LayerType.Dirt,
LayerType.Coconut, LayerType.Door, LayerType.Keys
},
Levels.SuperLevel2: {
LayerType.Blocks, LayerType.Goal, LayerType.Gold,
LayerType.Bluedoor, LayerType.Bluekeys, LayerType.Reddoor,
LayerType.Redkeys, LayerType.Rock, LayerType.Dirt,
LayerType.Coconut
},
Levels.MonsterLevel: {
LayerType.Blocks, LayerType.Goal, LayerType.Gold,
LayerType.Monster, LayerType.Rock
},
Levels.Causal7: {
LayerType.Blocks, LayerType.Goal, LayerType.Greencross,
LayerType.Bluecross, LayerType.Redcross, LayerType.Purplecross
},
Levels.CausalSuper: {
LayerType.Blocks, LayerType.Goal, LayerType.Super1,
LayerType.Super2, LayerType.Super3, LayerType.Super4,
LayerType.Super5, LayerType.Super6, LayerType.Super7
}
}
convert = {(use, [
layer for layer in LayerType if layer.name == name.split('_')[1]
][0])
for name, use in kwargs.items()
if name.split('_')[0] == "layer"}
self.layers: Layers = Layers(
batch, width, height, level, reset_chance, failed_actions_chance,
*[
layer for use, layer in convert
if use and (layer in self.uses[level])
])
for i in range(width):
for j in range(height):
for k in range(batch):
self.layers.all_items[k][(i, j)] = 0
self.layers.update(isFirstTime=True)
def train(random_seed=1234,
dim_word=256, # word vector dimensionality
ctx_dim=-1, # context vector dimensionality, auto set
dim=1000, # the number of LSTM units
n_layers_out=1,
n_layers_init=1,
encoder='none',
encoder_dim=100,
prev2out=False,
ctx2out=False,
patience=10,
max_epochs=5000,
dispFreq=100,
decay_c=0.,
alpha_c=0.,
alpha_entropy_r=0.,
lrate=0.01,
selector=False,
n_words=100000,
maxlen=100, # maximum length of the description
optimizer='adadelta',
clip_c=2.,
batch_size = 64,
valid_batch_size = 64,
save_model_dir='/data/lisatmp3/yaoli/exp/capgen_vid/attention/test/',
validFreq=10,
saveFreq=10, # save the parameters after every saveFreq updates
sampleFreq=10, # generate some samples after every sampleFreq updates
metric='blue',
dataset='youtube2text',
video_feature='googlenet',
use_dropout=False,
reload_=False,
from_dir=None,
K1=10,
K2=10,
OutOf=240,
verbose=True,
debug=True
):
rng_numpy, rng_theano = utils.get_two_rngs()
model_options = locals().copy()
model_options_c = locals().copy()
if 'self' in model_options:
del model_options['self']
with open('model_files/model_options.pkl', 'wb') as f:
pkl.dump(model_options, f)
with open('model_files/model_options_c3d.pkl', 'wb') as f:
pkl.dump(model_options_c, f)
# instance model
layers = Layers()
model = Model()
model_c = Model()
print 'Loading data'
engine = data_engine.Movie2Caption('attention', dataset,
video_feature,
batch_size, valid_batch_size,
maxlen, n_words,
K1, K2, OutOf)
model_options['ctx_dim'] = engine.ctx_dim
model_options_c['ctx_dim'] = engine.ctx_dim_c
model_options['n_words'] = engine.n_words
model_options_c['n_words'] = engine.n_words
print 'n_words:', model_options['n_words']
print model_options_c['dim'],model_options_c['ctx_dim']
# set test values, for debugging
idx = engine.kf_train[0]
[x_tv, mask_tv,
ctx_tv, ctx_mask_tv,
ctx_tv_c, ctx_mask_tv_c] = data_engine.prepare_data(
engine, [engine.train[index] for index in idx])
print 'init params'
t0 = time.time()
params = model.init_params(model_options)
params_c = model_c.init_params(model_options_c)
# reloading
model_saved = 'model_files/model_resnet.npz'
model_saved_c = 'model_files/model_c3d.npz'
assert os.path.isfile(model_saved)
print "Reloading model params..."
params = utils.load_params(model_saved, params)
params_c = utils.load_params(model_saved_c, params_c)
tparams = utils.init_tparams(params)
tparams_c = utils.init_tparams(params_c)
trng, use_noise, \
x, mask, ctx, mask_ctx, \
cost, extra = \
model.build_model(tparams, model_options)
alphas = extra[1]
betas = extra[2]
trng_c, use_noise_c, \
x_c, mask_c, ctx_c, mask_ctx_c, \
cost_c, extra_c = \
model_c.build_model(tparams_c, model_options_c)
alphas_c = extra_c[1]
betas_c = extra_c[2]
print 'buliding sampler'
f_init, f_next = model.build_sampler(tparams, model_options, use_noise, trng)
f_init_c, f_next_c = model_c.build_sampler(tparams_c, model_options_c, use_noise_c, trng_c)
# before any regularizer
print 'building f_log_probs'
f_log_probs = theano.function([x, mask, ctx, mask_ctx], -cost,
profile=False, on_unused_input='ignore')
f_log_probs_c = theano.function([x_c, mask_c, ctx_c, mask_ctx_c], -cost_c,
profile=False, on_unused_input='ignore')
bad_counter = 0
processes = None
queue = None
rqueue = None
shared_params = None
uidx = 0
uidx_best_blue = 0
uidx_best_valid_err = 0
estop = False
best_p = utils.unzip(tparams)
best_blue_valid = 0
best_valid_err = 999
alphas_ratio = []
for eidx in xrange(max_epochs):
n_samples = 0
train_costs = []
grads_record = []
print 'Epoch ', eidx
for idx in engine.kf_train:
tags = [engine.train[index] for index in idx]
n_samples += len(tags)
use_noise.set_value(1.)
pd_start = time.time()
x, mask, ctx, ctx_mask, ctx_c, ctx_mask_c = data_engine.prepare_data(
engine, tags)
#print 'x:',x.shape,'ctx:',ctx.shape,'ctx_c:',ctx_c.shape
pd_duration = time.time() - pd_start
if x is None:
print 'Minibatch with zero sample under length ', maxlen
continue
if numpy.mod(uidx, saveFreq) == 0:
pass
if numpy.mod(uidx, sampleFreq) == 0:
use_noise.set_value(0.)
print '------------- sampling from train ----------'
x_s = x
mask_s = mask
ctx_s = ctx
ctx_s_c = ctx_c
ctx_mask_s = ctx_mask
ctx_mask_s_c = ctx_mask_c
model.sample_execute_ensemble(engine, model_options,model_options_c, tparams,tparams_c,
f_init,f_init_c, f_next,f_next_c, x_s, ctx_s,
ctx_mask_s, ctx_s_c, ctx_mask_s_c, trng)
print '------------- sampling from valid ----------'
idx = engine.kf_valid[numpy.random.randint(1, len(engine.kf_valid) - 1)]
tags = [engine.valid[index] for index in idx]
x_s, mask_s, ctx_s, mask_ctx_s, ctx_s_c,mask_ctx_s_c = data_engine.prepare_data(engine, tags)
model.sample_execute_ensemble(engine, model_options,model_options_c, tparams,tparams_c,
f_init, f_init_c, f_next, f_next_c, x_s, ctx_s,
mask_ctx_s, ctx_s_c, mask_ctx_s_c, trng)
if validFreq != -1 and numpy.mod(uidx, validFreq) == 0:
current_params = utils.unzip(tparams)
use_noise.set_value(0.)
train_err = -1
train_perp = -1
valid_err = -1
valid_perp = -1
test_err = -1
test_perp = -1
mean_ranking = 0
blue_t0 = time.time()
scores, processes, queue, rqueue, shared_params = \
metrics.compute_score_ensemble(
model_type='attention',
model_archive=current_params,
options=model_options,
options_c=model_options_c,
engine=engine,
save_dir=save_model_dir,
beam=5, n_process=5,
whichset='both',
on_cpu=False,
processes=processes, queue=queue, rqueue=rqueue,
shared_params=shared_params, metric=metric,
one_time=False,
f_init=f_init, f_init_c=f_init_c, f_next=f_next, f_next_c= f_next_c, model=model
)
'''
{'blue': {'test': [-1], 'valid': [77.7, 60.5, 48.7, 38.5, 38.3]},
'alternative_valid': {'Bleu_3': 0.40702270203174923,
'Bleu_4': 0.29276570520368456,
'CIDEr': 0.25247168210607884,
'Bleu_2': 0.529069629270047,
'Bleu_1': 0.6804308797115253,
'ROUGE_L': 0.51083584331688392},
'meteor': {'test': [-1], 'valid': [0.282787550236724]}}
'''
valid_B1 = scores['valid']['Bleu_1']
valid_B2 = scores['valid']['Bleu_2']
valid_B3 = scores['valid']['Bleu_3']
valid_B4 = scores['valid']['Bleu_4']
valid_Rouge = scores['valid']['ROUGE_L']
valid_Cider = scores['valid']['CIDEr']
valid_meteor = scores['valid']['METEOR']
test_B1 = scores['test']['Bleu_1']
test_B2 = scores['test']['Bleu_2']
test_B3 = scores['test']['Bleu_3']
test_B4 = scores['test']['Bleu_4']
test_Rouge = scores['test']['ROUGE_L']
test_Cider = scores['test']['CIDEr']
test_meteor = scores['test']['METEOR']
print 'computing meteor/blue score used %.4f sec, '\
'blue score: %.1f, meteor score: %.1f'%(
time.time()-blue_t0, valid_B4, valid_meteor)
if test_B4>0.52 and test_meteor>0.32:
print 'Saving to %s...'%save_model_dir,
numpy.savez(
save_model_dir+'model_'+str(uidx)+'.npz',
**current_params)
print 'Train ', train_err, 'Valid ', valid_err, 'Test ', test_err, \
'best valid err so far',best_valid_err
print 'valid took %.2f sec'%(time.time() - t0_valid)
# end of validatioin
sys.exit()
if debug:
break
if estop:
break
if debug:
break
# end for loop over minibatches
print 'This epoch has seen %d samples, train cost %.2f'%(
n_samples, numpy.mean(train_costs))
# end for loop over epochs
print 'Optimization ended.'
if best_p is not None:
utils.zipp(best_p, tparams)
use_noise.set_value(0.)
valid_err = 0
test_err = 0
if not debug:
#if valid:
valid_err, valid_perp = model.pred_probs(
engine, 'valid', f_log_probs,
verbose=model_options['verbose'])
#if test:
#test_err, test_perp = self.pred_probs(
# 'test', f_log_probs,
# verbose=model_options['verbose'])
print 'stopped at epoch %d, minibatch %d, '\
'curent Train %.2f, current Valid %.2f, current Test %.2f '%(
eidx,uidx,numpy.mean(train_err),numpy.mean(valid_err),numpy.mean(test_err))
params = copy.copy(best_p)
numpy.savez(save_model_dir+'model_best.npz',
train_err=train_err,
valid_err=valid_err, test_err=test_err, history_errs=history_errs,
**params)
if history_errs != []:
history = numpy.asarray(history_errs)
best_valid_idx = history[:,6].argmin()
numpy.savetxt(save_model_dir+'train_valid_test.txt', history, fmt='%.4f')
print 'final best exp ', history[best_valid_idx]
return train_err, valid_err, test_err
def __init__(self, config):
self.config = config
self.layers = Layers(config)
def load_svg(self, filename):
paths, attributes, svg_attributes = svg2paths2(filename)
self.layers = Layers()
self.layers.load_layers(paths, attributes)
# 0- Load dataset
dataset = [[2.7810836, 2.550537003, 0], [1.465489372, 2.362125076, 0],
[3.396561688, 4.400293529, 0], [1.38807019, 1.850220317, 0],
[3.06407232, 3.005305973, 0], [7.627531214, 2.759262235, 1],
[5.332441248, 2.088626775, 1], [6.922596716, 1.77106367, 1],
[8.675418651, -0.242068655, 1], [7.673756466, 3.508563011, 1]]
X = [x[:-1] for x in dataset]
Y = [x[-1] for x in dataset]
# 1- initialise neural network
neural_network = Sequential()
# 2- create network layer
l = Layers()
layer_1 = l.dense_layer(output_dim=2,
input_dim=2,
init='random',
activation='sigmoid')
layer_2 = l.dense_layer(output_dim=2,
input_dim=2,
init='random',
activation='sigmoid')
print layer_1
# 3- add layer into neural network
neural_network.add_layer(layer_1)
neural_network.add_layer(layer_2)
network = neural_network.get_network()
print network
