def experiment2(experiment1, data,y,cnn_adr,reference_we,sent_pos,sent_neg,adr_lexicon):
"""
See Section 4.2 in ./final-report.pdf
"""
logger.info("\n\nLoading feature matrices for second experiment\n")
tokenized_tweets = list(data['tok_text'].values)
raw_text_tweet = list(data['raw_text'].values)
base_feature = {"LSA" : experiment1.pop("LSA") , "CNN-ADR" : experiment1.pop("CNN-ADR")}
X_neg = getNegFeatures(tokenized_tweets)
X_sent = getSentFeatures(tokenized_tweets,sent_pos,sent_pos)
X_adr = getADRlexFeatures(raw_text_tweet,adr_lexicon)
X_sent_specific = concatenate([X_sent,X_neg])
X_tot_extra = concatenate([X_sent_specific,X_adr])
extra_feature = {"ADR-LEX" : X_adr, "SENT" : X_sent_specific,"TOT-EXTRA" : X_tot_extra }
complete_feature = {}
for name_base_X,base_X in base_feature.items():
for name_extra_X,extra_X in extra_feature.items():
complete_feature["{0}-{1}".format(name_base_X,name_extra_X)] = concatenate([base_X,extra_X])
return complete_feature
def apply(self, sentence, sentence_mask):
state_below = self.table.apply(sentence)
# make sure state_below: n_steps * batch_size * embedding
if state_below.ndim == 2:
n_steps = state_below.shape[0]
embed = state_below.shape[1]
state_below = state_below.reshape((n_steps, 1, embed))
hiddens_forward = self.forward.apply(state_below, sentence_mask)
if sentence_mask is None:
hiddens_backward = self.backward.apply(state_below[::-1])
else:
hiddens_backward = self.backward.apply(state_below[::-1], sentence_mask[::-1])
training_c_components = []
training_c_components.append(hiddens_forward)
training_c_components.append(hiddens_backward[::-1])
#annotaitons = T.concatenate(training_c_components, axis=2)
annotaitons = concatenate(training_c_components, axis=training_c_components[0].ndim-1)
return annotaitons
def match(self, atomsLists):
"""Function matches atoms from atomsLists to self.lattice.positions
@return: match, referenceMatch, errors
match is list of indices of places for atoms match[atomIndex] = placeIndex
referenceMatch[atomIndex] = atomIndex on allAtomsList
errors is a list of (max error, average error)"""
errors = []
allAtoms = concatenate(atomsLists)
forbiddenPosList = []
forbiddenAtomsList = [
] #we'll need both of this to know which atom is banned
posCount = len(atomsLists[0])
match = [0 for _ in range(posCount)]
referenceMatch = [0 for _ in range(posCount)]
self._initPosSearch(len(allAtoms))
for _ in range(posCount):
atomIndex, placeIndex = self.matchPos(allAtoms, len(atomsLists),
forbiddenPosList,
forbiddenAtomsList)
if not placeIndex is None:
forbiddenPosList.append(placeIndex)
forbiddenAtomsList += [
(atomIndex + n * len(atomsLists[0])) % len(allAtoms)
for n in range(len(atomsLists))
]
if not placeIndex is None:
pos = self.lattice.positions[placeIndex]
errors.append(distance(pos.x0, allAtoms[atomIndex].x0))
if not placeIndex is None:
match[atomIndex % len(atomsLists[0])] = placeIndex
referenceMatch[atomIndex % len(atomsLists[0])] = atomIndex
return match, referenceMatch, errors
def next(self):
"""
Returns lattice containing projection of the next frame of xyzFile.
If file has finnished it returns None value
"""
frame = self.xyzFile.nextFrame()
if frame is None: return None
newFrame = XYZFrame()
newFrame.boxVectors = self.lattice.boxVectors
refFrame = XYZFrame()
refFrame.boxVectors = self.lattice.boxVectors
atomsLists = self.propagateAtomsThroughPbc(frame.atoms, frame.boxSize)
allAtoms = concatenate(atomsLists)
posCount = len(atomsLists[0])
match, referenceMatch, errors = self.match(atomsLists)
for atomIndex in range(posCount):
newFrame.atoms.append(
XYZAtom(atomsLists[0][atomIndex].symbol,
*self.lattice.positions[match[atomIndex]].x0))
for atomIndex in range(posCount):
refFrame.atoms.append(
XYZAtom(allAtoms[referenceMatch[atomIndex]].__repr__()))
refFrame.atoms[-1].x += 15
for atomIndex in range(len(allAtoms)):
refFrame.atoms.append(XYZAtom(allAtoms[atomIndex].__repr__()))
refFrame.atoms[-1].x += 30
return ProjectedFrame(newFrame, refFrame, errors)
def next(self):
"""
Returns lattice containing projection of the next frame of xyzFile.
If file has finnished it returns None value
"""
frame = self.xyzFile.nextFrame()
if frame is None: return None
newFrame = XYZFrame()
newFrame.boxVectors = self.lattice.boxVectors
refFrame = XYZFrame()
refFrame.boxVectors = self.lattice.boxVectors
atomsLists = self.propagateAtomsThroughPbc(frame.atoms, frame.boxSize)
allAtoms = concatenate(atomsLists)
posCount = len(atomsLists[0])
match, referenceMatch, errors = self.match(atomsLists)
for atomIndex in range(posCount):
newFrame.atoms.append(XYZAtom(atomsLists[0][atomIndex].symbol
, *self.lattice.positions[match[atomIndex]].x0))
for atomIndex in range(posCount):
refFrame.atoms.append(XYZAtom(allAtoms[referenceMatch[atomIndex]].__repr__()))
refFrame.atoms[-1].x += 15
for atomIndex in range(len(allAtoms)):
refFrame.atoms.append(XYZAtom(allAtoms[atomIndex].__repr__()))
refFrame.atoms[-1].x += 30
return ProjectedFrame(newFrame, refFrame, errors)
def match(self, atomsLists):
"""Function matches atoms from atomsLists to self.lattice.positions
@return: match, referenceMatch, errors
match is list of indices of places for atoms match[atomIndex] = placeIndex
referenceMatch[atomIndex] = atomIndex on allAtomsList
errors is a list of (max error, average error)"""
errors = []
allAtoms = concatenate(atomsLists)
forbiddenPosList = []
forbiddenAtomsList = [] #we'll need both of this to know which atom is banned
posCount = len(atomsLists[0])
match = [0 for _ in range(posCount)]
referenceMatch = [0 for _ in range(posCount)]
self._initPosSearch(len(allAtoms))
for _ in range(posCount):
atomIndex, placeIndex = self.matchPos(allAtoms, len(atomsLists)
, forbiddenPosList
, forbiddenAtomsList)
if not placeIndex is None:
forbiddenPosList.append(placeIndex)
forbiddenAtomsList += [(atomIndex + n * len(atomsLists[0])) % len(allAtoms)
for n in range(len(atomsLists))]
if not placeIndex is None:
pos = self.lattice.positions[placeIndex]
errors.append(distance(pos.x0, allAtoms[atomIndex].x0))
if not placeIndex is None:
match[atomIndex % len(atomsLists[0])] = placeIndex
referenceMatch[atomIndex % len(atomsLists[0])] = atomIndex
return match, referenceMatch, errors
def __replace_href(self, tag):
"""Function for replace html ref to []"""
if "<a href=" in str(tag.contents):
if len(tag.contents) > 1:
tag = concatenate(list(tag.contents))
else:
tag = str(tag.contents)[1:len(str(tag.contents)) - 1]
for k, v in self.rules.items():
tag = tag.replace(k, v)
else:
tag = tag.get_text()
return tag
def getToken (self):
"""
Создать элементы из прикрепленных файлов.
Отдельно картинки, отдельно все файлы
"""
attachesAll = []
attaches = Attachment (self.parser.page).attachmentFull
attaches.sort (self.sortByLength, reverse=True)
for attach in attaches:
fname = os.path.basename (attach)
if self.filterFile (fname):
attach = Literal (fname)
attachesAll.append (attach)
finalToken = Literal (self.attachString) + concatenate (attachesAll)
finalToken = finalToken.setParseAction (self.convertToLink)("attach")
return finalToken
def getToken (self):
"""
Создать элементы из прикрепленных файлов.
Отдельно картинки, отдельно все файлы
"""
attachesAll = []
attaches = Attachment (self.parser.page).attachmentFull
attaches.sort (self.sortByLength, reverse=True)
for attach in attaches:
fname = os.path.basename (attach)
if self.filterFile (fname):
attach = Literal (fname)
attachesAll.append (attach)
finalToken = Literal (self.attachString) + concatenate (attachesAll)
finalToken = finalToken.setParseAction (self.convertToLink)("attach")
return finalToken
def build_sampler(self):
# added by Longyue
x_hist = T.ltensor3()
x_hist_mask = T.tensor3()
annotations_1 = self.encoder_hist_1.apply_1(x_hist, x_hist_mask)
annotations_1 = annotations_1[-1]
annotations_2 = self.encoder_hist_2.apply_2(annotations_1)
annotations_3 = annotations_2[-1]
x = T.lmatrix()
# Build Networks
# src_mask is None
c = self.encoder.apply(x, None, annotations_3)
#init_context = ctx[0, :, -self.n_hids_src:]
# mean pooling
init_context = c.mean(0)
# added by Longyue
init_context = concatenate([init_context, annotations_3],
axis=annotations_3.ndim - 1)
init_state = self.decoder.create_init_state(init_context)
outs = [init_state, c, annotations_3]
if not self.with_attention:
outs.append(init_context)
# compile function
print 'Building compile_init_state_and_context function ...'
self.compile_init_and_context = theano.function(
[x, x_hist, x_hist_mask], outs, name='compile_init_and_context')
print 'Done'
y = T.lvector()
cur_state = T.matrix()
# if it is the first word, emb should be all zero, and it is indicated by -1
trg_emb = T.switch(y[:, None] < 0, T.alloc(0., 1, self.n_in_trg),
self.table_trg.apply(y))
# added by Zhaopeng Tu, 2016-06-09
# for with_attention=False
if self.with_attention and self.with_coverage:
cov_before = T.tensor3()
if self.coverage_type is 'linguistic':
print 'Building compile_fertility ...'
fertility = self.decoder._get_fertility(c)
fertility = T.addbroadcast(fertility, 1)
self.compile_fertility = theano.function(
[c], [fertility], name='compile_fertility')
print 'Done'
else:
fertility = None
else:
cov_before = None
fertility = None
# apply one step
# modified by Zhaopeng Tu, 2016-04-29
# [next_state, ctxs] = self.decoder.apply(state_below=trg_emb,
results = self.decoder.apply(
state_below=trg_emb,
init_state=cur_state,
# added by Zhaopeng Tu, 2016-06-09
init_context=None if self.with_attention else init_context,
c=c if self.with_attention else None,
hist=annotations_3, # added by Longyue
one_step=True,
# added by Zhaopeng Tu, 2016-04-27
cov_before=cov_before,
fertility=fertility)
next_state = results[0]
if self.with_attention:
ctxs, alignment = results[1], results[2]
if self.with_coverage:
cov = results[3]
else:
# if with_attention=False, we always use init_context as the source representation
ctxs = init_context
readout = self.decoder.readout(next_state, ctxs, trg_emb)
# maxout
if self.maxout_part > 1:
readout = self.decoder.one_step_maxout(readout)
# apply dropout
if self.dropout < 1.0:
readout = Dropout(self.trng, readout, 0, self.dropout)
# compute the softmax probability
next_probs = self.logistic_layer.get_probs(readout)
# sample from softmax distribution to get the sample
next_sample = self.trng.multinomial(pvals=next_probs).argmax(1)
# compile function
print 'Building compile_next_state_and_probs function ...'
inps = [y, cur_state]
if self.with_attention:
inps.append(c)
else:
inps.append(init_context)
# added by Longyue
inps.append(annotations_3)
outs = [next_probs, next_state, next_sample]
# added by Zhaopeng Tu, 2016-06-09
if self.with_attention:
outs.append(alignment)
# added by Zhaopeng Tu, 2016-04-29
if self.with_coverage:
inps.append(cov_before)
if self.coverage_type is 'linguistic':
inps.append(fertility)
outs.append(cov)
self.compile_next_state_and_probs = theano.function(
inps, outs, name='compile_next_state_and_probs')
print 'Done'
# added by Zhaopeng Tu, 2016-07-18
# for reconstruction
if self.with_reconstruction:
# Build Networks
# trg_mask is None
inverse_c = T.tensor3()
# mean pooling
inverse_init_context = inverse_c.mean(0)
inverse_init_state = self.inverse_decoder.create_init_state(
inverse_init_context)
outs = [inverse_init_state]
if not self.with_attention:
outs.append(inverse_init_context)
# compile function
print 'Building compile_inverse_init_state_and_context function ...'
self.compile_inverse_init_and_context = theano.function(
[inverse_c], outs, name='compile_inverse_init_and_context')
print 'Done'
src = T.lvector()
inverse_cur_state = T.matrix()
trg_mask = T.matrix()
# if it is the first word, emb should be all zero, and it is indicated by -1
src_emb = T.switch(src[:, None] < 0, T.alloc(0., 1, self.n_in_src),
self.table_src.apply(src))
# apply one step
# modified by Zhaopeng Tu, 2016-04-29
inverse_results = self.inverse_decoder.apply(
state_below=src_emb,
init_state=inverse_cur_state,
# added by Zhaopeng Tu, 2016-06-09
init_context=None
if self.with_attention else inverse_init_context,
c=inverse_c if self.with_attention else None,
c_mask=trg_mask,
one_step=True)
inverse_next_state = inverse_results[0]
if self.with_attention:
inverse_ctxs, inverse_alignment = inverse_results[
1], inverse_results[2]
else:
# if with_attention=False, we always use init_context as the source representation
inverse_ctxs = init_context
inverse_readout = self.inverse_decoder.readout(
inverse_next_state, inverse_ctxs, src_emb)
# maxout
if self.maxout_part > 1:
inverse_readout = self.inverse_decoder.one_step_maxout(
inverse_readout)
# apply dropout
if self.dropout < 1.0:
inverse_readout = Dropout(self.srng, inverse_readout, 0,
self.dropout)
# compute the softmax probability
inverse_next_probs = self.inverse_logistic_layer.get_probs(
inverse_readout)
# sample from softmax distribution to get the sample
inverse_next_sample = self.srng.multinomial(
pvals=inverse_next_probs).argmax(1)
# compile function
print 'Building compile_inverse_next_state_and_probs function ...'
inps = [src, trg_mask, inverse_cur_state]
if self.with_attention:
inps.append(inverse_c)
else:
inps.append(inverse_init_context)
outs = [
inverse_next_probs, inverse_next_state, inverse_next_sample
]
# added by Zhaopeng Tu, 2016-06-09
if self.with_attention:
outs.append(inverse_alignment)
self.compile_inverse_next_state_and_probs = theano.function(
inps, outs, name='compile_inverse_next_state_and_probs')
print 'Done'
def build_trainer(self, src, src_mask, src_hist, src_hist_mask, trg,
trg_mask, ite):
# added by Longyue
# checked by Zhaopeng: sentence dim = n_steps, hist_len, batch_size (4, 3, 25)
# hist = (bath_size, sent_num, sent_len) --.T-->
# hist = (sent_len, sent_num, bath_size) --lookup table-->
# (sent_len, sent_num, bath_size, word_emb) --reshape-->
# (sent_len, sent_num*bath_size, word_emb) --word-level rnn-->
# (sent_len, sent_num*bath_size, hidden_size) --reshape-->
# (sent_len, sent_num, bath_size, hidden_size) --[-1]-->
# (sent_num, bath_size, hidden_size) --sent-level rnn-->
# (sent_num, bath_size, hidden_size) --[-1]-->
# (bath_size, hidden_size) = cross-sent context vector
annotations_1 = self.encoder_hist_1.apply_1(src_hist, src_hist_mask)
annotations_1 = annotations_1[-1] # get last hidden states
annotations_2 = self.encoder_hist_2.apply_2(annotations_1)
annotations_3 = annotations_2[-1] # get last hidden states
#modified by Longyue
annotations = self.encoder.apply(src, src_mask, annotations_3)
# init_context = annotations[0, :, -self.n_hids_src:]
# modification #1
# mean pooling
init_context = (annotations *
src_mask[:, :, None]).sum(0) / src_mask.sum(0)[:, None]
#added by Longyue
init_context = concatenate([init_context, annotations_3],
axis=annotations_3.ndim - 1)
trg_emb = self.table_trg.apply(trg)
trg_emb_shifted = T.zeros_like(trg_emb)
trg_emb_shifted = T.set_subtensor(trg_emb_shifted[1:], trg_emb[:-1])
# modified by Longyue
hiddens, readout, alignment = self.decoder.run_pipeline(
state_below=trg_emb_shifted,
mask_below=trg_mask,
init_context=init_context,
c=annotations,
c_mask=src_mask,
hist=annotations_3)
# apply dropout
if self.dropout < 1.0:
logger.info('Apply dropout with p = {}'.format(self.dropout))
readout = Dropout(self.trng, readout, 1, self.dropout)
p_y_given_x = self.logistic_layer.get_probs(readout)
self.cost = self.logistic_layer.cost(p_y_given_x, trg,
trg_mask) / trg.shape[1]
# self.cost = theano.printing.Print('likilihood cost:')(self.cost)
# added by Zhaopeng Tu, 2016-07-12
# for reconstruction
if self.with_reconstruction:
# now hiddens is the annotations
inverse_init_context = (hiddens * trg_mask[:, :, None]
).sum(0) / trg_mask.sum(0)[:, None]
src_emb = self.table_src.apply(src)
src_emb_shifted = T.zeros_like(src_emb)
src_emb_shifted = T.set_subtensor(src_emb_shifted[1:],
src_emb[:-1])
inverse_hiddens, inverse_readout, inverse_alignment = self.inverse_decoder.run_pipeline(
state_below=src_emb_shifted,
mask_below=src_mask,
init_context=inverse_init_context,
c=hiddens,
c_mask=trg_mask)
# apply dropout
if self.dropout < 1.0:
# logger.info('Apply dropout with p = {}'.format(self.dropout))
inverse_readout = Dropout(self.srng, inverse_readout, 1,
self.dropout)
p_x_given_y = self.inverse_logistic_layer.get_probs(
inverse_readout)
self.reconstruction_cost = self.inverse_logistic_layer.cost(
p_x_given_y, src, src_mask) / src.shape[1]
# self.reconstruction_cost = theano.printing.Print('reconstructed cost:')(self.reconstruction_cost)
self.cost += self.reconstruction_cost * self.reconstruction_weight
self.L1 = sum(T.sum(abs(param)) for param in self.params)
self.L2 = sum(T.sum(param**2) for param in self.params)
params_regular = self.L1 * 1e-6 + self.L2 * 1e-6
# params_regular = theano.printing.Print('params_regular:')(params_regular)
# train cost
train_cost = self.cost + params_regular
# gradients
grads = T.grad(train_cost, self.params)
# apply gradient clipping here
grads = grad_clip(grads, self.clip_c)
# updates
updates = adadelta(self.params, grads)
# train function
# modified by Longyue
inps = [src, src_mask, src_hist, src_hist_mask, trg, trg_mask]
self.train_fn = theano.function(inps, [train_cost],
updates=updates,
name='train_function')
def build_sampler(tparams, options, trng, use_noise):
x = tensor.matrix('x', dtype='int64')
xr = x[::-1]
n_timesteps = x.shape[0]
n_samples = x.shape[1]
# word embedding (source), forward and backward
emb = tparams['Wemb'][x.flatten()]
emb = emb.reshape([n_timesteps, n_samples, options['dim_word']])
embr = tparams['Wemb'][xr.flatten()]
embr = embr.reshape([n_timesteps, n_samples, options['dim_word']])
# encoder
proj = get_layer(options['encoder'])[1](tparams, emb, options,
prefix='encoder')
projr = get_layer(options['encoder'])[1](tparams, embr, options,
prefix='encoder_r')
# concatenate forward and backward rnn hidden states
ctx = concatenate([proj[0], projr[0][::-1]], axis=proj[0].ndim - 1)
# get the input for decoder rnn initializer mlp
ctx_mean = ctx.mean(0)
# ctx_mean = concatenate([proj[0][-1],projr[0][-1]], axis=proj[0].ndim-2)
init_state = get_layer('ff')[1](tparams, ctx_mean, options,
prefix='ff_state', activ='tanh')
print 'Building f_init...',
outs = [init_state, ctx]
f_init = theano.function([x], outs, name='f_init', profile=profile)
print 'Done'
# x: 1 x 1
y = tensor.vector('y_sampler', dtype='int64')
init_state = tensor.matrix('init_state', dtype='float32')
# if it's the first word, emb should be all zero and it is indicated by -1
emb = tensor.switch(y[:, None] < 0,
tensor.alloc(0., 1, tparams['Wemb_dec'].shape[1]),
tparams['Wemb_dec'][y])
# apply one step of conditional gru with attention
proj = get_layer(options['decoder'])[1](tparams, emb, options,
prefix='decoder',
mask=None, context=ctx,
one_step=True,
init_state=init_state)
# get the next hidden state
next_state = proj[0]
# get the weighted averages of context for this target word y
ctxs = proj[1]
logit_lstm = get_layer('ff')[1](tparams, next_state, options,
prefix='ff_logit_lstm', activ='linear')
logit_prev = get_layer('ff')[1](tparams, emb, options,
prefix='ff_logit_prev', activ='linear')
logit_ctx = get_layer('ff')[1](tparams, ctxs, options,
prefix='ff_logit_ctx', activ='linear')
logit = tensor.tanh(logit_lstm + logit_prev + logit_ctx)
if options['use_dropout']:
logit = dropout_layer(logit, use_noise, trng)
logit = get_layer('ff')[1](tparams, logit, options,
prefix='ff_logit', activ='linear')
# compute the softmax probability
next_probs = tensor.nnet.softmax(logit)
# sample from softmax distribution to get the sample
next_sample = trng.multinomial(pvals=next_probs).argmax(1)
# compile a function to do the whole thing above, next word probability,
# sampled word for the next target, next hidden state to be used
print 'Building f_next..',
inps = [y, ctx, init_state]
outs = [next_probs, next_sample, next_state]
f_next = theano.function(inps, outs, name='f_next', profile=profile)
print 'Done'
return f_init, f_next
def cross(a,b):
""" Return list containing all combinations of elements of a and b."""
return concatenate([[(ai,bi) for bi in b] for ai in a])
def build_sampler(tparams, options, trng, use_noise):
if options['input_token_level'] == 'character':
x = tensor.tensor3('x', dtype='int64')
x_mask = tensor.tensor3('x_mask', dtype='float32')
n_characters = x.shape[0]
n_words = x.shape[1]
n_samples = x.shape[2]
# extract character embeddings
cemb = tparams['Cemb'][x.flatten()]
cemb = cemb.reshape(
[n_characters, n_words, n_samples, options['dim_char']])
# compute hidden states of character embeddings
cproj = get_layer('conv')[1](tparams,
cemb,
options,
options['char_emb_filters'],
prefix='char_emb_conv',
mask=x_mask)
word_rep = tensor.max(cproj, axis=0)
word_repr = word_rep[::-1]
w_mask = x_mask.sum(0)
w_mask = tensor.clip(w_mask, 0.0, 1.0)
wr_mask = w_mask[::-1]
else:
x = tensor.matrix('x', dtype='int64')
x_mask = tensor.matrix('x_mask', dtype='float32')
n_words = x.shape[0]
n_samples = x.shape[1]
# for the backward rnn, we just need to invert x and x_mask
xr = x[::-1] # reverse words
xr_mask = x_mask[::-1]
# extract word embeddings
wemb = tparams['Wemb'][x.flatten()]
wemb = wemb.reshape([n_words, n_samples, options['dim_word']])
wembr = tparams['Wemb'][xr.flatten()]
wembr = wembr.reshape([n_words, n_samples, options['dim_word']])
w_mask, wr_mask = x_mask, xr_mask
word_rep, word_repr = wemb, wembr
encoder_vars = [x, x_mask]
if options['enc_dir'] == 'forward' or \
options['enc_dir'] == 'bidir':
# hidden states for new word embeddings for the forward rnn
src_proj = get_layer(options['encoder'])[1](tparams,
word_rep,
options['encoder_dim'],
options,
prefix='word_encoder',
mask=w_mask)
if options['enc_dir'] == 'backward' or \
options['enc_dir'] == 'bidir':
# hidden states for new word embeddings for the backward rnn
src_projr = get_layer(options['encoder'])[1](tparams,
word_repr,
options['encoder_dim'],
options,
prefix='word_encoder_r',
mask=wr_mask)
if options['enc_dir'] == 'forward':
ctx = src_proj[-1][0]
ctx_mean = [state[0][-1] for state in src_proj]
elif options['enc_dir'] == 'backward':
ctx = src_projr[-1][0]
ctx_mean = [state[0][-1] for state in src_projr]
elif options['enc_dir'] == 'bidir':
# concatenate forward and backward rnn hidden states
ctx = concatenate([src_proj[-1][0], src_projr[-1][0][::-1]],
axis=src_proj[-1][0].ndim - 1)
ctx_mean = [
concatenate([fw_state[0][-1], bw_state[0][-1]],
axis=fw_state[0].ndim - 2)
for fw_state, bw_state in zip(src_proj, src_projr)
]
elif options['enc_dir'] == 'none':
ctx_mean = [(word_rep * x_mask[:, :, None]).sum(axis=0)]
ctx_mean = [ctx_mean[0] / x_mask.sum(axis=0)[:, None]] # if avg
ctx = word_rep
assert ctx_mean[0].ndim == 2
assert len(ctx_mean) == len(options['encoder_dim'])
init_state = get_layer('ff')[1](tparams,
ctx_mean,
options,
prefix='ff_label_state')
assert len(ctx_mean) == len(init_state)
init_state = tensor.stack(init_state)
# init_state: # layers x # samples x trg_hid_dim
LOGGER.info('Building f_init')
encoder_outs = [init_state, ctx]
f_init = theano.function(encoder_vars,
encoder_outs,
name='f_init',
profile=False)
f_inits = [f_init]
# y: 1 x 1
y = tensor.vector('y_sampler', dtype='int64')
init_state = tensor.tensor3('init_state', dtype='float32')
label_emb = tparams['Lemb'][y]
# if it's the first word, emb should be all zero and it is indicated by -1
label_emb = label_emb * (y[:, None] >= 0)
init_state_ = \
[
init_state[l] for l in xrange(len(options['decoder_dim']))
]
if options['enc_dir'] != 'none':
# apply one step of conditional gru with attention
label_proj = get_layer(options['decoder'])[1](tparams,
label_emb,
options['decoder_dim'],
options,
prefix='label_decoder',
mask=None,
context=ctx,
context_mask=w_mask,
one_step=True,
init_state=init_state_)
# next_label_state: # layers x # samples x rnn hid dim
next_label_state, ctxs, dec_alphas = label_proj
proj_h = next_label_state[-1]
else:
assert options['decoder'] == 'gru'
assert len(init_state_) == 1
label_proj = get_layer(options['decoder'])[1](
tparams,
label_emb,
options['decoder_dim'],
options,
prefix='label_decoder',
mask=None,
one_step=True,
init_state=init_state_[0])
assert type(label_proj) == list and len(label_proj) == 1
next_label_state = tensor.stack([label_proj[0][0]])
proj_h = label_proj[-1][0]
assert proj_h.ndim == label_emb.ndim, \
'expected {}: actual {}'.format(label_emb.ndim, proj_h.ndim)
# XXX note that all of the context vectors are same in this case.
ctxs = (ctx * x_mask[:, :, None]).sum(axis=0) / \
x_mask.sum(axis=0)[:, None]
# XXX no alignment matrix
dec_alphas = tensor.zeros((y.shape[0], x_mask.shape[0]))
label_logit_lstm = get_layer('ff')[1](tparams,
proj_h,
options,
prefix='ff_label_logit_lstm',
activ=None)
# characters in a label at t-1 to word at t
label_logit_prev = get_layer('ff')[1](tparams,
label_emb,
options,
prefix='ff_label_logit_prev',
activ=None)
label_logit_ctx = get_layer('ff')[1](tparams,
ctxs,
options,
prefix='ff_label_logit_ctx',
activ=None)
label_logit = tensor.tanh(label_logit_lstm + label_logit_prev +
label_logit_ctx)
label_logit = get_layer('ff')[1](tparams,
label_logit,
options,
prefix='ff_label_logit',
activ=None)
# compute the softmax probability
next_label_probs = tensor.nnet.softmax(label_logit)
# sample from softmax distribution to get the sample
next_label_sample = trng.multinomial(pvals=next_label_probs).argmax(1)
# compile a function to do the whole thing above, next label probability,
# sampled label for the next target, next hidden state to be used
LOGGER.info('Building f_label_next')
f_lsamp_inps = [x_mask, y, ctx, init_state]
f_lsamp_outs = [
next_label_probs, next_label_sample, next_label_state, dec_alphas
]
f_next = theano.function(f_lsamp_inps,
f_lsamp_outs,
name='f_next',
profile=False)
f_nexts = [f_next]
return f_inits, f_nexts
def build_model(tparams, options):
""" Build a computational graph for model training
Parameters:
-------
tparams : dict
Model parameters
options : dict
Model configurations
Returns:
-------
trng : Randomstream in Theano
use_noise : TheanoSharedVariable
encoder_vars : list
This return value contains TheanoVariables used to construct
part of the computational graph, especially used in the `encoder`.
decoder_vars : list
This return value contains TheanoVariables used to construct
part of the computational graph, especially used in the `decoder`.
opt_ret : dict
costs : list
A list of costs at word-level or
both word-level and character-level costs
"""
opt_ret = dict()
trng = MRG_RandomStreams(1234)
use_noise = theano.shared(numpy.float32(0.))
word_dp = options['word_drop_prob']
word_dropout = {'use_noise': use_noise, 'dp': word_dp, 'trng': trng}
if options['input_token_level'] == 'character':
# description string: #characters x #words x #samples
x = tensor.tensor3('x', dtype='int64')
x_mask = tensor.tensor3('x_mask', dtype='float32')
n_characters = x.shape[0]
n_words = x.shape[1]
n_samples = x.shape[2]
# extract character embeddings
cemb = tparams['Cemb'][x.flatten()]
cemb = cemb.reshape(
[n_characters, n_words, n_samples, options['dim_char']])
# compute hidden states of character embeddings in the source language
cproj = get_layer('conv')[1](tparams,
cemb,
options,
options['char_emb_filters'],
prefix='char_emb_conv',
mask=x_mask)
# XXX max over time: # n_chars x # n_words x # n_samples x dim
word_rep = tensor.max(cproj, axis=0)
word_repr = word_rep[::-1]
assert word_rep.ndim == 3
# NOTE w_mask should be a matrix of elements 1 or 0
w_mask = x_mask.sum(0)
w_mask = tensor.clip(w_mask, 0.0, 1.0)
wr_mask = w_mask[::-1]
else:
# description string: #words x #samples
x = tensor.matrix('x', dtype='int64')
x_mask = tensor.matrix('x_mask', dtype='float32')
n_words = x.shape[0]
n_samples = x.shape[1]
# for the backward rnn, we just need to invert x and x_mask
xr = x[::-1] # reverse words
xr_mask = x_mask[::-1]
# extract word embeddings
wemb = tparams['Wemb'][x.flatten()]
wemb = wemb.reshape([n_words, n_samples, options['dim_word']])
wembr = tparams['Wemb'][xr.flatten()]
wembr = wembr.reshape([n_words, n_samples, options['dim_word']])
w_mask, wr_mask = x_mask, xr_mask
word_rep, word_repr = wemb, wembr
if options.get('fixed_embeddings', False):
LOGGER.info('NOTE: word embeddings are NOT going to be updated!')
word_rep = theano.gradient.zero_grad(word_rep)
word_repr = theano.gradient.zero_grad(word_repr)
y = tensor.matrix('y', dtype='int64')
y_mask = tensor.matrix('y_mask', dtype='float32')
encoder_vars = [x, x_mask] # collection of varaibles used in the encoder
decoder_vars = [y, y_mask] # used in the decoder
n_labels = y.shape[0]
if options['enc_dir'] == 'forward' or \
options['enc_dir'] == 'bidir':
# hidden states for new word embeddings for the forward rnn
src_proj = get_layer(options['encoder'])[1](tparams,
word_rep,
options['encoder_dim'],
options,
prefix='word_encoder',
mask=w_mask,
**word_dropout)
if options['enc_dir'] == 'backward' or \
options['enc_dir'] == 'bidir':
# hidden states for new word embeddings for the backward rnn
src_projr = get_layer(options['encoder'])[1](tparams,
word_repr,
options['encoder_dim'],
options,
prefix='word_encoder_r',
mask=wr_mask,
**word_dropout)
if options['enc_dir'] == 'forward':
ctx = src_proj[-1][0]
ctx_mean = [state[0][-1] for state in src_proj]
elif options['enc_dir'] == 'backward':
ctx = src_projr[-1][0]
ctx_mean = [state[0][-1] for state in src_projr]
elif options['enc_dir'] == 'bidir':
# context will be the concatenation of forward and backward rnns
ctx = concatenate([src_proj[-1][0], src_projr[-1][0][::-1]],
axis=src_proj[-1][0].ndim - 1)
ctx_mean = [
concatenate([fw_state[0][-1], bw_state[0][-1]],
axis=fw_state[0].ndim - 2)
for fw_state, bw_state in zip(src_proj, src_projr)
]
elif options['enc_dir'] == 'none':
# since there is no source encoder, the following list contains only
# a single element, which coressponds to a matrix of N x D where N is
# the number of instances in a batch and D is the dimensionality of
# instance vectors.
ctx_mean = [(word_rep * x_mask[:, :, None]).sum(axis=0)]
ctx_mean = [ctx_mean[0] / x_mask.sum(axis=0)[:, None]] # if avg
assert ctx_mean[0].ndim == 2
assert type(ctx_mean) is list
if options['enc_dir'] != 'none':
assert len(ctx_mean) == len(options['encoder_dim'])
# word embedding (target)
label_emb = tparams['Lemb'][y.flatten()]
label_emb = label_emb.reshape([n_labels, n_samples, options['dim_label']])
# We will shift the target sequence one time step
# to the right. This is done because of the bi-gram connections in the
# readout and decoder rnn. The first target will be all zeros and we will
# not condition on the last output.
label_emb_shifted = tensor.zeros_like(label_emb)
label_emb_shifted = tensor.set_subtensor(label_emb_shifted[1:],
label_emb[:-1])
label_emb = label_emb_shifted
# initial decoder state
init_state = get_layer('ff')[1](tparams,
ctx_mean,
options,
prefix='ff_label_state')
assert len(ctx_mean) == len(init_state)
if options['enc_dir'] != 'none':
# decoder - pass through the decoder conditional gru with attention
label_proj = get_layer(options['decoder'])[1](tparams,
label_emb,
options['decoder_dim'],
options,
prefix='label_decoder',
mask=y_mask,
context=ctx,
context_mask=w_mask,
one_step=False,
init_state=init_state,
**word_dropout)
# proj_h: hidden states of the decoder gru
# ctxs: weighted averages of context, generated by attention module
# dec_alphas: weights (alignment matrix)
# num trg words x batch size x num src words
proj_h_all, ctxs, opt_ret['dec_alphas'] = label_proj
proj_h = proj_h_all[-1] # pick activations of the last hidden layer
else:
assert options['decoder'] == 'gru'
label_proj = get_layer(options['decoder'])[1](tparams,
label_emb,
options['decoder_dim'],
options,
prefix='label_decoder',
mask=y_mask,
one_step=False,
init_state=init_state,
**word_dropout)
proj_h = label_proj[-1][0]
# XXX note that all of the context vectors are same in this case.
ctxs = tensor.tile(ctx_mean[0], (proj_h.shape[0], 1, 1))
# XXX no alignment matrix
opt_ret['dec_alphas'] = tensor.zeros(
(y.shape[0], y.shape[1], x.shape[0]))
# compute word probabilities
# hidden at t to word at t
label_logit_lstm = get_layer('ff')[1](tparams,
proj_h,
options,
prefix='ff_label_logit_lstm',
activ=None)
# combined representation of label at t-1 to label at t
label_logit_prev = get_layer('ff')[1](tparams,
label_emb,
options,
prefix='ff_label_logit_prev',
activ=None)
# context at t to label at t
label_logit_ctx = get_layer('ff')[1](tparams,
ctxs,
options,
prefix='ff_label_logit_ctx',
activ=None)
label_logit = tensor.tanh(label_logit_lstm + label_logit_prev +
label_logit_ctx)
if options['use_dropout']:
label_logit = dropout_layer(label_logit, use_noise, word_dp, trng)
label_logit = get_layer('ff')[1](tparams,
label_logit,
options,
prefix='ff_label_logit',
activ=None)
label_logit_shp = label_logit.shape
label_logit = label_logit.reshape(
[label_logit_shp[0] * label_logit_shp[1], label_logit_shp[2]])
label_probs = tensor.nnet.softmax(label_logit)
# compute cost for labels
y_flat = y.flatten()
y_flat_idx = tensor.arange(y_flat.shape[0]) * \
options['n_labels'] + y_flat
label_cost = -tensor.log(label_probs.flatten()[y_flat_idx])
label_cost = label_cost.reshape([y.shape[0], y.shape[1]])
label_cost = (label_cost * y_mask).sum(0)
label_cost.name = 'label_cost'
costs = [label_cost]
return trng, use_noise, encoder_vars, decoder_vars, opt_ret, costs
def evaluate(self, sess, take):
digest = Bunch(encoded=None,
reconstructed=None,
source=None,
loss=.0,
eval_loss=.0,
dumb_loss=.0)
blurred = inp.apply_gaussian(self.test_set, self._get_blur_sigma())
# Encode
for i, batch in enumerate(self._batch_generator(blurred,
shuffle=False)):
encoding = self.encode.eval(feed_dict={self.input: batch[0]})
digest.encoded = ut.concatenate(digest.encoded, encoding)
# Save encoding for visualization
encoded_no_nan = np.nan_to_num(digest.encoded)
self.embedding_assign.eval(
feed_dict={self.embedding_test_ph: encoded_no_nan})
try:
self.embedding_saver.save(sess,
self.get_checkpoint_path() + EMB_SUFFIX)
except:
ut.print_info("Unexpected error: %s" % str(sys.exc_info()[0]),
color=33)
# Calculate expected evaluation
expected = digest.encoded[1:-1] * 2 - digest.encoded[:-2]
average = 0.5 * (digest.encoded[1:-1] + digest.encoded[:-2])
digest.size = len(expected)
# evaluation summaries
self.summary_writer.add_summary(self.eval_summs.eval(
feed_dict={self.blur_ph: self._get_blur_sigma()}),
global_step=self.get_past_epochs())
# evaluation losses
for p in self._batch_permutation_generator(digest.size, shuffle=False):
digest.loss += self.eval_loss.eval(
feed_dict={
self.encoding: digest.encoded[p + 2],
self.target: blurred[p + 2]
})
digest.eval_loss += self.eval_loss.eval(feed_dict={
self.encoding: expected[p],
self.target: blurred[p + 2]
})
digest.dumb_loss += self.loss_ae.eval(feed_dict={
self.input: blurred[p],
self.target: blurred[p + 2]
})
# for batch in self._batch_generator(blurred, batches=1):
# digest.source = batch[1][:take]
# digest.reconstructed = self.decode.eval(feed_dict={self.input: batch[0]})[:take]
# Reconstruction visualizations
for p in self._batch_permutation_generator(digest.size,
shuffle=True,
batches=1):
self.visualization_batch_perm = self.visualization_batch_perm if self.visualization_batch_perm is not None else p
p = self.visualization_batch_perm
digest.source = self.eval_decode.eval(
feed_dict={self.encoding: expected[p]})[:take]
digest.source = blurred[(p + 2)[:take]]
digest.reconstructed = self.eval_decode.eval(
feed_dict={self.encoding: average[p]})[:take]
self._eval_image_summaries(blurred[p], digest.encoded[p],
average[p], expected[p])
digest.dumb_loss = guard_nan(digest.dumb_loss)
digest.eval_loss = guard_nan(digest.eval_loss)
digest.loss = guard_nan(digest.loss)
return digest
def download_video(cls, page_url, title, dest='video'):
"""从视频页面获取到视频链接进行下载"""
res = get_page(page_url, **{'verify': False})
# 从网页源码获取视频链接
origin_txt = re.findall(
r'<script>window.__playinfo__=(\{.*?\})</script>', res.text,
re.S)[0]
origin_json = json.loads(origin_txt, encoding='utf-8')
urls = origin_json['durl']
# 下载视频
size = 0
chunk = 1024
content_size = sum([i['size'] for i in urls])
print('文件大小： %0.2fMB' % (content_size / chunk / 1024))
# 创建存放视频临时文件夹
create_folder(os.path.join(BASE_DIR, title))
start = time.time()
# 循环下载视频
for i, data in enumerate(urls):
url = data['url']
header = {
'Origin': 'https://www.bilibili.com',
'Referer': page_url,
}
try:
# 请求视频链接
response = get_page(url,
header=header,
**{
'verify': False,
'stream': True
})
video_path = title + '/' + '{}.mp4'.format(i)
# 下载视频
with open(video_path, 'wb') as file:
for item in response.iter_content(chunk):
file.write(item)
file.flush()
size += len(item)
print('\r' + '[下载进度]：%s %0.2f%%' %
('>' * int(size * 50 / content_size),
float(size / content_size) * 100),
end='')
except Exception as e:
print(e)
stop = time.time()
print('\n' + '视频下载完成，耗时%.2f秒' % (stop - start))
# 合并视频
dest = os.path.join(BASE_DIR, dest)
video_save_path = concatenate(title=title, dest=dest)
video = {'title': title, 'url': page_url, 'path': video_save_path}
return video
def cross(a,b):
""" Return list containing all combinations of elements of a and b."""
return concatenate([[(ai,bi) for bi in b] for ai in a])
def next(self):
"""
Returns lattice containing projection of the next frame of xyzFile.
If file has finnished it returns None value
"""
frame = self.xyzFile.nextFrame()
if frame is None: return None
newFrame = XYZFrame()
newFrame.boxVectors = self.lattice.boxVectors
refFrame = XYZFrame()
refFrame.boxVectors = self.lattice.boxVectors
atomsLists = self.propagateAtomsThroughPbc(frame.atoms, frame.boxSize)
allAtoms = concatenate(atomsLists)
#posCount = len(atomsLists[0])
errors = []
differences = []
#match, referenceMatch, errors = self.match(atomsLists)
atoms = []
# print self._radius
for pos in self.lattice.positions:
ACounter = 0
BCounter = 0
for atom in allAtoms:
dist = distance(atom.x0, pos.x0)
atoms.append(atom)
if atom.symbol == "DLPC":
try:
ACounter += 1. / (dist**2)
except:
ACounter = 1e308
else:
if atom.symbol == "DOPC":
try:
BCounter += 1. / (dist**2)
except:
BCounter = 1e308
differences.append((ACounter - BCounter, pos.x0))
# if ACounter > BCounter: symbol = "DSPC"
# else: symbol = "A"
# newFrame.atoms.append(XYZAtom(symbol, *pos.x0))
errors.append(0.0)
num = 0
for atom in atoms:
refFrame.atoms.append(XYZAtom(atom.symbol, *atom.x0))
refFrame.atoms[-1].x += 17
for atom in allAtoms:
refFrame.atoms.append(XYZAtom(atom.symbol, *atom.x0))
refFrame.atoms[-1].x += 48
for diff, pos in sorted(differences, key=lambda tup: tup[0]):
num += 1
if num < 129:
symbol = "DOPC"
else:
symbol = "DPPC"
newFrame.atoms.append(XYZAtom(symbol, *pos))
#for atomIndex in range(posCount):
# newFrame.atoms.append(XYZAtom(atomsLists[0][atomIndex].symbol
# , *self.lattice.positions[match[atomIndex]].x0))
return ProjectedFrame(newFrame, refFrame, errors)
def sample(self,
material: int,
quantity: int = 1) -> Tuple[Tensor, Tuple[Tensor, SVBRDF]]:
'''
Returns a Dataset tuple derived from the given texture with the specified batch size.
Args:
material: Index of the texture to sample from this Dataset.
quantity: Number of samples to include in the returned batch.
Returns:
Tensor [B, 3, R, C] of cropped front-parallel "renderings" of the indicated texture which are suitable for
consumption by an SVBRDF autoencoder, Tensor [B, R, C, 3] representing the ground-truth normal maps of the
texture samples, and an SVBRDF embedded with the ground-truth parameter values for the texture samples.
'''
assert 0 <= material < len(
self.textures
), f'Material {material} falls outside the half-open range [0, {len(self.textures)}).'
# Load the normals and SVBRDF parameters of the texture only once; hitting the disk is expensive!
texture = self.textures[material]
texture_normals = self._load_normals(texture)
texture_parameters = self._load_parameters(texture)
# The texture and crop dimensions are, of course, the same for each sample.
num_texture_rows = self._dims['Texture'][0]
num_texture_cols = self._dims['Texture'][1]
num_crop_rows = self._dims['Crop'][0]
num_crop_cols = self._dims['Crop'][1]
# Initializing the batch Tensors with an empty Tensor() invokes special behaviour in utils.concatenate().
batch_inputs = Tensor()
batch_normals = Tensor()
batch_parameters = Tensor()
for sample in range(quantity):
lights, viewer = self._lights, self._viewer
# Wrapping or reflection crops are not supported so an embedded rectangle will suffice.
row_crop, col_crop = utils.sample_embedded_rectangle(
num_outer_rows=num_texture_rows,
num_inner_rows=num_crop_rows,
num_outer_cols=num_texture_cols,
num_inner_cols=num_crop_cols)
# The SVBRDF is cloned here to avoid polluting the global SVBRDF with arbitrary parameter values.
svbrdf = self._svbrdf.clone()
svbrdf.parameters = texture_parameters[:, row_crop, col_crop]
normals = texture_normals[:, row_crop, col_crop]
# It should not matter if each picture in the batch was produced from a different rendering context.
for transform in self._transforms:
normals, svbrdf.parameters, lights, viewer = transform.apply(
normals, svbrdf.parameters, lights, viewer)
# The surface radiance can be interpreted as a perspective rendering from the location of the Viewer.
radiance = shader.shade(surface=self._surface,
normals=normals,
lights=lights,
viewer=viewer,
svbrdf=svbrdf)
inputs = torch.cat([radiance, self._radial_distance_field],
dim=3).permute(0, 3, 1, 2)
# Unlike torch.cat(), utils.concatenate() gracefully handles empty tensors in the first argument.
batch_normals = utils.concatenate(batch_normals, normals)
batch_parameters = utils.concatenate(batch_parameters,
svbrdf.parameters)
batch_inputs = utils.concatenate(batch_inputs, inputs)
# Clone the SVBRDF (again) to avoid overwriting parameter values in a multiprocessing context.
batch_svbrdf = self._svbrdf.clone()
batch_svbrdf.parameters = batch_parameters
# The structure of the Dataset tuple suggests an association between the normals and SVBRDF.
return batch_inputs, (batch_normals, batch_svbrdf)
def build_sampler(tparams, options, trng):
x = tensor.matrix('x', dtype='int64')
xr = x[::-1]
n_timesteps = x.shape[0]
n_samples = x.shape[1]
# word embedding (source), forward and backward
emb = tparams['Wemb'][x.flatten()]
emb = emb.reshape([n_timesteps, n_samples, options['dim_word_src']])
embr = tparams['Wemb'][xr.flatten()]
embr = embr.reshape([n_timesteps, n_samples, options['dim_word_src']])
# encoder
proj = get_layer(options['encoder'])[1](tparams,
emb,
options,
prefix='encoder')
projr = get_layer(options['encoder'])[1](tparams,
embr,
options,
prefix='encoder_r')
# concatenate forward and backward rnn hidden states
ctx = concatenate([proj[0], projr[0][::-1]], axis=proj[0].ndim - 1)
# get the input for decoder rnn initializer mlp
# ctx_mean = ctx.mean(0)
ctx_mean = concatenate([proj[0][-1], projr[0][-1]], axis=proj[0].ndim - 2)
init_state = get_layer('ff')[1](tparams,
ctx_mean,
options,
prefix='ff_state',
activ=tensor.tanh)
LOGGER.info('Building f_init')
outs = [init_state, ctx]
f_init = theano.function([x], outs, name='f_init', profile=False)
# x: 1 x 1
y = tensor.vector('y_sampler', dtype='int64')
init_state = tensor.matrix('init_state', dtype='float32')
# if it's the first word, emb should be all zero and it is indicated by -1
emb = tensor.switch(y[:, None] < 0,
tensor.alloc(0., 1, tparams['Wemb_dec'].shape[1]),
tparams['Wemb_dec'][y])
# apply one step of conditional gru with attention
proj = get_layer(options['decoder'])[1](tparams,
emb,
options,
prefix='decoder',
mask=None,
context=ctx,
one_step=True,
init_state=init_state)
# get the next hidden state
next_state = proj[0]
# get the weighted averages of context for this target word y
ctxs = proj[1]
dec_alphas = proj[2]
logit_lstm = get_layer('ff')[1](tparams,
next_state,
options,
prefix='ff_logit_lstm',
activ=None)
logit_prev = get_layer('ff')[1](tparams,
emb,
options,
prefix='ff_logit_prev',
activ=None)
logit_ctx = get_layer('ff')[1](tparams,
ctxs,
options,
prefix='ff_logit_ctx',
activ=None)
logit = tensor.tanh(logit_lstm + logit_prev + logit_ctx)
logit = get_layer('ff')[1](tparams,
logit,
options,
prefix='ff_logit',
activ=None)
# compute the softmax probability
next_probs = tensor.nnet.softmax(logit)
# sample from softmax distribution to get the sample
next_sample = trng.multinomial(pvals=next_probs).argmax(1)
# compile a function to do the whole thing above, next word probability,
# sampled word for the next target, next hidden state to be used
LOGGER.info('Building f_next')
inps = [y, ctx, init_state]
outs = [next_probs, next_sample, next_state, dec_alphas]
f_next = theano.function(inps, outs, name='f_next', profile=False)
return f_init, f_next
def build_model(tparams, options):
opt_ret = dict()
trng = MRG_RandomStreams(1234)
use_noise = theano.shared(numpy.float32(0.))
# description string: #words x #samples
x = tensor.matrix('x', dtype='int64')
x_mask = tensor.matrix('x_mask', dtype='float32')
y = tensor.matrix('y', dtype='int64')
y_mask = tensor.matrix('y_mask', dtype='float32')
# for the backward rnn, we just need to invert x and x_mask
xr = x[::-1]
xr_mask = x_mask[::-1]
n_timesteps = x.shape[0]
n_timesteps_trg = y.shape[0]
n_samples = x.shape[1]
# word embedding for forward rnn (source)
emb = tparams['Wemb'][x.flatten()]
emb = emb.reshape([n_timesteps, n_samples, options['dim_word_src']])
proj = get_layer(options['encoder'])[1](tparams,
emb,
options,
prefix='encoder',
mask=x_mask)
# word embedding for backward rnn (source)
embr = tparams['Wemb'][xr.flatten()]
embr = embr.reshape([n_timesteps, n_samples, options['dim_word_src']])
projr = get_layer(options['encoder'])[1](tparams,
embr,
options,
prefix='encoder_r',
mask=xr_mask)
# context will be the concatenation of forward and backward rnns
ctx = concatenate([proj[0], projr[0][::-1]], axis=proj[0].ndim - 1)
# mean of the context (across time) will be used to initialize decoder rnn
# ctx_mean = (ctx * x_mask[:, :, None]).sum(0) / x_mask.sum(0)[:, None]
# or you can use the last state of forward + backward encoder rnns
ctx_mean = concatenate([proj[0][-1], projr[0][-1]], axis=proj[0].ndim - 2)
# initial decoder state
init_state = get_layer('ff')[1](tparams,
ctx_mean,
options,
prefix='ff_state',
activ=tensor.tanh)
# word embedding (target), we will shift the target sequence one time step
# to the right. This is done because of the bi-gram connections in the
# readout and decoder rnn. The first target will be all zeros and we will
# not condition on the last output.
emb = tparams['Wemb_dec'][y.flatten()]
emb = emb.reshape([n_timesteps_trg, n_samples, options['dim_word_trg']])
emb_shifted = tensor.zeros_like(emb)
emb_shifted = tensor.set_subtensor(emb_shifted[1:], emb[:-1])
emb = emb_shifted
# decoder - pass through the decoder conditional gru with attention
proj = get_layer(options['decoder'])[1](tparams,
emb,
options,
prefix='decoder',
mask=y_mask,
context=ctx,
context_mask=x_mask,
one_step=False,
init_state=init_state)
# hidden states of the decoder gru
proj_h = proj[0]
# weighted averages of context, generated by attention module
ctxs = proj[1]
# weights (alignment matrix)
opt_ret['dec_alphas'] = proj[2]
# compute word probabilities
logit_lstm = get_layer('ff')[1](tparams,
proj_h,
options,
prefix='ff_logit_lstm',
activ=None)
logit_prev = get_layer('ff')[1](tparams,
emb,
options,
prefix='ff_logit_prev',
activ=None)
logit_ctx = get_layer('ff')[1](tparams,
ctxs,
options,
prefix='ff_logit_ctx',
activ=None)
logit = tensor.tanh(logit_lstm + logit_prev + logit_ctx)
if options['use_dropout']:
logit = dropout_layer(logit, use_noise, trng)
logit = get_layer('ff')[1](tparams,
logit,
options,
prefix='ff_logit',
activ=None)
logit_shp = logit.shape
probs = tensor.nnet.softmax(
logit.reshape([logit_shp[0] * logit_shp[1], logit_shp[2]]))
# cost
y_flat = y.flatten()
y_flat_idx = tensor.arange(y_flat.shape[0]) * options['n_words'] + y_flat
cost = -tensor.log(probs.flatten()[y_flat_idx])
cost = cost.reshape([y.shape[0], y.shape[1]])
cost = (cost * y_mask).sum(0)
return trng, use_noise, x, x_mask, y, y_mask, opt_ret, cost
def next(self):
"""
Returns lattice containing projection of the next frame of xyzFile.
If file has finnished it returns None value
"""
frame = self.xyzFile.nextFrame()
if frame is None: return None
newFrame = XYZFrame()
newFrame.boxVectors = self.lattice.boxVectors
refFrame = XYZFrame()
refFrame.boxVectors = self.lattice.boxVectors
atomsLists = self.propagateAtomsThroughPbc(frame.atoms, frame.boxSize)
allAtoms = concatenate(atomsLists)
#posCount = len(atomsLists[0])
errors = []
differences = []
#match, referenceMatch, errors = self.match(atomsLists)
atoms = []
# print self._radius
for pos in self.lattice.positions:
ACounter = 0
BCounter = 0
for atom in allAtoms:
dist = distance( atom.x0, pos.x0 )
atoms.append(atom)
if atom.symbol == "DLPC":
try:
ACounter += 1. / ( dist**2 )
except:
ACounter = 1e308
else:
if atom.symbol == "DOPC":
try:
BCounter += 1. / ( dist**2 )
except:
BCounter = 1e308
differences.append( (ACounter - BCounter, pos.x0) )
# if ACounter > BCounter: symbol = "DSPC"
# else: symbol = "A"
# newFrame.atoms.append(XYZAtom(symbol, *pos.x0))
errors.append(0.0)
num = 0
for atom in atoms:
refFrame.atoms.append(XYZAtom(atom.symbol, *atom.x0))
refFrame.atoms[-1].x += 17
for atom in allAtoms:
refFrame.atoms.append(XYZAtom(atom.symbol, *atom.x0))
refFrame.atoms[-1].x += 48
for diff, pos in sorted(differences, key = lambda tup: tup[0]):
num += 1
if num < 129:
symbol = "DOPC"
else: symbol = "DPPC"
newFrame.atoms.append(XYZAtom(symbol, *pos))
#for atomIndex in range(posCount):
# newFrame.atoms.append(XYZAtom(atomsLists[0][atomIndex].symbol
# , *self.lattice.positions[match[atomIndex]].x0))
return ProjectedFrame(newFrame, refFrame, errors)
def build_model(tparams, options):
"""
Computation graph for the model
"""
opt_ret = dict()
use_noise = theano.shared(numpy.asarray(1., dtype=theano.config.floatX))
try:
trng = RandomStreams(1234, use_cuda=True)
except:
print "Could not apply use_cuda==True in RandonStreams ..."
trng = RandomStreams(1234)
xs = []
xmasks = []
langs = options['langs']
for lang in langs:
# description string: #words x #samples
x_lang = tensor.matrix('x_%s' % lang, dtype='int64')
mask_lang = tensor.matrix('mask_%s' % lang, dtype='float32')
xs.append(x_lang)
xmasks.append(mask_lang)
xs_r = []
xmasks_r = []
if options['bidirectional_enc']:
for i, lang in enumerate(langs):
x_lang = xs[i]
mask_lang = xmasks[i]
# reverse
x_lang_r = x_lang[::-1]
mask_lang_r = mask_lang[::-1]
xs_r.append(x_lang_r)
xmasks_r.append(mask_lang_r)
sents_all = []
im = tensor.matrix('im', dtype='float32')
n_samples = im.shape[0]
for i, lang in enumerate(langs):
x_lang = xs[i]
mask_lang = xmasks[i]
n_timesteps_lang = x_lang.shape[0]
n_samples_lang = x_lang.shape[1]
if options['use_dropout']:
# dropout probs for the word embeddings
retain_probability_emb = 1 - options['dropout_embedding']
# dropout probs for the RNN hidden states
retain_probability_hidden = 1 - options['dropout_hidden']
# dropout probs for the source words
retain_probability_source = 1 - options['dropout_source']
# hidden states
rec_dropout = shared_dropout_layer(
(2, n_samples_lang, options['dim']), use_noise, trng,
retain_probability_hidden)
rec_dropout_r = shared_dropout_layer(
(2, n_samples_lang, options['dim']), use_noise, trng,
retain_probability_hidden)
# word embeddings
emb_dropout = shared_dropout_layer(
(2, n_samples_lang, options['dim_word']), use_noise, trng,
retain_probability_emb)
emb_dropout_r = shared_dropout_layer(
(2, n_samples_lang, options['dim_word']), use_noise, trng,
retain_probability_emb)
# source words
source_dropout = shared_dropout_layer(
(n_timesteps_lang, n_samples_lang, 1), use_noise, trng,
retain_probability_source)
source_dropout = tensor.tile(source_dropout,
(1, 1, options['dim_word']))
else:
# hidden states
rec_dropout = theano.shared(numpy.array([1.] * 2, dtype='float32'))
rec_dropout_r = theano.shared(
numpy.array([1.] * 2, dtype='float32'))
# word embeddings
emb_dropout = theano.shared(numpy.array([1.] * 2, dtype='float32'))
emb_dropout_r = theano.shared(
numpy.array([1.] * 2, dtype='float32'))
# Word embedding (for a particular language `lang`)
# forward
emb_lang = tparams['Wemb_%s' % lang][x_lang.flatten()]
emb_lang = emb_lang.reshape(
[n_timesteps_lang, n_samples_lang, options['dim_word']])
if options['use_dropout']:
emb_lang *= source_dropout
if options['bidirectional_enc']:
x_lang_r = xs_r[i]
mask_lang_r = xmasks_r[i]
# backward lang encoder
emb_lang_r = tparams['Wemb_%s' % lang][x_lang_r.flatten()]
emb_lang_r = emb_lang_r.reshape(
[n_timesteps_lang, n_samples_lang, options['dim_word']])
if options['use_dropout']:
emb_lang_r *= source_dropout[::-1]
# Encode sentence in language `lang`
if options['encoder_%s' % lang] == 'bow':
sents_lang = (emb_lang * mask_lang[:, :, None]).sum(0)
else:
# iteratively push input from first hidden layer until the last
for i in range(int(options['n_enc_hidden_layers'])):
layer_name_prefix = 'encoder_%s_%i' % (lang, i)
# if first hidden layer use wembs, otherwise output of previous hidden layer
layer_below = emb_lang if i == 0 else layer_below[0]
# do not apply dropout on word embeddings layer
#if options['use_dropout'] and i>0:
# layer_below = dropout_layer(layer_below, use_noise, trng, prob=options['dropout_prob'])
layer_below = get_layer(options['encoder_%s' % lang])[1](
tparams,
layer_below,
options,
None,
prefix=layer_name_prefix,
mask=mask_lang,
emb_dropout=emb_dropout,
rec_dropout=rec_dropout)
if i == int(options['n_enc_hidden_layers']) - 1:
# sentence embeddings (projections) are the output of the last hidden layer
proj_lang = layer_below
# apply forward and backward steps and concatenate both
if options['bidirectional_enc']:
# concatenate forward and backward pass RNNs
# iteratively push input from first hidden layer until the last
for i in range(int(options['n_enc_hidden_layers'])):
layer_name_prefix = 'encoder_%s_r_%i' % (lang, i)
# if first hidden layer use wembs, else output of prev hidden layer
layer_below = emb_lang_r if i == 0 else layer_below[0]
# do not apply dropout on word embeddings layer
#if options['use_dropout'] and i>0:
# layer_below = dropout_layer(layer_below, use_noise, trng, prob=options['dropout_prob'])
layer_below = get_layer(options['encoder_%s' % lang])[1](
tparams,
layer_below,
options,
None,
prefix=layer_name_prefix,
mask=mask_lang_r,
emb_dropout=emb_dropout_r,
rec_dropout=rec_dropout_r)
if i == int(options['n_enc_hidden_layers']) - 1:
# sentence embeddings (projections) are the output of the last hidden layer
proj_lang_r = layer_below
# use the last state of forward + backward encoder rnns
sents_lang = concatenate(
[proj_lang[0][-1], proj_lang_r[0][-1]],
axis=proj_lang[0].ndim - 2)
else:
sents_lang = proj_lang[0][-1]
if options['use_dropout']:
sents_lang *= shared_dropout_layer(
(n_samples_lang, options['dim']), use_noise, trng,
retain_probability_hidden)
# project sentences into multimodal space
sents_mm = get_layer('ff')[1](tparams,
sents_lang,
options,
prefix='ff_sentence_mm',
activ='linear')
if options['attention_type'] == 'dot':
sents_mm = l2norm(sents_mm)
if options['use_dropout']:
sents_mm *= shared_dropout_layer(
(n_samples_lang, options['dim_multimodal']), use_noise, trng,
retain_probability_hidden)
sents_all.append(sents_mm)
# Encode images
images = get_layer('ff')[1](tparams,
im,
options,
prefix='ff_image_mm',
activ='linear')
if options['attention_type'] == 'dot':
images = l2norm(images)
if options['use_dropout']:
images *= shared_dropout_layer((n_samples, options['dim_multimodal']),
use_noise, trng,
retain_probability_hidden)
# Compute loss
lambda_img_sent = options['lambda_img_sent']
lambda_sent_sent = options['lambda_sent_sent']
if options['use_all_costs']:
cost = contrastive_loss_all(tparams, options, images, sents_all,
lambda_img_sent, lambda_sent_sent)
else:
cost = contrastive_loss(tparams, options, images, sents_all)
# return flattened inputs
inps = []
inps.extend(xs)
inps.extend(xmasks)
inps.append(im)
return trng, inps, cost
def build_model(tparams, options):
opt_ret = dict()
trng = RandomStreams(1234)
use_noise = theano.shared(np.float32(0.))
# description string: #words x #samples
x = tensor.matrix('x', dtype='int64')
x_mask = tensor.matrix('x_mask', dtype='float32')
y = tensor.matrix('y', dtype='int64')
y_mask = tensor.matrix('y_mask', dtype='float32')
# for the backward rnn, we just need to invert x and x_mask
xr = x[::-1]
xr_mask = x_mask[::-1]
n_timesteps = x.shape[0]
n_timesteps_trg = y.shape[0]
n_samples = x.shape[1]
# word embedding for forward rnn (source)
emb = tparams['Wemb'][x.flatten()]
emb = emb.reshape([n_timesteps, n_samples, options['dim_word']])
proj = get_layer(options['encoder'])[1](tparams, emb, options,
prefix='encoder',
mask=x_mask)
# word embedding for backward rnn (source)
embr = tparams['Wemb'][xr.flatten()]
embr = embr.reshape([n_timesteps, n_samples, options['dim_word']])
projr = get_layer(options['encoder'])[1](tparams, embr, options,
prefix='encoder_r',
mask=xr_mask)
# context will be the concatenation of forward and backward rnns
ctx = concatenate([proj[0], projr[0][::-1]], axis=proj[0].ndim - 1)
# mean of the context (across time) will be used to initialize decoder rnn
ctx_mean = (ctx * x_mask[:, :, None]).sum(0) / x_mask.sum(0)[:, None]
# or you can use the last state of forward + backward encoder rnns
# ctx_mean = concatenate([proj[0][-1], projr[0][-1]], axis=proj[0].ndim-2)
# initial decoder state
init_state = get_layer('ff')[1](tparams, ctx_mean, options,
prefix='ff_state', activ='tanh')
# word embedding (target), we will shift the target sequence one time step
# to the right. This is done because of the bi-gram connections in the
# readout and decoder rnn. The first target will be all zeros and we will
# not condition on the last output.
emb = tparams['Wemb_dec'][y.flatten()]
emb = emb.reshape([n_timesteps_trg, n_samples, options['dim_word']])
emb_shifted = tensor.zeros_like(emb)
emb_shifted = tensor.set_subtensor(emb_shifted[1:], emb[:-1])
emb = emb_shifted
# decoder - pass through the decoder conditional gru with attention
proj = get_layer(options['decoder'])[1](tparams, emb, options,
prefix='decoder',
mask=y_mask, context=ctx,
context_mask=x_mask,
one_step=False,
init_state=init_state)
# hidden states of the decoder gru
proj_h = proj[0] # n_timestep * n_sample * dim
# weighted averages of context, generated by attention module
ctxs = proj[1]
# weights (alignment matrix)
opt_ret['dec_alphas'] = proj[2]
# compute word probabilities
logit_lstm = get_layer('ff')[1](tparams, proj_h, options,
prefix='ff_logit_lstm', activ='linear')
logit_prev = get_layer('ff')[1](tparams, emb, options,
prefix='ff_logit_prev', activ='linear')
logit_ctx = get_layer('ff')[1](tparams, ctxs, options,
prefix='ff_logit_ctx', activ='linear')
logit = tensor.tanh(logit_lstm + logit_prev + logit_ctx) # n_timestep * n_sample * dim_word
if options['use_dropout']:
logit = dropout_layer(logit, use_noise, trng)
logit = get_layer('ff')[1](tparams, logit, options,
prefix='ff_logit', activ='linear') # n_timestep * n_sample * n_words
logit_shp = logit.shape
probs = tensor.nnet.softmax(logit.reshape([logit_shp[0] * logit_shp[1],
logit_shp[2]]))
# cost
y_flat = y.flatten()
y_flat_idx = tensor.arange(y_flat.shape[0]) * options['n_words'] + y_flat
cost = -tensor.log(probs.flatten()[y_flat_idx])
cost = cost.reshape([y.shape[0], y.shape[1]])
cost = (cost * y_mask).sum(0)
return trng, use_noise, x, x_mask, y, y_mask, opt_ret, cost, ctx_mean
def build_sentence_encoders(tparams, options):
"""
Sentence encoder only to be used at test time
"""
opt_ret = dict()
trng = RandomStreams(1234)
#xs, masks, sents_all = [], [], []
in_outs = []
langs = options['langs']
for lang in langs:
# description string: #words x #samples
# forward
x = tensor.matrix('x_%s' % lang, dtype='int64')
mask = tensor.matrix('x_mask_%s' % lang, dtype='float32')
n_timesteps = x.shape[0]
n_samples = x.shape[1]
# Word embedding (forward)
emb = tparams['Wemb_%s' % lang][x.flatten()].reshape(
[n_timesteps, n_samples, options['dim_word']])
if options['bidirectional_enc']:
# backward RNN
x_r = x[::-1]
mask_r = mask[::-1]
emb_r = tparams['Wemb_%s' % lang][x_r.flatten()].reshape(
[n_timesteps, n_samples, options['dim_word']])
if options['use_dropout']:
retain_probability_emb = 1 - options['dropout_embedding']
retain_probability_hidden = 1 - options['dropout_hidden']
retain_probability_source = 1 - options['dropout_source']
rec_dropout = theano.shared(
numpy.array([retain_probability_hidden] * 2, dtype='float32'))
rec_dropout_r = theano.shared(
numpy.array([retain_probability_hidden] * 2, dtype='float32'))
emb_dropout = theano.shared(
numpy.array([retain_probability_emb] * 2, dtype='float32'))
emb_dropout_r = theano.shared(
numpy.array([retain_probability_emb] * 2, dtype='float32'))
source_dropout = theano.shared(
numpy.float32(retain_probability_source))
emb *= source_dropout
if options['bidirectional_enc']:
embr *= source_dropout
else:
rec_dropout = theano.shared(numpy.array([1.] * 2, dtype='float32'))
rec_dropout_r = theano.shared(
numpy.array([1.] * 2, dtype='float32'))
emb_dropout = theano.shared(numpy.array([1.] * 2, dtype='float32'))
emb_dropout_r = theano.shared(
numpy.array([1.] * 2, dtype='float32'))
# Encode sentences
if options['encoder_%s' % lang] == 'bow':
sents = (emb * mask[:, :, None]).sum(0)
else:
# iteratively push input from first hidden layer until the last
for i in range(int(options['n_enc_hidden_layers'])):
layer_name_prefix = 'encoder_%s_%i' % (lang, i)
# if first layer input are wembs, otherwise input will be output of last hidden layer
layer_below = emb if i == 0 else layer_below[0]
layer_below = get_layer(options['encoder_%s' % lang])[1](
tparams,
layer_below,
options,
None,
prefix=layer_name_prefix,
mask=mask,
emb_dropout=emb_dropout,
rec_dropout=rec_dropout)
if i == int(options['n_enc_hidden_layers']) - 1:
# sentence embeddings (projections) are the output of the last hidden layer
proj = layer_below
if options['bidirectional_enc']:
for i in range(int(options['n_enc_hidden_layers'])):
layer_name_prefix = 'encoder_%s_r_%i' % (lang, i)
# if first layer input are wembs, otherwise input will be output of last hidden layer
layer_below = emb_r if i == 0 else layer_below[0]
layer_below = get_layer(options['encoder_%s' % lang])[1](
tparams,
layer_below,
options,
None,
prefix=layer_name_prefix,
mask=mask_r,
emb_dropout=emb_dropout_r,
rec_dropout=rec_dropout_r)
if i == int(options['n_enc_hidden_layers']) - 1:
# sentence embeddings (projections) are the output of the last hidden layer
proj_r = layer_below
# use last hidden state of forward and backward RNNs
sents = concatenate([proj[0][-1], proj_r[0][-1]],
axis=proj[0].ndim - 2)
else:
sents = proj[0][-1]
if options['use_dropout']:
sents *= shared_dropout_layer((n_samples, options['dim']),
use_noise, trng,
retain_probability_hidden)
# project sentences into multimodal space
sents_mm = get_layer('ff')[1](tparams,
sents,
options,
prefix='ff_sentence_mm',
activ='linear')
if not 'attention_type' in options or options[
'attention_type'] == 'dot':
sents_mm = l2norm(sents_mm)
if options['use_dropout']:
sents_mm *= shared_dropout_layer(
(n_samples, options['dim_multimodal']), use_noise, trng,
retain_probability_hidden)
# outputs per language
in_outs.append(([x, mask], sents_mm))
return trng, in_outs
def build_encoder(self,
dim_emb=0,
factors=1,
dim_per_factor=(),
encoder_layers=1,
encoder='gru',
dropout=False,
dropout_word=0.,
dropout_emb=0.,
dropout_rec=0.,
trng=None,
use_noise=True,
**kwargs):
"""
Build the bi-directional encoder (bi-gru / bi-lstm)
MUST match nmt.model.build_encoder for multitasking
:param dim_emb:
:param factors:
:param dim_per_factor:
:param encoder_layers:
:param encoder: gru or lstm
:param dropout:
:param dropout_word:
:param dropout_emb:
:param dropout_rec:
:param trng:
:param use_noise: apply dropout (use_noise) or not (test)
:param kwargs:
:return:
"""
logger.warn('Building encoder - use_noise: {}'.format(use_noise))
assert sum(dim_per_factor) == dim_emb, 'sum dim_per_factor != dim_emb'
dropout = dropout and use_noise # no dropout during test time
# input to forward rnn (#factors x #words x #batch_size)
x = tensor.tensor3('x_word', dtype='int64')
x.tag.test_value = np.random.randint(1000, size=(1, 3, 2))
# input to the masking function (#words x #batch_size)
# mask is set to 0 when we are padding the input
# see prepare_batch() for more details
x_mask = tensor.matrix('x_mask', dtype=floatX)
x_mask.tag.test_value = np.ones([3, 2]).astype('float32')
x_mask.tag.test_value[2, 0] = 0
x_mask.tag.test_value[1, 1] = 0
x_mask.tag.test_value[2, 1] = 0
if kwargs['verbose'] and kwargs['compute_test_values'] == 'warn':
logger.warn(x.tag.test_value)
logger.warn(x_mask.tag.test_value)
# input to backward rnn (x and x_mask reversed)
x_bw = x[:, ::-1]
x_mask_bw = x_mask[::-1]
if kwargs['verbose']:
logger.warn(x_bw.tag.test_value)
logger.warn(x_mask_bw.tag.test_value)
n_timesteps = x.shape[1] # length of longest sentence in batch
batch_size = x.shape[2] # size of this batch (can vary!)
# forward RNN
# first build the forward embeddings
# we do the concatenate() in the case that we have multiple factors
# for the input data. Not currently used in this code but inherited
# from nematus.
emb_fw = []
for factor in range(factors):
emb_fw.append(self.theano_params[embedding_name(factor)][
x[factor].flatten()])
emb_fw = concatenate(emb_fw, axis=1)
emb_fw = emb_fw.reshape([n_timesteps, batch_size, dim_emb])
# drop out whole words by zero-ing their embeddings
if dropout and dropout_word > 0.:
logger.warn('Using word dropout (p={})'.format(dropout_word))
p = 1 - dropout_word
word_drop = inv_dropout_mask((n_timesteps, batch_size, 1), trng, p)
word_drop = tensor.tile(word_drop, (1, 1, dim_emb))
emb_fw *= word_drop
# now build the forward rnn layer
fw_layers = [
get_layer(encoder)[1](self.theano_params,
emb_fw,
trng=trng,
prefix='enc_fw_0',
mask=x_mask,
dropout=dropout,
dropout_W=dropout_emb,
dropout_U=dropout_rec)
]
# backward rnn
# first build the backward embeddings
# Same deal with the concatenate() of the factors as emb_fw.
emb_bw = []
for factor in range(factors):
emb_bw.append(self.theano_params[embedding_name(factor)][
x_bw[factor].flatten()])
emb_bw = concatenate(emb_bw, axis=1)
emb_bw = emb_bw.reshape([n_timesteps, batch_size, dim_emb])
# drop out the same words as above in forward rnn
if dropout and dropout_word > 0.:
logger.warn('Also dropping bw words (p={})'.format(dropout_word))
emb_bw *= word_drop[::-1]
# now build the backward rnn layer
bw_layers = [
get_layer(encoder)[1](self.theano_params,
emb_bw,
trng=trng,
prefix='enc_bw_0',
mask=x_mask_bw,
dropout=dropout,
dropout_W=dropout_emb,
dropout_U=dropout_rec)
]
# add additional layers if Deep Encoder is specified
for i in range(1,
encoder_layers): # add additional layers if specified
input_states = concatenate(
(fw_layers[i - 1][0], bw_layers[i - 1][0][::-1]), axis=2)
fw_layers.append(
get_layer(encoder)[1](self.theano_params,
input_states,
trng=trng,
prefix='enc_fw_{}'.format(i),
mask=x_mask,
dropout=dropout,
dropout_W=dropout_emb,
dropout_U=dropout_rec))
bw_layers.append(
get_layer(encoder)[1](self.theano_params,
input_states[::-1],
trng=trng,
prefix='enc_bw_{}'.format(i),
mask=x_mask_bw,
dropout=dropout,
dropout_W=dropout_emb,
dropout_U=dropout_rec))
# combine final layers of forward and backward RNNs
# this is a concatenated vector, which means it has dim=dim*2
# [::-1] means reverse the bw_layers
states = concatenate((fw_layers[-1][0], bw_layers[-1][0][::-1]),
axis=2)
if kwargs['verbose']:
logger.info("encoder_states shape {}".format(
states.tag.test_value.shape))
if kwargs['mean_birnn']:
# calculate a 2D vector over time but summing over axis 0
mlp_input = (states *
x_mask[:, :, None]).sum(0) / x_mask.sum(0)[:, None]
elif kwargs['final_birnn']:
mlp_input = concatenate(
(fw_layers[-1][-1][-1, :, :], bw_layers[-1][-1][-1, :, :]),
axis=fw_layers[-1][-1].ndim - 2)
if kwargs['verbose']:
logger.info("FW LAYERS shape {}".format(
fw_layers[-1][0][-1, :, :].tag.test_value.shape))
logger.info(fw_layers[-1][0][-1, :, :].tag.test_value)
logger.info("BW LAYERS shape {}".format(
bw_layers[-1][0][-1, :, :].tag.test_value.shape))
logger.info(bw_layers[-1][0][-1, :, :].tag.test_value)
logger.info("CONCAT shape {}".format(
mlp_input.tag.test_value.shape))
logger.info(mlp_input.tag.test_value)
# Why don't we return the reverse mask?
return x, x_mask, states, mlp_input
