def _step(x_prev, h_prev, c_prev):
preact = np.dot(h_prev, params[_p(prefix, 'U')]) + \
np.dot(x_prev, params[_p(prefix, 'W')]) + \
np.dot(z, params[_p(prefix, 'C')]) + params[_p(prefix, 'b')]
i = sigmoid(_slice(preact, 0, n_h))
f = sigmoid(_slice(preact, 1, n_h))
o = sigmoid(_slice(preact, 2, n_h))
c = np.tanh(_slice(preact, 3, n_h))
c = f * c_prev + i * c
h = o * np.tanh(c)
y = np.dot(h, Vhid) + params['bhid']
return y, h, c
def predict_lab(z, params, prefix='dis_decoder'):
""" z: size of (n_z, 1)
"""
n_h2 = params[_p(prefix, 'W1')].shape[0]
n_label = params[_p(prefix, 'V1')].shape[0]
def sigmoid(x):
return 1 / (1 + np.exp(-x))
def softmax(x):
"""Compute softmax values for each sets of scores in x."""
e_x = np.exp(x - np.max(x))
return e_x / np.sum(e_x, axis=1).reshape(x.shape[0], 1)
n_test = params['n_test']
hidden_2_out = sigmoid(
np.dot(z, params[_p(prefix, 'W1')].T) + params[_p(prefix, 'b1')])
y_recons = sigmoid(
np.dot(hidden_2_out, params[_p(prefix, 'V1')].T) +
params[_p(prefix, 'c1')])
y_recons = softmax(y_recons)
print "y_recons shape:" + str(y_recons.shape)
return y_recons
def find_sent_embedding(n_words=21102, img_w=300, img_h=48, feature_maps=200,
filter_hs=[3,4,5],n_x=300, n_h=600):
options = {}
options['n_words'] = n_words
options['img_w'] = img_w
options['img_h'] = img_h
options['feature_maps'] = feature_maps
options['filter_hs'] = filter_hs
options['n_x'] = n_x
options['n_h'] = n_h
filter_w = img_w
filter_shapes = []
pool_sizes = []
for filter_h in filter_hs:
filter_shapes.append((feature_maps, 1, filter_h, filter_w))
pool_sizes.append((img_h-filter_h+1, img_w-filter_w+1))
options['filter_shapes'] = filter_shapes
options['pool_sizes'] = pool_sizes
params = init_params(options)
tparams = init_tparams(params)
data = np.load('./bookcorpus_result.npz')
for kk, pp in params.iteritems():
params[kk] = data[kk]
for kk, pp in params.iteritems():
tparams[kk].set_value(params[kk])
x = tensor.matrix('x', dtype='int32')
layer0_input = tparams['Wemb'][tensor.cast(x.flatten(),dtype='int32')].reshape((x.shape[0],1,x.shape[1],tparams['Wemb'].shape[1]))
layer1_inputs = []
for i in xrange(len(options['filter_hs'])):
filter_shape = options['filter_shapes'][i]
pool_size = options['pool_sizes'][i]
conv_layer = encoder(tparams, layer0_input,filter_shape, pool_size,prefix=_p('cnn_encoder',i))
layer1_input = conv_layer
layer1_inputs.append(layer1_input)
layer1_input = tensor.concatenate(layer1_inputs,1)
f_embed = theano.function([x], layer1_input, name='f_embed')
return f_embed, params
def find_sent_embedding(whole,
n_words=21102,
img_w=300,
img_h=48,
feature_maps=200,
filter_hs=[3, 4, 5],
n_x=300,
n_h=600):
options = {}
options['n_words'] = n_words
options['img_w'] = img_w
options['img_h'] = img_h
options['feature_maps'] = feature_maps
options['filter_hs'] = filter_hs
options['n_x'] = n_x
options['n_h'] = n_h
filter_w = img_w
filter_shapes = []
pool_sizes = []
for filter_h in filter_hs:
filter_shapes.append((feature_maps, 1, filter_h, filter_w))
pool_sizes.append((img_h - filter_h + 1, img_w - filter_w + 1))
options['filter_shapes'] = filter_shapes
options['pool_sizes'] = pool_sizes
params = init_params(options)
tparams = init_tparams(params)
data = np.load('./bookcorpus_result.npz')
for kk, pp in params.iteritems():
params[kk] = data[kk]
for kk, pp in params.iteritems():
tparams[kk].set_value(params[kk])
x = tensor.matrix('x', dtype='int32')
layer0_input = tparams['Wemb'][tensor.cast(x.flatten(),
dtype='int32')].reshape(
(x.shape[0], 1, x.shape[1],
tparams['Wemb'].shape[1]))
layer1_inputs = []
for i in xrange(len(options['filter_hs'])):
filter_shape = options['filter_shapes'][i]
pool_size = options['pool_sizes'][i]
conv_layer = encoder(tparams,
layer0_input,
filter_shape,
pool_size,
prefix=_p('cnn_encoder', i))
layer1_input = conv_layer
layer1_inputs.append(layer1_input)
layer1_input = tensor.concatenate(layer1_inputs, 1)
f_embed = theano.function([x], layer1_input, name='f_embed')
kf = get_minibatches_idx(len(whole), 100)
sent_emb = np.zeros((len(whole), 600))
for i, train_index in kf:
sents = [whole[t] for t in train_index]
x = prepare_data_for_cnn(sents)
sent_emb[train_index[0]:train_index[-1] + 1] = f_embed(x)
if i % 500 == 0:
print i,
np.savez('./bookcorpus_embedding.npz', sent_emb=sent_emb)
return sent_emb
def predict(z, params, beam_size, max_step, prefix='decoder'):
""" z: size of (n_z, 1)
"""
n_h = params[_p(prefix,'U')].shape[0]
def _slice(_x, n, dim):
return _x[n*dim:(n+1)*dim]
def sigmoid(x):
return 1/(1+np.exp(-x))
Vhid = np.dot(params['Vhid'],params['Wemb'].T)
def _step(x_prev, h_prev, c_prev):
preact = np.dot(h_prev, params[_p(prefix, 'U')]) + \
np.dot(x_prev, params[_p(prefix, 'W')]) + \
np.dot(z, params[_p(prefix, 'C')]) + params[_p(prefix, 'b')]
i = sigmoid(_slice(preact, 0, n_h))
f = sigmoid(_slice(preact, 1, n_h))
o = sigmoid(_slice(preact, 2, n_h))
c = np.tanh(_slice(preact, 3, n_h))
c = f * c_prev + i * c
h = o * np.tanh(c)
y = np.dot(h, Vhid) + params['bhid']
return y, h, c
h0 = np.tanh(np.dot(z, params[_p(prefix, 'C0')]) + params[_p(prefix, 'b0')])
y0 = np.dot(h0, Vhid) + params['bhid']
c0 = np.zeros(h0.shape)
maxy0 = np.amax(y0)
e0 = np.exp(y0 - maxy0) # for numerical stability shift into good numerical range
p0 = e0 / np.sum(e0)
y0 = np.log(1e-20 + p0) # and back to log domain
beams = []
nsteps = 1
# generate the first word
top_indices = np.argsort(-y0) # we do -y because we want decreasing order
for i in xrange(beam_size):
wordix = top_indices[i]
# log probability, indices of words predicted in this beam so far, and the hidden and cell states
beams.append((y0[wordix], [wordix], h0, c0))
# perform BEAM search.
if beam_size > 1:
# generate the rest n words
while True:
beam_candidates = []
for b in beams:
ixprev = b[1][-1] if b[1] else 0 # start off with the word where this beam left off
if ixprev == 0 and b[1]:
# this beam predicted end token. Keep in the candidates but don't expand it out any more
beam_candidates.append(b)
continue
(y1, h1, c1) = _step(params['Wemb'][ixprev], b[2], b[3])
y1 = y1.ravel() # make into 1D vector
maxy1 = np.amax(y1)
e1 = np.exp(y1 - maxy1) # for numerical stability shift into good numerical range
p1 = e1 / np.sum(e1)
y1 = np.log(1e-20 + p1) # and back to log domain
top_indices = np.argsort(-y1) # we do -y because we want decreasing order
for i in xrange(beam_size):
wordix = top_indices[i]
beam_candidates.append((b[0] + y1[wordix], b[1] + [wordix], h1, c1))
beam_candidates.sort(reverse = True) # decreasing order
beams = beam_candidates[:beam_size] # truncate to get new beams
nsteps += 1
if nsteps >= max_step: # bad things are probably happening, break out
break
# strip the intermediates
predictions = [(b[0], b[1]) for b in beams]
else:
nsteps = 1
h = h0
# generate the first word
top_indices = np.argsort(-y0) # we do -y because we want decreasing order
ixprev = top_indices[0]
predix = [ixprev]
predlogprob = y0[ixprev]
while True:
(y1, h) = _step(params['Wemb'][ixprev], h)
ixprev, ixlogprob = ymax(y1)
predix.append(ixprev)
predlogprob += ixlogprob
nsteps += 1
if nsteps >= max_step:
break
predictions = [(predlogprob, predix)]
return predictions
def find_sent_embedding(testset,
params_file_path,
n_words=22153,
period=887,
img_w=300,
img_h=148,
feature_maps=300,
filter_hs=[3, 4, 5],
n_x=300,
n_h=500,
n_h2=200,
p_lambda_q=0,
p_lambda_fm=1,
n_codes=2,
max_epochs=16,
lr_d=0.0001,
lr_g=0.00006,
kde_sigma=1.,
batch_size=64,
valid_batch_size=64,
dim_mmd=32,
dispFreq=10,
encodeDisFreq=1,
Large=1e3,
validFreq=5000,
saveFreq=1000):
data = np.load(params_file_path)
# n_h2 = data['dis_decoder_W1'].shape[0]
options = {}
options['n_words'] = n_words
options['img_w'] = img_w
options['img_h'] = img_h
options['feature_maps'] = feature_maps
options['filter_hs'] = filter_hs #band width
options['n_x'] = n_x
options['n_h'] = n_h
options['n_h2'] = n_h2
options['max_epochs'] = max_epochs
options['batch_size'] = batch_size
options['valid_batch_size'] = valid_batch_size
options['dispFreq'] = dispFreq
options['validFreq'] = validFreq
options['saveFreq'] = saveFreq
options['encodeDisFreq'] = encodeDisFreq
options['cnn_activation'] = 'tanh'
# options['label_sizes'] = n_label
# options['input_shape'] = (n_h2, n_dis*len(options['filter_hs']))
# options['pred_shape'] = (n_label, n_h2)
# options['dis_encoder_input_shape'] = (n_h2, options['feature_maps']*len(options['filter_hs'])+n_label)
# options['dis_encoder_pred_shape'] = (n_dis*len(options['filter_hs']), n_h2)
# options['gan_input_shape'] = (n_h2, n_gan)
# options['gan_pred_shape'] = (n_dis*len(options['filter_hs']), n_h2)
# options['pred_d_shape'] = (2, n_h2)
filter_w = img_w
filter_shapes = []
pool_sizes = []
for filter_h in filter_hs:
filter_shapes.append((feature_maps, 1, filter_h, filter_w))
pool_sizes.append((img_h - filter_h + 1, img_w - filter_w + 1))
options['filter_shapes'] = filter_shapes
options['pool_sizes'] = pool_sizes
# params = init_params(options)
tparams = OrderedDict()
# tparams = init_tparams(data)
for kk, pp in data.iteritems():
tparams[kk] = theano.shared(data[kk], name=kk)
# for kk, pp in params.iteritems():
# params[kk] = data[kk]
#
# for kk, pp in params.iteritems():
# tparams[kk].set_value(params[kk])
x = tensor.matrix('x', dtype='int32')
layer0_input = tparams['Wemb'][tensor.cast(x.flatten(),
dtype='int32')].reshape(
(x.shape[0], 1, x.shape[1],
tparams['Wemb'].shape[1]))
layer1_inputs = []
for i in xrange(len(options['filter_hs'])):
filter_shape = options['filter_shapes'][i]
pool_size = options['pool_sizes'][i]
conv_layer = encoder(tparams,
layer0_input,
filter_shape,
pool_size,
options,
prefix=_p('cnn_d', i))
layer1_input = conv_layer
layer1_inputs.append(layer1_input)
layer1_input = tensor.concatenate(layer1_inputs, 1)
f_embed = theano.function([x], [layer1_input], name='f_embed')
#print testset
#x = prepare_data_for_cnn(testset)
[sent_emb] = f_embed(testset)
#print str(testset.shape) + " " + str(x.shape) + " " + str(sent_emb.shape) + " " + str(sent_emb_dis.shape)
return sent_emb
