def train_time_steps(x, y_true, h_prev, ct_prev, dimensionality, h_size,
z_size, lstm_instance):
timeStep = models.TimeStepModel()
lstm_instance = lstm.lstm_layer(h_size, z_size)
timeStep.ht_s[-1] = np.copy(h_prev)
timeStep.ct_s[-1] = np.copy(ct_prev)
loss = 0
for i in range(len(x)):
timeStep.x_s[i] = np.zeros((dimensionality, 1))
timeStep.x_s[i][y_true[i]] = 1
timeStep = lstm.passes.forward_pass(timeStep, timeStep.x_s[i],
timeStep.h_s[i - 1],
timeStep.ct_s[i - 1],
lstm_instance, True)
loss += -np.log(timeStep.y_s[i][y_true[i], 0])
print(loss)
lstm_instance = lstm.passes.reset_grad(lstm_instance)
dht_next = np.zeros_like(timeStep.ht_s[0])
dct_next = np.zeros_like(timeStep.ct_s[0])
x_reverse = x.reverse()
for i in range(len(x_reverse)):
dht_next, dct_next, timeStep = lstm.passes.backward_pass(
timeStep, i, y_true, dht_next, dct_next, lstm_instance, True,
h_size)
lstm_instance = lstm.passes.clip_grad(lstm_instance)
return loss, timeStep.ht_s[len(x) - 1], timeStep.ct_s[len(x) -
1], lstm_instance
def build_model(shared_params, options, use_noise = None):
trng = RandomStreams(SEED)
# Used for dropout.
if use_noise is None:
use_noise = theano.shared(lstm_layer.numpy_floatX(0.))
x = tensor.matrix('x', dtype = 'int64')
x_mask = tensor.tensor3('x_mask', dtype=config.floatX)
x_time = tensor.tensor3('x_time', dtype=config.floatX)
method = tensor.matrix('method', dtype=config.floatX)
y = tensor.vector('y', dtype='int64')
adm_list = tensor.tensor3('adm_list', dtype='int64')
adm_mask = tensor.tensor3('adm_mask', dtype=config.floatX)
n_steps = x.shape[0]
n_samples = x.shape[1]
n_words = adm_list.shape[2]
# compute mean vector for diagnosis & proc/medi
for i in range(2):
adm_words = adm_list[i][x.flatten()]
word_mask = adm_mask[i][x.flatten()]
if 'drin' in options['reg']:
word_mask = lstm_layer.dropout_layer(word_mask, use_noise, trng, 0.8)
emb_vec = shared_params['Wemb'][adm_words.flatten()].reshape([n_steps*n_samples,
n_words,
options['dim_emb']])
mean_vec = (emb_vec * word_mask[:, :, None]).sum(axis=1)
if options['embed'] == 'mean':
mean_vec = mean_vec / word_mask.sum(axis=1)[:, None]
elif options['embed'] == 'sum':
mean_vec = mean_vec / tensor.sqrt(word_mask.sum(axis=1))[:, None]
elif options['embed'] == 'max':
mean_vec = (emb_vec * word_mask[:, :, None]).max(axis=1)
elif options['embed'] == 'sqrt':
mean_vec = mean_vec / tensor.sqrt(abs(mean_vec))
emb_vec = mean_vec.reshape([n_steps, n_samples, options['dim_emb']])
if 'drfeat' in options['reg']:
emb_vec = lstm_layer.dropout_layer(emb_vec, use_noise, trng, 0.8)
if i == 0: emb_dia = emb_vec
else: emb_pm = emb_vec
proj = lstm_layer.lstm_layer(shared_params, options, emb_dia, emb_pm, x_mask, x_time[0], method)
# weighted mean of hidden states - weighted funcion: 1/log(x_mask)
weight = x_mask[:, 0] / (method + tensor.log(x_time[1]/30.0 + 1))# / tensor.log(x_time[1])
weight0 = weight * x_mask[:, 3]
weight1 = weight * x_mask[:, 4]
weight2 = weight
hidd_0 = tensor.sum(proj * weight0[:, :, None], axis=0) / tensor.sum(weight0, axis=0)[:, None]
hidd_1 = tensor.sum(proj * weight1[:, :, None], axis=0) / tensor.sum(weight1, axis=0)[:, None]
hidd_2 = tensor.sum(proj * weight2[:, :, None], axis=0) / tensor.sum(weight2, axis=0)[:, None]
hidd = tensor.concatenate([hidd_0, hidd_1, hidd_2], axis=1)
if 'drhid' in options['reg']:
hidd = lstm_layer.dropout_layer(hidd, use_noise, trng, 0.9)
# pool the hidden state to a neural network
hid1 = tensor.dot(hidd, shared_params['U1']) + shared_params['b1']
hid1 = tensor.nnet.sigmoid(hid1)
if 'drhid' in options['reg']:
hid1 = lstm_layer.dropout_layer(hid1, use_noise, trng, 0.5)
pred = tensor.nnet.softmax(tensor.dot(hid1, shared_params['U2']) + shared_params['b2'])
f_pred = theano.function(inputs = [x, x_mask, x_time, method, adm_list, adm_mask],
outputs = pred.argmax(axis=1), name = 'f_pred')
esp = 1e-8
if pred.dtype == 'float16': esp = 1e-6
cost = -tensor.log(pred[tensor.arange(n_samples), y] + esp).mean()
if 'norm' in options['reg']:
cost += options['L1_reg'] * lstm_layer.L1_reg(shared_params) + options['L2_reg'] * lstm_layer.L2_reg(shared_params)
return x, x_mask, x_time, method, y, adm_list, adm_mask, f_pred, cost, use_noise
def build_model(shared_params, options, use_noise=None):
trng = RandomStreams(SEED)
# Used for dropout.
if use_noise is None:
use_noise = theano.shared(lstm_layer.numpy_floatX(0.))
x = tensor.matrix('x', dtype='int64')
x_mask = tensor.tensor3('x_mask', dtype=config.floatX)
x_time = tensor.matrix('x_time', dtype=config.floatX)
method = tensor.matrix('method', dtype=config.floatX)
y = tensor.matrix('y', dtype='int64')
adm_list = tensor.tensor3('adm_list', dtype='int64')
adm_mask = tensor.tensor3('adm_mask', dtype=config.floatX)
n_steps = x.shape[0]
n_samples = x.shape[1]
n_words = adm_list.shape[2]
# compute mean vector for diagnosis & proc/medi
for i in range(2):
adm_words = adm_list[i][x.flatten()]
word_mask = adm_mask[i][x.flatten()]
if 'drin' in options['reg']:
word_mask = lstm_layer.dropout_layer(word_mask, use_noise, trng,
0.8)
emb_vec = shared_params['Wemb'][adm_words.flatten()].reshape(
[n_steps * n_samples, n_words, options['dim_emb']])
mean_vec = (emb_vec * word_mask[:, :, None]).sum(axis=1)
if options['embed'] == 'mean':
mean_vec = mean_vec / word_mask.sum(axis=1)[:, None]
elif options['embed'] == 'sum':
mean_vec = mean_vec / tensor.sqrt(word_mask.sum(axis=1))[:, None]
elif options['embed'] == 'max':
mean_vec = (emb_vec * word_mask[:, :, None]).max(axis=1)
elif options['embed'] == 'sqrt':
mean_vec = mean_vec / tensor.sqrt(abs(mean_vec))
emb_vec = mean_vec.reshape([n_steps, n_samples, options['dim_emb']])
if 'drfeat' in options['reg']:
emb_vec = lstm_layer.dropout_layer(emb_vec, use_noise, trng, 0.8)
if i == 0: emb_dia = emb_vec
else: emb_pm = emb_vec
proj = lstm_layer.lstm_layer(shared_params, options, emb_dia, emb_pm,
x_mask, x_time, method)
hid1 = tensor.dot(proj, shared_params['V1']).flatten().reshape(
[n_steps * n_samples, options['dim_emb']]) + shared_params['c1']
#hid1 = tensor.nnet.sigmoid(hid1)
hid2 = tensor.dot(hid1, shared_params['V2']) + shared_params['c2']
prob = tensor.nnet.softmax(hid2).flatten()
esp = 1e-8
if prob.dtype == 'float16': esp = 1e-6
pred = -tensor.log(prob + esp)
sorted_idx = pred.reshape([n_steps, n_samples,
options['n_diag']]).argsort(axis=2)
f_pred = theano.function(
inputs=[x, x_mask, x_time, method, adm_list, adm_mask],
outputs=sorted_idx,
name='f_pred')
next_diag = adm_list[0][y.flatten()]
diag_mask = adm_mask[0][y.flatten()]
diag = next_diag + (tensor.arange(next_diag.shape[0]) *
options['n_diag'])[:, None]
diag_pred = pred[diag.flatten()].reshape(
[next_diag.shape[0], next_diag.shape[1]])
diag_pred = (diag_pred * diag_mask).sum(axis=1)
diag_pred = diag_pred / diag_mask.sum(axis=1)
diag_pred = (diag_pred.reshape([n_steps, n_samples]) *
x_mask[:, 0]).sum(axis=0)
diag_pred = diag_pred / x_mask[:, 0].sum(axis=0)
cost = tensor.mean(diag_pred)
if 'norm' in options['reg']:
cost += options['L1_reg'] * lstm_layer.L1_reg(shared_params) + options[
'L2_reg'] * lstm_layer.L2_reg(shared_params)
return x, x_mask, x_time, method, y, adm_list, adm_mask, f_pred, cost, use_noise
def test_neuron_init():
new_neuron = lstm_layer(1, 1)
assert sum(new_neuron.cell_bias.values) != 0
def build_model(shared_params, options, use_noise = None):
trng = RandomStreams(SEED)
# Used for dropout.
if use_noise is None:
use_noise = theano.shared(lstm_layer.numpy_floatX(0.))
x = tensor.matrix('x', dtype = 'int64')
x_mask = tensor.tensor3('x_mask', dtype=config.floatX)
x_time = tensor.tensor3('x_time', dtype=config.floatX)
method = tensor.matrix('method', dtype=config.floatX)
y = tensor.matrix('y', dtype='int64')
adm_list = tensor.tensor3('adm_list', dtype='int64')
adm_mask = tensor.tensor3('adm_mask', dtype=config.floatX)
n_steps = x.shape[0]
n_samples = x.shape[1]
n_words = adm_list.shape[2]
# compute mean vector for diagnosis & proc/medi
for i in range(2):
adm_words = adm_list[i][x.flatten()]
word_mask = adm_mask[i][x.flatten()]
if 'drin' in options['reg']:
word_mask = lstm_layer.dropout_layer(word_mask, use_noise, trng, 0.8)
emb_vec = shared_params['Wemb'][adm_words.flatten()].reshape([n_steps*n_samples,
n_words,
options['dim_emb']])
mean_vec = (emb_vec * word_mask[:, :, None]).sum(axis=1)
if options['embed'] == 'mean':
mean_vec = mean_vec / word_mask.sum(axis=1)[:, None]
elif options['embed'] == 'sum':
mean_vec = mean_vec / tensor.sqrt(word_mask.sum(axis=1))[:, None]
elif options['embed'] == 'max':
mean_vec = (emb_vec * word_mask[:, :, None]).max(axis=1)
elif options['embed'] == 'sqrt':
mean_vec = mean_vec / tensor.sqrt(abs(mean_vec))
emb_vec = mean_vec.reshape([n_steps, n_samples, options['dim_emb']])
if 'drfeat' in options['reg']:
emb_vec = lstm_layer.dropout_layer(emb_vec, use_noise, trng, 0.8)
if i == 0: emb_dia = emb_vec
else: emb_pm = emb_vec
proj = lstm_layer.lstm_layer(shared_params, options, emb_dia, emb_pm, x_mask, x_time[0], method)
# pool the hidden state to a neural network
hidd = tensor.dot(proj, shared_params['U1']) + shared_params['b1']
hidd = tensor.nnet.sigmoid(hidd)
if 'drhid' in options['reg']:
hidd = lstm_layer.dropout_layer(hidd, use_noise, trng, 0.5)
hid1 = tensor.dot(hidd, shared_params['U2']) + shared_params['b2']
prob = tensor.nnet.softmax(hid1.flatten().reshape([n_steps * n_samples, options['dim_y']]))
prob_3D = prob.flatten().reshape([n_steps, n_samples, options['dim_y']])
f_pred = theano.function(inputs = [x, x_mask, x_time, method, adm_list, adm_mask],
outputs = prob_3D.argmax(axis=2), name = 'f_pred')
f_prob = theano.function([x, x_mask, x_time, method, adm_list, adm_mask], prob_3D)
esp = 1e-8
if prob.dtype == 'float16': esp = 1e-6
log_loss = -tensor.log(prob[tensor.arange(n_steps*n_samples), y.flatten()] + esp)
m_flat = x_mask[:, 0].flatten()
cost = (log_loss * m_flat).sum() / m_flat.sum()
if 'norm' in options['reg']:
cost += options['L1_reg'] * lstm_layer.L1_reg(shared_params) + options['L2_reg'] * lstm_layer.L2_reg(shared_params)
return x, x_mask, x_time, method, y, adm_list, adm_mask, f_pred, f_prob, cost, use_noise
def build_model(shared_params, options, use_noise = None):
trng = RandomStreams(SEED)
# Used for dropout.
if use_noise is None:
use_noise = theano.shared(lstm_layer.numpy_floatX(0.))
x = tensor.matrix('x', dtype='int64')
x_mask = tensor.tensor3('x_mask', dtype=config.floatX)
x_time = tensor.matrix('x_time', dtype=config.floatX)
method = tensor.matrix('method', dtype=config.floatX)
y = tensor.matrix('y', dtype='int64')
adm_list = tensor.tensor3('adm_list', dtype='int64')
adm_mask = tensor.tensor3('adm_mask', dtype=config.floatX)
n_steps = x.shape[0]
n_samples = x.shape[1]
n_words = adm_list.shape[2]
# compute mean vector for diagnosis & proc/medi
for i in range(2):
adm_words = adm_list[i][x.flatten()]
word_mask = adm_mask[i][x.flatten()]
if 'drin' in options['reg']:
word_mask = lstm_layer.dropout_layer(word_mask, use_noise, trng, 0.8)
emb_vec = shared_params['Wemb'][adm_words.flatten()].reshape([n_steps*n_samples,
n_words,
options['dim_emb']])
mean_vec = (emb_vec * word_mask[:, :, None]).sum(axis=1)
if options['embed'] == 'mean':
mean_vec = mean_vec / word_mask.sum(axis=1)[:, None]
elif options['embed'] == 'sum':
mean_vec = mean_vec / tensor.sqrt(word_mask.sum(axis=1))[:, None]
elif options['embed'] == 'max':
mean_vec = (emb_vec * word_mask[:, :, None]).max(axis=1)
elif options['embed'] == 'sqrt':
mean_vec = mean_vec / tensor.sqrt(abs(mean_vec))
emb_vec = mean_vec.reshape([n_steps, n_samples, options['dim_emb']])
if 'drfeat' in options['reg']:
emb_vec = lstm_layer.dropout_layer(emb_vec, use_noise, trng, 0.8)
if i == 0: emb_dia = emb_vec
else: emb_pm = emb_vec
proj = lstm_layer.lstm_layer(shared_params, options, emb_dia, emb_pm, x_mask, x_time, method, 0)
hid1 = tensor.dot(proj, shared_params['V1']).flatten().reshape([n_steps * n_samples, options['dim_emb']]) + shared_params['c1']
#hid1 = tensor.nnet.sigmoid(hid1)
hid2 = tensor.dot(hid1, shared_params['V2']) + shared_params['c2']
prob = tensor.nnet.softmax(hid2).flatten()
esp = 1e-8
if prob.dtype == 'float16': esp = 1e-6
pred = -tensor.log(prob + esp)
sorted_idx = pred.reshape([n_steps, n_samples, options['n_pm']]).argsort(axis=2)
f_pred = theano.function(inputs=[x, x_mask, x_time, method, adm_list, adm_mask], outputs=sorted_idx, name='f_pred')
curr_pm = adm_list[1][y.flatten()] - options['n_diag']
pm_mask = adm_mask[1][y.flatten()]
pm = curr_pm + (tensor.arange(curr_pm.shape[0]) * options['n_pm'])[:, None]
pm_pred = pred[pm.flatten()].reshape([curr_pm.shape[0], curr_pm.shape[1]])
pm_pred = (pm_pred * pm_mask).sum(axis=1)
pm_pred = pm_pred / pm_mask.sum(axis=1)
pm_pred = (pm_pred.reshape([n_steps, n_samples]) * x_mask[:, 0]).sum(axis=0)
pm_pred = pm_pred / x_mask[:, 0].sum(axis=0)
cost = tensor.mean(pm_pred)
if 'norm' in options['reg']:
cost += options['L1_reg'] * lstm_layer.L1_reg(shared_params) + options['L2_reg'] * lstm_layer.L2_reg(shared_params)
return x, x_mask, x_time, method, y, adm_list, adm_mask, f_pred, cost, use_noise
def build_model(tparams, options):
trng = RandomStreams(options['SEED'])
# Used for dropout.
use_noise = theano.shared(numpy_floatX(options['dropout_val']))
# input sentences, size of n_steps * n_samples
x = tensor.matrix('x', dtype='int64')
# the corresponding masks padding zeros
mask = tensor.matrix('mask', dtype=config.floatX)
# size of n_samples * n_z
z = tensor.tensor3('z', dtype=config.floatX)
z_score = tensor.matrix('z_score', dtype=config.floatX)
s = tensor.matrix('s', dtype=config.floatX)
z1 = dropout(z, trng, use_noise)
s = dropout(s, trng, use_noise)
n_steps = x.shape[0] # the sentence length in this mini-batch
n_samples = x.shape[1] # the number of sentences in this mini-batch
n_x = tparams['Wemb'].shape[1] # the dimension of the word embedding
# size of n_steps,n_samples,n_x
emb = tparams['Wemb'][x.flatten()].reshape([n_steps, n_samples, n_x])
emb = dropout(emb, trng, use_noise)
# visual feature: 1 * n_samples * n_x
z0 = tensor.dot(z1[:, 0, :], tparams['Tv']).dimshuffle('x', 0, 1)
bos = (tensor.dot(tparams['bos'], tparams['Ts']) +
tensor.dot(s, tparams['Ta'])).dimshuffle('x', 0, 1)
# n_steps * n_samples * n_x
# feed visual feature z only at the first step.
emb_input = tensor.dot(emb, tparams['Ts']) + tensor.dot(s, tparams['Ta'])
emb_input = tensor.concatenate((z0, bos, emb_input[:n_steps - 1]))
# n_steps * n_samples
# z has the same mask as the first word.
mask0 = mask[0].dimshuffle('x', 0)
mask1 = mask[0].dimshuffle('x', 0)
mask_input = tensor.concatenate((mask0, mask1, mask[:n_steps - 1]))
# decoding the sentence vector z back into the original sentence
h_decoder, _ = lstm_layer(tparams, emb_input, mask_input, prefix='lstm')
h_decoder = dropout(h_decoder, trng, use_noise)
h_decoder = h_decoder[1:]
#re-z
pred_z = tensor.tanh(
tensor.dot(h_decoder, tparams['task_W']) + tparams['task_b'])
pred_z_cos = z_score[None, :, :] - (
pred_z[:, :, None, :] * z[None, :, :, :]).sum(axis=3) / tensor.sqrt(
tensor.sqr(pred_z).sum(axis=2))[:, :, None] / tensor.sqrt(
tensor.sqr(z).sum(axis=2))[None, :, :]
shape = h_decoder.shape
h_decoder = h_decoder.reshape((shape[0] * shape[1], shape[2]))
Vhid = tensor.dot(tparams['Vhid'], tparams['Wemb'].T)
pred_x = tensor.dot(h_decoder, Vhid) + tparams['bhid']
pred = tensor.nnet.softmax(pred_x)
x_vec = x.reshape((shape[0] * shape[1], ))
index = tensor.arange(shape[0] * shape[1])
pred_word = pred[index, x_vec]
mask_word = mask.reshape((shape[0] * shape[1], ))
index_list = theano.tensor.eq(mask_word, 1.).nonzero()[0]
pred_word = pred_word[index_list]
# the cross-entropy loss
cost1 = -tensor.log(pred_word + 1e-6).sum() / n_samples
# the cosine hinge loss
pred_z_cos = pred_z_cos.reshape((shape[0] * shape[1], pred_z_cos.shape[2]))
pred_z_cos = tensor.maximum(pred_z_cos[index_list], 0.)
cost2 = pred_z_cos.sum() / n_samples
cost = cost1 + cost2
return use_noise, x, mask, s, z, z_score, cost, cost1, cost2