def build(self):
"""build the model. This method should be called after self.add_data.
"""
x_sym = sparse.csr_matrix('x', dtype = 'float32')
y_sym = T.imatrix('y')
g_sym = T.imatrix('g')
gy_sym = T.vector('gy')
ind_sym = T.ivector('ind')
l_x_in = lasagne.layers.InputLayer(shape = (None, self.x.shape[1]), input_var = x_sym)
l_g_in = lasagne.layers.InputLayer(shape = (None, 2), input_var = g_sym)
l_ind_in = lasagne.layers.InputLayer(shape = (None, ), input_var = ind_sym)
l_gy_in = lasagne.layers.InputLayer(shape = (None, ), input_var = gy_sym)
num_ver = max(self.graph.keys()) + 1
l_emb_in = lasagne.layers.SliceLayer(l_g_in, indices = 0, axis = 1)
l_emb_in = lasagne.layers.EmbeddingLayer(l_emb_in, input_size = num_ver, output_size = self.embedding_size)
l_emb_out = lasagne.layers.SliceLayer(l_g_in, indices = 1, axis = 1)
if self.neg_samp > 0:
l_emb_out = lasagne.layers.EmbeddingLayer(l_emb_out, input_size = num_ver, output_size = self.embedding_size)
l_emd_f = lasagne.layers.EmbeddingLayer(l_ind_in, input_size = num_ver, output_size = self.embedding_size, W = l_emb_in.W)
l_x_hid = layers.SparseLayer(l_x_in, self.y.shape[1], nonlinearity = lasagne.nonlinearities.softmax)
if self.use_feature:
l_emd_f = layers.DenseLayer(l_emd_f, self.y.shape[1], nonlinearity = lasagne.nonlinearities.softmax)
l_y = lasagne.layers.ConcatLayer([l_x_hid, l_emd_f], axis = 1)
l_y = layers.DenseLayer(l_y, self.y.shape[1], nonlinearity = lasagne.nonlinearities.softmax)
else:
l_y = layers.DenseLayer(l_emd_f, self.y.shape[1], nonlinearity = lasagne.nonlinearities.softmax)
py_sym = lasagne.layers.get_output(l_y)
loss = lasagne.objectives.categorical_crossentropy(py_sym, y_sym).mean()
if self.layer_loss and self.use_feature:
hid_sym = lasagne.layers.get_output(l_x_hid)
loss += lasagne.objectives.categorical_crossentropy(hid_sym, y_sym).mean()
emd_sym = lasagne.layers.get_output(l_emd_f)
loss += lasagne.objectives.categorical_crossentropy(emd_sym, y_sym).mean()
if self.neg_samp == 0:
l_gy = layers.DenseLayer(l_emb_in, num_ver, nonlinearity = lasagne.nonlinearities.softmax)
pgy_sym = lasagne.layers.get_output(l_gy)
g_loss = lasagne.objectives.categorical_crossentropy(pgy_sym, lasagne.layers.get_output(l_emb_out)).sum()
else:
l_gy = lasagne.layers.ElemwiseMergeLayer([l_emb_in, l_emb_out], T.mul)
pgy_sym = lasagne.layers.get_output(l_gy)
g_loss = - T.log(T.nnet.sigmoid(T.sum(pgy_sym, axis = 1) * gy_sym)).sum()
params = [l_emd_f.W, l_emd_f.b, l_x_hid.W, l_x_hid.b, l_y.W, l_y.b] if self.use_feature else [l_y.W, l_y.b]
if self.update_emb:
params = lasagne.layers.get_all_params(l_y)
updates = lasagne.updates.sgd(loss, params, learning_rate = self.learning_rate)
self.train_fn = theano.function([x_sym, y_sym, ind_sym], loss, updates = updates, on_unused_input = 'ignore')
self.test_fn = theano.function([x_sym, ind_sym], py_sym, on_unused_input = 'ignore')
self.l = [l_gy, l_y]
g_params = lasagne.layers.get_all_params(l_gy, trainable = True)
g_updates = lasagne.updates.sgd(g_loss, g_params, learning_rate = self.g_learning_rate)
self.g_fn = theano.function([g_sym, gy_sym], g_loss, updates = g_updates, on_unused_input = 'ignore')
def buildModel(self):
print(' -- Building...')
x_init = sparse.csr_matrix('x', dtype='float32')
y_init = T.imatrix('y')
gx_init = sparse.csr_matrix('gx', dtype='float32')
gy_init = T.ivector('gy')
gz_init = T.vector('gz')
mask_init = T.fmatrix('subMask')
# step train
x_input = lgl.InputLayer(shape=(None, self.x.shape[1]),
input_var=x_init)
x_to_label = layers.SparseLayer(x_input, self.y.shape[1],
nonlinearity=lg.nonlinearities.softmax)
x_to_emd = layers.SparseLayer(x_input, self.embedding_size)
W = x_to_emd.W
x_to_emd = layers.DenseLayer(x_to_emd, self.y.shape[1],
nonlinearity=lg.nonlinearities.softmax)
x_concat = lgl.ConcatLayer([x_to_label, x_to_emd], axis=1)
x_concat = layers.DenseLayer(x_concat, self.y.shape[1],
nonlinearity=lg.nonlinearities.softmax)
pred = lgl.get_output(x_concat)
step_loss = lgo.categorical_crossentropy(pred, y_init).mean()
hid_loss = lgl.get_output(x_to_label)
step_loss += lgo.categorical_crossentropy(hid_loss, y_init).mean()
emd_loss = lgl.get_output(x_to_emd)
step_loss += lgo.categorical_crossentropy(emd_loss, y_init).mean()
step_params = lgl.get_all_params(x_concat)
step_updates = lg.updates.sgd(step_loss, step_params,
learning_rate=self.step_learning_rate)
self.step_train = theano.function([x_init, y_init], step_loss,
updates=step_updates)
self.test_fn = theano.function([x_init], pred)
# supervised train
gx_input = lgl.InputLayer(shape=(None, self.x.shape[1]),
input_var=gx_init)
gx_to_emd = layers.SparseLayer(gx_input, self.embedding_size, W=W)
gx_to_emd = lgl.DenseLayer(gx_to_emd, self.num_ver,
nonlinearity=lg.nonlinearities.softmax)
gx_pred = lgl.get_output(gx_to_emd)
g_loss = lgo.categorical_crossentropy(gx_pred, gy_init).sum()
sup_params = lgl.get_all_params(gx_to_emd)
sup_updates = lg.updates.sgd(g_loss, sup_params,
learning_rate=self.sup_learning_rate)
self.sup_train = theano.function([gx_init, gy_init, gz_init], g_loss,
updates=sup_updates,
on_unused_input='ignore')
# handle lstm input
cross_entropy = lgo.categorical_crossentropy(gx_pred, gy_init)
cross_entropy = T.reshape(cross_entropy, (1, self.subpath_num), ndim=None)
mask_input = lgl.InputLayer(shape=(None, self.window_size + 1),
input_var=mask_init)
sub_path_batch1 = sparse.csr_matrix('x', dtype='float32')
sub_path_input1 = lgl.InputLayer(shape=(None, self.x.shape[1]),
input_var=sub_path_batch1)
sub_path_batch2 = sparse.csr_matrix('x', dtype='float32')
sub_path_input2 = lgl.InputLayer(shape=(None, self.x.shape[1]),
input_var=sub_path_batch2)
sub_path_batch3 = sparse.csr_matrix('x', dtype='float32')
sub_path_input3 = lgl.InputLayer(shape=(None, self.x.shape[1]),
input_var=sub_path_batch3)
sub_path_batch4 = sparse.csr_matrix('x', dtype='float32')
sub_path_input4 = lgl.InputLayer(shape=(None, self.x.shape[1]),
input_var=sub_path_batch4)
sub_path_emd1 = layers.SparseLayer(sub_path_input1, self.embedding_size,
W=W)
sub_path_emd1 = T.reshape(lgl.get_output(sub_path_emd1),
(self.subpath_num, 1, self.embedding_size))
sub_path_emd2 = layers.SparseLayer(sub_path_input2,
self.embedding_size, W=W)
sub_path_emd2 = T.reshape(lgl.get_output(sub_path_emd2),
(self.subpath_num, 1, self.embedding_size))
sub_path_emd3 = layers.SparseLayer(sub_path_input3, self.embedding_size,
W=W)
sub_path_emd3 = T.reshape(lgl.get_output(sub_path_emd3),
(self.subpath_num, 1, self.embedding_size))
sub_path_emd4 = layers.SparseLayer(sub_path_input4, self.embedding_size,
W=W)
sub_path_emd4 = T.reshape(lgl.get_output(sub_path_emd4),
(self.subpath_num, 1, self.embedding_size))
sub_path_concat = T.concatenate([sub_path_emd1, sub_path_emd2,
sub_path_emd3, sub_path_emd4], axis=1)
sub_path_concat_layer = lgl.InputLayer(shape=(None, self.window_size + 1,
self.embedding_size),
input_var=sub_path_concat)
# lstm layer
lstm_layer = lgl.LSTMLayer(sub_path_concat_layer,
self.lstm_hidden_units,
grad_clipping=3,
mask_input=mask_input)
# handle path weight
max1 = T.mean(lgl.get_output(lstm_layer), axis=1)
max2 = T.mean(max1, axis=1)
max2_init = T.fcol('max2')
max2_init = T.reshape(max2, ((self.subpath_num, 1)))
max2_input = lgl.InputLayer(shape=(self.subpath_num, 1),
input_var=max2_init)
max2_input = lgl.BatchNormLayer(max2_input)
path_weight = lgl.get_output(max2_input)
path_weight = lg.nonlinearities.sigmoid(path_weight)
path_weight = 1 + 0.3 * path_weight
# unsupervised train
reweight_loss = T.dot(cross_entropy, path_weight)[0][0]
lstm_params = lgl.get_all_params(lstm_layer, trainable=True)
lstm_updates = lg.updates.sgd(reweight_loss, lstm_params,
learning_rate=0.01)
self.lstm_fn = theano.function([gx_init, gy_init, gz_init,
sub_path_batch1, sub_path_batch2,
sub_path_batch3, sub_path_batch4,
mask_init],
reweight_loss,
updates=lstm_updates,
on_unused_input='ignore')
alpha_updates = lg.updates.sgd(reweight_loss, sup_params,
learning_rate=0.001)
self.alpha_fn = theano.function([gx_init, gy_init, gz_init,
sub_path_batch1, sub_path_batch2,
sub_path_batch3, sub_path_batch4,
mask_init],
reweight_loss,
updates=alpha_updates,
on_unused_input='ignore')
print(' -- Done!')
def build(self):
"""build the model. This method should be called after self.add_data.
"""
x_sym = sparse.csr_matrix('x', dtype='float32')
self.x_sym = x_sym
y_sym = T.imatrix('y')
gx_sym = sparse.csr_matrix('gx', dtype='float32')
gy_sym = T.ivector('gy')
gz_sym = T.vector('gz')
l_x_in = lasagne.layers.InputLayer(shape=(None, self.x.shape[1]),
input_var=x_sym)
l_gx_in = lasagne.layers.InputLayer(shape=(None, self.x.shape[1]),
input_var=gx_sym)
l_gy_in = lasagne.layers.InputLayer(shape=(None, ), input_var=gy_sym)
l_x_1 = layers.SparseLayer(l_x_in,
self.y.shape[1],
nonlinearity=lasagne.nonlinearities.softmax)
l_x_2 = layers.SparseLayer(l_x_in, self.embedding_size)
W = l_x_2.W
l_x_2 = layers.DenseLayer(l_x_2,
self.y.shape[1],
nonlinearity=lasagne.nonlinearities.softmax)
if self.use_feature:
l_x = lasagne.layers.ConcatLayer([l_x_1, l_x_2], axis=1)
l_x = layers.DenseLayer(
l_x,
self.y.shape[1],
nonlinearity=lasagne.nonlinearities.softmax)
else:
l_x = l_x_2
l_gx = layers.SparseLayer(l_gx_in, self.embedding_size, W=W)
if self.neg_samp > 0:
l_gy = lasagne.layers.EmbeddingLayer(
l_gy_in,
input_size=self.num_ver,
output_size=self.embedding_size)
l_gx = lasagne.layers.ElemwiseMergeLayer([l_gx, l_gy], T.mul)
pgy_sym = lasagne.layers.get_output(l_gx)
g_loss = -T.log(T.nnet.sigmoid(
T.sum(pgy_sym, axis=1) * gz_sym)).sum()
else:
l_gx = lasagne.layers.DenseLayer(
l_gx,
self.num_ver,
nonlinearity=lasagne.nonlinearities.softmax)
pgy_sym = lasagne.layers.get_output(l_gx)
g_loss = lasagne.objectives.categorical_crossentropy(
pgy_sym, gy_sym).sum()
self.l = [l_x, l_gx]
py_sym = lasagne.layers.get_output(l_x)
loss = lasagne.objectives.categorical_crossentropy(py_sym,
y_sym).mean()
if self.layer_loss and self.use_feature:
hid_sym = lasagne.layers.get_output(l_x_1)
loss += lasagne.objectives.categorical_crossentropy(
hid_sym, y_sym).mean()
emd_sym = lasagne.layers.get_output(l_x_2)
loss += lasagne.objectives.categorical_crossentropy(
emd_sym, y_sym).mean()
params = [l_x_1.W, l_x_1.b, l_x_2.W, l_x_2.b, l_x.W, l_x.b
] if self.use_feature else [l_x.W, l_x.b]
if self.update_emb:
params = lasagne.layers.get_all_params(l_x)
updates = lasagne.updates.sgd(loss,
params,
learning_rate=self.learning_rate)
self.train_fn = theano.function([x_sym, y_sym], loss, updates=updates)
g_params = lasagne.layers.get_all_params(l_gx)
g_updates = lasagne.updates.sgd(g_loss,
g_params,
learning_rate=self.g_learning_rate)
self.g_fn = theano.function([gx_sym, gy_sym, gz_sym],
g_loss,
updates=g_updates,
on_unused_input='ignore')
self.test_fn = theano.function([x_sym], py_sym)
def buildModel(self):
print(' -- Building...')
x_init = sparse.csr_matrix('x', dtype='float32')
y_init = T.imatrix('y')
g_init = T.imatrix('g')
ind_init = T.ivector('ind')
sub_path_init = T.imatrix('subPathsBatch')
mask_init = T.fmatrix('subMask')
# step train
x_input = lgl.InputLayer(shape=(None, self.x.shape[1]),
input_var=x_init)
g_input = lgl.InputLayer(shape=(None, 2), input_var=g_init)
ind_input = lgl.InputLayer(shape=(None, ), input_var=ind_init)
pair_second = lgl.SliceLayer(g_input, indices=1, axis=1)
pair_first = lgl.SliceLayer(g_input, indices=0, axis=1)
pair_first_emd = lgl.EmbeddingLayer(pair_first,
input_size=self.num_ver,
output_size=self.embedding_size)
emd_to_numver = layers.DenseLayer(
pair_first_emd,
self.num_ver,
nonlinearity=lg.nonlinearities.softmax)
index_emd = lgl.EmbeddingLayer(ind_input,
input_size=self.num_ver,
output_size=self.embedding_size,
W=pair_first_emd.W)
x_to_ydim = layers.SparseLayer(x_input,
self.y.shape[1],
nonlinearity=lg.nonlinearities.softmax)
index_emd = layers.DenseLayer(index_emd,
self.y.shape[1],
nonlinearity=lg.nonlinearities.softmax)
concat_two = lgl.ConcatLayer([x_to_ydim, index_emd], axis=1)
concat_two = layers.DenseLayer(concat_two,
self.y.shape[1],
nonlinearity=lg.nonlinearities.softmax)
concat_two_output = lgl.get_output(concat_two)
step_loss = lgo.categorical_crossentropy(concat_two_output,
y_init).mean()
hid_loss = lgl.get_output(x_to_ydim)
step_loss += lgo.categorical_crossentropy(hid_loss, y_init).mean()
emd_loss = lgl.get_output(index_emd)
step_loss += lgo.categorical_crossentropy(emd_loss, y_init).mean()
step_params = [
index_emd.W, index_emd.b, x_to_ydim.W, x_to_ydim.b, concat_two.W,
concat_two.b
]
step_updates = lg.updates.sgd(step_loss,
step_params,
learning_rate=self.step_learning_rate)
self.step_train = theano.function([x_init, y_init, ind_init],
step_loss,
updates=step_updates,
on_unused_input='ignore')
self.test_fn = theano.function([x_init, ind_init],
concat_two_output,
on_unused_input='ignore')
# supervised train
fc_output = lgl.get_output(emd_to_numver)
pair_second_output = lgl.get_output(pair_second)
sup_loss = lgo.categorical_crossentropy(fc_output,
pair_second_output).sum()
sup_params = lgl.get_all_params(emd_to_numver, trainable=True)
sup_updates = lg.updates.sgd(sup_loss,
sup_params,
learning_rate=self.sup_learning_rate)
self.sup_train = theano.function([g_init],
sup_loss,
updates=sup_updates,
on_unused_input='ignore')
cross_entropy = lgo.categorical_crossentropy(fc_output,
pair_second_output)
cross_entropy = T.reshape(cross_entropy, (1, self.unsup_batch_size),
ndim=None)
mask_input = lgl.InputLayer(shape=(None, self.window_size + 1),
input_var=mask_init)
subPath_in = lgl.InputLayer(shape=(None, self.window_size + 1),
input_var=sub_path_init)
sub_path_emd = lgl.EmbeddingLayer(subPath_in,
input_size=self.num_ver,
output_size=self.embedding_size,
W=pair_first_emd.W)
lstm_layer = lgl.LSTMLayer(sub_path_emd,
self.lstm_hidden_units,
grad_clipping=3,
mask_input=mask_input)
# handle path weight
max1 = T.mean(lgl.get_output(lstm_layer), axis=1)
max2 = T.mean(max1, axis=1)
max2_init = T.fcol('max2')
max2_init = T.reshape(max2, ((self.subpath_num, 1)))
max2_input = lgl.InputLayer(shape=(self.subpath_num, 1),
input_var=max2_init)
max2_input = lgl.BatchNormLayer(max2_input)
path_weight = lgl.get_output(max2_input)
path_weight = lg.nonlinearities.sigmoid(path_weight)
path_weight = 1 + 0.3 * path_weight
# unsupervised train
reweight_loss = T.dot(cross_entropy, path_weight)[0][0]
lstm_params_all = lgl.get_all_params(lstm_layer, trainable=True)
lstm_params = list(set(lstm_params_all).difference(set(sup_params)))
lstm_updates = lg.updates.sgd(reweight_loss,
lstm_params,
learning_rate=0.01)
self.lstm_fn = theano.function([sub_path_init, g_init, mask_init],
reweight_loss,
updates=lstm_updates,
on_unused_input='ignore')
alpha_updates = lg.updates.sgd(reweight_loss,
sup_params,
learning_rate=0.001)
self.alpha_fn = theano.function([sub_path_init, g_init, mask_init],
reweight_loss,
updates=alpha_updates,
on_unused_input='ignore')
print(' -- Done!')