def build(self): """build the model. This method should be called after self.add_data. """ x_sym = sparse.csr_matrix('x', dtype = 'float32') y_sym = T.imatrix('y') g_sym = T.imatrix('g') gy_sym = T.vector('gy') ind_sym = T.ivector('ind') l_x_in = lasagne.layers.InputLayer(shape = (None, self.x.shape[1]), input_var = x_sym) l_g_in = lasagne.layers.InputLayer(shape = (None, 2), input_var = g_sym) l_ind_in = lasagne.layers.InputLayer(shape = (None, ), input_var = ind_sym) l_gy_in = lasagne.layers.InputLayer(shape = (None, ), input_var = gy_sym) num_ver = max(self.graph.keys()) + 1 l_emb_in = lasagne.layers.SliceLayer(l_g_in, indices = 0, axis = 1) l_emb_in = lasagne.layers.EmbeddingLayer(l_emb_in, input_size = num_ver, output_size = self.embedding_size) l_emb_out = lasagne.layers.SliceLayer(l_g_in, indices = 1, axis = 1) if self.neg_samp > 0: l_emb_out = lasagne.layers.EmbeddingLayer(l_emb_out, input_size = num_ver, output_size = self.embedding_size) l_emd_f = lasagne.layers.EmbeddingLayer(l_ind_in, input_size = num_ver, output_size = self.embedding_size, W = l_emb_in.W) l_x_hid = layers.SparseLayer(l_x_in, self.y.shape[1], nonlinearity = lasagne.nonlinearities.softmax) if self.use_feature: l_emd_f = layers.DenseLayer(l_emd_f, self.y.shape[1], nonlinearity = lasagne.nonlinearities.softmax) l_y = lasagne.layers.ConcatLayer([l_x_hid, l_emd_f], axis = 1) l_y = layers.DenseLayer(l_y, self.y.shape[1], nonlinearity = lasagne.nonlinearities.softmax) else: l_y = layers.DenseLayer(l_emd_f, self.y.shape[1], nonlinearity = lasagne.nonlinearities.softmax) py_sym = lasagne.layers.get_output(l_y) loss = lasagne.objectives.categorical_crossentropy(py_sym, y_sym).mean() if self.layer_loss and self.use_feature: hid_sym = lasagne.layers.get_output(l_x_hid) loss += lasagne.objectives.categorical_crossentropy(hid_sym, y_sym).mean() emd_sym = lasagne.layers.get_output(l_emd_f) loss += lasagne.objectives.categorical_crossentropy(emd_sym, y_sym).mean() if self.neg_samp == 0: l_gy = layers.DenseLayer(l_emb_in, num_ver, nonlinearity = lasagne.nonlinearities.softmax) pgy_sym = lasagne.layers.get_output(l_gy) g_loss = lasagne.objectives.categorical_crossentropy(pgy_sym, lasagne.layers.get_output(l_emb_out)).sum() else: l_gy = lasagne.layers.ElemwiseMergeLayer([l_emb_in, l_emb_out], T.mul) pgy_sym = lasagne.layers.get_output(l_gy) g_loss = - T.log(T.nnet.sigmoid(T.sum(pgy_sym, axis = 1) * gy_sym)).sum() params = [l_emd_f.W, l_emd_f.b, l_x_hid.W, l_x_hid.b, l_y.W, l_y.b] if self.use_feature else [l_y.W, l_y.b] if self.update_emb: params = lasagne.layers.get_all_params(l_y) updates = lasagne.updates.sgd(loss, params, learning_rate = self.learning_rate) self.train_fn = theano.function([x_sym, y_sym, ind_sym], loss, updates = updates, on_unused_input = 'ignore') self.test_fn = theano.function([x_sym, ind_sym], py_sym, on_unused_input = 'ignore') self.l = [l_gy, l_y] g_params = lasagne.layers.get_all_params(l_gy, trainable = True) g_updates = lasagne.updates.sgd(g_loss, g_params, learning_rate = self.g_learning_rate) self.g_fn = theano.function([g_sym, gy_sym], g_loss, updates = g_updates, on_unused_input = 'ignore')
def buildModel(self): print(' -- Building...') x_init = sparse.csr_matrix('x', dtype='float32') y_init = T.imatrix('y') gx_init = sparse.csr_matrix('gx', dtype='float32') gy_init = T.ivector('gy') gz_init = T.vector('gz') mask_init = T.fmatrix('subMask') # step train x_input = lgl.InputLayer(shape=(None, self.x.shape[1]), input_var=x_init) x_to_label = layers.SparseLayer(x_input, self.y.shape[1], nonlinearity=lg.nonlinearities.softmax) x_to_emd = layers.SparseLayer(x_input, self.embedding_size) W = x_to_emd.W x_to_emd = layers.DenseLayer(x_to_emd, self.y.shape[1], nonlinearity=lg.nonlinearities.softmax) x_concat = lgl.ConcatLayer([x_to_label, x_to_emd], axis=1) x_concat = layers.DenseLayer(x_concat, self.y.shape[1], nonlinearity=lg.nonlinearities.softmax) pred = lgl.get_output(x_concat) step_loss = lgo.categorical_crossentropy(pred, y_init).mean() hid_loss = lgl.get_output(x_to_label) step_loss += lgo.categorical_crossentropy(hid_loss, y_init).mean() emd_loss = lgl.get_output(x_to_emd) step_loss += lgo.categorical_crossentropy(emd_loss, y_init).mean() step_params = lgl.get_all_params(x_concat) step_updates = lg.updates.sgd(step_loss, step_params, learning_rate=self.step_learning_rate) self.step_train = theano.function([x_init, y_init], step_loss, updates=step_updates) self.test_fn = theano.function([x_init], pred) # supervised train gx_input = lgl.InputLayer(shape=(None, self.x.shape[1]), input_var=gx_init) gx_to_emd = layers.SparseLayer(gx_input, self.embedding_size, W=W) gx_to_emd = lgl.DenseLayer(gx_to_emd, self.num_ver, nonlinearity=lg.nonlinearities.softmax) gx_pred = lgl.get_output(gx_to_emd) g_loss = lgo.categorical_crossentropy(gx_pred, gy_init).sum() sup_params = lgl.get_all_params(gx_to_emd) sup_updates = lg.updates.sgd(g_loss, sup_params, learning_rate=self.sup_learning_rate) self.sup_train = theano.function([gx_init, gy_init, gz_init], g_loss, updates=sup_updates, on_unused_input='ignore') # handle lstm input cross_entropy = lgo.categorical_crossentropy(gx_pred, gy_init) cross_entropy = T.reshape(cross_entropy, (1, self.subpath_num), ndim=None) mask_input = lgl.InputLayer(shape=(None, self.window_size + 1), input_var=mask_init) sub_path_batch1 = sparse.csr_matrix('x', dtype='float32') sub_path_input1 = lgl.InputLayer(shape=(None, self.x.shape[1]), input_var=sub_path_batch1) sub_path_batch2 = sparse.csr_matrix('x', dtype='float32') sub_path_input2 = lgl.InputLayer(shape=(None, self.x.shape[1]), input_var=sub_path_batch2) sub_path_batch3 = sparse.csr_matrix('x', dtype='float32') sub_path_input3 = lgl.InputLayer(shape=(None, self.x.shape[1]), input_var=sub_path_batch3) sub_path_batch4 = sparse.csr_matrix('x', dtype='float32') sub_path_input4 = lgl.InputLayer(shape=(None, self.x.shape[1]), input_var=sub_path_batch4) sub_path_emd1 = layers.SparseLayer(sub_path_input1, self.embedding_size, W=W) sub_path_emd1 = T.reshape(lgl.get_output(sub_path_emd1), (self.subpath_num, 1, self.embedding_size)) sub_path_emd2 = layers.SparseLayer(sub_path_input2, self.embedding_size, W=W) sub_path_emd2 = T.reshape(lgl.get_output(sub_path_emd2), (self.subpath_num, 1, self.embedding_size)) sub_path_emd3 = layers.SparseLayer(sub_path_input3, self.embedding_size, W=W) sub_path_emd3 = T.reshape(lgl.get_output(sub_path_emd3), (self.subpath_num, 1, self.embedding_size)) sub_path_emd4 = layers.SparseLayer(sub_path_input4, self.embedding_size, W=W) sub_path_emd4 = T.reshape(lgl.get_output(sub_path_emd4), (self.subpath_num, 1, self.embedding_size)) sub_path_concat = T.concatenate([sub_path_emd1, sub_path_emd2, sub_path_emd3, sub_path_emd4], axis=1) sub_path_concat_layer = lgl.InputLayer(shape=(None, self.window_size + 1, self.embedding_size), input_var=sub_path_concat) # lstm layer lstm_layer = lgl.LSTMLayer(sub_path_concat_layer, self.lstm_hidden_units, grad_clipping=3, mask_input=mask_input) # handle path weight max1 = T.mean(lgl.get_output(lstm_layer), axis=1) max2 = T.mean(max1, axis=1) max2_init = T.fcol('max2') max2_init = T.reshape(max2, ((self.subpath_num, 1))) max2_input = lgl.InputLayer(shape=(self.subpath_num, 1), input_var=max2_init) max2_input = lgl.BatchNormLayer(max2_input) path_weight = lgl.get_output(max2_input) path_weight = lg.nonlinearities.sigmoid(path_weight) path_weight = 1 + 0.3 * path_weight # unsupervised train reweight_loss = T.dot(cross_entropy, path_weight)[0][0] lstm_params = lgl.get_all_params(lstm_layer, trainable=True) lstm_updates = lg.updates.sgd(reweight_loss, lstm_params, learning_rate=0.01) self.lstm_fn = theano.function([gx_init, gy_init, gz_init, sub_path_batch1, sub_path_batch2, sub_path_batch3, sub_path_batch4, mask_init], reweight_loss, updates=lstm_updates, on_unused_input='ignore') alpha_updates = lg.updates.sgd(reweight_loss, sup_params, learning_rate=0.001) self.alpha_fn = theano.function([gx_init, gy_init, gz_init, sub_path_batch1, sub_path_batch2, sub_path_batch3, sub_path_batch4, mask_init], reweight_loss, updates=alpha_updates, on_unused_input='ignore') print(' -- Done!')
def build(self): """build the model. This method should be called after self.add_data. """ x_sym = sparse.csr_matrix('x', dtype='float32') self.x_sym = x_sym y_sym = T.imatrix('y') gx_sym = sparse.csr_matrix('gx', dtype='float32') gy_sym = T.ivector('gy') gz_sym = T.vector('gz') l_x_in = lasagne.layers.InputLayer(shape=(None, self.x.shape[1]), input_var=x_sym) l_gx_in = lasagne.layers.InputLayer(shape=(None, self.x.shape[1]), input_var=gx_sym) l_gy_in = lasagne.layers.InputLayer(shape=(None, ), input_var=gy_sym) l_x_1 = layers.SparseLayer(l_x_in, self.y.shape[1], nonlinearity=lasagne.nonlinearities.softmax) l_x_2 = layers.SparseLayer(l_x_in, self.embedding_size) W = l_x_2.W l_x_2 = layers.DenseLayer(l_x_2, self.y.shape[1], nonlinearity=lasagne.nonlinearities.softmax) if self.use_feature: l_x = lasagne.layers.ConcatLayer([l_x_1, l_x_2], axis=1) l_x = layers.DenseLayer( l_x, self.y.shape[1], nonlinearity=lasagne.nonlinearities.softmax) else: l_x = l_x_2 l_gx = layers.SparseLayer(l_gx_in, self.embedding_size, W=W) if self.neg_samp > 0: l_gy = lasagne.layers.EmbeddingLayer( l_gy_in, input_size=self.num_ver, output_size=self.embedding_size) l_gx = lasagne.layers.ElemwiseMergeLayer([l_gx, l_gy], T.mul) pgy_sym = lasagne.layers.get_output(l_gx) g_loss = -T.log(T.nnet.sigmoid( T.sum(pgy_sym, axis=1) * gz_sym)).sum() else: l_gx = lasagne.layers.DenseLayer( l_gx, self.num_ver, nonlinearity=lasagne.nonlinearities.softmax) pgy_sym = lasagne.layers.get_output(l_gx) g_loss = lasagne.objectives.categorical_crossentropy( pgy_sym, gy_sym).sum() self.l = [l_x, l_gx] py_sym = lasagne.layers.get_output(l_x) loss = lasagne.objectives.categorical_crossentropy(py_sym, y_sym).mean() if self.layer_loss and self.use_feature: hid_sym = lasagne.layers.get_output(l_x_1) loss += lasagne.objectives.categorical_crossentropy( hid_sym, y_sym).mean() emd_sym = lasagne.layers.get_output(l_x_2) loss += lasagne.objectives.categorical_crossentropy( emd_sym, y_sym).mean() params = [l_x_1.W, l_x_1.b, l_x_2.W, l_x_2.b, l_x.W, l_x.b ] if self.use_feature else [l_x.W, l_x.b] if self.update_emb: params = lasagne.layers.get_all_params(l_x) updates = lasagne.updates.sgd(loss, params, learning_rate=self.learning_rate) self.train_fn = theano.function([x_sym, y_sym], loss, updates=updates) g_params = lasagne.layers.get_all_params(l_gx) g_updates = lasagne.updates.sgd(g_loss, g_params, learning_rate=self.g_learning_rate) self.g_fn = theano.function([gx_sym, gy_sym, gz_sym], g_loss, updates=g_updates, on_unused_input='ignore') self.test_fn = theano.function([x_sym], py_sym)
def buildModel(self): print(' -- Building...') x_init = sparse.csr_matrix('x', dtype='float32') y_init = T.imatrix('y') g_init = T.imatrix('g') ind_init = T.ivector('ind') sub_path_init = T.imatrix('subPathsBatch') mask_init = T.fmatrix('subMask') # step train x_input = lgl.InputLayer(shape=(None, self.x.shape[1]), input_var=x_init) g_input = lgl.InputLayer(shape=(None, 2), input_var=g_init) ind_input = lgl.InputLayer(shape=(None, ), input_var=ind_init) pair_second = lgl.SliceLayer(g_input, indices=1, axis=1) pair_first = lgl.SliceLayer(g_input, indices=0, axis=1) pair_first_emd = lgl.EmbeddingLayer(pair_first, input_size=self.num_ver, output_size=self.embedding_size) emd_to_numver = layers.DenseLayer( pair_first_emd, self.num_ver, nonlinearity=lg.nonlinearities.softmax) index_emd = lgl.EmbeddingLayer(ind_input, input_size=self.num_ver, output_size=self.embedding_size, W=pair_first_emd.W) x_to_ydim = layers.SparseLayer(x_input, self.y.shape[1], nonlinearity=lg.nonlinearities.softmax) index_emd = layers.DenseLayer(index_emd, self.y.shape[1], nonlinearity=lg.nonlinearities.softmax) concat_two = lgl.ConcatLayer([x_to_ydim, index_emd], axis=1) concat_two = layers.DenseLayer(concat_two, self.y.shape[1], nonlinearity=lg.nonlinearities.softmax) concat_two_output = lgl.get_output(concat_two) step_loss = lgo.categorical_crossentropy(concat_two_output, y_init).mean() hid_loss = lgl.get_output(x_to_ydim) step_loss += lgo.categorical_crossentropy(hid_loss, y_init).mean() emd_loss = lgl.get_output(index_emd) step_loss += lgo.categorical_crossentropy(emd_loss, y_init).mean() step_params = [ index_emd.W, index_emd.b, x_to_ydim.W, x_to_ydim.b, concat_two.W, concat_two.b ] step_updates = lg.updates.sgd(step_loss, step_params, learning_rate=self.step_learning_rate) self.step_train = theano.function([x_init, y_init, ind_init], step_loss, updates=step_updates, on_unused_input='ignore') self.test_fn = theano.function([x_init, ind_init], concat_two_output, on_unused_input='ignore') # supervised train fc_output = lgl.get_output(emd_to_numver) pair_second_output = lgl.get_output(pair_second) sup_loss = lgo.categorical_crossentropy(fc_output, pair_second_output).sum() sup_params = lgl.get_all_params(emd_to_numver, trainable=True) sup_updates = lg.updates.sgd(sup_loss, sup_params, learning_rate=self.sup_learning_rate) self.sup_train = theano.function([g_init], sup_loss, updates=sup_updates, on_unused_input='ignore') cross_entropy = lgo.categorical_crossentropy(fc_output, pair_second_output) cross_entropy = T.reshape(cross_entropy, (1, self.unsup_batch_size), ndim=None) mask_input = lgl.InputLayer(shape=(None, self.window_size + 1), input_var=mask_init) subPath_in = lgl.InputLayer(shape=(None, self.window_size + 1), input_var=sub_path_init) sub_path_emd = lgl.EmbeddingLayer(subPath_in, input_size=self.num_ver, output_size=self.embedding_size, W=pair_first_emd.W) lstm_layer = lgl.LSTMLayer(sub_path_emd, self.lstm_hidden_units, grad_clipping=3, mask_input=mask_input) # handle path weight max1 = T.mean(lgl.get_output(lstm_layer), axis=1) max2 = T.mean(max1, axis=1) max2_init = T.fcol('max2') max2_init = T.reshape(max2, ((self.subpath_num, 1))) max2_input = lgl.InputLayer(shape=(self.subpath_num, 1), input_var=max2_init) max2_input = lgl.BatchNormLayer(max2_input) path_weight = lgl.get_output(max2_input) path_weight = lg.nonlinearities.sigmoid(path_weight) path_weight = 1 + 0.3 * path_weight # unsupervised train reweight_loss = T.dot(cross_entropy, path_weight)[0][0] lstm_params_all = lgl.get_all_params(lstm_layer, trainable=True) lstm_params = list(set(lstm_params_all).difference(set(sup_params))) lstm_updates = lg.updates.sgd(reweight_loss, lstm_params, learning_rate=0.01) self.lstm_fn = theano.function([sub_path_init, g_init, mask_init], reweight_loss, updates=lstm_updates, on_unused_input='ignore') alpha_updates = lg.updates.sgd(reweight_loss, sup_params, learning_rate=0.001) self.alpha_fn = theano.function([sub_path_init, g_init, mask_init], reweight_loss, updates=alpha_updates, on_unused_input='ignore') print(' -- Done!')