def test_all(): dataset = MovieLens('./ml-latest-small/ratings.csv', 'train', 60, 5, .8) model = BERT4REC(total_items=10000, emb_dims=32, num_heads=1, dropout_rate=.8, learning_rate=1e-3) print(model) test_dataset = DataLoader(dataset, batch_size=1) for i, data in enumerate(test_dataset): x, y_label, mask = data y_pred = model(x) y_pred = y_pred.view(-1, y_pred.size(2)) y_label = y_label.view(-1) x = x.view(-1) mask = mask.view(-1) print(mask, mask.shape) print(x, x.shape) print(x.shape, y_label.shape, y_pred.shape) loss = masked_cross_entropy(y_pred, y_label, mask) acc = masked_accuracy(y_pred, y_label, mask) recall = masked_recall_at_k(y_pred, y_label, mask, 10) print(loss, acc, recall) break
def set_accuracy(self): """ 准确率 """ with tf.name_scope('accuracy'): self.accuracy = metrics.masked_accuracy(self.outputs, self.labels, self.labels_mask) self.report = [tf.argmax(self.labels, 1), tf.argmax(self.outputs, 1)]
def test(self): y_train, train_mask = self.y_test, self.test_mask hidden = forward_hidden(self.adj, self.features, self.weight_hidden, activation=lambda x: np.maximum(x, 0)) outputs = forward_hidden(self.adj, hidden, self.weight_outputs) loss = forward_cross_entropy_loss(outputs, y_train, train_mask) loss += self.weight_decay * l2_loss(self.weight_hidden) acc = masked_accuracy(outputs, y_train, train_mask) return loss, acc
def loss_sum(scores, lbl_in, msk_in, neg_msk, weight_decay, coefs, emb): loss_basic = masked_accuracy(scores, lbl_in, msk_in, neg_msk) para_decode = tf.compat.v1.get_collection( tf.compat.v1.GraphKeys.GLOBAL_VARIABLES, scope="deco") loss_basic += weight_decay * tf.nn.l2_loss(para_decode) return loss_basic
def loss_overall(scores, lbl_in, msk_in, neg_msk, weight_decay, emb): loss_basic = masked_accuracy(scores, lbl_in, msk_in, neg_msk) para_decode = tf.compat.v1.get_collection( tf.compat.v1.GraphKeys.GLOBAL_VARIABLES, scope="deco_revised") loss_basic += weight_decay * tf.nn.l2_loss(para_decode[0]) loss_basic += weight_decay * tf.nn.l2_loss(para_decode[1]) loss_basic += weight_decay * tf.nn.l2_loss(para_decode[2]) return loss_basic
def _accuracy(self): if self.multilabel: self.accuracy = metrics.masked_accuracy_multilabel( self.outputs, self.placeholders['labels'], self.placeholders['labels_mask']) else: self.accuracy = metrics.masked_accuracy( self.outputs, self.placeholders['labels'], self.placeholders['labels_mask'])
def train_loop(sess, fetches, feed_dict, epoch): outs = sess.run(fetches, feed_dict=feed_dict) print("Epoch:", '%04d' % (epoch + 1), "train_loss=", "{:.5f}".format(outs[1])) preds = outs[2] labels = outs[3] labels[labels > 0] = 1 f1_micro, f1_macro = masked_accuracy(preds, labels) print('f1_micro, f1_macro', f1_micro, f1_macro) return f1_micro, f1_macro
def train(self): # outputs的输出形状(2708,16),即(batch, hidden) self.hidden = forward_hidden(self.adj, self.features, self.weight_hidden, activation=lambda x: np.maximum(x, 0)) # outputs的输出形状(2708,7),即(batch, classes) self.outputs = forward_hidden(self.adj, self.hidden, self.weight_outputs) # 正向传播过程中的loss loss = forward_cross_entropy_loss(self.outputs, self.y_train, self.train_mask) # 正则项 weight_decay_loss = self.weight_decay * l2_loss(self.weight_hidden) loss += weight_decay_loss # 计算准确率 acc = masked_accuracy(self.outputs, self.y_train, self.train_mask) return loss, acc
def eval_f1(outs, x, y, name): y_preds = outs[x] y_labels = outs[y] y_labels[y_labels > 0] = 1 f1_micro, f1_macro = masked_accuracy(y_preds, y_labels) if name == 'validation': f1_micros_valid.append(f1_micro) f1_macros_valid.append(f1_macro) print(name, 'f1_micro, f1_macro', f1_micro, f1_macro, np.argmax(f1_micros_valid), np.argmax(f1_macros_valid)) else: f1_micros_test.append(f1_micro) f1_macros_test.append(f1_macro) print(name, 'f1_micro, f1_macro', f1_micro, f1_macro, np.argmax(f1_micros_test), np.argmax(f1_macros_test))
def eval_f1(outs, x, y, name): global f1_micro_max, f1_macro_max y_preds = outs[x] y_labels = outs[y] y_labels[y_labels > 0] = 1 f1_micro, f1_macro = masked_accuracy(y_preds, y_labels) if name == 'validation': f1_micros_valid.append(f1_micro) f1_macros_valid.append(f1_macro) print(name, 'f1_micro, f1_macro', f1_micro, f1_macro, np.argmax(f1_micros_valid), np.argmax(f1_macros_valid)) else: f1_micros_test.append(f1_micro) f1_macros_test.append(f1_macro) print(name, 'f1_micro, f1_macro', f1_micro, f1_macro, np.argmax(f1_micros_test), np.argmax(f1_macros_test)) f1_micro_max = max(f1_micro_max, f1_micro) f1_macro_max = max(f1_macro_max, f1_macro) print('f1_micro_max', f1_micro_max, 'f1_macro_max', f1_macro_max) np.save( '{}/../exp/{}_result_{}'.format(current_folder, FLAGS.eval, FLAGS.train_ratio), (f1_micro_max, f1_macro_max))
def _accuracy(self): if self.categorical: self.accuracy = metrics.masked_accuracy(self.outputs, self.placeholders['labels'], self.placeholders['labels_mask'])
def _accuracy(self): self.accuracy = masked_accuracy(self.outputs, self.placeholders['labels'], self.placeholders['labels_mask'])
def accuracy(outputs, labels, is_gcn=True): accuracy = masked_accuracy(outputs, labels) return accuracy
def train(train_arr, test_arr, mask_neg, label_neg): # training params batch_size = 1 nb_epochs = 200 lr = 0.005 l2_coef = 5e-4 weight_decay = 5e-4 hid_units = [8] n_heads = [1, 1] residual = False nonlinearity = tf.nn.elu model = HeteGAT #print('Dataset: ' + dataset) print('----- Opt. hyperparams -----') print('lr: ' + str(lr)) print('l2_coef: ' + str(l2_coef)) print('----- Archi. hyperparams -----') print('nb. layers: ' + str(len(hid_units))) print('nb. units per layer: ' + str(hid_units)) print('nb. attention heads: ' + str(n_heads)) print('residual: ' + str(residual)) print('nonlinearity: ' + str(nonlinearity)) interaction_list, adj_list, fea_list, y_train, y_test, train_mask, test_mask, labels = load_data( train_arr, test_arr) nb_nodes = fea_list[0].shape[0] ft_size = fea_list[0].shape[1] fea_list = [fea[np.newaxis] for fea in fea_list] adj_list = [adj[np.newaxis] for adj in adj_list] interaction_list = [inter[np.newaxis] for inter in interaction_list] biases_list = [ process.adj_to_bias(adj, [nb_nodes], nhood=1) for adj in adj_list ] print('build graph...') with tf.Graph().as_default(): with tf.name_scope('input'): ftr_in_list = [ tf.placeholder(dtype=tf.float32, shape=(batch_size, nb_nodes, ft_size), name='ftr_in_{}'.format(i)) for i in range(len(fea_list)) ] bias_in_list = [ tf.placeholder(dtype=tf.float32, shape=(batch_size, nb_nodes, nb_nodes), name='bias_in_{}'.format(i)) for i in range(len(biases_list)) ] inter_in_list = [ tf.placeholder(dtype=tf.float32, shape=(batch_size, nb_nodes, nb_nodes), name='inter_in_{}'.format(i)) for i in range(len(interaction_list)) ] lbl_in = tf.placeholder(dtype=tf.int32, shape=(237529, batch_size), name='lbl_in') msk_in = tf.placeholder(dtype=tf.int32, shape=(237529, batch_size), name='msk_in') neg_msk = tf.placeholder(dtype=tf.int32, shape=(237529, batch_size), name='neg_msk') attn_drop = tf.placeholder(dtype=tf.float32, shape=(), name='attn_drop') ffd_drop = tf.placeholder(dtype=tf.float32, shape=(), name='ffd_drop') is_train = tf.placeholder(dtype=tf.bool, shape=(), name='is_train') # forward final_embedding, att_val, embed_list = model.encoder( ftr_in_list, nb_nodes, is_train, attn_drop, ffd_drop, bias_mat_list=bias_in_list, inter_mat_list=inter_in_list, hid_units=hid_units, n_heads=n_heads, residual=residual, activation=nonlinearity) logits = model.decoder(final_embedding) # cal masked_loss loss = masked_accuracy(logits, lbl_in, msk_in, neg_msk) para_decode = tf.compat.v1.get_collection( tf.compat.v1.GraphKeys.GLOBAL_VARIABLES, scope="deco") loss += weight_decay * tf.nn.l2_loss(para_decode) para_encode = tf.compat.v1.get_collection( tf.compat.v1.GraphKeys.GLOBAL_VARIABLES, scope="enco_second") loss += weight_decay * tf.nn.l2_loss(para_encode[0]) loss += weight_decay * tf.nn.l2_loss(para_encode[1]) accuracy = masked_accuracy(logits, lbl_in, msk_in, neg_msk) # optimzie train_op = model.training(loss, lr, l2_coef) #saver = tf.train.Saver() init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) print("Start to train") with tf.Session() as sess: sess.run(init_op) train_loss_avg = 0 train_acc_avg = 0 #neg_mask, label_neg = generate_mask(labels, len(train_arr)) neg_mask = mask_neg label_neg = label_neg for epoch in range(nb_epochs): t = time.time() tr_step = 0 tr_size = fea_list[0].shape[0] # ================ training ============ while tr_step * batch_size < tr_size: fd1 = { i: d[tr_step * batch_size:(tr_step + 1) * batch_size] for i, d in zip(ftr_in_list, fea_list) } fd2 = { i: d[tr_step * batch_size:(tr_step + 1) * batch_size] for i, d in zip(bias_in_list, biases_list) } fd4 = { i: d[tr_step * batch_size:(tr_step + 1) * batch_size] for i, d in zip(inter_in_list, interaction_list) } fd3 = { lbl_in: y_train, msk_in: train_mask, neg_msk: neg_mask, is_train: True, attn_drop: 0.0, ffd_drop: 0.0 } fd = fd1 fd.update(fd2) fd.update(fd4) fd.update(fd3) _, loss_value_tr, acc_tr, att_val_train, embed = sess.run( [train_op, loss, accuracy, att_val, embed_list], feed_dict=fd) train_loss_avg += loss_value_tr train_acc_avg += acc_tr tr_step += 1 print( 'Epoch: %04d | Training: loss = %.5f, acc = %.5f, time = %.5f' % ((epoch + 1), loss_value_tr, acc_tr, time.time() - t)) print("Finish traing.") # ================ test ======================================================================================================== ts_size = fea_list[0].shape[0] ts_step = 0 ts_loss = 0.0 ts_acc = 0.0 while ts_step * batch_size < ts_size: fd1 = { i: d[ts_step * batch_size:(ts_step + 1) * batch_size] for i, d in zip(ftr_in_list, fea_list) } fd2 = { i: d[ts_step * batch_size:(ts_step + 1) * batch_size] for i, d in zip(bias_in_list, biases_list) } fd4 = { i: d[ts_step * batch_size:(ts_step + 1) * batch_size] for i, d in zip(inter_in_list, interaction_list) } fd3 = { lbl_in: y_test, msk_in: test_mask, neg_msk: neg_mask, is_train: False, attn_drop: 0.0, ffd_drop: 0.0 } fd = fd1 fd.update(fd2) fd.update(fd4) fd.update(fd3) out_come, loss_value_ts, acc_ts, jhy_final_embedding, embed = sess.run( [logits, loss, accuracy, final_embedding, embed_list], feed_dict=fd) ts_loss += loss_value_ts ts_acc += acc_ts ts_step += 1 print('Test loss:', ts_loss / ts_step, '; Test accuracy:', ts_acc / ts_step) out_come = out_come.reshape((1373, 173)) test_negative_samples = test_negative_sample( labels, len(test_arr), neg_mask.reshape((1373, 173))) test_labels, scores = ROC(out_come, labels, test_arr, test_negative_samples) return test_labels, scores sess.close()
def train(train_arr, test_arr): # training params batch_size = 1 nb_epochs = 200 lr = 0.005 l2_coef = 0.0005 weight_decay = 5e-4 hid_units = [8] n_heads = [4, 1] residual = False nonlinearity = tf.nn.elu model = GAT print('----- Opt. hyperparams -----') print('lr: ' + str(lr)) print('l2_coef: ' + str(l2_coef)) print('----- Archi. hyperparams -----') print('nb. layers: ' + str(len(hid_units))) print('nb. units per layer: ' + str(hid_units)) print('nb. attention heads: ' + str(n_heads)) print('residual: ' + str(residual)) print('nonlinearity: ' + str(nonlinearity)) print('model: ' + str(model)) interaction, features, y_train, y_test, train_mask, test_mask, labels = load_data(train_arr, test_arr) nb_nodes = features.shape[0] ft_size = features.shape[1] features = features[np.newaxis] interaction = interaction[np.newaxis] biases = adj_to_bias(interaction, [nb_nodes], nhood=1) nd = np.max(labels[:,0]) nm = np.max(labels[:,1]) nd = nd.astype(np.int32) nm = nm.astype(np.int32) entry_size = nd * nm with tf.Graph().as_default(): with tf.name_scope('input'): feature_in = tf.compat.v1.placeholder(dtype=tf.float32, shape=(batch_size, nb_nodes, ft_size)) bias_in = tf.compat.v1.placeholder(dtype=tf.float32, shape=(batch_size, nb_nodes, nb_nodes)) lbl_in = tf.compat.v1.placeholder(dtype=tf.int32, shape=(entry_size, batch_size)) msk_in = tf.compat.v1.placeholder(dtype=tf.int32, shape=(entry_size, batch_size)) neg_msk = tf.compat.v1.placeholder(dtype=tf.int32, shape=(entry_size,batch_size)) attn_drop = tf.compat.v1.placeholder(dtype=tf.float32, shape=()) ffd_drop = tf.compat.v1.placeholder(dtype=tf.float32, shape=()) is_train = tf.compat.v1.placeholder(dtype=tf.bool, shape=()) final_embedding, coefs = model.encoder(feature_in, nb_nodes, is_train, attn_drop, ffd_drop, bias_mat=bias_in, hid_units=hid_units, n_heads=n_heads, residual=residual, activation=nonlinearity) scores = model.decoder(final_embedding, nd) loss = model.loss_sum(scores, lbl_in, msk_in, neg_msk, weight_decay, coefs, final_embedding) accuracy = masked_accuracy(scores, lbl_in, msk_in, neg_msk) train_op = model.training(loss, lr, l2_coef) init_op = tf.group(tf.compat.v1.global_variables_initializer(), tf.compat.v1.local_variables_initializer()) with tf.compat.v1.Session() as sess: sess.run(init_op) train_loss_avg = 0 train_acc_avg = 0 for epoch in range(nb_epochs): t = time.time() ########## train ############## tr_step = 0 tr_size = features.shape[0] neg_mask, label_neg = generate_mask(labels, len(train_arr)) while tr_step * batch_size < tr_size: _, loss_value_tr, acc_tr = sess.run([train_op, loss, accuracy], feed_dict={ feature_in: features[tr_step*batch_size:(tr_step+1)*batch_size], bias_in: biases[tr_step*batch_size:(tr_step+1)*batch_size], lbl_in: y_train, msk_in: train_mask, neg_msk: neg_mask, is_train: True, attn_drop: 0.1, ffd_drop: 0.1}) train_loss_avg += loss_value_tr train_acc_avg += acc_tr tr_step += 1 print('Epoch: %04d | Training: loss = %.5f, acc = %.5f, time = %.5f' % ((epoch+1), loss_value_tr,acc_tr, time.time()-t)) print("Finish traing.") ########### test ############ ts_size = features.shape[0] ts_step = 0 ts_loss = 0.0 ts_acc = 0.0 print("Start to test") while ts_step * batch_size < ts_size: out_come, emb, coef, loss_value_ts, acc_ts = sess.run([scores, final_embedding, coefs, loss, accuracy], feed_dict={ feature_in: features[ts_step*batch_size:(ts_step+1)*batch_size], bias_in: biases[ts_step*batch_size:(ts_step+1)*batch_size], lbl_in: y_test, msk_in: test_mask, neg_msk: neg_mask, is_train: False, attn_drop: 0.0, ffd_drop: 0.0}) ts_loss += loss_value_ts ts_acc += acc_ts ts_step += 1 print('Test loss:', ts_loss/ts_step, '; Test accuracy:', ts_acc/ts_step) out_come = out_come.reshape((nd,nm)) test_negative_samples = test_negative_sample(labels,len(test_arr),neg_mask.reshape((nd,nm))) test_labels, score = ROC(out_come,labels, test_arr,test_negative_samples) return test_labels, score sess.close()
def train(train_arr, test_arr, cv, args, labels): batch_size = 1 l2_coef = 0.0005 hid_units = [8] #nb_epochs = 600 #lr = 0.005 #weight_decay = 1e-4 #n_heads = 2 nb_epochs = args.n_epoch lr = args.learning_rate weight_decay = args.weight_decay n_heads = args.n_head residual = False nonlinearity = tf.nn.elu model = GAT print('----- Opt. hyperparams -----') print('lr: ' + str(lr)) print('l2_coef: ' + str(l2_coef)) print('----- Archi. hyperparams -----') print('nb. layers: ' + str(len(hid_units))) print('nb. units per layer: ' + str(hid_units)) print('nb. attention heads: ' + str(n_heads)) print('residual: ' + str(residual)) print('nonlinearity: ' + str(nonlinearity)) print('model: ' + str(model)) interaction_local_list, features_list, y_train, y_test, train_mask, test_mask, interaction_global_list = load_data(train_arr, test_arr, cv, args, labels) nb_nodes = features_list[0].shape[0] ft_size = features_list[0].shape[1] features_list = [feature[np.newaxis] for feature in features_list] biases_local_list = [sparse_to_tuple(interaction) for interaction in interaction_local_list] biases_global_list = [sparse_to_tuple(interaction) for interaction in interaction_global_list] #n = 6375 n = args.n_node entry_size = n * n with tf.Graph().as_default(): with tf.name_scope('input'): feature_in_list = [tf.placeholder(dtype=tf.float32, shape=(batch_size, nb_nodes, ft_size), name='ftr_in_{}'.format(i)) for i in range(len(features_list))] bias_in_local_list = [tf.compat.v1.sparse_placeholder(tf.float32, name='ftr_in_{}'.format(i)) for i in range(len(biases_local_list))] bias_in_global_list = [tf.compat.v1.sparse_placeholder(tf.float32, name='ftr_in_{}'.format(i)) for i in range(len(biases_global_list))] lbl_in = tf.compat.v1.placeholder(dtype=tf.int32, shape=(entry_size, batch_size)) msk_in = tf.compat.v1.placeholder(dtype=tf.int32, shape=(entry_size, batch_size)) neg_msk = tf.compat.v1.placeholder(dtype=tf.int32, shape=(entry_size,batch_size)) attn_drop = tf.compat.v1.placeholder(dtype=tf.float32, shape=()) ffd_drop = tf.compat.v1.placeholder(dtype=tf.float32, shape=()) is_train = tf.compat.v1.placeholder(dtype=tf.bool, shape=()) final_embedding = model.encoder(feature_in_list, nb_nodes, is_train, attn_drop, ffd_drop, bias_mat_local_list = bias_in_local_list, bias_mat_global_list = bias_in_global_list, hid_units=hid_units, n_heads = n_heads, residual=residual, activation=nonlinearity) pro_matrix = model.decoder_revised(final_embedding) loss = model.loss_overall(pro_matrix, lbl_in, msk_in, neg_msk, weight_decay, final_embedding) accuracy = masked_accuracy(pro_matrix, lbl_in, msk_in, neg_msk) train_op = model.training(loss, lr, l2_coef) init_op = tf.group(tf.compat.v1.global_variables_initializer(), tf.compat.v1.local_variables_initializer()) neg_mask = generate_mask(labels, len(train_arr), args.n_node) #start to train with tf.compat.v1.Session() as sess: sess.run(init_op) train_loss_avg = 0 train_acc_avg = 0 for epoch in range(nb_epochs): t = time.time() ########## train ############## tr_step = 0 tr_size = features_list[0].shape[0] while tr_step * batch_size < tr_size: fd1 = {i: d[tr_step * batch_size:(tr_step + 1) * batch_size] for i, d in zip(feature_in_list, features_list)} fd2 = {bias_in_local_list[i]: biases_local_list[i] for i in range(len(biases_local_list))} fd3 = {bias_in_global_list[i]: biases_global_list[i] for i in range(len(biases_global_list))} fd4 = {lbl_in: y_train, msk_in: train_mask, neg_msk: neg_mask, is_train: True, attn_drop: args.dropout, ffd_drop: args.dropout} fd = fd1 fd.update(fd2) fd.update(fd3) fd.update(fd4) _, loss_value_tr, acc_tr = sess.run([train_op, loss, accuracy], feed_dict=fd) train_loss_avg += loss_value_tr train_acc_avg += acc_tr tr_step += 1 print('Epoch: %04d | Training: loss = %.5f, acc = %.5f, time = %.5f' % ((epoch+1), loss_value_tr,acc_tr, time.time()-t)) print("Finish traing.") ########### test ############ ts_size = features_list[0].shape[0] ts_step = 0 ts_loss = 0.0 ts_acc = 0.0 print("Start to test") while ts_step * batch_size < ts_size: fd1 = {i: d[ts_step * batch_size:(ts_step + 1) * batch_size] for i, d in zip(feature_in_list, features_list)} fd2 = {bias_in_local_list[i]: biases_local_list[i] for i in range(len(biases_local_list))} fd3 = {bias_in_global_list[i]: biases_global_list[i] for i in range(len(biases_global_list))} fd4 = {lbl_in: y_test, msk_in: test_mask, neg_msk: neg_mask, is_train: False, attn_drop: 0.0, ffd_drop: 0.0} fd = fd1 fd.update(fd2) fd.update(fd3) fd.update(fd4) score_matrix, loss_value_ts, acc_ts = sess.run([pro_matrix, loss, accuracy], feed_dict=fd) ts_loss += loss_value_ts ts_acc += acc_ts ts_step += 1 print('Test loss:', ts_loss/ts_step, '; Test accuracy:', ts_acc/ts_step) score_matrix = score_matrix.reshape((n,n)) test_positive_samples = test_positive_sample(labels, test_arr) test_negative_samples = test_negative_sample(labels,len(test_arr),neg_mask.reshape((n,n))) test_labels, test_scores = ROC(score_matrix,labels, test_arr,test_negative_samples) test_samples = np.vstack((test_positive_samples, test_negative_samples)) return test_labels, test_scores, test_samples sess.close()
def _accuracy(self): #pdb.set_trace() #print("##### model accuracy part ####") self.accuracy = masked_accuracy(self.outputs, self.placeholders['labels'], self.placeholders['labels_mask'])