def test_dot(): import operator sa = sparse.random((3, 4, 5)) sb = sparse.random((5, 6)) a = sa.todense() b = sb.todense() assert_eq(a.dot(b), sa.dot(sb)) assert_eq(np.dot(a, b), sparse.dot(sa, sb)) if hasattr(operator, 'matmul'): # Basic equivalences assert_eq(eval("a @ b"), eval("sa @ sb")) assert_eq(eval("sa @ sb"), sparse.dot(sa, sb))
def test_dot(ab, a_format, b_format, a_kwargs, b_kwargs): a_shape, b_shape = ab if len(a_shape) == 1: a_kwargs = {} if len(b_shape) == 1: b_kwargs = {} sa = sparse.random(a_shape, density=0.5, format=a_format, **a_kwargs) sb = sparse.random(b_shape, density=0.5, format=b_format, **b_kwargs) a = sa.todense() b = sb.todense() e = np.dot(a, b) assert_eq(e, sa.dot(sb)) assert_eq(e, sparse.dot(sa, sb)) assert_eq(e, sparse.dot(a, sb)) assert_eq(e, sparse.dot(a, sb))
def test_dot_type(a_dense, b_dense, o_type, a, b): if a_dense: a = a.todense() if b_dense: b = b.todense() assert isinstance(sparse.dot(a, b), o_type)
def test_dot(): a = random_x((3, 4, 5)) b = random_x((5, 6)) sa = COO.from_numpy(a) sb = COO.from_numpy(b) assert_eq(a.dot(b), sa.dot(sb)) assert_eq(np.dot(a, b), sparse.dot(sa, sb))
def dimensionReduction(d): importFiles2() dimArray = randomTable(61067, d) temp = sparse.dot(dimArray) global main main = temp allCosineNN() filename = 'dimRed2' + str(d) + '.png' print filename makeHeatMap(baselineCosineNN, groups, 'Blues', filename)
def test_dot_type(a_dense, b_dense, o_type): a = sparse.random((3, 4), density=0.8) b = sparse.random((4, 5), density=0.8) if a_dense: a = a.todense() if b_dense: b = b.todense() assert isinstance(sparse.dot(a, b), o_type)
def test_dot(a_shape, b_shape): sa = sparse.random(a_shape, density=0.5) sb = sparse.random(b_shape, density=0.5) a = sa.todense() b = sb.todense() assert_eq(a.dot(b), sa.dot(sb)) assert_eq(np.dot(a, b), sparse.dot(sa, sb)) if hasattr(operator, 'matmul'): # Basic equivalences assert_eq(operator.matmul(a, b), operator.matmul(sa, sb))
def test_dot(a_shape, b_shape, a_format, b_format, a_comp_axes, b_comp_axes): if a_format == "coo" or len(a_shape) == 1: a_comp_axes = None if b_format == "coo" or len(b_shape) == 1: b_comp_axes = None sa = sparse.random( a_shape, density=0.5, format=a_format, compressed_axes=a_comp_axes ) sb = sparse.random( b_shape, density=0.5, format=b_format, compressed_axes=b_comp_axes ) a = sa.todense() b = sb.todense() assert_eq(a.dot(b), sa.dot(sb)) assert_eq(np.dot(a, b), sparse.dot(sa, sb)) assert_eq(sparse.dot(sa, b), sparse.dot(a, sb)) assert_eq(np.dot(a, b), sparse.dot(sa, sb)) # Basic equivalences assert_eq(operator.matmul(a, b), operator.matmul(sa, sb))
def test_nonzero_fv(self): xs = sparse.random((2, 3), density=0.5, fill_value=1) ys = sparse.random((3, 4), density=0.5) with pytest.raises(ValueError): sparse.dot(xs, ys)
def reconstruct(x, sparse, radius): b_approx = sparse.dot(x) b_image = b_approx.reshape((2 * radius + 1, 2 * radius + 1)) b_image = np.clip(b_image, 0, 255) return b_image
def reconstruct(x, sparse, radius): b_approx = sparse.dot(x) b_image = b_approx.reshape((2*radius+1, 2*radius+1)) b_image = np.clip(b_image, 0, 255) return b_image
def train(): # train GCN first adj_norm_sparse_csr = adj_norm_sparse.tocsr() # sizes = [FLAGS.gcn_hidden1, FLAGS.gcn_hidden2, n_class] # surrogate_model = GCN.GCN(sizes, adj_norm_sparse_csr, features_csr, with_relu=True, name="surrogate", gpu_id=gpu_id) # surrogate_model.train(adj_norm_sparse_csr, split_train, split_val, node_labels) # ori_acc = surrogate_model.test(split_unlabeled, node_labels, adj_norm_sparse_csr) testacc, valid_acc = GCN.run(FLAGS.dataset, adj_orig, name="original") cos = sp.dot(features_csr.transpose(), features_csr) norm = spnorm(features_csr, axis=1) norm_mat = norm.dot(norm.transpose()) if_drop_edge = True ## set the checkpoint path checkpoints_dir_base = "./checkpoints" current_time = datetime.datetime.now().strftime("%y%m%d%H%M%S") checkpoints_dir = os.path.join(checkpoints_dir_base, current_time, current_time) ############ global_steps = tf.get_variable('global_step', trainable=False, initializer=0) new_learning_rate = tf.train.exponential_decay(FLAGS.learn_rate_init, global_step=global_steps, decay_steps=10000, decay_rate=0.98) new_learn_rate_value = FLAGS.learn_rate_init ## set the placeholders placeholders = { 'features': tf.sparse_placeholder(tf.float32, name="ph_features"), 'adj': tf.sparse_placeholder(tf.float32, name="ph_adj"), 'adj_orig': tf.sparse_placeholder(tf.float32, name="ph_orig"), 'dropout': tf.placeholder_with_default(0., shape=(), name="ph_dropout"), # 'node_labels': tf.placeholder(tf.float32, name = "ph_node_labels"), # 'node_ids' : tf.placeholder(tf.float32, name = "ph_node_ids") } # build models model = None if model_str == "gae_gan": model = gaegan(placeholders, num_features, num_nodes, features_nonzero, new_learning_rate) model.build_model() pos_weight = float(adj.shape[0] * adj.shape[0] - adj.sum()) / adj.sum() norm = adj.shape[0] * adj.shape[0] / float( (adj.shape[0] * adj.shape[0] - adj.sum()) * 2) opt = 0 # Optimizer with tf.name_scope('optimizer'): if model_str == 'gae_gan': opt = Optimizergaegan( preds=tf.reshape(model.x_tilde, [-1]), labels=tf.reshape( tf.sparse_tensor_to_dense(placeholders['adj_orig'], validate_indices=False), [-1]), #comm_label=placeholders["comm_label"], model=model, num_nodes=num_nodes, pos_weight=pos_weight, norm=norm, global_step=global_steps, new_learning_rate=new_learning_rate) # init the sess sess = tf.Session() sess.run(tf.global_variables_initializer()) saver = "" var_list = tf.global_variables() var_list = [ var for var in var_list if ("encoder" in var.name) or ('generate' in var.name) ] saver = tf.train.Saver(var_list, max_to_keep=10) if if_save_model: os.mkdir(os.path.join(checkpoints_dir_base, current_time)) saver.save(sess, checkpoints_dir) # save the graph if restore_trained_our: checkpoints_dir_our = "./checkpoints" checkpoints_dir_our = os.path.join(checkpoints_dir_our, FLAGS.trained_our_path) # checkpoints_dir_meta = os.path.join(checkpoints_dir_base, FLAGS.trained_our_path, # FLAGS.trained_our_path + ".meta") #saver.restore(sess,tf.train.latest_checkpoint(checkpoints_dir_our)) saver.restore( sess, os.path.join("./checkpoints", "191215231708", "191215231708-1601")) print("model_load_successfully") # else: # if not restore the original then restore the base dis one. # checkpoints_dir_base = os.path.join("./checkpoints/base", FLAGS.trained_base_path) # saver.restore(sess, tf.train.latest_checkpoint(checkpoints_dir_base)) feed_dict = construct_feed_dict(adj_norm, adj_label, features, placeholders) feed_dict.update({placeholders['dropout']: FLAGS.dropout}) # pred_dis_res = model.vaeD_tilde.eval(session=sess, feed_dict=feed_dict) #### save new_adj without norm############# if restore_trained_our: modified_adj = get_new_adj(feed_dict, sess, model) modified_adj = sp.csr_matrix(modified_adj) sp.save_npz("transfer_new/transfer_1216_1/qq_5000_gaegan_new.npz", modified_adj) sp.save_npz("transfer_new/transfer_1216_1/qq_5000_gaegan_ori.npz", adj_orig) print("save the loaded adj") # print("before training generator") ##################################################### ##################################################### G_loss_min = 1000 for epoch in range(FLAGS.epochs): t = time.time() # run Encoder's optimizer #sess.run(opt.encoder_min_op, feed_dict=feed_dict) # run G optimizer on trained model if restore_trained_our: sess.run(opt.G_min_op, feed_dict=feed_dict) else: # it is the new model if epoch < FLAGS.epochs: sess.run(opt.G_min_op, feed_dict=feed_dict) # ## ## if epoch % 50 == 0: print("Epoch:", '%04d' % (epoch + 1), "time=", "{:.5f}".format(time.time() - t)) G_loss, laplacian_para, new_learn_rate_value = sess.run( [opt.G_comm_loss, opt.reg, new_learning_rate], feed_dict=feed_dict) #new_adj = get_new_adj(feed_dict, sess, model) new_adj = model.new_adj_output.eval(session=sess, feed_dict=feed_dict) temp_pred = new_adj.reshape(-1) #temp_ori = adj_norm_sparse.todense().A.reshape(-1) temp_ori = adj_label_sparse.todense().A.reshape(-1) mutual_info = normalized_mutual_info_score(temp_pred, temp_ori) print( "Step: %d,G: loss=%.7f ,Lap_para: %f ,info_score = %.6f, LR=%.7f" % (epoch, G_loss, laplacian_para, mutual_info, new_learn_rate_value)) ## here is the debug part of the model################################# laplacian_mat, reg_trace, reg_log, reward_ratio = sess.run( [ opt.reg_mat, opt.reg_trace, opt.reg_log, opt.new_percent_softmax ], feed_dict=feed_dict) print("lap_mat is:") print(np.diag(laplacian_mat)) print("reg_trace is:") print(reg_trace) print("reg_log is:") print(reg_log) print("reward_percentage") print(reward_ratio) ########################################## #';# check the D_loss_min if (G_loss < G_loss_min) and (epoch > 1000) and (if_save_model): saver.save(sess, checkpoints_dir, global_step=epoch, write_meta_graph=False) print("min G_loss new") if G_loss < G_loss_min: G_loss_min = G_loss if (epoch % 200 == 1) and if_save_model: saver.save(sess, checkpoints_dir, global_step=epoch, write_meta_graph=False) print("Epoch:", '%04d' % (epoch + 1), "time=", "{:.5f}".format(time.time() - t)) saver.save(sess, checkpoints_dir, global_step=FLAGS.epochs, write_meta_graph=True) print("Optimization Finished!") feed_dict.update({placeholders['dropout']: 0}) new_adj = get_new_adj(feed_dict, sess, model) new_adj = new_adj - np.diag(np.diag(new_adj)) new_adj_sparse = sp.csr_matrix(new_adj) print((abs(new_adj_sparse - new_adj_sparse.T) > 1e-10).nnz == 0) # new_adj_norm, new_adj_norm_sparse = preprocess_graph(new_adj) # new_adj_norm_sparse_csr = new_adj_norm_sparse.tocsr() # modified_model = GCN.GCN(sizes, new_adj_norm_sparse_csr, features_csr, with_relu=True, name="surrogate", gpu_id=gpu_id) # modified_model.train(new_adj_norm_sparse_csr, split_train, split_val, node_labels) # modified_acc = modified_model.test(split_unlabeled, node_labels, new_adj_norm_sparse_csr) testacc_new, valid_acc_new = GCN.run(FLAGS.dataset, new_adj_sparse, name="modified") new_adj = get_new_adj(feed_dict, sess, model) new_adj = new_adj - np.diag(np.diag(new_adj)) new_adj_sparse = sp.csr_matrix(new_adj) testacc_new2, valid_acc_new = GCN.run(FLAGS.dataset, new_adj_sparse, name="modified") new_adj = get_new_adj(feed_dict, sess, model) new_adj = new_adj - np.diag(np.diag(new_adj)) new_adj_sparse = sp.csr_matrix(new_adj) testacc_new3, valid_acc_new = GCN.run(FLAGS.dataset, new_adj_sparse, name="modified") #np.save("./data/hinton/hinton_new_adj_48_0815.npy", new_adj) #roc_score, ap_score = get_roc_score(test_edges, test_edges_false,feed_dict, sess, model) ##### The final results #### print("*" * 30) print("the final results:\n") print("The original acc is: ") print(testacc) print("*#" * 15) print("The modified acc is : ") print(testacc_new) print("*#" * 15) print("The modified acc is : ") print(testacc_new2) print("*#" * 15) print("The modified acc is : ") print(testacc_new3) return new_adj, testacc, testacc_new, testacc_new2, testacc_new3