def make_laplacian_directory(fname_input): levs = 3 A = genfromtxt(fname_input, delimiter=',') A_ = sparse.csr_matrix(A).astype(np.float32) graphs, perm = coarsening.coarsen(A_, levels=levs, self_connections=False) L_list = [graph.laplacian(g, normalized=True) for g in graphs] L = L_list[0].astype(np.float32) #pdb.set_trace() return L, perm, levs, L_list
def coarsening_pooling(self, normalize=True): adj = scipy.sparse.csr_matrix(self.adjacency_matrix) for i in range(len(self.pooling_sizes)): adj_coarsened, pooling_matrices = self._coarserning_pooling_( adj, self.pooling_sizes[i], normalize) pooling_matrices = np.array(pooling_matrices) self.graphs.append(adj_coarsened) self.layer2pooling_matrices[i] = pooling_matrices adj = scipy.sparse.csr_matrix(adj_coarsened) #num_nodes_before_final = adj_coarsened.shape[0] #if num_nodes_before_final < 4: #num_nodes_before_final = 4 num_nodes_before_final = 4 pooling_matrices_final = [ sp.lil_matrix((adj_coarsened.shape[0], 1)) for i in range(num_nodes_before_final) ] if adj_coarsened.shape[0] > 1: L_i = graph.laplacian(adj_coarsened, normalize) lamb_i, U_i = graph.fourier(L_i) for j in range(num_nodes_before_final): if j < adj_coarsened.shape[0]: if U_i[0, j] < 0: pooling_matrices_final[j][:, 0] = -U_i[:, j].reshape( -1, 1) else: pooling_matrices_final[j][:, 0] = U_i[:, j].reshape(-1, 1) else: if U_i[0, adj_coarsened.shape[0] - 1] < 0: pooling_matrices_final[ j][:, 0] = -U_i[:, adj_coarsened.shape[0] - 1].reshape(-1, 1) else: pooling_matrices_final[j][:, 0] = U_i[:, adj_coarsened. shape[0] - 1].reshape(-1, 1) else: for j in range(num_nodes_before_final): pooling_matrices_final[j][:, 0] = adj_coarsened.reshape(-1, 1) self.layer2pooling_matrices[i + 1] = pooling_matrices_final
def coarsen(A, levels): graphs, parents = coarsening.metis(A, levels) # Coarsen a graph multiple times using Graclus variation of the METIS algorithm. # Basically, we randomly sort the nodes, we iterate on them and we decided to group each node # with the neighbor having highest w_ij * 1/(\sum_k w_ik) + w_ij * 1/(\sum_k w_kj) # i.e. highest sum of probabilities to randomly walk from i to j and from j to i. # We thus favour strong connections (i.e. the ones with high weight wrt all the others for both nodes) # in the choice of the neighbor of each node. # Construction is done a priori, so we have one graph for all the samples! # graphs = list of spare adjacency matrices (it contains in position # 0 the original graph) # parents = list of numpy arrays (every array in position i contains # the mapping from graph i to graph i+1, i.e. the idx of # node i in the coarsed graph -> that is, the idx of its cluster) perms = coarsening.compute_perm(parents) # Return a list of indices to reorder the adjacency and data matrices so # that two consecutive nodes correspond to neighbors that should be collapsed # to produce the coarsed version of the graph. # Fake nodes are appended for each node which is not grouped with anybody else laplacians = [] for i, A in enumerate(graphs): M, M = A.shape # We remove self-connections created by metis. A = A.tocoo() A.setdiag(0) if i < levels: # if we have to pool the graph A = coarsening.perm_adjacency(A, perms[i]) # matrix A is here extended with the fakes nodes # in order to do an efficient pooling operation # in tensorflow as it was a 1D pooling A = A.tocsr() A.eliminate_zeros() Mnew, Mnew = A.shape print('Layer {0}: M_{0} = |V| = {1} nodes ({2} added), |E| = {3} edges'.format(i, Mnew, Mnew - M, A.nnz // 2)) L = graph.laplacian(A, normalized=normalized_laplacian) laplacians.append(L) return laplacians, perms[0] if len(perms) > 0 else None
while os.path.exists(full_tb_path): run_number += 1 full_tb_path = tb_path+"train_run"+str(run_number) with tf.Session() as sess: tf.global_variables_initializer().run() writer = tf.summary.FileWriter(full_tb_path, sess.graph) self.op_summary = tf.summary.merge_all() for i in range(iterations): print i next_batch = self.inputs.next_batch(self.batch_size) inputs = {"data:0": next_batch[0], "labels:0": next_batch[1], "do_rate:0": .5} sess.run(self.op_train, feed_dict=inputs) if i%10 == 0: writer.add_summary(sess.run(self.op_summary, feed_dict=inputs), i) print i if __name__ == "__main__": test_data = datasets.test_singles() #train_data = datasets.train_singles() coords = np.asarray([[float(cord) for cord in re.split("\n | ", line)] for line in open("234_coords_3column.txt")]) dist, idx = graph.distance_scipy_spatial(coords, k=10, metric="euclidean") adj = graph.adjacency(dist, idx).astype(np.float64) L = graph.laplacian(adj, normalized=True) comp_graph = tf.Graph() with comp_graph.as_default(): A = deep_cgnn(comp_graph, L, test_data, batch_size=1) A.train(100)
def prepare_graphs(): p = os.path.join(os.getcwd(), '../survivalnet/data/Brain_Integ.mat') D = sio.loadmat(p) T = np.asarray([t[0] for t in D['Survival']]) O = 1 - np.asarray([c[0] for c in D['Censored']]) X = D['Integ_X'] #[:,1855:] X = (X - np.mean(X, axis=0)) / np.std(X, axis=0) fold_size = int(10 * len(X) / 100) X_train, T_train, O_train = X[2 * fold_size:], T[2 * fold_size:], O[2 * fold_size:] X_test, T_test, O_test = X[:fold_size], T[:fold_size], O[:fold_size] X_val, T_val, O_val = X[fold_size:2 * fold_size], T[fold_size:2 * fold_size], O[fold_size:2 * fold_size] print_log('train and test shapes:' + str(X_train.shape) + str(X_test.shape)) if not LOAD_A: start = time.time() dist, idx = graph.distance_scipy_spatial(X_train.T, k=4, metric='euclidean') print_log('graph constructed:' + str(dist.shape) + str(idx.shape) + ' in ' + str(time.time() - start)) A = graph.adjacency(dist, idx).astype(np.float32) d = X.shape[1] assert A.shape == (d, d) np.savez('A_sml', data=A.data, indices=A.indices, indptr=A.indptr, shape=A.shape) print('d = |V| = {}, k|V| < |E| = {}'.format(d, A.nnz)) #plt.spy(A, markersize=2, color='black'); #plt.savefig('tmp.png') else: start = time.time() loader = np.load('A.npz') A = csr_matrix((loader['data'], loader['indices'], loader['indptr']), shape=loader['shape']) print_log('graph loaded:' + ' in ' + str(time.time() - start)) print('adjacency matrix type and shape: ', A.__class__, A.shape) start = time.time() graphs, perm = coarsening.coarsen(A, levels=CL, self_connections=False) print_log('graph coarsened:' + ' in ' + str(time.time() - start)) X_train = coarsening.perm_data(X_train, perm) X_val = coarsening.perm_data(X_val, perm) X_test = coarsening.perm_data(X_test, perm) print_log('train and test shapes:' + str(X_train.shape) + str(X_test.shape)) L = [graph.laplacian(A, normalized=True) for A in graphs] #graph.plot_spectrum(L) n_train = len(X_train) params = dict() params['dir_name'] = 'demo' params['num_epochs'] = 2000 params['batch_size'] = int(len(X_train) / 1.0) params['eval_frequency'] = 10 # Building blocks. params['filter'] = 'chebyshev5' params['brelu'] = 'b1relu' params['pool'] = 'apool1' # Architecture. params['F'] = [8, 8, 8] # Number of graph convolutional filters. params['K'] = [9, 9, 9] # Polynomial orders. params['p'] = [2, 2, 2] # Pooling sizes. params['M'] = [128, 1] # Output dimensionality of fully connected layers. # Optimization. params['regularization'] = 0 params['dropout'] = 1 params['learning_rate'] = 1e-4 params['decay_rate'] = 0.999 params['momentum'] = 0 params['decay_steps'] = n_train / params['batch_size'] model = models.cgcnn(L, **params) accuracy, loss, t_step = model.cox_fit(X_train, T_train, O_train, X_val, T_val, O_val)
def train(method, view_com, n_views, k, m, n_epoch, batch_size, pairs, labels, coords, subj, data, data_type, i_fold): str_params = view_com + '_k' + str(k) + '_m' + str(m) + '_' obj_params = 'softmax' print(str_params) print('Construct ROI graphs...') t_start = time.process_time() # A = grid_graph(86, corners=False) # A = graph.replace_random_edges(A, 0) coo1, coo2, coo3 = coords.shape # coo2 is the roi dimension features = np.zeros([coo1 * coo3, coo2]) for i in range(coo3): features[coo1 * i:coo1 * (i + 1), :] = coords[:, :, i] dist, idx = graph.distance_scipy_spatial(np.transpose(features), k=10, metric='euclidean') A = graph.adjacency(dist, idx).astype(np.float32) if method == '2gcn': graphs, perm = coarsening.coarsen(A, levels=FLAGS.coarsening_levels, self_connections=False) L = [graph.laplacian(A, normalized=True) for A in graphs] data = coarsening.perm_data1(data, perm, True) else: graphs = list() graphs.append(A) L = [graph.laplacian(A, normalized=True)] print('Execution time: {:.2f}s'.format(time.process_time() - t_start)) # graph.plot_spectrum(L) del A print('Set parameters...') mp = model_perf() # Architecture. common = {} common['dir_name'] = 'ppmi/' common['num_epochs'] = n_epoch common['batch_size'] = batch_size common['eval_frequency'] = 5 * common['num_epochs'] common['patience'] = 5 common['regularization'] = 5e-3 common['dropout'] = 1 common['learning_rate'] = 1e-2 common['decay_rate'] = 0.95 common['momentum'] = 0.9 common['n_views'] = n_views common['view_com'] = view_com # common['brelu'] = 'b1relu' # common['pool'] = 'mpool1' print('Get feed pairs and labels...') train_pairs, train_y, val_x, val_y, test_pairs, test_y = get_feed_data( data, subj, pairs, labels, method) C = max(train_y) + 1 common['decay_steps'] = train_pairs.shape[0] / (common['batch_size'] * 5) if method == 'fnn': str_params += 'siamese_' name = 'mvfnn' params = common.copy() params['method'] = 'fnn' params['fin'] = 1 params['F'] = [m] params['K'] = [1] params['p'] = [1] params['M'] = [C] params['dir_name'] += name mp.test(models.siamese_fnn(L, **params), name, params, data, train_pairs, train_y, val_x, val_y, test_pairs, test_y) if method == '2fnn': str_params += 'siamese_layer2_' name = 'mvfnn2' params = common.copy() params['method'] = 'fnn' params['fin'] = 1 params['F'] = [m] params['K'] = [1] params['p'] = [1] params['M'] = [64, C] params['dir_name'] += name mp.test(models.siamese_fnn(L, **params), name, params, data, train_pairs, train_y, val_x, val_y, test_pairs, test_y) if method == 'gcn': # str_params += 'b_max_eu_' name = 'mvgcn' params = common.copy() params['method'] = 'gcn' params['F'] = [m] # filters params['K'] = [k] # supports params['p'] = [1] params['M'] = [C] params['fin'] = val_x.shape[3] params['dir_name'] += name params['filter'] = 'chebyshev5' params['brelu'] = 'b2relu' params['pool'] = 'apool1' mp.test(models.siamese_m_cgcnn(L, **params), name, params, data, train_pairs, train_y, val_x, val_y, test_pairs, test_y) # Common hyper-parameters for LeNet5-like networks. if method == '2gcn': str_params += 'p4_fc64_' name = 'mvgcn2' params = common.copy() params['method'] = '2gcn' params['F'] = [m, 64] # filters params['K'] = [k, k] # supports params['p'] = [4, 4] params['M'] = [512, C] params['fin'] = val_x.shape[3] params['dir_name'] += name params['filter'] = 'chebyshev5' params['brelu'] = 'b2relu' params['pool'] = 'apool1' mp.test(models.siamese_m_cgcnn(L, **params), name, params, data, train_pairs, train_y, val_x, val_y, test_pairs, test_y) # mp.save(data_type) method_type = method + '_' mp.fin_result(method_type + data_type + str_params + obj_params, i_fold)
# Connections are only vertical or horizontal on the grid. # Corner vertices are connected to 2 neightbors only. if corners: import scipy.sparse A = A.toarray() A[A < A.max()/1.5] = 0 A = scipy.sparse.csr_matrix(A) print('{} edges'.format(A.nnz)) print("{} > {} edges".format(A.nnz//2, FLAGS.number_edges*m**2//2)) return A t_start = time.process_time() A = grid_graph(28, corners=False) A = graph.replace_random_edges(A, 0) L = graph.laplacian(A, normalized=True) print('Execution time: {:.2f}s'.format(time.process_time() - t_start)) # graph.plot_spectrum(L) del A gcnn = GCNN(L, 10, 10, 10) batch_size = 200 epochs = 20 prefetch_size = 1 train_data_copy = train_data.map(lambda x: tf.expand_dims(x, 1)).batch(batch_size).prefetch(prefetch_size) train_labels_copy = train_labels.batch(batch_size).prefetch(prefetch_size) train_data = train_data.map(lambda x: tf.expand_dims(x, 1)).batch(batch_size).prefetch(prefetch_size) train_labels = train_labels.batch(batch_size).prefetch(prefetch_size) val_data = val_data.map(lambda x: tf.expand_dims(x, 1)).batch(batch_size).prefetch(prefetch_size) val_labels = val_labels.batch(batch_size).prefetch(prefetch_size)
return (x, y) def MAELoss(out, truth): err = out.data.numpy() - truth.data.numpy() MAE = np.average(np.abs(err)) return MAE train_data = PopDataset(t1=20, t2=10) train_loader = Data.DataLoader(dataset=train_data, batch_size=15, shuffle=True) mex = pd.read_csv('E:/NN/Metro_train/Metro_roadMap.csv', index_col=0) a = mex.as_matrix() w = sp.csc_matrix(a).astype(float) L = laplacian(w) class Net(nn.Module): def __init__(self, L, input_dim, hidden_dim, k, num_layers, batch_first=True, bias=True, return_all_layers=True): super(Net, self).__init__() self.L = L self.input_dim = input_dim
def train(modality, method, data_type, distance, k, fdim, nhops, mem_size, code_size, n_words, edim, n_epoch, batch_size, pairs, labels, coords, data, records, i_fold): str_params = '_' + modality + '_' + distance + '_k' + str(k) + '_fdim' + str(fdim) + '_nhops' + str(nhops) + '_memsize' + str(mem_size) + '_codesize' + str(code_size) + '_nwords' + str(n_words) + '_edim' + str(edim) print ('Construct ROI graphs...') t_start = time.process_time() coo1, coo2, coo3 = coords.shape # coo2 is the roi dimension features = np.zeros([coo1*coo3, coo2]) for i in range(coo3): features[coo1*i:coo1*(i+1), :] = coords[:, :, i] dist, idx = graph.distance_scipy_spatial(np.transpose(features), k=10, metric='euclidean') A = graph.adjacency(dist, idx).astype(np.float32) if method == '2gcn': graphs, perm = coarsening.coarsen(A, levels=FLAGS.coarsening_levels, self_connections=False) L = [graph.laplacian(A, normalized=True) for A in graphs] data = coarsening.perm_data1(data, perm) else: graphs = list() graphs.append(A) L = [graph.laplacian(A, normalized=True)] print ('The number of GCN layers: ', len(L)) print('Execution time: {:.2f}s'.format(time.process_time() - t_start)) # graph.plot_spectrum(L) del A print ('Set parameters...') mp = model_perf(i_fold, get_pair_label(pairs, labels)) # Architecture. common = {} common['dir_name'] = 'ppmi/' common['num_epochs'] = n_epoch common['batch_size'] = batch_size common['eval_frequency'] = 5 * common['num_epochs'] common['patience'] = 5 common['regularization'] = 1e-2 common['dropout'] = 1 common['learning_rate'] = 5e-3 common['decay_rate'] = 0.95 common['momentum'] = 0.9 common['init_std'] = 5e-2 print ('Get feed pairs and labels...') train_pairs, val_pairs, test_pairs = pairs train_x, train_y, val_x, val_y, test_x, test_y = get_feed_data(data, pairs, labels, method) train_r, val_r, test_r = get_feed_records(records, pairs, mem_size, code_size, method) C = max(train_y)+1 common['decay_steps'] = train_x.shape[0] / common['batch_size'] if method == 'MemGCN': # str_params += '' name = 'cgconv_softmax' params = common.copy() params['method'] = method params['p'] = [1] # pooling size params['M'] = [C] params['K'] = k # support number params['nhops'] = nhops # hop number params['fdim'] = fdim # filters dimension params['edim'] = edim # embeddings dimension params['mem_size'] = mem_size # the length of sequential records params['code_size'] = code_size # the size of one record params['n_words'] = n_words # feature dimension params['distance'] = distance params['fin'] = train_x.shape[2] params['dir_name'] += name params['filter'] = 'chebyshev5' params['brelu'] = 'b2relu' params['pool'] = 'apool1' mp.test(models.siamese_cgcnn_mem(L, **params), name, params, data, records, train_x, train_r, train_y, val_x, val_r, val_y, test_x, test_r, test_y, train_pairs, test_pairs) # mp.save(data_type) method_type = method + '_' mp.fin_result(method_type + data_type + str_params, i_fold)
# Corner vertices are connected to 2 neightbors only. if corners: import scipy.sparse A = A.toarray() A[A < A.max()/1.5] = 0 A = scipy.sparse.csr_matrix(A) print('{} edges'.format(A.nnz)) print("{} > {} edges".format(A.nnz//2, FLAGS.number_edges*m**2//2)) return A t_start = timeit.default_timer() A = grid_graph(58, corners=False) A = graph.replace_random_edges(A, 0) graphs, perm = coarsening.coarsen(A, levels=FLAGS.coarsening_levels, self_connections=False) L = [graph.laplacian(A, normalized=True) for A in graphs] print('Execution time: {:.2f}s'.format(timeit.default_timer() - t_start)) graph.plot_spectrum(L) del A # # Data with gzip.open('../data/syn_adhd_train_data_quater_scale.pkl.gz', 'rb') as f: X_train = pickle.load(f) X = zip(*X_train)[0] labels = zip(*X_train)[1] X = np.asarray(X) y = np.asarray(labels)
def fGraph(inFIDDic, series_size=3, is_distance=True): # # 1 get the label of this sample. k = list(inFIDDic.keys())[0] #print(inFIDDic) inFIDDic[k][0] label = 1 if inFIDDic[k][0] == 3 else 0 # # 2 get the feature vector of vertices. # representing the building object (one vertice) by a feature vector, # Fourer expansion or Geometry description. # the number of buildings (vertices) in one building group (a sample) subObject_size = len(inFIDDic[k][1]) XY_coords, vertice, coords = [], [], [] for i in range(0, subObject_size): # one building in the sample. subObject = inFIDDic[k][1][i] [density] = inFIDDic[k][2][i] # Calculate the basic indicators of polygon. # Geometry descriptors: area, peri, SMBR_area [[CX, CY], area, peri] = geoutils.get_basic_parametries_of_Poly(subObject) XY_coords.append([CX, CY, i]) compactness = area / math.pow(0.282 * peri, 2) OBB, SMBR_area = geoutils.mininumAreaRectangle(subObject) orientation = OBB.Orientation() length_width = OBB.e1 / OBB.e0 if OBB.e0 > OBB.e1 else OBB.e0 / OBB.e1 area_radio = area / (OBB.e1 * OBB.e0 * 4) # print("area={}, peri={}, SMBR_area={}".format(area, peri, SMBR_area)) # Five basic indices. Faster geometry_vector = [ orientation, area, length_width, area_radio, compactness ] # More indices and more slowly. if True: # preparatory work uniform_coords = np.array([[(j[0] - CX), (j[1] - CY)] for j in subObject]) uniform_size = len(uniform_coords) # Closing the polygon. if uniform_coords[0][0] - uniform_coords[uniform_size - 1][ 0] != 0 or uniform_coords[0][1] - uniform_coords[ uniform_size - 1][1] != 0: print('Closing!') uniform_coords.append(uniform_coords[0]) # Part One. Size indicators: CONVEX_area, MEAN_radius, LONG_chord convexHull = ConvexHull(uniform_coords) CONVEX_area = convexHull.area sum_radius, size_radius, MEAN_radius, LONG_chord = 0, 0, 0, 0 for j in range(0, uniform_size - 1): sum_radius += math.sqrt( uniform_coords[j][0] * uniform_coords[j][0] + uniform_coords[j][1] * uniform_coords[j][1]) size_radius += 1 if size_radius != 0: MEAN_radius = sum_radius / size_radius pairwise_distances, index_j, index_h = sklearn.metrics.pairwise.pairwise_distances( uniform_coords[convexHull.vertices], metric="euclidean", n_jobs=1), 0, 0 for j in range(0, len(pairwise_distances)): for h in range(j, len(pairwise_distances)): if (pairwise_distances[j, h] > LONG_chord): LONG_chord, index_j, index_h = pairwise_distances[ j, h], j, h SECOND_chord, index_p, index_q = 0, 0, 0 for j in range(0, len(pairwise_distances)): for h in range(j, len(pairwise_distances)): if pairwise_distances[j, h] > SECOND_chord: if j != index_j and h != index_h: SECOND_chord, index_p, index_q = pairwise_distances[ j, h], j, h # Part two. Orientation indicators: LONGEDGE_orien, SMBR_orien, WEIGHT_orien from_longedge, to_longedge = uniform_coords[convexHull.vertices[ index_j]], uniform_coords[convexHull.vertices[index_h]] LONGEDGE_orien = abs( math.atan2(from_longedge[0] - to_longedge[0], from_longedge[1] - to_longedge[1])) from_secondedge, to_secondedge = uniform_coords[ convexHull.vertices[index_p]], uniform_coords[ convexHull.vertices[index_q]] SENCONDEDGE_orien = abs( math.atan2(from_secondedge[0] - to_secondedge[0], from_secondedge[1] - to_secondedge[1])) #LONGEDGE_dis = math.sqrt((from_longedge[0]-to_longedge[0])*(from_longedge[0]-to_longedge[0]) + (from_longedge[1]-to_longedge[1])*(from_longedge[1]-to_longedge[1])) SECONDEDGE_dis = math.sqrt( (from_secondedge[0] - to_secondedge[0]) * (from_secondedge[0] - to_secondedge[0]) + (from_secondedge[1] - to_secondedge[1]) * (from_secondedge[1] - to_secondedge[1])) BISSECTOR_orien = (LONGEDGE_orien * LONG_chord + SENCONDEDGE_orien * SECONDEDGE_dis) / ( LONG_chord + SECONDEDGE_dis) # print("LONG_width={}, LONGEDGE_dis={}".format(LONG_width, LONGEDGE_dis)) SMBR_orien, WALL_orien, WEIGHT_orien = orientation, 0, 0 # Calculate vertical width agaist long cord. # line equation: longedge_a, longedge_b, longedge_c = geoutils2.get_equation( from_longedge, to_longedge) LONG_width, up_offset, down_offset = 0, longedge_c, longedge_c for j in range(0, uniform_size - 1): crossing_product = longedge_a * uniform_coords[j][ 0] + longedge_b * uniform_coords[j][1] if crossing_product + up_offset < 0: up_offset = -crossing_product if crossing_product + down_offset > 0: down_offset = -crossing_product longedge_square = math.sqrt(longedge_a * longedge_a + longedge_b * longedge_b) if longedge_square == 0: LONG_width = abs(up_offset - down_offset) else: LONG_width = abs(up_offset - down_offset) / longedge_square edge_orien_weight, edge_length_sun, edge_tuple, candidate_max = 0, 0, [], 0 for j in range(0, uniform_size - 1): dx, dy = uniform_coords[j + 1][0] - uniform_coords[j][ 0], uniform_coords[j + 1][1] - uniform_coords[j][1] edge_orien = (math.atan2(dx, dy) + math.pi) % (math.pi / 2.0) # edge_orien = (math.atan2(dx, dy) + 2*math.pi) % math.pi # edge_orien = math.atan2(dx, dy) edge_length = math.sqrt(dx * dx + dy * dy) edge_orien_weight += edge_length * edge_orien edge_length_sun += edge_length edge_tuple.append([edge_orien, edge_length]) # add test code. # print("edge_length={}, edge_orien={}".format(edge_length, edge_orien*180/math.pi)) WALL_orien = edge_orien_weight / edge_length_sun for j in range(0, 90): candidate_orien, candidate_weight = j * math.pi / 180, 0 for j in range(0, len(edge_tuple)): if abs(edge_tuple[j][0] - candidate_orien) < math.pi / 24: candidate_weight += ( math.pi / 24 - abs(edge_tuple[j][0] - candidate_orien) ) * edge_tuple[j][1] / (math.pi / 24) if candidate_weight > candidate_max: candidate_max, WEIGHT_orien = candidate_weight, candidate_orien # Part three. shape indices: Diameter-Perimeter-Area- measurements RIC_compa, IPQ_compa, FRA_compa = area / peri, 4 * math.pi * area / ( peri * peri), 1 - math.log(area) * .5 / math.log(peri) GIB_compa, Div_compa = 2 * math.sqrt( math.pi * area) / LONG_chord, 4 * area / (LONG_chord * peri) # Part four. shape indices: Related shape # fit_Ellipse = geoutils2.fitEllipse(np.array(uniform_coords)[:,0], np.array(uniform_coords)[:,1]) # ellipse_axi = geoutils2.ellipse_axis_length(fit_Ellipse) # elongation, ellipticity, concavity = length_width, ellipse_axi[0]/ellipse_axi[1] if ellipse_axi[1] != 0 else 1, area/convexHull.area elongation, ellipticity, concavity = length_width, LONG_width / LONG_chord, area / convexHull.area radius, standard_circle, enclosing_circle = math.sqrt( area / math.pi), [], geoutils2.make_circle(uniform_coords) for j in range(0, 60): standard_circle.append([ math.cos(2 * math.pi * j / 100) * radius, math.sin(2 * math.pi * j / 100) * radius ]) standard_intersection = Polygon(uniform_coords).intersection( Polygon(standard_circle)) standard_union = Polygon(uniform_coords).union( Polygon(standard_circle)) DCM_index = area / (math.pi * enclosing_circle[2] * enclosing_circle[2]) BOT_index = 1 - standard_intersection.area / area closest_length, closest_sun, closest_size, BOY_measure = [], 0, 0, 0 for j in range(0, 40): x, y = math.cos(2 * math.pi * j / 40) * peri, math.sin( 2 * math.pi * j / 40) * peri closest_point, is_test = geoutils2.find_intersection( uniform_coords, [x, y]) if is_test: print("k={}, i={}, j={}".format(k, i, j)) # plt.plot([0, closest_point[0]], [0, closest_point[1]]) # debug if closest_point is not None: # plt.plot([0, closest_point[0]], [0, closest_point[1]]) # debug closest_length.append( math.sqrt(closest_point[0] * closest_point[0] + closest_point[1] * closest_point[1])) closest_sun += math.sqrt( closest_point[0] * closest_point[0] + closest_point[1] * closest_point[1]) closest_size += 1 #else: # print("Maybe the centerpoint is not in the polygon.") for j in closest_length: BOY_measure += abs(100 * j / closest_sun - 100 / closest_size) BOY_index = 1 - BOY_measure / 200 #print("BOY_index={}".format(BOY_index)) # Part six. shape indices: Dispersion of elements / components of area M02, M20, M11 = 0, 0, 0 for j in range(0, uniform_size - 1): M02 += (uniform_coords[j][1]) * (uniform_coords[j][1]) M20 += (uniform_coords[j][0]) * (uniform_coords[j][0]) M11 += (uniform_coords[j][0]) * (uniform_coords[j][1]) Eccentricity = ((M02 + M20) * (M02 + M20) + 4 * M11) / area geometry_vector = [CX, CY, area, peri, LONG_chord, MEAN_radius, \ SMBR_orien, LONGEDGE_orien, BISSECTOR_orien, WEIGHT_orien, \ RIC_compa, IPQ_compa, FRA_compa, GIB_compa, Div_compa, \ elongation, ellipticity, concavity, DCM_index, BOT_index, BOY_index, \ M11, Eccentricity, \ density] geometry_vector = [CX, CY, area, peri, MEAN_radius, \ SMBR_orien, WEIGHT_orien, \ IPQ_compa, FRA_compa, elongation, concavity, BOT_index, \ density] vertice.append(geometry_vector) coords.append(subObject) # # 3 get the adjacency graph of the building group (one sample). # KNN, MST, Delaunay = 1, 2, 3. vertice = np.array(vertice) points = np.array(XY_coords) adjacency = np.zeros((subObject_size, subObject_size)) # print(points[:,0:2]) tri = Delaunay(points[:, 0:2]) for i in range(0, tri.nsimplex): if i > tri.neighbors[i, 2]: adjacency[tri.vertices[i, 0], tri.vertices[i, 1]] = 1 adjacency[tri.vertices[i, 1], tri.vertices[i, 0]] = 1 if i > tri.neighbors[i, 0]: adjacency[tri.vertices[i, 1], tri.vertices[i, 2]] = 1 adjacency[tri.vertices[i, 2], tri.vertices[i, 1]] = 1 if i > tri.neighbors[i, 1]: adjacency[tri.vertices[i, 2], tri.vertices[i, 0]] = 1 adjacency[tri.vertices[i, 0], tri.vertices[i, 2]] = 1 adjacency = scipy.sparse.coo_matrix(adjacency, shape=(subObject_size, subObject_size)) distances = sklearn.metrics.pairwise.pairwise_distances(points[:, 0:2], metric='euclidean') if False: # Distance matrix. is it necessary to be normalized? idx = np.argsort(distances)[:, 1:1 + 1] distances.sort() distances = graph.adjacency(distances[:, 1:1 + 1], idx) adjacency = scipy.sparse.coo_matrix( np.ones((subObject_size, subObject_size)), shape=(subObject_size, subObject_size)).multiply(distances) print(distances.toarray()) # adjacency = adjacency.multiply(distances) else: adjacency = adjacency.multiply(distances) if False: adjacency = scipy.sparse.csgraph.minimum_spanning_tree(adjacency) adjacency = scipy.sparse.csr_matrix(adjacency).toarray() adjacency += adjacency.T - np.diag(adjacency.diagonal()) # print(adjacency) else: # distances = sklearn.metrics.pairwise.pairwise_distances(points[:,0:2], metric="euclidean", n_jobs=1) adjacency = scipy.sparse.csr_matrix(adjacency).toarray() if False: file = r'C:\Users\Administrator\Desktop\ahtor\thesis\gcnn\data\_used_bk.txt' file = "./data/_config_22.txt" conc = np.loadtxt(file) #print((vertice[0,3] - conc[0,1]) / conc[1,1]) vertice_shape = vertice[:, 2:].shape vertice[:, 2:] -= np.tile(conc[0, :], vertice_shape[0]).reshape(vertice_shape) vertice[:, 2:] /= np.tile(conc[1, :], vertice_shape[0]).reshape(vertice_shape) if len(vertice) < 128 and False: vertice = np.pad(vertice, ((0, 128 - len(vertice)), (0, 0)), 'constant', constant_values=(0)) adjacency = np.pad(adjacency, ((0, 128 - adjacency.shape[0]), (0, 128 - adjacency.shape[0])), 'constant', constant_values=(0)) laplacian = graph.laplacian(scipy.sparse.csr_matrix(adjacency), normalized=True, rescaled=True) print(vertice.shape) print(adjacency.shape) print(laplacian.shape) return coords, np.array(vertice), np.array(adjacency), laplacian, np.array( label)
def _coarserning_pooling_(self, adjacency_matrix, pooling_size, normalize=False): num_nodes = adjacency_matrix[:, 0].shape[0] A_dense = adjacency_matrix.todense() num_clusters = int(num_nodes / pooling_size) if num_clusters == 0: num_clusters = num_clusters + 1 sc = SpectralClustering(n_clusters=num_clusters, affinity='precomputed', n_init=10) sc.fit(A_dense) clusters = dict() for inx, label in enumerate(sc.labels_): if label not in clusters: clusters[label] = [] clusters[label].append(inx) num_clusters = len(clusters) num_nodes_in_largest_clusters = 0 for label in clusters: if len(clusters[label]) >= num_nodes_in_largest_clusters: num_nodes_in_largest_clusters = len(clusters[label]) if num_nodes_in_largest_clusters <= 5: num_nodes_in_largest_clusters = 5 num_nodes_in_largest_clusters = 5 Adjacencies_per_cluster = [ adjacency_matrix[clusters[label], :][:, clusters[label]] for label in range(len(clusters)) ] ######## Get inter matrix A_int = sp.lil_matrix(adjacency_matrix) for i in range(len(clusters)): zero_list = list(set(range(num_nodes)) - set(clusters[i])) for j in clusters[i]: A_int[j, zero_list] = 0 A_int[zero_list, j] = 0 ######## Getting adjacenccy matrix wuith only external links A_ext = adjacency_matrix - A_int ######## Getting cluster vertex indicate matrix row_inds = [] col_inds = [] data = [] for i in clusters: for j in clusters[i]: row_inds.append(j) col_inds.append(i) data.append(1) Omega = sp.coo_matrix((data, (row_inds, col_inds))) A_coarsened = np.dot(np.dot(np.transpose(Omega), A_ext), Omega) ########## Constructing pooling matrix pooling_matrices = [ sp.lil_matrix((num_nodes, num_clusters)) for i in range(num_nodes_in_largest_clusters) ] for i in clusters: adj = Adjacencies_per_cluster[i] if len(clusters[i]) > 1: L_i = graph.laplacian(adj, normalize) lamb_i, U_i = graph.fourier(L_i) for j in range(num_nodes_in_largest_clusters): if j < len(clusters[i]): if U_i[0, j] < 0: pooling_matrices[j][clusters[i], i] = -U_i[:, j].reshape(-1, 1) else: pooling_matrices[j][clusters[i], i] = U_i[:, j].reshape(-1, 1) else: if U_i[0, len(clusters[i]) - 1] < 0: pooling_matrices[j][clusters[i], i] = -U_i[:, len(clusters[i]) - 1].reshape(-1, 1) else: pooling_matrices[j][clusters[i], i] = U_i[:, len(clusters[i]) - 1].reshape(-1, 1) else: for j in range(num_nodes_in_largest_clusters): pooling_matrices[j][clusters[i], i] = adj.reshape(-1, 1) return A_coarsened, pooling_matrices
def __init__(self, config, rng): self.config = config self.rng = rng self.model_dir = config.model_dir self.gpu_memory_fraction = config.gpu_memory_fraction self.checkpoint_secs = config.checkpoint_secs self.log_step = config.log_step self.num_epoch = config.num_epochs self.stop_win_size = config.stop_win_size self.stop_early = config.stop_early ## import data Loader ##ir batch_size = config.batch_size server_name = config.server_name mode = config.mode target = config.target sample_rate = config.sample_rate win_size = config.win_size hist_range = config.hist_range s_month = config.s_month e_month = config.e_month e_date = config.e_date s_date = config.s_date data_rm = config.data_rm coarsening_level = config.coarsening_level cnn_mode = config.conv is_coarsen = config.is_coarsen self.data_loader = BatchLoader(server_name, mode, target, sample_rate, win_size, hist_range, s_month, s_date, e_month, e_date, data_rm, batch_size, coarsening_level, cnn_mode, is_coarsen) actual_node = self.data_loader.adj.shape[0] if config.conv == 'gcnn': graphs = self.data_loader.graphs if config.is_coarsen: L = [ graph.laplacian(A, normalized=config.normalized) for A in graphs ] else: L = [ graph.laplacian(self.data_loader.adj, normalized=config.normalized) ] * len(graphs) elif config.conv == 'cnn': L = [actual_node] tmp_node = actual_node while tmp_node > 0: tmp_node = int(tmp_node / 2) L.append(tmp_node) else: raise ValueError("Unsupported config.conv {}".format(config.conv)) tf.reset_default_graph() ## define model ## self.model = Model(config, L, actual_node) ## model saver / summary writer ## self.saver = tf.train.Saver() self.model_saver = tf.train.Saver(self.model.model_vars) self.summary_writer = tf.summary.FileWriter(self.model_dir) # Checkpoint # meta file: describes the saved graph structure, includes # GraphDef, SaverDef, and so on; then apply # tf.train.import_meta_graph('/tmp/model.ckpt.meta'), # will restore Saver and Graph. # index file: it is a string-string immutable # table(tensorflow::table::Table). Each key is a name of a tensor # and its value is a serialized BundleEntryProto. # Each BundleEntryProto describes the metadata of a # tensor: which of the "data" files contains the content of a tensor, # the offset into that file, checksum, some auxiliary data, etc. # # data file: it is TensorBundle collection, save the values of all variables. sv = tf.train.Supervisor(logdir=self.model_dir, is_chief=True, saver=self.saver, summary_op=None, summary_writer=self.summary_writer, save_summaries_secs=300, save_model_secs=self.checkpoint_secs, global_step=self.model.model_step) gpu_options = tf.GPUOptions( per_process_gpu_memory_fraction=self.gpu_memory_fraction, allow_growth=True) # seems to be not working sess_config = tf.ConfigProto(allow_soft_placement=True, gpu_options=gpu_options) # self.sess = sv.prepare_or_wait_for_session(config=sess_config)