Exemplo n.º 1
0
def make_laplacian_directory(fname_input):
    levs = 3
    A = genfromtxt(fname_input, delimiter=',')
    A_ = sparse.csr_matrix(A).astype(np.float32)

    graphs, perm = coarsening.coarsen(A_, levels=levs, self_connections=False)
    L_list = [graph.laplacian(g, normalized=True) for g in graphs]
    L = L_list[0].astype(np.float32)
    #pdb.set_trace()
    return L, perm, levs, L_list
Exemplo n.º 2
0
 def _coarsen(self, dense_node_inputs, adj):
     assert ('metis_' in FLAGS.coarsening)
     self.num_level = get_coarsen_level()
     assert (self.num_level >= 1)
     graphs, perm = coarsen(sp.csr_matrix(adj), levels=self.num_level,
                            self_connections=False)
     permuted_padded_dense_node_inputs = perm_data(
         dense_node_inputs.T, perm).T
     self.sparse_permuted_padded_dense_node_inputs = self._preprocess_inputs(
         sp.csr_matrix(permuted_padded_dense_node_inputs))
     self.coarsened_laplacians = []
     for g in graphs:
         self.coarsened_laplacians.append([self._preprocess_adj(g.todense())])
     assert (len(self.coarsened_laplacians) == self.num_laplacians * self.num_level + 1)
Exemplo n.º 3
0
 def __coarsen(self, at_depth):
     assert(at_depth > 0)
     if not self.is_leaf() and at_depth > 1:
         self.__depth -= 1
         for c in self.get_children():
             if c.get_depth() is at_depth-1:
                 c.__coarsen(at_depth-1)
     elif not self.is_leaf() and at_depth == 1:
         vertex_set = self.get_dataset()
         coarsened_set = coarsen(vertex_set)
         self.__depth = 0
         self.__children = None
         self.__data = coarsened_set
     else:
         raise Exception("something went wrong...")
Exemplo n.º 4
0
def first_coarsen(node_features, n):
    """
    Returns a graph object based on the coordinates of the tract
    Input:
    node_features - An array (n x 3) with the 3D coordinates of each point
    n - The number of points sampled on each tract
    """
    row = np.array(list(range(n - 1)) +
                   list(range(1, n)))  # row[i] connects to col[i]
    col = np.array(list(range(1, n)) + list(range(n - 1)))
    data = np.array([float(1) for i in range(2 * (n - 1))])  # Unit weights
    A = scipy.sparse.csr_matrix((data, (row, col)),
                                shape=(n, n)).astype(np.float32)
    coarsening_levels = 2
    L, perm = coarsen(A, coarsening_levels)
    vert = perm_data(node_features.transpose(), perm)

    return vert.transpose().astype('float16'), L, perm
Exemplo n.º 5
0
    def prepare(self):
        # split train and test
        self.training_events = self.dataset[:self.n_training_events]
        self.test_events = self.dataset[self.n_training_events:]
        # set up model
        self.model = Graph_ConvNet(self.options)
        self.grid_side = self.options['grid_side']
        self.number_edges = self.options['number_edges']
        self.metric = self.options['metric']
        # create graph of Euclidean grid
        self.Grid = grid_graph(self.grid_side, self.number_edges, self.metric)

        # Compute coarsened graphs
        self.L, self.perm = coarsen(self.Grid, self.options['coarsening'])
        lmax = []
        for i in range(self.options['coarsening']):
            lmax.append(lmax_L(self.L[i]))
        self.options['D'] = max(perm) + 1
        self.options['FC1Fin'] = self.options['CL2_F'] * \
            (self.options['D'] // 16)
Exemplo n.º 6
0
def build_coarse_graphs(mesh_face, joint_num, skeleton, flip_pairs, levels=9):
    joint_adj = build_adj(joint_num, skeleton, flip_pairs)
    # Build graph
    mesh_adj = build_graph(mesh_face, mesh_face.max() + 1)
    graph_Adj, graph_L, graph_perm = coarsen(mesh_adj, levels=levels)
    input_Adj = sp.csr_matrix(joint_adj)
    input_Adj.eliminate_zeros()
    input_L = laplacian(input_Adj, normalized=True)

    graph_L[-1] = input_L
    graph_Adj[-1] = input_Adj

    # Compute max eigenvalue of graph Laplacians, rescale Laplacian
    graph_lmax = []
    renewed_lmax = []
    for i in range(levels):
        graph_lmax.append(lmax_L(graph_L[i]))
        graph_L[i] = rescale_L(graph_L[i], graph_lmax[i])
    #     renewed_lmax.append(lmax_L(graph_L[i]))

    return graph_Adj, graph_L, graph_perm, perm_index_reverse(graph_perm[0])
from coarsening import lmax_L
from coarsening import perm_data
from coarsening import rescale_L

# Construct graph
t_start = time.time()
grid_side = 28
number_edges = 4
metric = 'euclidean'
A = grid_graph(grid_side, number_edges,
               metric)  # create graph of Euclidean grid

# Compute coarsened graphs
coarsening_levels = 4

L, perm = coarsen(A, coarsening_levels)

# Compute max eigenvalue of graph Laplacians
lmax = []
for i in range(coarsening_levels):
    lmax.append(lmax_L(L[i]))
print('lmax: ' + str([lmax[i] for i in range(coarsening_levels)]))

# Reindex nodes to satisfy a binary tree structure
train_data = perm_data(train_data, perm)
val_data = perm_data(val_data, perm)
test_data = perm_data(test_data, perm)

print(train_data.shape)
print(val_data.shape)
print(test_data.shape)
Exemplo n.º 8
0
    def __init__(self,
                 sname,
                 mode,
                 target,
                 sample_rate,
                 win_size,
                 hist_range,
                 s_month,
                 s_date,
                 e_month,
                 e_date,
                 data_rm,
                 batch_size=-1,
                 coarsening_level=4,
                 conv_mode='gcnn',
                 is_coarsen=True):

        base_dir = os.path.join('.', 'data', sname)
        if target == 'avg':
            data_dir = os.path.join(
                base_dir, '{}_{}'.format(sample_rate, win_size), mode, target,
                '{}_{}-{}_{}'.format(s_date, s_month, e_date,
                                     e_month), 'rm{}'.format(data_rm))
        else:
            data_dir = os.path.join(
                base_dir, '{}_{}'.format(sample_rate, win_size), mode, target,
                '{}_{}_{}'.format(hist_range[0], hist_range[-1] + 1,
                                  hist_range[1] - hist_range[0]),
                '{}_{}-{}_{}'.format(s_date, s_month, e_date,
                                     e_month), 'rm{}'.format(data_rm))

        dict_normal_fname = os.path.join(data_dir, 'dict_normal.pickle')
        train_data_dict_fname = os.path.join(data_dir,
                                             'train_data_dict.pickle')
        validate_data_dict_fname = os.path.join(data_dir,
                                                'validate_data_dict.pickle')
        Adj_fname = os.path.join(base_dir, 'edge_adj.pickle')

        if not os.path.exists(dict_normal_fname) or \
                not os.path.exists(train_data_dict_fname) or \
                not os.path.exists(validate_data_dict_fname) or \
                not os.path.exists(Adj_fname):
            print("Creating Data...")
            self.data_generator(base_dir, data_dir, sname, mode, target,
                                sample_rate, win_size, hist_range, s_month,
                                s_date, e_month, e_date, data_rm)

        print("Loading data...")
        adj = pklLoad(Adj_fname)
        dict_normal = pklLoad(dict_normal_fname)
        train_data_dict = pklLoad(train_data_dict_fname)
        validate_data_dict = pklLoad(validate_data_dict_fname)

        if target == 'avg':
            self.y_scaler = dict_normal['velocity']
        else:
            self.y_scaler = None

        train_data = train_data_dict['velocity_x']
        train_labels = train_data_dict['velocity_y']
        train_label_weight = train_data_dict['weight_y']
        train_counts = train_data_dict['count_y']
        train_vel_lists = train_data_dict['vel_list']
        cat_train = train_data_dict['cat']
        con_train = train_data_dict['con']

        test_data = validate_data_dict['velocity_x']
        test_labels = validate_data_dict['velocity_y']
        test_labels_weight = validate_data_dict['weight_y']
        test_counts = validate_data_dict['count_y']
        test_vel_lists = validate_data_dict['vel_list']
        cat_test = validate_data_dict['cat']
        con_test = validate_data_dict['con']

        # self.mean_y = mean_gt(train_data)
        # self.mean_y[np.isnan(self.mean_y)] = 0.0
        # train_data = fill_mean(train_data, self.mean_y)
        # train_labels = fill_mean(train_labels, self.mean_y)
        # train_label_weight = np.ones(train_label_weight.shape)
        #
        # test_data = fill_mean(test_data, self.mean_y)

        if conv_mode == 'gcnn':
            perm_file = os.path.join(base_dir, 'adj_perm.pickle')
            graph_file = os.path.join(base_dir, 'perm_graphs.pickle')
            if os.path.exists(perm_file) and os.path.exists(graph_file):
                self.perm = pklLoad(perm_file)
                self.graphs = pklLoad(graph_file)
            else:
                self.graphs, self.perm = coarsening.coarsen(
                    adj, levels=coarsening_level, self_connections=False)
                pklSave(perm_file, self.perm)
                pklSave(graph_file, self.graphs)
            if is_coarsen:
                train_data = coarsening.perm_data_hist(train_data, self.perm)
                test_data = coarsening.perm_data_hist(test_data, self.perm)

        val_x, test_x, val_y, test_y, \
        val_y_weight, test_y_weight, \
        val_con, test_con, val_cat, \
        test_cat, val_count, test_count, \
        val_vel_list, test_vel_list = \
            train_test_split(test_data, test_labels, test_labels_weight,
                             con_test, cat_test, test_counts,
                             test_vel_lists, test_size=0.8)

        print("Reshaping tensors...")
        self.all_batches = []
        self.sizes = []
        # Split train, val, test data into batches
        self.construct_batches(train_data, train_labels, train_label_weight,
                               cat_train, con_train, train_counts,
                               train_vel_lists, batch_size)
        self.construct_batches(val_x, val_y, val_y_weight, val_cat, val_con,
                               val_count, val_vel_list, batch_size)
        self.construct_batches(test_x, test_y, test_y_weight, test_cat,
                               test_con, test_count, test_vel_list, batch_size)

        self.adj = adj
        self.batch_idx = [0, 0, 0]
        print(
            "data load done. Number of batches in train: %d, val: %d, test: %d"
            % (self.sizes[0], self.sizes[1], self.sizes[2]))
Exemplo n.º 9
0
def prepare_graphs():
    p = os.path.join(os.getcwd(), '../survivalnet/data/Brain_Integ.mat')
    D = sio.loadmat(p)
    T = np.asarray([t[0] for t in D['Survival']])
    O = 1 - np.asarray([c[0] for c in D['Censored']])
    X = D['Integ_X']  #[:,1855:]
    X = (X - np.mean(X, axis=0)) / np.std(X, axis=0)
    fold_size = int(10 * len(X) / 100)
    X_train, T_train, O_train = X[2 * fold_size:], T[2 *
                                                     fold_size:], O[2 *
                                                                    fold_size:]
    X_test, T_test, O_test = X[:fold_size], T[:fold_size], O[:fold_size]
    X_val, T_val, O_val = X[fold_size:2 *
                            fold_size], T[fold_size:2 *
                                          fold_size], O[fold_size:2 *
                                                        fold_size]
    print_log('train and test shapes:' + str(X_train.shape) +
              str(X_test.shape))
    if not LOAD_A:
        start = time.time()
        dist, idx = graph.distance_scipy_spatial(X_train.T,
                                                 k=4,
                                                 metric='euclidean')
        print_log('graph constructed:' + str(dist.shape) + str(idx.shape) +
                  ' in ' + str(time.time() - start))
        A = graph.adjacency(dist, idx).astype(np.float32)
        d = X.shape[1]
        assert A.shape == (d, d)
        np.savez('A_sml',
                 data=A.data,
                 indices=A.indices,
                 indptr=A.indptr,
                 shape=A.shape)
        print('d = |V| = {}, k|V| < |E| = {}'.format(d, A.nnz))
    #plt.spy(A, markersize=2, color='black');
    #plt.savefig('tmp.png')
    else:
        start = time.time()
        loader = np.load('A.npz')
        A = csr_matrix((loader['data'], loader['indices'], loader['indptr']),
                       shape=loader['shape'])
        print_log('graph loaded:' + ' in ' + str(time.time() - start))
        print('adjacency matrix type and shape: ', A.__class__, A.shape)
    start = time.time()
    graphs, perm = coarsening.coarsen(A, levels=CL, self_connections=False)
    print_log('graph coarsened:' + ' in ' + str(time.time() - start))
    X_train = coarsening.perm_data(X_train, perm)
    X_val = coarsening.perm_data(X_val, perm)
    X_test = coarsening.perm_data(X_test, perm)
    print_log('train and test shapes:' + str(X_train.shape) +
              str(X_test.shape))
    L = [graph.laplacian(A, normalized=True) for A in graphs]
    #graph.plot_spectrum(L)

    n_train = len(X_train)
    params = dict()
    params['dir_name'] = 'demo'
    params['num_epochs'] = 2000
    params['batch_size'] = int(len(X_train) / 1.0)
    params['eval_frequency'] = 10

    # Building blocks.
    params['filter'] = 'chebyshev5'
    params['brelu'] = 'b1relu'
    params['pool'] = 'apool1'

    # Architecture.
    params['F'] = [8, 8, 8]  # Number of graph convolutional filters.
    params['K'] = [9, 9, 9]  # Polynomial orders.
    params['p'] = [2, 2, 2]  # Pooling sizes.
    params['M'] = [128, 1]  # Output dimensionality of fully connected layers.

    # Optimization.
    params['regularization'] = 0
    params['dropout'] = 1
    params['learning_rate'] = 1e-4
    params['decay_rate'] = 0.999
    params['momentum'] = 0
    params['decay_steps'] = n_train / params['batch_size']
    model = models.cgcnn(L, **params)
    accuracy, loss, t_step = model.cox_fit(X_train, T_train, O_train, X_val,
                                           T_val, O_val)
Exemplo n.º 10
0
def train(method, view_com, n_views, k, m, n_epoch, batch_size, pairs, labels,
          coords, subj, data, data_type, i_fold):
    str_params = view_com + '_k' + str(k) + '_m' + str(m) + '_'
    obj_params = 'softmax'
    print(str_params)

    print('Construct ROI graphs...')
    t_start = time.process_time()
    # A = grid_graph(86, corners=False)
    # A = graph.replace_random_edges(A, 0)
    coo1, coo2, coo3 = coords.shape  # coo2 is the roi dimension
    features = np.zeros([coo1 * coo3, coo2])
    for i in range(coo3):
        features[coo1 * i:coo1 * (i + 1), :] = coords[:, :, i]
    dist, idx = graph.distance_scipy_spatial(np.transpose(features),
                                             k=10,
                                             metric='euclidean')
    A = graph.adjacency(dist, idx).astype(np.float32)

    if method == '2gcn':
        graphs, perm = coarsening.coarsen(A,
                                          levels=FLAGS.coarsening_levels,
                                          self_connections=False)
        L = [graph.laplacian(A, normalized=True) for A in graphs]
        data = coarsening.perm_data1(data, perm, True)
    else:
        graphs = list()
        graphs.append(A)
        L = [graph.laplacian(A, normalized=True)]

    print('Execution time: {:.2f}s'.format(time.process_time() - t_start))
    # graph.plot_spectrum(L)
    del A

    print('Set parameters...')
    mp = model_perf()
    # Architecture.
    common = {}
    common['dir_name'] = 'ppmi/'
    common['num_epochs'] = n_epoch
    common['batch_size'] = batch_size
    common['eval_frequency'] = 5 * common['num_epochs']
    common['patience'] = 5
    common['regularization'] = 5e-3
    common['dropout'] = 1
    common['learning_rate'] = 1e-2
    common['decay_rate'] = 0.95
    common['momentum'] = 0.9
    common['n_views'] = n_views
    common['view_com'] = view_com
    # common['brelu']          = 'b1relu'
    # common['pool']           = 'mpool1'

    print('Get feed pairs and labels...')
    train_pairs, train_y, val_x, val_y, test_pairs, test_y = get_feed_data(
        data, subj, pairs, labels, method)
    C = max(train_y) + 1
    common['decay_steps'] = train_pairs.shape[0] / (common['batch_size'] * 5)

    if method == 'fnn':
        str_params += 'siamese_'
        name = 'mvfnn'
        params = common.copy()
        params['method'] = 'fnn'
        params['fin'] = 1
        params['F'] = [m]
        params['K'] = [1]
        params['p'] = [1]
        params['M'] = [C]
        params['dir_name'] += name
        mp.test(models.siamese_fnn(L, **params), name, params, data,
                train_pairs, train_y, val_x, val_y, test_pairs, test_y)

    if method == '2fnn':
        str_params += 'siamese_layer2_'
        name = 'mvfnn2'
        params = common.copy()
        params['method'] = 'fnn'
        params['fin'] = 1
        params['F'] = [m]
        params['K'] = [1]
        params['p'] = [1]
        params['M'] = [64, C]
        params['dir_name'] += name
        mp.test(models.siamese_fnn(L, **params), name, params, data,
                train_pairs, train_y, val_x, val_y, test_pairs, test_y)

    if method == 'gcn':
        # str_params += 'b_max_eu_'
        name = 'mvgcn'
        params = common.copy()
        params['method'] = 'gcn'
        params['F'] = [m]  # filters
        params['K'] = [k]  # supports
        params['p'] = [1]
        params['M'] = [C]
        params['fin'] = val_x.shape[3]
        params['dir_name'] += name
        params['filter'] = 'chebyshev5'
        params['brelu'] = 'b2relu'
        params['pool'] = 'apool1'
        mp.test(models.siamese_m_cgcnn(L, **params), name, params, data,
                train_pairs, train_y, val_x, val_y, test_pairs, test_y)

    # Common hyper-parameters for LeNet5-like networks.
    if method == '2gcn':
        str_params += 'p4_fc64_'
        name = 'mvgcn2'
        params = common.copy()
        params['method'] = '2gcn'
        params['F'] = [m, 64]  # filters
        params['K'] = [k, k]  # supports
        params['p'] = [4, 4]
        params['M'] = [512, C]
        params['fin'] = val_x.shape[3]
        params['dir_name'] += name
        params['filter'] = 'chebyshev5'
        params['brelu'] = 'b2relu'
        params['pool'] = 'apool1'
        mp.test(models.siamese_m_cgcnn(L, **params), name, params, data,
                train_pairs, train_y, val_x, val_y, test_pairs, test_y)

    # mp.save(data_type)
    method_type = method + '_'
    mp.fin_result(method_type + data_type + str_params + obj_params, i_fold)
Exemplo n.º 11
0
def train(modality, method, data_type, distance, k, fdim, nhops, mem_size, code_size, n_words, edim, n_epoch, batch_size, pairs, labels, coords, data, records, i_fold):
    str_params = '_' + modality + '_' + distance + '_k' + str(k) + '_fdim' + str(fdim) + '_nhops' + str(nhops) + '_memsize' + str(mem_size) + '_codesize' + str(code_size) + '_nwords' + str(n_words) + '_edim' + str(edim)

    print ('Construct ROI graphs...')
    t_start = time.process_time()
    coo1, coo2, coo3 = coords.shape # coo2 is the roi dimension
    features = np.zeros([coo1*coo3, coo2])
    for i in range(coo3):
        features[coo1*i:coo1*(i+1), :] = coords[:, :, i]
    dist, idx = graph.distance_scipy_spatial(np.transpose(features), k=10, metric='euclidean')
    A = graph.adjacency(dist, idx).astype(np.float32)
    if method == '2gcn':
        graphs, perm = coarsening.coarsen(A, levels=FLAGS.coarsening_levels, self_connections=False)
        L = [graph.laplacian(A, normalized=True) for A in graphs]
        data = coarsening.perm_data1(data, perm)
    else:
        graphs = list()
        graphs.append(A)
        L = [graph.laplacian(A, normalized=True)]
    print ('The number of GCN layers: ', len(L))
    print('Execution time: {:.2f}s'.format(time.process_time() - t_start))
    # graph.plot_spectrum(L)
    del A

    print ('Set parameters...')
    mp = model_perf(i_fold, get_pair_label(pairs, labels))
    # Architecture.
    common = {}
    common['dir_name']       = 'ppmi/'
    common['num_epochs']     = n_epoch
    common['batch_size']     = batch_size
    common['eval_frequency'] = 5 * common['num_epochs']
    common['patience']       = 5
    common['regularization'] = 1e-2
    common['dropout']        = 1
    common['learning_rate']  = 5e-3
    common['decay_rate']     = 0.95
    common['momentum']       = 0.9
    common['init_std']       = 5e-2

    print ('Get feed pairs and labels...')
    train_pairs, val_pairs, test_pairs = pairs
    train_x, train_y, val_x, val_y, test_x, test_y = get_feed_data(data, pairs, labels, method)
    train_r, val_r, test_r = get_feed_records(records, pairs, mem_size, code_size, method)
    C = max(train_y)+1
    common['decay_steps']    = train_x.shape[0] / common['batch_size']

    if method == 'MemGCN':
        # str_params += ''
        name = 'cgconv_softmax'
        params = common.copy()
        params['method'] = method
        params['p']              = [1] # pooling size
        params['M']              = [C]
        params['K']              = k    # support number
        params['nhops']          = nhops # hop number
        params['fdim']           = fdim # filters dimension
        params['edim']           = edim # embeddings dimension
        params['mem_size']       = mem_size # the length of sequential records
        params['code_size']      = code_size # the size of one record
        params['n_words']        = n_words # feature dimension
        params['distance']       = distance
        params['fin'] = train_x.shape[2]
        params['dir_name'] += name
        params['filter'] = 'chebyshev5'
        params['brelu'] = 'b2relu'
        params['pool'] = 'apool1'
        mp.test(models.siamese_cgcnn_mem(L, **params), name, params,
                        data, records, train_x, train_r, train_y,
                        val_x, val_r, val_y, test_x, test_r, test_y,
                        train_pairs, test_pairs)

    # mp.save(data_type)
    method_type = method + '_'
    mp.fin_result(method_type + data_type + str_params, i_fold)
Exemplo n.º 12
0
    # Connections are only vertical or horizontal on the grid.
    # Corner vertices are connected to 2 neightbors only.
    if corners:
        import scipy.sparse
        A = A.toarray()
        A[A < A.max()/1.5] = 0
        A = scipy.sparse.csr_matrix(A)
        print('{} edges'.format(A.nnz))

    print("{} > {} edges".format(A.nnz//2, FLAGS.number_edges*m**2//2))
    return A

t_start = timeit.default_timer()
A = grid_graph(58, corners=False)
A = graph.replace_random_edges(A, 0)
graphs, perm = coarsening.coarsen(A, levels=FLAGS.coarsening_levels, self_connections=False)
L = [graph.laplacian(A, normalized=True) for A in graphs]
print('Execution time: {:.2f}s'.format(timeit.default_timer() - t_start))
graph.plot_spectrum(L)
del A


# # Data


with gzip.open('../data/syn_adhd_train_data_quater_scale.pkl.gz', 'rb') as f:  
    X_train = pickle.load(f)
X = zip(*X_train)[0]
labels = zip(*X_train)[1]
X = np.asarray(X)
y = np.asarray(labels)
Exemplo n.º 13
0
import coarsening
import tensorflow as tf
import numpy as np
import scipy as spy

# A=[[(0,1),0.3],
#    [(0,2),0.4],
#    [(0,3),0.3],
#    [(1,2),0.3],
#    [(1,3),0.3],
#    [(1,4),0.4],
#    [(2,0),0.4],
#    [(2,3),0.3],
#    [(2,4),0.3],
#    [(3,0),0.3],
#    [(3,1),0.4],
#    [(3,4),0.3],
#    [(4,0),0.3],
#    [(4,2),0.4],
#    [(4,3),0.3]]
A = spy.sparse.rand(5, 5, 0.3, 'coo', None)
print(A)

graphs, perm = coarsening.coarsen(A, 1, False)
print('graphs:')
print(graphs)
print('perm:')
print(perm)