def main(_): # Initialize tempdir if tf.gfile.Exists(FLAGS.log_dir): tf.gfile.DeleteRecursively(FLAGS.log_dir) tf.gfile.MakeDirs(FLAGS.log_dir) # Initialize data G = graphs.ErdosRenyi(FLAGS.num_vertices, 0.1, seed=42) G.compute_laplacian("normalized") L = initialize_laplacian_tensor(G.W) W = (G.W).astype(np.float32) train_data, train_labels = generate_wave_samples(FLAGS.num_train, W, T=FLAGS.num_frames, sigma=FLAGS.sigma) train_mb_source = MinibatchSource(train_data, train_labels) test_data, test_labels = generate_wave_samples(FLAGS.num_test, W, T=FLAGS.num_frames, sigma=FLAGS.sigma) # Run training and evaluation loop run_training(train_mb_source, L, test_data, test_labels)
def main(_): global EPOCH_SIZE # Initialize tempdir if FLAGS.action == "eval" and FLAGS.read_dir is not None: FLAGS.log_dir = FLAGS.read_dir else: FLAGS.log_dir = os.path.join(FLAGS.log_dir, FLAGS.model_type) exp_n = _last_exp( FLAGS.log_dir) + 1 if FLAGS.action == "train" else _last_exp( FLAGS.log_dir) FLAGS.log_dir = os.path.join(FLAGS.log_dir, "exp_" + str(exp_n)) print(FLAGS.log_dir) # Initialize data G = create_graph(DATASET_FILE) G.compute_laplacian("normalized") if FLAGS.action == "train": if tf.gfile.Exists(FLAGS.log_dir): tf.gfile.DeleteRecursively(FLAGS.log_dir) tf.gfile.MakeDirs(FLAGS.log_dir) # Prepare pooling num_levels = _number_of_pooling_levels(FLAGS.vertex_poolings) error = True while error: try: adjacencies, perm = coarsen( G.A, levels=num_levels) # Coarsens in powers of 2 error = False except IndexError: error = True continue np.save(os.path.join(FLAGS.log_dir, "ordering"), perm) L = [initialize_laplacian_tensor(A) for A in adjacencies] L = keep_pooling_laplacians(L, FLAGS.vertex_poolings) elif FLAGS.action == "eval": perm = np.load(os.path.join(FLAGS.log_dir, "ordering.npy")) train_mb_source, test_mb_source = create_train_test_mb_sources( DATASET_FILE, FLAGS.test_size, perm=perm, n_samples=FLAGS.n_samples) EPOCH_SIZE = train_mb_source.dataset_length if FLAGS.action == "train": params = vars(FLAGS) with open(os.path.join(FLAGS.log_dir, "params.json"), "w") as f: json.dump(params, f) # Run training and evaluation loop print("Training model...") run_training(L, train_mb_source, test_mb_source) elif FLAGS.action == "eval": print("Evaluating model...") run_eval(test_mb_source) else: raise ValueError("No valid action selected")
def main(_): # Initialize tempdir if tf.gfile.Exists(FLAGS.log_dir): tf.gfile.DeleteRecursively(FLAGS.log_dir) tf.gfile.MakeDirs(FLAGS.log_dir) x = gen_data(interval=5, future=7, graphDataPath=FLAGS.graphPath) sparsed = pd.read_csv('NY/data/' + FLAGS.graphPath, index_col=0) W = sparsed.values - np.identity(len(sparsed)) G = graphs.Graph(W) G.compute_laplacian("normalized") L = initialize_laplacian_tensor(G.W) W = (G.W).astype(np.float32) print('{} nodes, {} edges'.format(G.N, G.Ne)) scaler = StandardScaler() scaler.fit(x) seqs = np.array(list(split_seq(x.index.values, 19))) np.random.shuffle(seqs) print('{} total samples'.format(len(seqs))) split = int(len(seqs) * FLAGS.train_ratio) train_ds = DataLoader(x, seqs[:split], transform=scaler) train_dl = data.DataLoader(train_ds, batch_size=10, shuffle=True, num_workers=4) test_ds = DataLoader(x, seqs[split:], transform=scaler) test_dl = data.DataLoader(test_ds, batch_size=10, shuffle=True, num_workers=4) # Run training and evaluation loop run_training(train_dl, L, test_dl)
def main(_): global EPOCH_SIZE # Initialize tempdir if FLAGS.action == "eval" and FLAGS.read_dir is not None: FLAGS.log_dir = FLAGS.read_dir else: FLAGS.log_dir = os.path.join(FLAGS.log_dir, FLAGS.model_type) exp_n = _last_exp( FLAGS.log_dir) + 1 if FLAGS.action == "train" else _last_exp( FLAGS.log_dir) FLAGS.log_dir = os.path.join(FLAGS.log_dir, "exp_" + str(exp_n)) print(FLAGS.log_dir) # Initialize data X, y = load_data() G = create_spatial_eeg_graph(MONTAGE, q=FLAGS.q, k=FLAGS.k) # G = create_data_eeg_graph(MONTAGE, X) G.compute_laplacian("normalized") if FLAGS.action == "train": if tf.gfile.Exists(FLAGS.log_dir): tf.gfile.DeleteRecursively(FLAGS.log_dir) tf.gfile.MakeDirs(FLAGS.log_dir) # Prepare pooling num_levels = _number_of_pooling_levels(FLAGS.vertex_poolings) error = True while error: try: adjacencies, perm = coarsen( G.A, levels=num_levels) # Coarsens in powers of 2 error = False except IndexError: error = True continue np.save(os.path.join(FLAGS.log_dir, "ordering"), perm) L = [initialize_laplacian_tensor(A) for A in adjacencies] L = keep_pooling_laplacians(L, FLAGS.vertex_poolings) elif FLAGS.action == "eval": perm = np.load(os.path.join(FLAGS.log_dir, "ordering.npy")) X = perm_data(X, perm) train_samples, test_samples = train_validation_test_split( len(y), FLAGS.margin, FLAGS.test_size) if FLAGS.action == "train": EPOCH_SIZE = train_samples.shape[0] print(EPOCH_SIZE) train_mb_source = EyeMinibatchSource(X, y, train_samples, margin=FLAGS.margin, repeat=True) EPOCH_SIZE = train_mb_source.dataset_length test_mb_source = EyeMinibatchSource(X, y, test_samples, margin=FLAGS.margin, repeat=False) if FLAGS.action == "train": params = vars(FLAGS) with open(os.path.join(FLAGS.log_dir, "params.json"), "w") as f: json.dump(params, f) # Run training and evaluation loop print("Training model...") run_training(L, train_mb_source, test_mb_source) elif FLAGS.action == "eval": print("Evaluating model...") run_eval(test_mb_source) else: raise ValueError("No valid action selected")
:param L: List of coarsened laplacians :param poolings: Pooling values :return: New list with only useful laplacians """ j = 0 Lnew = list() Lnew.append(L[0]) for p in poolings: j += int(np.log2(p)) if p > 1 else 0 Lnew.append(L[j]) return Lnew if __name__ == '__main__': from pygsp import graphs from synthetic_data.data_generation import generate_spectral_samples from graph_utils.laplacian import initialize_laplacian_tensor N = 100 G = graphs.Community(N) X, labels = generate_spectral_samples(5, 128, G, f_h=20, f_l=50, lambda_h=80, lambda_l=10) graphs, perm = coarsen(G.A, levels=3) X = perm_data(X, perm) num_levels = np.log2(np.prod([2, 4, 1])).astype(int) adjacencies, perm = coarsen(G.A, levels=num_levels) # Coarsens in powers of 2 L = [initialize_laplacian_tensor(A) for A in adjacencies] print(len(L)) L = keep_pooling_laplacians(L, [2, 4]) print([L.get_shape() for L in L])
def main(_): # Initialize tempdir if FLAGS.action == "eval" and FLAGS.read_dir is not None: FLAGS.log_dir = FLAGS.read_dir else: FLAGS.log_dir = os.path.join(FLAGS.log_dir, FLAGS.model_type) exp_n = _last_exp( FLAGS.log_dir) + 1 if FLAGS.action == "train" else _last_exp( FLAGS.log_dir) FLAGS.log_dir = os.path.join(FLAGS.log_dir, "exp_" + str(exp_n)) print(FLAGS.log_dir) if FLAGS.action == "train": if tf.io.gfile.exists(FLAGS.log_dir): tf.gfile.DeleteRecursively(FLAGS.log_dir) tf.io.gfile.makedirs(FLAGS.log_dir) # Initialize data G = graphs.Community(FLAGS.num_vertices, seed=FLAGS.seed) G.compute_laplacian("normalized") # Save graph np.save(os.path.join(FLAGS.log_dir, "graph_weights"), G.W.todense()) # Prepare pooling num_levels = _number_of_pooling_levels(FLAGS.vertex_poolings) error = True while error: try: adjacencies, perm = coarsen( G.A, levels=num_levels) # Coarsens in powers of 2 error = False except IndexError: error = True continue np.save(os.path.join(FLAGS.log_dir, "ordering"), perm) L = [initialize_laplacian_tensor(A) for A in adjacencies] L = keep_pooling_laplacians(L, FLAGS.vertex_poolings) elif FLAGS.action == "eval": W = np.load(os.path.join(FLAGS.log_dir, "graph_weights.npy")) G = graphs.Graph(W) G.compute_laplacian("normalized") perm = np.load(os.path.join(FLAGS.log_dir, "ordering.npy")) if FLAGS.action == "train": #G_nx = nx.from_numpy_matrix(G.W.todense()) if FLAGS.load_data: #num test can't exceed saved array size train_data, train_labels = load_from_npy('train', FLAGS.num_train) else: random.seed(FLAGS.seed) train_data, train_labels = epidemics_generator( g_nx=nx.from_numpy_matrix(G.W.todense()), batch_size=FLAGS.num_train, timesteps=FLAGS.num_frames, initial_nodes=random.sample(range(FLAGS.num_vertices), int(FLAGS.num_vertices / 10))) save_to_npy('train', train_data, train_labels) train_data = perm_data(train_data, perm) train_mb_source = MinibatchSource(train_data, train_labels, repeat=True) if FLAGS.load_data: #num test can't exceed saved array size test_data, test_labels = load_from_npy('test', FLAGS.num_test) else: test_data, test_labels = epidemics_generator( g_nx=nx.from_numpy_matrix(G.W.todense()), batch_size=FLAGS.num_test, timesteps=FLAGS.num_frames, initial_nodes=random.sample(range(FLAGS.num_vertices), int(FLAGS.num_vertices / 10))) save_to_npy('test', test_data, test_labels) test_data = perm_data(test_data, perm) test_mb_source = MinibatchSource(test_data, test_labels, repeat=False) if FLAGS.action == "train": params = vars(FLAGS) with open(os.path.join(FLAGS.log_dir, "params.json"), "w") as f: json.dump(params, f) # Run training and evaluation loop print("Training model...") run_training(L, train_mb_source, test_mb_source) elif FLAGS.action == "eval": print("Evaluating model...") run_eval(test_mb_source) else: raise ValueError("No valid action selected")
def main(_): # Initialize tempdir if FLAGS.action == "eval" and FLAGS.read_dir is not None: FLAGS.log_dir = FLAGS.read_dir else: FLAGS.log_dir = os.path.join(FLAGS.log_dir, FLAGS.model_type) exp_n = _last_exp( FLAGS.log_dir) + 1 if FLAGS.action == "train" else _last_exp( FLAGS.log_dir) FLAGS.log_dir = os.path.join(FLAGS.log_dir, "exp_" + str(exp_n)) print(FLAGS.log_dir) if FLAGS.action == "train": if tf.gfile.Exists(FLAGS.log_dir): tf.gfile.DeleteRecursively(FLAGS.log_dir) tf.gfile.MakeDirs(FLAGS.log_dir) # Initialize data G = graphs.Community(FLAGS.num_vertices, seed=FLAGS.seed) #, world_density = 0.1) G.compute_laplacian("normalized") # Save graph np.save(os.path.join(FLAGS.log_dir, "graph_weights"), G.W.todense()) # Prepare pooling num_levels = _number_of_pooling_levels(FLAGS.vertex_poolings) error = True while error: try: adjacencies, perm = coarsen( G.A, levels=num_levels) # Coarsens in powers of 2 error = False except IndexError: error = True continue np.save(os.path.join(FLAGS.log_dir, "ordering"), perm) L = [initialize_laplacian_tensor(A) for A in adjacencies] L = keep_pooling_laplacians(L, FLAGS.vertex_poolings) #0 graph if FLAGS.zeros == True: for i in range(len(L)): L[i] *= 0 print(FLAGS.zeros) print("0 Graph \n\n") elif FLAGS.action == "eval": W = np.load(os.path.join(FLAGS.log_dir, "graph_weights.npy")) G = graphs.Graph(W) G.compute_laplacian("normalized") perm = np.load(os.path.join(FLAGS.log_dir, "ordering.npy")) if FLAGS.action == "train": train_data, train_labels = generate_spectral_samples_hard( N=FLAGS.num_train // FLAGS.num_classes, G=G, T=FLAGS.num_frames, f_h=FLAGS.f_h, f_l=FLAGS.f_h, lambda_h=FLAGS.lambda_h, lambda_l=FLAGS.lambda_l, sigma=FLAGS.sigma, sigma_n=FLAGS.sigma_n) train_data = perm_data(train_data, perm) train_mb_source = MinibatchSource(train_data, train_labels, repeat=True) test_data, test_labels = generate_spectral_samples_hard( N=FLAGS.num_test // FLAGS.num_classes, G=G, T=FLAGS.num_frames, f_h=FLAGS.f_h, f_l=FLAGS.f_h, lambda_h=FLAGS.lambda_h, lambda_l=FLAGS.lambda_l, sigma=FLAGS.sigma, sigma_n=FLAGS.sigma_n) test_data = perm_data(test_data, perm) test_mb_source = MinibatchSource(test_data, test_labels, repeat=False) if FLAGS.action == "train": params = vars(FLAGS) with open(os.path.join(FLAGS.log_dir, "params.json"), "w") as f: json.dump(params, f) # Run training and evaluation loop print("Training model...") run_training(L, train_mb_source, test_mb_source) elif FLAGS.action == "eval": print("Evaluating model...") run_eval(test_mb_source) else: raise ValueError("No valid action selected")
def main(_): # Initialize tempdir if FLAGS.action == "eval" and FLAGS.read_dir is not None: FLAGS.log_dir = FLAGS.read_dir else: FLAGS.log_dir = os.path.join(FLAGS.log_dir, FLAGS.model_type) exp_n = _last_exp( FLAGS.log_dir) + 1 if FLAGS.action == "train" else _last_exp( FLAGS.log_dir) FLAGS.log_dir = os.path.join(FLAGS.log_dir, "exp_" + str(exp_n)) print(FLAGS.log_dir) # Make graph and fill with data x = gen_data(interval=FLAGS.interval, graphDataPath=FLAGS.graphPath) x = (x - x.mean(axis=0)) / x.std(axis=0) print(x.shape) #Normalize them #scaler = StandardScaler() #scaler.fit(x) #Data loading seqs = np.array( list( split_seq(x.index.values, FLAGS.num_frames + FLAGS.num_frames_test))) np.random.seed() np.random.shuffle(seqs) print('{} total samples'.format(len(seqs))) split = int(len(seqs) * FLAGS.train_ratio) #x.values = scaler.transform(x) train_ds = DataLoader(x, seqs[:split], train_num=FLAGS.num_frames, test_num=FLAGS.num_frames_test, transform=None) train_dl = data.DataLoader(train_ds, batch_size=FLAGS.batch_size, shuffle=True, num_workers=4) test_ds = DataLoader(x, seqs[split:], train_num=FLAGS.num_frames, test_num=FLAGS.num_frames_test, transform=None) test_dl = data.DataLoader(test_ds, batch_size=FLAGS.batch_size, shuffle=False, num_workers=4) if FLAGS.action == "train": if tf.io.gfile.exists(FLAGS.log_dir): tf.gfile.DeleteRecursively(FLAGS.log_dir) tf.io.gfile.makedirs(FLAGS.log_dir) sparsed = pd.read_csv('NY/data/' + FLAGS.graphPath, index_col=0) W = sparsed.values np.fill_diagonal(W, 0) #W = sparsed.values - np.identity(len(sparsed)) G = graphs.Graph(W) #G = graphs.Community(len(sparsed), seed=0) G.compute_laplacian("normalized") L = initialize_laplacian_tensor(G.W) W = (G.W).astype(np.float32) #print(W) print('{} nodes, {} edges'.format(G.N, G.Ne)) #Log to file params = vars(FLAGS) with open(os.path.join(FLAGS.log_dir, "params.json"), "w") as f: json.dump(params, f) # Run training and evaluation loop print("Training model...") run_training(train_dl, L, test_dl) #run eval results, truths = run_eval(test_dl) np.save('results', results) np.save('truths', truths) elif FLAGS.action == "eval": #x,y,prediction,phase,dropout,metric,metric_opt,loss = setup(L) results, truths = run_eval(test_dl) np.save('results', results) np.save('truths', truths)