def train_and_evaluate(args): # Build model model = model_fn( first_layer=[args.inp_kernel, args.inp_window], gcn_layers=[args.n_GC_units] * args.n_GC_layers, conv_layer_filters=[int(k) for k in args.conv_kernels.split(',')], conv_layer_windows=[int(k) for k in args.conv_windows.split(',')], nBins=1250, lr=args.lr, nMarks=args.n_marks, verbose=1 ) # Load chromosomes st_ed_pos = chromosome_sizes[args.cell_line] chromosomes = ['chr' + elm for elm in args.chrs.split(',')] if args.chrs.lower() != 'all' else st_ed_pos.keys() # Load resolutions resolutions = [int(elm) for elm in args.inp_resolutions.split(',')] # Load epigenetic data epi_names = ['ATAC_seq', 'CTCF', 'H3K4me1', 'H3K4me3', 'H3K9ac', 'H3K27ac', 'H3K27me3', 'H3K36me3'] epigenetic_data = load_epigenetic_data(args.cell_line, list(st_ed_pos.keys()), epi_names) # epigenetic_data = load_processed_epigenetic_data_2(chromosomes, epi_names, args.cell_line) for i in range(args.epochs): print('Epoch', i, ':') t1 = time.time() for (epi, hics), micros in generate_batches(args.cell_line, chromosomes, resolutions, epi_names, epigenetic_data, args.batch_size, args.inp_window): t2 = time.time() print(' - Loading data:', t2 - t1, 's') model.train_on_batch([hics, epi], micros) t3 = time.time() print(' - Training:', t3 - t2, 's') mse = model.evaluate([hics, epi], micros, batch_size=args.batch_size, verbose=0) t1 = time.time() print(' - Evaluating:', t1 - t3, 's') print(' - MSE:', mse) v0 = 0 for (epi, hics), micros in generate_batches(args.cell_line, ['chr2'], resolutions, epi_names, epigenetic_data, args.batch_size, args.inp_window): mse = model.evaluate([hics, epi], micros, batch_size=args.batch_size, verbose=0) print(' - Validation:', v0, mse) v0 += 1 if i % args.checkpoint_frequency == 0: model.save_weights('temp_model_{0}.h5'.format(i))
def validate_keras(self, max_count=-1): print('validating') valid_total_loss = 0 count = 0 batch_count = 0 n_val_batch_iters = self.num_val_examples // self.batch_size valid_datagen = utils.generate_batches(filename=self.valid_file, seqlen=self.seq_len, vocab=self.vocab, batch_size=self.batch_size) for _ in range(n_val_batch_iters): x_val_batch, y_val_batch = next(valid_datagen) print('=', end='', flush=True) val_loss = self.model.test_on_batch(x_val_batch, y_val_batch) valid_total_loss = valid_total_loss + val_loss count += len(x_val_batch) batch_count += 1 if max_count > 0 and count > max_count: break print('.') valid_total_loss /= batch_count return valid_total_loss
def main(_): # Get config config = utils.load_config(FLAGS.config) config['training']['batch_size'] = 1 # Get checkpoint dir ckpt_dir = os.path.join(config['training']['ckpt_dir'], config['experiment_name']) ckpt = os.path.join(ckpt_dir, 'model.ckpt') # Build graph results = [] g = tf.Graph() with g.as_default(): tf.logging.info('Creating graph') build_graph(config) saver = tf.train.Saver() with tf.Session() as sess: saver.restore(sess, ckpt) sess.run([tf.local_variables_initializer()]) for i, batch in enumerate( utils.generate_batches('test', config, mate=True)): batch, product = batch try: results.append((greedy_table_search(sess, batch, config), product)) except tf.errors.InvalidArgumentError: # A bad JPEG continue if not (i + 1) % 10000: # Setup output writer with open(FLAGS.output, 'w') as f: json.dump(results, f) break
def run_model(train_data, train_labels, test_data, test_labels): x = tf.placeholder(tf.float32, [None, 784]) y_ = tf.placeholder(tf.float32, [None, 10]) y_pred, keep_prob = cnn(x) with tf.name_scope('loss'): cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_pred) cross_entropy = tf.reduce_mean(cross_entropy) with tf.name_scope('adam_optimizer'): train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy) with tf.name_scope('accuracy'): correct_prediction = tf.equal(tf.argmax(y_pred, 1), tf.argmax(y_, 1)) correct_prediction = tf.cast(correct_prediction, tf.float32) accuracy = tf.reduce_mean(correct_prediction) graph_location = tempfile.mkdtemp() print('Saving graph to: %s' % graph_location) train_writer = tf.summary.FileWriter(graph_location) train_writer.add_graph(tf.get_default_graph()) accuracies = list() iterations = list() with tf.Session() as sess: sess.run(tf.global_variables_initializer()) for epoch in range(0, 1): print('epoch', epoch) # regenerate batches in each epoch batched = utils.generate_batches(train_data, train_labels, batch_size=50) i = 0 for batch in batched: if i % 100 == 0: train_accuracy = accuracy.eval(feed_dict={ x: batch[0], y_: batch[1], keep_prob: 1.0 }) print('step %d, training accuracy %g' % (i, train_accuracy)) accuracies.append(train_accuracy) iterations.append(epoch * len(batched) + i) print(iterations[-1]) i += 1 train_step.run(feed_dict={ x: batch[0], y_: batch[1], keep_prob: 0.5 }) print('test accuracy %g' % accuracy.eval(feed_dict={ x: test_data, y_: test_labels, keep_prob: 1.0 })) print(accuracies) print(iterations)
def train(test_data, test_labels): x = tf.placeholder(tf.float32, [None, 784]) W = tf.Variable(tf.zeros([784, 10])) b = tf.Variable(tf.zeros([10])) y = tf.matmul(x, W) + b y_ = tf.placeholder(tf.float32, [None, 10]) cross_entropy = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)) train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy) sess = tf.InteractiveSession() tf.global_variables_initializer().run() for epoch in range(0, 10): print('epoch', epoch) batched = utils.generate_batches(train_data, train_labels, batch_size=100) i = 0 for batch in batched: if i % 10 == 0: print('running batch', i, '...') sess.run(train_step, feed_dict={x: batch[0], y_: batch[1]}) i += 1 return sess, y, y_, x
def main(): # create the model model = create_model() # compile with set config model.compile(optimizer='adam', loss='categorical_crossentropy') # train the model model.fit_generator(generate_batches(train_data, all_tags, word_to_id, tag_to_id), len(train_data) / batch_size, callbacks=[EvaluateAccuracy()], epochs=5) # final test accuracy acc = compute_test_accuracy(model) print("\nFinal accuracy: %.5f" % acc)
def main(_): # Get config config = utils.load_config(FLAGS.config) config['training']['batch_size'] = 1 # Setup output writer output_file = open(FLAGS.output, 'w', newline='') output_writer = csv.writer(output_file, delimiter=',') # Get checkpoint dir ckpt_dir = os.path.join(config['training']['ckpt_dir'], config['experiment_name']) ckpt = os.path.join(ckpt_dir, 'model.ckpt') # Build graph g = tf.Graph() with g.as_default(): tf.logging.info('Creating graph') build_graph(config) saver = tf.train.Saver() metric_op = tf.get_collection('rank_metrics') update_op = tf.get_collection('rank_updates') with tf.Session() as sess: saver.restore(sess, ckpt) sess.run([tf.local_variables_initializer()]) for i, batch in enumerate(utils.generate_batches('test', config)): batch, uris = batch # TODO: Remove try: attr_query_id = batch['attr_query_ids:0'][0] correct_value_id = batch['correct_value_ids:0'][0] rank, scores = sess.run(['rank:0', 'scores:0'], feed_dict=batch) output_writer.writerow([uris[0], attr_query_id, correct_value_id, rank[0], *scores[0]]) except tf.errors.InvalidArgumentError: # A bad JPEG continue # Close writer output_file.close()
def fetch_all(self): self.server.select('inbox') status, indices = self.server.search(None, 'ALL') if status != 'OK': print("Failed to perform initial search. Abort.") exit() indices = indices[0].split() if self.verbose: print(f"Found {len(indices)} emails.") self.total_messages = len(indices) batches = generate_batches(1, len(indices) - 1, 50) for batch in batches: print(f'Downloading emails {batch[0]} through {batch[1]}') status, response = self.server.fetch(f'{batch[0]}:{batch[1]}', '(RFC822)') for item in response: if len(item) == 2: decoded = item[1].decode('ISO-8859-1') self.messages.append(parse_email(decoded))
def main(): args = parser.parse_args() device = torch.device( "cuda:" + str(args.device)) if torch.cuda.is_available() else torch.device("cpu") Gs, class_labels = load_data(args.dataset, args.use_node_labels) if args.use_node_labels: Gs, feat_dim = process_node_labels(Gs) else: feat_dim = 1 enc = LabelEncoder() class_labels = enc.fit_transform(class_labels) unique_labels = np.unique(class_labels) n_classes = unique_labels.size y = [np.array(class_labels[i]) for i in range(class_labels.size)] kf = KFold(n_splits=10, shuffle=True) it = 0 accs = list() for train_index, test_index in kf.split(y): it += 1 idx = np.random.permutation(train_index) train_index = idx[:int(idx.size * 0.9)].tolist() val_index = idx[int(idx.size * 0.9):].tolist() n_train = len(train_index) n_val = len(val_index) n_test = len(test_index) Gs_train = [Gs[i] for i in train_index] y_train = [y[i] for i in train_index] Gs_val = [Gs[i] for i in val_index] y_val = [y[i] for i in val_index] Gs_test = [Gs[i] for i in test_index] y_test = [y[i] for i in test_index] adj_train, features_train, idx_train, y_train = generate_batches( Gs_train, args.use_node_labels, feat_dim, y_train, args.batch_size, args.radius, device, shuffle=True) adj_val, features_val, idx_val, y_val = generate_batches( Gs_val, args.use_node_labels, feat_dim, y_val, args.batch_size, args.radius, device) adj_test, features_test, idx_test, y_test = generate_batches( Gs_test, args.use_node_labels, feat_dim, y_test, args.batch_size, args.radius, device) n_train_batches = ceil(n_train / args.batch_size) n_val_batches = ceil(n_val / args.batch_size) n_test_batches = ceil(n_test / args.batch_size) model = k_hop_GraphNN(feat_dim, args.hidden_dim, n_classes, args.dropout, args.radius, device).to(device) optimizer = optim.Adam(model.parameters(), lr=args.lr) def train(epoch, adj, features, idx, y): optimizer.zero_grad() output = model(adj, features, idx) loss_train = F.cross_entropy(output, y) loss_train.backward() model.clip_grad(5) optimizer.step() return output, loss_train def test(adj, features, idx, y): output = model(adj, features, idx) loss_test = F.cross_entropy(output, y) return output, loss_test best_val_acc = 0 for epoch in range(args.epochs): start = time.time() model.train() train_loss = AverageMeter() train_acc = AverageMeter() # Train for one epoch for i in range(n_train_batches): output, loss = train(epoch, adj_train[i], features_train[i], idx_train[i], y_train[i]) train_loss.update(loss.data.item(), output.size(0)) train_acc.update(accuracy(output.data, y_train[i].data), output.size(0)) # Evaluate on validation set model.eval() val_loss = AverageMeter() val_acc = AverageMeter() for i in range(n_val_batches): output, loss = test(adj_val[i], features_val[i], idx_val[i], y_val[i]) val_loss.update(loss.data.item(), output.size(0)) val_acc.update(accuracy(output.data, y_val[i].data), output.size(0)) # Print results print("Cross-val iter:", '%02d' % it, "epoch:", '%03d' % (epoch + 1), "train_loss=", "{:.5f}".format(train_loss.avg), "train_acc=", "{:.5f}".format(train_acc.avg), "val_loss=", "{:.5f}".format( val_loss.avg), "val_acc=", "{:.5f}".format(val_acc.avg), "time=", "{:.5f}".format(time.time() - start)) # Remember best accuracy and save checkpoint is_best = val_acc.avg >= best_val_acc best_val_acc = max(val_acc.avg, best_val_acc) if is_best: torch.save( { 'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict(), }, 'model_best.pth.tar') print("Optimization finished!") # Testing test_loss = AverageMeter() test_acc = AverageMeter() print("Loading checkpoint!") checkpoint = torch.load('model_best.pth.tar') model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) model.eval() print("epoch:", epoch) for i in range(n_test_batches): output, loss = test(adj_test[i], features_test[i], idx_test[i], y_test[i]) test_loss.update(loss.data.item(), output.size(0)) test_acc.update(accuracy(output.data, y_test[i].data), output.size(0)) accs.append(test_acc.avg.cpu().numpy()) # Print results print("test_loss=", "{:.5f}".format(test_loss.avg), "test_acc=", "{:.5f}".format(test_acc.avg)) print() print("avg_test_acc=", "{:.5f}".format(np.mean(accs))) return np.mean(accs)
def main(): start = timer() if (os.path.isfile("data/tweets" + str(max_example) + ".npy") and os.path.isfile("data/emojis" + str(max_example) + ".npy")): tweets = np.load("data/tweets" + str(max_example) + ".npy").tolist() emojis = np.load("data/emojis" + str(max_example) + ".npy").tolist() else: tweets, emojis = utils.load_data(path='data/final_train', max_example=max_example) np.save("data/tweets" + str(max_example) + ".npy", np.array(tweets)) np.save("data/emojis" + str(max_example) + ".npy", np.array(emojis)) if (os.path.isfile("data/dev_tweets" + str(max_dev_example) + ".npy") and os.path.isfile("data/dev_emojis" + str(max_dev_example) + ".npy")): dev_tweets = np.load("data/dev_tweets" + str(max_dev_example) + ".npy").tolist() dev_emojis = np.load("data/dev_emojis" + str(max_dev_example) + ".npy").tolist() else: dev_tweets, dev_emojis = utils.load_data(max_example=max_dev_example) np.save("data/dev_tweets" + str(max_dev_example) + ".npy", np.array(dev_tweets)) np.save("data/dev_emojis" + str(max_dev_example) + ".npy", np.array(dev_emojis)) start1 = timer() print(start1 - start) word_dict = utils.build_dict(tweets) # embeddings = utils.generate_embeddings(word_dict, dim=300, pretrained_path='data/glove.6B.300d.txt') embeddings = utils.generate_embeddings(word_dict, dim=300, pretrained_path=None) end0 = timer() print(end0 - start1) x, y = utils.vectorize(tweets, emojis, word_dict) dev_x, dev_y = utils.vectorize(dev_tweets, dev_emojis, word_dict) end1 = timer() print(end1 - end0) batch_size, input_size, hidden_size, output_size, layers = 32, 300, 200, 20, 1 all_train = utils.generate_batches(x, y, batch_size=batch_size) all_dev = utils.generate_batches(dev_x, dev_y, batch_size=batch_size) end2 = timer() print(end2 - end1) # set the parameters # batch_size, input_size, hidden_size, output_size, layers = 64, 50, 200, 20, 1 vocabulary_size = len(embeddings) if run_GRU: print("running GRU...") # initialize the model model = GRU_Classifier(vocabulary_size, input_size, hidden_size, output_size, layers, run_BD_GRU) model.word_embeddings.weight.data = torch.FloatTensor( embeddings.tolist()) if torch.cuda.is_available(): model.cuda() (model.word_embeddings.weight.data).cuda() loss_function = nn.CrossEntropyLoss() if torch.cuda.is_available(): loss_function.cuda() optimizer = optim.Adam(model.parameters(), lr=global_learning_rate) epoch_num = 500 it = 0 best_dev_acc = 0 best_f1 = 0 # model training for epoch in range(epoch_num): np.random.shuffle(all_train) for idx, (mb_x, mb_y, mb_lengths) in enumerate(all_train): # sort the input in descending order according to sentence length # This is required by nn.utils.rnn.pack_padded_sequence sorted_index = len_value_argsort(mb_lengths) mb_x = [mb_x[i] for i in sorted_index] mb_y = [mb_y[i] for i in sorted_index] mb_lengths = [mb_lengths[i] for i in sorted_index] print('#Examples = %d, max_seq_len = %d' % (len(mb_x), len(mb_x[0]))) mb_x = Variable(torch.from_numpy(np.array(mb_x, dtype=np.int64)), requires_grad=False) if torch.cuda.is_available(): mb_x = mb_x.cuda() y_pred = model(mb_x.t(), mb_lengths) mb_y = Variable(torch.from_numpy(np.array(mb_y, dtype=np.int64)), requires_grad=False) if torch.cuda.is_available(): mb_y = mb_y.cuda() loss = loss_function(y_pred, mb_y) # print('epoch ', epoch, 'batch ', idx, 'loss ', loss.data[0]) optimizer.zero_grad() loss.backward(retain_graph=True) optimizer.step() it += 1 if it % 100 == 0: # every 100 updates, check dev accuracy correct = 0 n_examples = 0 ground_truth = [] predicted = [] for idx, (d_x, d_y, d_lengths) in enumerate(all_dev): ground_truth += d_y n_examples += len(d_x) sorted_index = len_value_argsort(d_lengths) d_x = [d_x[i] for i in sorted_index] d_y = [d_y[i] for i in sorted_index] d_lengths = [d_lengths[i] for i in sorted_index] d_x = Variable(torch.from_numpy( np.array(d_x, dtype=np.int64)), requires_grad=False) if torch.cuda.is_available(): d_x = d_x.cuda() # use pytorch way to calculate the correct count d_y = Variable(torch.from_numpy( np.array(d_y, dtype=np.int64)), requires_grad=False) if torch.cuda.is_available(): d_y = d_y.cuda() y_pred = model(d_x.t(), d_lengths) predicted += list( torch.max(y_pred, 1)[1].view(d_y.size()).data) correct += (torch.max(y_pred, 1)[1].view( d_y.size()).data == d_y.data).sum() dev_acc = correct / n_examples f1 = f1_score(ground_truth, predicted, average='macro') print("Dev Accuracy: %f, F1 Score: %f" % (dev_acc, f1)) if f1 > best_f1: best_f1 = f1 print("Best F1 Score: %f" % best_f1) gru_output = open('./out/gru_best', 'w') gru_output.write(str(ground_truth) + '\n') gru_output.write(str(predicted) + '\n') gru_output.write(str(best_f1) + ' ' + str(dev_acc)) gru_output.close() if dev_acc > best_dev_acc: best_dev_acc = dev_acc print("Best Dev Accuracy: %f" % best_dev_acc) if run_LSTM: print("Running LSTM...") model = LSTM_Classifier(vocabulary_size, input_size, hidden_size, output_size, layers, run_BD_LSTM) model.word_embeddings.weight.data = torch.FloatTensor( embeddings.tolist()) if torch.cuda.is_available(): model.cuda() (model.word_embeddings.weight.data).cuda() loss_function = nn.CrossEntropyLoss() if torch.cuda.is_available(): loss_function.cuda() optimizer = optim.Adam(model.parameters(), lr=global_learning_rate) it = 0 best_dev_acc = 0 best_f1 = 0 epoch_num = 500 # train LSTM for epoch in range(epoch_num): np.random.shuffle(all_train) for idx, (mb_x, mb_y, mb_lengths) in enumerate(all_train): sorted_index = len_value_argsort(mb_lengths) mb_x = [mb_x[i] for i in sorted_index] mb_y = [mb_y[i] for i in sorted_index] mb_lengths = [mb_lengths[i] for i in sorted_index] print('#Examples = %d, max_seq_len = %d' % (len(mb_x), len(mb_x[0]))) mb_x = Variable(torch.from_numpy(np.array(mb_x, dtype=np.int64)), requires_grad=False) if torch.cuda.is_available(): mb_x = mb_x.cuda() y_pred = model(mb_x.t(), mb_lengths) mb_y = Variable(torch.from_numpy(np.array(mb_y, dtype=np.int64)), requires_grad=False) if torch.cuda.is_available(): mb_y = mb_y.cuda() loss = loss_function(y_pred, mb_y) # print('epoch ', epoch, 'batch ', idx, 'loss ', loss.data[0]) optimizer.zero_grad() loss.backward(retain_graph=True) optimizer.step() it += 1 if it % 100 == 0: # every 100 updates, check dev accuracy correct = 0 n_examples = 0 ground_truth = [] predicted = [] for idx, (d_x, d_y, d_lengths) in enumerate(all_dev): ground_truth += d_y n_examples += len(d_x) sorted_index = len_value_argsort(d_lengths) d_x = [d_x[i] for i in sorted_index] d_y = [d_y[i] for i in sorted_index] d_lengths = [d_lengths[i] for i in sorted_index] d_x = Variable(torch.from_numpy( np.array(d_x, dtype=np.int64)), requires_grad=False) if torch.cuda.is_available(): d_x = d_x.cuda() d_y = Variable(torch.from_numpy( np.array(d_y, dtype=np.int64)), requires_grad=False) if torch.cuda.is_available(): d_y = d_y.cuda() y_pred = model(d_x.t(), d_lengths) predicted += list( torch.max(y_pred, 1)[1].view(d_y.size()).data) correct += (torch.max(y_pred, 1)[1].view( d_y.size()).data == d_y.data).sum() dev_acc = correct / n_examples f1 = f1_score(ground_truth, predicted, average='macro') print("Dev Accuracy: %f, F1 Score: %f" % (dev_acc, f1)) if f1 > best_f1: best_f1 = f1 print("Best F1 Score: %f" % best_f1) lstm_output = open('./out/lstm_best', 'w') lstm_output.write(str(ground_truth) + '\n') lstm_output.write(str(predicted) + '\n') lstm_output.write(str(best_f1) + ' ' + str(dev_acc)) lstm_output.close() if dev_acc > best_dev_acc: best_dev_acc = dev_acc print("Best Dev Accuracy: %f" % best_dev_acc)
def train(self, X, y, X_valid=None, y_valid=None, batch_size=200, n_epoch=40, batch_seed=0, verbose=True): """Train the neural network with data. Args: X: numpy.ndarray Input data of size `n` (sample size) by `p` (data dimension). y: numpy.ndarray (binary) One-hot labels of size `n` (sample size) by `k` (# classes). X_valid: numpy.ndarray Optional validation data matrix. If provided with `y_valid`, current validation error rate is stored in the model. y_valid: numpy.ndarray Optional validation outcome vector. If provided with `X_valid`, current validation error rate is stored in the model. batch_size: int Size of random batches of the input data. n_epoch: int Number of epochs to train on the input data. batch_seed: int First random seed for batch selection. verbose: bool If true (default), report training updates per epoch to stdout. Returns: nn: NN Trained `NN` model. """ assert self.layers[0].n_in == X.shape[1] assert X.shape[0] == y.shape[0] n = X.shape[0] n_batches = int(np.ceil(n / batch_size)) if verbose: print( '|-------|---------------------------|---------------------------|' ) print( '| Epoch | Training | Validation |' ) print( '|-------|---------------------------|---------------------------|' ) print( '| # | Error | Cross-Ent | Error | Cross-Ent |' ) print( '|-------|---------------------------|---------------------------|' ) for t in range(n_epoch): for i, batch in enumerate(\ generate_batches(n, batch_size, batch_seed + t)): # Forward propagation (last h is output prob) h = X[batch, :] for l in self.layers: h = l.fprop(h, update_units=True) # Backpropagation grad = -(y[batch, :] - h) for l in self.layers[::-1]: grad = l.bprop(grad) self.epoch += 1 for l in self.layers: l.learning_rate.epoch = self.epoch # Errors training_error = self.compute_error(X, y) training_loss = self.compute_cross_entropy(X, y) self.training_error.append((self.epoch, training_error)) self.training_loss.append((self.epoch, training_loss)) if X_valid is not None and y_valid is not None: validation_error = self.compute_error(X_valid, y_valid) validation_loss = self.compute_cross_entropy(X_valid, y_valid) self.validation_error.append((self.epoch, validation_error)) self.validation_loss.append((self.epoch, validation_loss)) if verbose: print('| {:3d} | {:.5f} | {:.5f} | {:.5f} | {:.5f} |'.\ format(self.epoch, training_error, training_loss, validation_error, validation_loss)) if self.early_stopping: if (self.epoch >= 40 and self.validation_loss[-2][1] < validation_loss and self.validation_loss[-3][1] < validation_loss and self.validation_loss[-4][1] < validation_loss): print('======Early stopping: validation loss increase at epoch {:3d}======'.\ format(self.epoch)) break else: if verbose: print('| {:3d} | {:.5f} | {:.5f} | | |'.\ format(self.epoch, training_error, training_loss)) if verbose: print( '|-------|---------------------------|---------------------------|' ) return self
def train_keras(self): tf.set_random_seed(1) # self.session.run(tf.global_variables_initializer()) sys.stdout.write('\n Global variables initialized \n') n_batch_iters = self.num_train_examples // self.batch_size batch_eval_thresh = self.print_freq // self.batch_size for epoch in range(self.epochs): # Hack to continue training from previous model # epoch = epoch + 1 train_datagen = utils.generate_batches(filename=self.train_file, seqlen=self.seq_len, vocab=self.vocab, batch_size=self.batch_size) print('starting training epoch {}'.format(epoch + 1)) train_loss = 0 count = 0 batch_count = 0 for batch_idx in range(n_batch_iters): x_batch, y_batch = next(train_datagen) train_loss_ = self.model.train_on_batch(x_batch, y_batch) count += len(x_batch) batch_count += 1 train_loss += train_loss_ update_loss = train_loss / batch_count update_prpl = math.exp(update_loss) sys.stdout.write( '\rloss = {0:8.3f} - perplexity = {1:8.3f} - train iters = {2} / {3} ' .format(update_loss, update_prpl, batch_count, n_batch_iters)) if (batch_count > 0) and (batch_count % batch_eval_thresh == 0): print( '\n>> Update: Average train loss : {} - Average train prpl : {}\n' .format(update_loss, update_prpl)) valid_loss = self.validate_keras( max_count=self.print_freq // 16) print( '\n>>Update: Average validation loss : {} - Average valdiation prpl : {}\n' .format(valid_loss, math.exp(valid_loss))) # Epoch summaries print('current count:', count) train_loss /= batch_count valid_total_loss = self.validate_keras( max_count=self.num_val_examples) print() print('Epoch {} Summary:'.format(epoch + 1)) print('=' * 80) print( "(epoch {0:5}) train loss = {1:8.3f} ({2:8.3f}); valid loss = {3:8.3f} ({4:8.3f})" .format(epoch, train_loss, math.exp(train_loss), valid_total_loss, math.exp(valid_total_loss))) print('=' * 80) print() # Save epoch model here self.model.save( 'keras_lstm_lm_epoch{0}_val-prpl{1:8.3f}_v2.h5'.format( epoch + 21, math.exp(valid_total_loss)))
features_test = list() subgraphs_test = list() subgraphs_tmp = [subgraphs[i] for i in test_index] c = 0 for s in subgraphs_tmp: l = list() for i in s: adj_test.append(adj[i]) features_test.append(features[i]) l.append(c) c += 1 subgraphs_test.append(l) y_test = [y[i] for i in test_index] adj_train, adj_s_train, features_train, shapes_train, y_train = generate_batches( adj_train, features_train, subgraphs_train, y_train, args.batch_size, args.use_master_node, args.graph_of_sentences) adj_val, adj_s_val, features_val, shapes_val, y_val = generate_batches( adj_val, features_val, subgraphs_val, y_val, args.batch_size, args.use_master_node, args.graph_of_sentences) adj_test, adj_s_test, features_test, shapes_test, y_test = generate_batches( adj_test, features_test, subgraphs_test, y_test, args.batch_size, args.use_master_node, args.graph_of_sentences) n_train_batches = ceil(n_train / args.batch_size) n_val_batches = ceil(n_val / args.batch_size) n_test_batches = ceil(n_test / args.batch_size) # Model and optimizer model = MPAD(embeddings.shape[1], args.message_passing_layers, args.hidden, args.penultimate, nclass, args.dropout, embeddings,
layer.trainable = False model.compile(optimizer='rmsprop', loss='categorical_crossentropy') print("Start training...") import glob files = glob.glob(train_path + '/*/*jpg') samples = len(files) if data_augmentation: model.fit_generator(generate_batches_with_augmentation( train_path, batch_size, validation_split, augmented_data), verbose=1, epochs=epochs, callbacks=[checkpoint]) else: model.fit_generator(generate_batches(train_path, batch_size), epochs=epochs, steps_per_epoch=samples // batch_size, verbose=1, callbacks=[checkpoint]) print("Saving...") model.save(model_path + "/save_model_stage1.h5") # print ("Visualization...") # for i, layer in enumerate(base_model.layers): # print(i, layer.name) if epochs_after_unfreeze > 0: print("Unfreezing all layers...") for i in range(len(model.layers)):
# Load input data and train model on the input data # Create complete training set triples_data = pickle.load( open((config.data_dir) + '/input_train_triples.pkl', 'rb')) cornell_data = pickle.load( open((config.data_dir) + '/input_train_cornell.pkl', 'rb')) twitter_data = pickle.load( open((config.data_dir) + '/input_train_twitter.pkl', 'rb')) train_data = [ triples_data[0] + cornell_data[0] + twitter_data[0], triples_data[1] + cornell_data[1] + twitter_data[1] ] eval_data = pickle.load( open((config.data_dir) + '/input_test_triples.pkl', 'rb')) train_batches = generate_batches(train_data, batch_size=config.batch_size, num_epochs=config.n_epochs) eval_batches = generate_batches(eval_data, batch_size=config.batch_size, num_epochs=1) # model.train(train_batches, eval_batches, verbose=True) # # Evaluate perplexity of trained model on validation data # model_dir = 'runs/1496648182' # model_file = '/model-12000' # # print(model.evaluate(eval_batches, model_dir=model_dir, model_file=model_file)) # # Infer outputs on a subset of test data model_dir = 'runs/149664818' model_file = '/model-12000' checkpoint_file = model_dir + model_file
def __init__(self, sess, data_type='synthetic', batch_size=10, Xtrain=None, Ytrain=None, Xtest=None, Ytest=None, Ytest_und=None, Sig=None, M0=None, Params=None): # Initialize the object self.sess = sess # Input shape if data_type == 'synthetic': self.height_MR = 70 self.width_MR = 70 self.channels = 10 # Number of PLDs or time-points else: self.height_MR = 40 self.width_MR = 40 self.channels = 7 self.output_dim = 2 # number of ASL parameters to estimate (CBF, ATT) self.data_type = data_type # Train Data self.Xtrain = Xtrain # self.Ytrain = Ytrain # Test Data self.Xtest = Xtest self.Ytest = Ytest self.Ytest_und = Ytest_und # Number of residual blocks in the generator self.n_residual_blocks = 8 # Depth of dual paths in the generator self.depth_dual = 4 # Weight decay self.wd = 0.0005 self.df = 64 # Batch normalization flag self.bn_g = False # For generator self.bn_d = True # For discriminator # Lambda value for loss terms self.lam_lp = 1.0 self.lam_asl = 3.0 # 3.0 is the optimal self.lam_adv = 0.5 print( "[ LP loss coef: %f, ASL loss coef: %f, Adversarial loss coef: %f ]" % (self.lam_lp, self.lam_asl, self.lam_adv)) # Learning rate self.learning_rate = 1e-3 self.batch_size = batch_size # Norm of reconstruction term self.l_num = 1 # Get the generic ASL parameters self.common_vars = ut.load_variables(Params) # Call the batch generation for every iteration of GAN self.data_generator = ut.generate_batches(Xtrain, Ytrain, Sig, M0, self.batch_size) # Build GAN model self.build_model()
dataset.set_split('test') test_bar = tqdm(desc='split=test', total=dataset.get_num_batches(args.batch_size), position=1, leave=True) if args.train: try: for epoch_index in range(args.num_epochs): train_state['epoch_index'] = epoch_index # Iterate over training dataset # setup: batch generator, set loss and acc to 0, set train mode on dataset.set_split('train') batch_generator = generate_batches(dataset, batch_size=args.batch_size, device=args.device) running_loss = 0.0 running_acc = 0.0 classifier.train() for batch_index, batch_dict in enumerate(batch_generator): # the training routine is these 5 steps: # -------------------------------------- # step 1. zero the gradients optimizer.zero_grad() # step 2. compute the output # print(f'Classifier is \n {classifier}') if args.architecture_type == 'RNN':
def main(_): # Load config config = utils.load_config(FLAGS.config) ckpt_dir = os.path.join(config['training']['ckpt_dir'], config['experiment_name']) ckpt = os.path.join(ckpt_dir, 'model.ckpt') log_dir = os.path.join(config['training']['log_dir'], config['experiment_name']) if not os.path.exists(ckpt_dir): tf.logging.info('Creating checkpoint directory: %s' % ckpt_dir) os.mkdir(ckpt_dir) if not os.path.exists(log_dir): tf.logging.info('Creating log directory: %s' % log_dir) config_path = os.path.join(ckpt_dir, 'config.yaml') if os.path.exists(config_path): tf.logging.info('Existing configuration file detected.') if config != utils.load_config(config_path): if FLAGS.force: warnings.warn( 'Specified configuration does not match ' 'existing configuration in checkpoint directory. ' 'Forcing overwrite of existing configuration.') shutil.copyfile(FLAGS.config, config_path) else: raise ValueError('Specified configuration does not match ' 'existing configuration in checkpoint ' 'directory.') else: tf.logging.info('No existing configuration found. Copying config file ' 'to "%s".' % config_path) shutil.copyfile(FLAGS.config, config_path) g = tf.Graph() with g.as_default(): tf.logging.info('Creating graph') build_graph(config) global_step = tf.train.get_or_create_global_step() saver = tf.train.Saver(max_to_keep=5) init_fn = get_init_fn(config) total_loss = tf.losses.get_total_loss() trainable_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) trainable_vars = [ var for var in trainable_vars if 'vgg' not in var.name ] learning_rate = tf.train.exponential_decay( learning_rate=config['training']['initial_learning_rate'], global_step=global_step, decay_steps=config['training']['decay_steps'], decay_rate=config['training']['decay_rate'], staircase=True) optimizer = tf.train.AdamOptimizer(learning_rate) train_op = slim.learning.create_train_op( total_loss, optimizer, clip_gradient_norm=config['training']['gradient_clipping'], variables_to_train=trainable_vars, summarize_gradients=True) summary_op = tf.summary.merge_all() eval_logger = tf.summary.FileWriter(log_dir) metric_op = tf.get_collection('metrics') update_op = tf.get_collection('updates') streaming_vars = [ i for i in tf.local_variables() if 'streaming' in i.name ] reset_op = [tf.variables_initializer(streaming_vars)] with tf.Session() as sess: sess.run([ tf.global_variables_initializer(), tf.local_variables_initializer() ]) init_fn(sess) # Responsible for restoring variables / warm starts # Generate data loop. for feed_dict, uris in utils.generate_batches('train', config, mc_attr=True): try: i, _ = sess.run([global_step, train_op], feed_dict=feed_dict) except tf.errors.InvalidArgumentError: # Encountered a bad JPEG continue if not i % config['training']['log_frequency']: try: loss = sess.run(total_loss, feed_dict=feed_dict) except tf.errors.InvalidArgumentError: # Encountered a bad JPEG continue tf.logging.info('Iteration %i - Loss: %0.4f' % (i, loss)) if not i % config['training']['save_frequency']: tf.logging.info('Saving checkpoint for iteration %i' % i) saver.save(sess, ckpt) sess.run(reset_op) # Evaluate on test data. for feed_dict, uris in utils.generate_batches( 'val', config, mc_attr=True): try: sess.run(update_op, feed_dict=feed_dict) except tf.errors.InvalidArgumentError: continue print(sess.run(metric_op)) # Write summaries. summary = sess.run(summary_op, feed_dict=feed_dict) eval_logger.add_summary(summary, i) if i >= config['training']['max_steps']: tf.logging.info('Training complete') sys.exit(0)
def train_3grams_layered_model(train_datasets, dev_datasets, test_datasets, model_path, model_ID=0, epochs=NUM_EPOCHS, batch_size=BATCH_SIZE, hidden_size=HIDDEN_SIZE, lr=LEARNING_RATE, dropout=DROPOUT, rec_dropout=REC_DROPOUT): """ Builds and trains a 2 layers 3-grams model with the given hyper-parameters and training options. :param train_datasets: Tuple (input file, label file) of the training set :param dev_datasets: Tuple (input file, label file) of the development set :param test_datasets: Tuple (input file, label file) of the test set :param model_path: Path to file to be used for saving the trained model :param model_ID: self-explanatory :param epochs: Number of epochs the model has to train for :param batch_size: self-explanatory :param hidden_size: Number of hidden units of the model :param lr: Learning rate :param dropout: Dropout probability for each unit :param rec_dropout: Recurrent dropout probability for each unit :return: tuple (last recorded accuracy on the training set, last recorded accuracy on dev set, accuracy on test set) """ print("Creating model...") x_1grams, x_2grams, x_3grams, y, \ keep_pr, recurrent_keep_pr, \ lengths, train, \ loss, preds = m.get_3grams_layered_model( pretrained_emb_1grams=emb_matrix_1grams, pretrained_emb_2grams=emb_matrix_2grams, pretrained_emb_3grams=emb_matrix_3grams, hidden_size=hidden_size, layers=1, y_size=len(labels_to_idx), learning_rate=lr ) model_params = "ID:%d, units:%d, lr:%.5f, dr:%.2f, rec_dr: %.2f]" % (model_ID, hidden_size, lr, dropout, rec_dropout) saver = tf.train.Saver() with \ tf.Session() as sess,\ tf.summary.FileWriter("logging/3grams_2layers_model %s" % model_params, sess.graph) as train_logger: sess.run(tf.global_variables_initializer()) # keep track of last accuracies recorded to allow greedy optimization accuracies = [] print("Starting training...") for epoch in range(epochs): time = "[%s]" % (str(datetime.now())) print("Epoch %d %s" % (epoch, time)) accumulated_loss = 0 accumulated_acc = 0 iterations = 0 for batch_inputs,\ batch_labels,\ batch_lengths in u.generate_batches(dataset_input=train_datasets[0], dataset_label=train_datasets[1], batch_size=batch_size, label_to_idx=labels_to_idx, ngram_features=[1, 2, 3], word_to_idx=[word_to_idx_1grams, word_to_idx_2grams, word_to_idx_3grams]): _, loss_val, preds_val = sess.run( [train, loss, preds], feed_dict={x_1grams: batch_inputs[0], x_2grams: batch_inputs[1], x_3grams: batch_inputs[2], y: batch_labels, lengths: batch_lengths, keep_pr: 1.0-dropout, recurrent_keep_pr: 1.0-rec_dropout} ) accumulated_loss += loss_val accumulated_acc += compute_accuracy(preds_val, batch_labels) iterations += 1 accumulated_loss /= iterations accumulated_acc /= iterations print("\t- Train set:\tLoss: %.5f\tAcc: %.5f" % (accumulated_loss, accumulated_acc)) # only store the last accuracy recorded if epoch == epochs-1: accuracies.append(accumulated_acc) add_summary(train_logger, "train loss", accumulated_loss, epoch) add_summary(train_logger, "train acc", accumulated_acc, epoch) # dev set accumulated_loss = 0 accumulated_acc = 0 iterations = 0 for batch_inputs,\ batch_labels,\ batch_lengths in u.generate_batches(dataset_input=dev_datasets[0], dataset_label=dev_datasets[1], batch_size=batch_size, label_to_idx=labels_to_idx, ngram_features=[1, 2, 3], word_to_idx=[word_to_idx_1grams, word_to_idx_2grams, word_to_idx_3grams]): loss_val, preds_val = sess.run( [loss, preds], feed_dict={x_1grams: batch_inputs[0], x_2grams: batch_inputs[1], x_3grams: batch_inputs[2], y: batch_labels, lengths: batch_lengths, keep_pr: 1.0, recurrent_keep_pr: 1.0} ) accumulated_loss += loss_val accumulated_acc += compute_accuracy(preds_val, batch_labels) iterations += 1 accumulated_loss /= iterations accumulated_acc /= iterations print("\t- Dev set:\t\tLoss: %.5f\tAcc: %.5f" % (accumulated_loss, accumulated_acc)) # only store the last accuracy recorded if epoch == epochs - 1: accuracies.append(accumulated_acc) # save model periodically if epoch % 10 == 0: saver.save(sess, model_path) time = "[%s]" % (str(datetime.now())) print("Model saved: %s" % time) add_summary(train_logger, "dev loss", accumulated_loss, epoch) add_summary(train_logger, "dev acc", accumulated_acc, epoch) # save model after training ended saver.save(sess, model_path) time = "[%s]" % (str(datetime.now())) print("Training ended %s.\nEvaluating model on test set..." % time) accumulated_loss = 0 accumulated_acc = 0 iterations = 0 for batch_inputs, \ batch_labels, \ batch_lengths in u.generate_batches(dataset_input=test_datasets[0], dataset_label=test_datasets[1], batch_size=batch_size, label_to_idx=labels_to_idx, ngram_features=[1, 2, 3], word_to_idx=[word_to_idx_1grams, word_to_idx_2grams, word_to_idx_3grams], to_shuffle=False): loss_val, preds_val = sess.run( [loss, preds], feed_dict={x_1grams: batch_inputs[0], x_2grams: batch_inputs[1], x_3grams: batch_inputs[2], y: batch_labels, lengths: batch_lengths, keep_pr: 1.0, recurrent_keep_pr: 1.0} ) accumulated_loss += loss_val accumulated_acc += compute_accuracy(preds_val, batch_labels) iterations += 1 accumulated_loss /= iterations accumulated_acc /= iterations print("Test set:\tLoss: %.5f\tAcc: %.5f" % (accumulated_loss, accumulated_acc)) add_summary(train_logger, "test loss", accumulated_loss, epoch) add_summary(train_logger, "test acc", accumulated_acc, epoch) # always store accuracy on test set accuracies.append(accumulated_acc) return accuracies
n_val = len(val_index) n_test = len(test_index) adj_train = [adj_lst[i] for i in train_index] features_train = [features_lst[i] for i in train_index] y_train = [y[i] for i in train_index] adj_val = [adj_lst[i] for i in val_index] features_val = [features_lst[i] for i in val_index] y_val = [y[i] for i in val_index] adj_test = [adj_lst[i] for i in test_index] features_test = [features_lst[i] for i in test_index] y_test = [y[i] for i in test_index] adj_train, features_train, graph_indicator_train, y_train = generate_batches( adj_train, features_train, y_train, args.batch_size, device) adj_val, features_val, graph_indicator_val, y_val = generate_batches( adj_val, features_val, y_val, args.batch_size, device) adj_test, features_test, graph_indicator_test, y_test = generate_batches( adj_test, features_test, y_test, args.batch_size, device) n_train_batches = ceil(n_train / args.batch_size) n_val_batches = ceil(n_val / args.batch_size) n_test_batches = ceil(n_test / args.batch_size) model = RW_NN(features_dim, args.max_step, args.hidden_graphs, args.size_hidden_graphs, args.hidden_dim, args.penultimate_dim, args.normalize, n_classes, args.dropout, device).to(device) optimizer = optim.Adam(model.parameters(), lr=args.lr) scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=50, gamma=0.5)
def create_multiple_link_clicks_ads( self, accountid, pageid, name, titles, bodies, urls, image_paths, targeting, optimization_goal, billing_event, bid_amount, daily_budget=None, lifetime_budget=None, start_time=None, end_time=None, ): """ There are 7 steps in this sample: 1. Create a campaign 2. Create an ad set 3. Upload images 4. Make combinations of specified creative elements 5. Prepare an API batch 6. For each creative combination, add a call to the batch to create a new ad 7. Execute API batch Params: * `accountid` is your Facebook AdAccount id * `name` is a string of basename of your ads. * `page` is the Facebook page used to publish the ads * `titles` array of strings of what user will see as ad title * `bodies` array of strings of what user will see as ad body * `image_paths` array of image file paths * `targeting` is a JSON string specifying targeting info of your ad See [Targeting Specs](https://developers.facebook.com/docs/marketing-api/targeting-specs) for details. * `optimization_goal` the optimization goal for the adsets * `billing_event` what you want to pay for * `bid_amount` how much you want to pay per billing event * `daily_budget` is integer number in your currency. E.g., if your currency is USD, dailybudget=1000 says your budget is 1000 USD * `lifetime_budget` lifetime budget for created ads * `start_time` when the campaign should start * `end_time` when the campaign should end """ my_account = AdAccount(fbid=accountid) """ Take different title body url and image paths, create a batch of ads based on the permutation of these elements """ # Check for bad specs if daily_budget is None: if lifetime_budget is None: raise TypeError( 'One of daily_budget or lifetime_budget must be defined.') elif end_time is None: raise TypeError( 'If lifetime_budget is defined, end_time must be defined.') """ Step 1: Create new campaign with WEBSITE_CLICKS objective See [Campaign Group](https://developers.facebook.com/docs/marketing-api/reference/ad-campaign-group) for further details on the API used here. """ campaign = Campaign(parent_id=accountid) campaign[Campaign.Field.name] = name + ' Campaign' campaign[Campaign.Field.objective] = \ Campaign.Objective.link_clicks campaign.remote_create(params={ 'status': Campaign.Status.paused, }) """ Step 2: Create AdSet using specified optimization goal, billing event and bid. See [AdSet](https://developers.facebook.com/docs/marketing-api/reference/ad-campaign) for further details on the API used here. """ # Create ad set ad_set = AdSet(parent_id=accountid) ad_set[AdSet.Field.campaign_id] = campaign.get_id_assured() ad_set[AdSet.Field.name] = name + ' AdSet' ad_set[AdSet.Field.optimization_goal] = optimization_goal ad_set[AdSet.Field.billing_event] = billing_event ad_set[AdSet.Field.bid_amount] = bid_amount if daily_budget: ad_set[AdSet.Field.daily_budget] = daily_budget else: ad_set[AdSet.Field.lifetime_budget] = lifetime_budget if end_time: ad_set[AdSet.Field.end_time] = end_time if start_time: ad_set[AdSet.Field.start_time] = start_time ad_set[AdSet.Field.targeting] = targeting ad_set.remote_create() """ Step 3: Upload images and get image hashes for use in ad creative. See [Ad Image](https://developers.facebook.com/docs/marketing-api/reference/ad-image#Creating) for further details on the API used here. """ # Upload the images first one by one image_hashes = [] for image_path in image_paths: img = AdImage(parent_id=accountid) img[AdImage.Field.filename] = image_path img.remote_create() image_hashes.append(img.get_hash()) ADGROUP_BATCH_CREATE_LIMIT = 10 ads_created = [] def callback_failure(response): raise response.error() """ Step 4: Using itertools.product get combinations of creative elements. """ for creative_info_batch in generate_batches( itertools.product(titles, bodies, urls, image_hashes), ADGROUP_BATCH_CREATE_LIMIT): """ Step 5: Create an API batch so we can create all ad creatives with one HTTP request. See [Batch Requests](https://developers.facebook.com/docs/graph-api/making-multiple-requests#simple) for further details on batching API calls. """ api_batch = my_account.get_api_assured().new_batch() for title, body, url, image_hash in creative_info_batch: # Create the ad """ Step 6: For each combination of creative elements, add to the batch an API call to create a new Ad and specify the creative inline. See [AdGroup](https://developers.facebook.com/docs/marketing-api/adgroup/) for further details on creating Ads. """ ad = Ad(parent_id=accountid) ad[Ad.Field.name] = name + ' Ad' ad[Ad.Field.adset_id] = ad_set.get_id_assured() ad[Ad.Field.creative] = { AdCreative.Field.object_story_spec: { "page_id": pageid, "link_data": { "message": body, "link": url, "caption": title, "image_hash": image_hash } }, } ad.remote_create(batch=api_batch, failure=callback_failure) ads_created.append(ad) """ Step 7: Execute the batched API calls See [Batch Requests](https://developers.facebook.com/docs/graph-api/making-multiple-requests#simple) for further details on batching API calls. """ api_batch.execute() return [campaign, ad_set, ads_created]
def main(_): # Load config config = utils.load_config(FLAGS.config) ckpt_dir = os.path.join(config['training']['ckpt_dir'], config['experiment_name']) ckpt = os.path.join(ckpt_dir, 'av.ckpt') log_dir = os.path.join(config['training']['log_dir'], config['experiment_name']) if not os.path.exists(ckpt_dir): tf.logging.info('Creating checkpoint directory: %s' % ckpt_dir) os.mkdir(ckpt_dir) if not os.path.exists(log_dir): tf.logging.info('Creating log directory: %s' % log_dir) g = tf.Graph() with g.as_default(): tf.logging.info('Creating graph') build_graph(config) global_step = tf.train.get_or_create_global_step() saver = tf.train.Saver(max_to_keep=5) init_fn = get_init_fn(config) total_loss = tf.losses.get_total_loss() optimizer = tf.train.AdamOptimizer(config['training']['learning_rate']) train_op = slim.learning.create_train_op( total_loss, optimizer, clip_gradient_norm=config['training']['gradient_clipping'], summarize_gradients=True) summary_op = tf.summary.merge_all() eval_logger = tf.summary.FileWriter(log_dir) metric_op = tf.get_collection('metrics') update_op = tf.get_collection('updates') streaming_vars = [ i for i in tf.local_variables() if 'streaming' in i.name ] reset_op = [tf.variables_initializer(streaming_vars)] with tf.Session() as sess: sess.run([ tf.global_variables_initializer(), tf.local_variables_initializer() ]) init_fn(sess) # Responsible for restoring variables / warm starts # Evaluate on test data. for feed_dict, uris in utils.generate_batches('val', config): sess.run(update_op, feed_dict=feed_dict) print(sess.run(metric_op)) # Write summaries. summary = sess.run(summary_op, feed_dict=feed_dict) eval_logger.add_summary(summary, 0) # Generate data loop. for feed_dict, uris in utils.generate_batches('train', config): try: i, _ = sess.run([global_step, train_op], feed_dict=feed_dict) except tf.errors.InvalidArgumentError: # Encountered a bad JPEG continue if not i % config['training']['log_frequency']: loss = sess.run(total_loss, feed_dict=feed_dict) tf.logging.info('Iteration %i - Loss: %0.4f' % (i, loss)) if not i % config['training']['save_frequency']: tf.logging.info('Saving checkpoint for iteration %i' % i) saver.save(sess, ckpt) sess.run(reset_op) # Evaluate on test data. for feed_dict, uris in utils.generate_batches( 'val', config): sess.run(update_op, feed_dict=feed_dict) print(sess.run(metric_op)) # Write summaries. summary = sess.run(summary_op, feed_dict=feed_dict) eval_logger.add_summary(summary, i) if i >= config['training']['max_steps']: tf.logging.info('Training complete') sys.exit(0)
n_test = len(test_index) adj_train = [adj[i] for i in train_index] features_train = [features[i] for i in train_index] y_train = [y[i] for i in train_index] adj_val = [adj[i] for i in val_index] features_val = [features[i] for i in val_index] y_val = [y[i] for i in val_index] adj_test = [adj[i] for i in test_index] features_test = [features[i] for i in test_index] y_test = [y[i] for i in test_index] adj_train, features_train, batch_n_graphs_train, y_train = generate_batches( adj_train, features_train, y_train, args.batch_size, args.use_master_node) adj_val, features_val, batch_n_graphs_val, y_val = generate_batches( adj_val, features_val, y_val, args.batch_size, args.use_master_node) adj_test, features_test, batch_n_graphs_test, y_test = generate_batches( adj_test, features_test, y_test, args.batch_size, args.use_master_node) n_train_batches = ceil(n_train / args.batch_size) n_val_batches = ceil(n_val / args.batch_size) n_test_batches = ceil(n_test / args.batch_size) # Model and optimizer model = MPAD(embeddings.shape[1], args.message_passing_layers, args.hidden, args.penultimate, nclass, args.dropout, embeddings, args.use_master_node)
def train_model( classifier, dataset, loss_func, optimizer, args, ): train_state = make_train_state(args) for epoch_index in range(args.num_epochs): train_state['epoch_index'] = epoch_index dataset.set_split('train') batch_generator = generate_batches(dataset, batch_size=args.batch_size, device=args.device) running_loss = 0.0 running_acc = 0.0 classifier.train() batch_generator = tqdm(batch_generator) batch_generator.set_description( f"E:({epoch_index+1}/{args.num_epochs})") for batch_index, batch_dict in enumerate(batch_generator): optimizer.zero_grad() y_pred = classifier(batch_dict['x_data'].float()) loss = loss_func(y_pred, batch_dict['y_target'].float()) loss_batch = loss.item() running_loss += (loss_batch - running_loss) / (batch_index + 1) loss.backward() optimizer.step() acc_batch = compute_accuracy(y_pred, batch_dict['y_target']) running_acc += (acc_batch - running_acc) / (batch_index + 1) batch_generator.set_postfix(train_acc=running_acc, train_loss=running_loss) train_state['train_acc'].append(running_acc) train_state['train_loss'].append(running_loss) dataset.set_split('valid') batch_generator = generate_batches(dataset, batch_size=args.batch_size, device=args.device) batch_generator = tqdm(batch_generator) batch_generator.set_description( f"E:({epoch_index+1}/{args.num_epochs})") running_loss = 0.0 running_acc = 0.0 classifier.eval() for batch_index, batch_dict in enumerate(batch_generator): y_pred = classifier(batch_dict['x_data'].float()) loss = loss_func(y_pred, batch_dict['y_target'].float()) loss_batch = loss.item() running_loss += (loss_batch - running_loss) / (batch_index + 1) acc_batch = compute_accuracy(y_pred, batch_dict['y_target']) running_acc += (acc_batch - running_acc) / (batch_index + 1) batch_generator.set_postfix(val_acc=running_acc, val_loss=running_loss) train_state['val_acc'].append(running_acc) train_state['val_loss'].append(running_loss) if args.save_model: if not os.path.isdir(args.model_save_dir): os.makedirs(args.model_save_dir) now = datetime.now() file_path = os.path.join(args.model_save_dir, datetime.strftime(now, '%y%m%d_%H%M%S.pth')) torch.save(classifier.state_dict(), file_path) print(f"Model has been saved: {file_path}")
def predict(input_path, output_path, resources_path): """ This is the skeleton of the prediction function. The predict function will build your model, load the weights from the checkpoint and write a new file (output_path) with your predictions in the BIES format. The resources folder should contain everything you need to make the predictions. It is the "resources" folder in your submission. N.B. DO NOT HARD CODE PATHS IN HERE. Use resource_path instead, otherwise we will not be able to run the code. :param input_path: the path of the input file to predict. :param output_path: the path of the output file (where you save your predictions) :param resources_path: the path of the resources folder containing your model and stuff you might need. :return: None """ print("Loading embeddings...") emb_path_1grams = resources_path + "/train/embeddings_1grams.utf8" emb_path_2grams = resources_path + "/train/embeddings_2grams.utf8" word_to_idx_1grams, idx_to_word_1grams, emb_matrix_1grams = u.read_embeddings( emb_path_1grams) word_to_idx_2grams, idx_to_word_2grams, emb_matrix_2grams = u.read_embeddings( emb_path_2grams) labels_to_idx, idx_to_labels = u.get_label_dictionaries() print("Done.") tf.reset_default_graph() x_1grams, x_2grams, y, \ keep_pr, recurrent_keep_pr, \ lengths, train, \ loss, preds = m.get_layered_model(pretrained_emb_1grams=emb_matrix_1grams, pretrained_emb_2grams=emb_matrix_2grams, hidden_size=96, layers=1, y_size=len(labels_to_idx), learning_rate=0.005) model_path = resources_path + "/2layers_model_cityu/base_model.ckpt" print("Loading model saved in path: %s" % model_path) saver = tf.train.Saver() with tf.Session() as sess: saver.restore(sess, model_path) with open(output_path, mode='w', encoding='utf-8') as preds_file: pass print("\nGenerating predictions...") predictions = [] with open(output_path, mode='a', encoding='utf-8') as preds_file: for batch_inputs, \ batch_labels, \ batch_lengths in u.generate_batches(dataset_input=input_path, dataset_label="", batch_size=32, label_to_idx=labels_to_idx, ngram_features=[1, 2], word_to_idx=[word_to_idx_1grams, word_to_idx_2grams], to_shuffle=False, testing=True): preds_val = sess.run( [preds], feed_dict={ x_1grams: batch_inputs[0], x_2grams: batch_inputs[1], lengths: batch_lengths, keep_pr: 1.0, recurrent_keep_pr: 1.0 }) for p in preds_val[0]: p = p[1:np.count_nonzero(p) - 1] p = p.tolist() # default to "S" if some special tag (either '-' or '<PAD>') is predicted p = [ idx_to_labels[c] if c > 1 else idx_to_labels[5] for c in p ] predictions.append(p) if len(predictions) == 128: preds_file.writelines("%s\n" % ''.join(p) for p in predictions) predictions = [] if len(predictions) > 0: preds_file.writelines("%s\n" % ''.join(p) for p in predictions) print("Done.\nYour predictions have been stored in path: %s" % output_path)