def train_cnn(): """Training CNN model.""" # Load sentences, labels, and training parameters # logger.info('✔︎ Loading data...') # logger.info('✔︎ Training data processing...') # train_data = dh.load_data_and_labels(FLAGS.training_data_file, FLAGS.embedding_dim) # logger.info('✔︎ Validation data processing...') # validation_data = dh.load_data_and_labels(FLAGS.validation_data_file, FLAGS.embedding_dim) # logger.info('Recommended padding Sequence length is: {0}'.format(FLAGS.pad_seq_len)) # logger.info('✔︎ Training data padding...') # x_train_front, x_train_behind, y_train = dh.pad_data(train_data, FLAGS.pad_seq_len) # logger.info('✔︎ Validation data padding...') # x_validation_front, x_validation_behind, y_validation = dh.pad_data(validation_data, FLAGS.pad_seq_len) # Build vocabulary # VOCAB_SIZE = dh.load_vocab_size(FLAGS.embedding_dim) # pretrained_word2vec_matrix = dh.load_word2vec_matrix(VOCAB_SIZE, FLAGS.embedding_dim) pretrained_word2vec_matrix = None # Build a graph and cnn object with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=True, log_device_placement=False) session_conf.gpu_options.allow_growth = FLAGS.gpu_options_allow_growth sess = tf.Session(config=session_conf) timestamp = str(int(time.time())) with sess.as_default(): batch_loader = MulBatchLoader("data/train.data", FLAGS.batch_size, "runs/"+timestamp+"/") if mode == "cnn": cnn = TwoLangTextCNN( sequence_length=batch_loader.max_len, num_classes=FLAGS.num_classes, vocab_size_en=batch_loader.vocab_size_en, vocab_size_zh=batch_loader.vocab_size_zh, fc_hidden_size=FLAGS.fc_hidden_size, embedding_size=FLAGS.embedding_dim, embedding_type=FLAGS.embedding_type, filter_sizes=list(map(int, FLAGS.filter_sizes.split(","))), num_filters=FLAGS.num_filters, l2_reg_lambda=FLAGS.l2_reg_lambda, pretrained_embedding=pretrained_word2vec_matrix) elif mode == "rnn": cnn = SiameseLSTM( sequence_length=batch_loader.max_len, num_classes=FLAGS.num_classes, vocab_size_en=batch_loader.vocab_size_en, vocab_size_zh=batch_loader.vocab_size_zh, fc_hidden_size=FLAGS.fc_hidden_size, embedding_size=FLAGS.embedding_dim, embedding_type=FLAGS.embedding_type, l2_reg_lambda=FLAGS.l2_reg_lambda, pretrained_embedding=pretrained_word2vec_matrix) # Define training procedure # learning_rate = tf.train.exponential_decay(learning_rate=FLAGS.learning_rate, global_step=cnn.global_step, # decay_steps=FLAGS.decay_steps, decay_rate=FLAGS.decay_rate, # staircase=True) with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)): optimizer = tf.train.AdamOptimizer(args.learning_rate) grads, vars = zip(*optimizer.compute_gradients(cnn.loss)) grads, _ = tf.clip_by_global_norm(grads, clip_norm=FLAGS.norm_ratio) train_op = optimizer.apply_gradients(zip(grads, vars), global_step=cnn.global_step, name="train_op") # Keep track of gradient values and sparsity (optional) # grad_summaries = [] # for g, v in zip(grads, vars): # if g is not None: # grad_hist_summary = tf.summary.histogram("{0}/grad/hist".format(v.name), g) # sparsity_summary = tf.summary.scalar("{0}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g)) # grad_summaries.append(grad_hist_summary) # grad_summaries.append(sparsity_summary) # grad_summaries_merged = tf.summary.merge(grad_summaries) # Output directory for models and summaries if FLAGS.train_or_restore == 'R': MODEL = input("☛ Please input the checkpoints model you want to restore, " "it should be like(1490175368): ") # The model you want to restore while not (MODEL.isdigit() and len(MODEL) == 10): MODEL = input('✘ The format of your input is illegal, please re-input: ') logger.info('✔︎ The format of your input is legal, now loading to next step...') checkpoint_dir = 'runs/' + MODEL + '/checkpoints/' out_dir = os.path.abspath(os.path.join(os.path.curdir, "runs", MODEL)) logger.info("✔︎ Writing to {0}\n".format(out_dir)) else: out_dir = os.path.abspath(os.path.join(os.path.curdir, "runs", timestamp)) logger.info("✔︎ Writing to {0}\n".format(out_dir)) # Summaries for loss and accuracy loss_summary = tf.summary.scalar("loss", cnn.loss) acc_summary = tf.summary.scalar("accuracy", cnn.accuracy) # Train summaries # train_summary_op = tf.summary.merge([loss_summary, acc_summary, grad_summaries_merged]) train_summary_op = tf.summary.merge([loss_summary, acc_summary]) train_summary_dir = os.path.join(out_dir, "summaries", "train") train_summary_writer = tf.summary.FileWriter(train_summary_dir, sess.graph) # Validation summaries validation_summary_op = tf.summary.merge([loss_summary, acc_summary]) validation_summary_dir = os.path.join(out_dir, "summaries", "validation") validation_summary_writer = tf.summary.FileWriter(validation_summary_dir, sess.graph) saver = tf.train.Saver(tf.global_variables(), max_to_keep=FLAGS.num_checkpoints) if FLAGS.train_or_restore == 'R': # Load cnn model logger.info("✔ Loading model...") checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir) logger.info(checkpoint_file) # Load the saved meta graph and restore variables saver = tf.train.import_meta_graph("{0}.meta".format(checkpoint_file)) saver.restore(sess, checkpoint_file) else: checkpoint_dir = os.path.abspath(os.path.join(out_dir, "checkpoints")) if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) # Embedding visualization config config = projector.ProjectorConfig() embedding_conf = config.embeddings.add() embedding_conf.tensor_name = 'embedding' embedding_conf.metadata_path = FLAGS.metadata_file projector.visualize_embeddings(train_summary_writer, config) projector.visualize_embeddings(validation_summary_writer, config) # Save the embedding visualization saver.save(sess, os.path.join(out_dir, 'embedding', 'embedding.ckpt')) current_step = sess.run(cnn.global_step) def train_step(x_batch_front, x_batch_behind, y_batch, epoch): """A single training step""" feed_dict = { cnn.input_x_en: x_batch_front, cnn.input_x_zh: x_batch_behind, cnn.input_y: y_batch, cnn.dropout_keep_prob: FLAGS.dropout_keep_prob, cnn.is_training: True } _, step, summaries, loss, accuracy = sess.run( [train_op, cnn.global_step, train_summary_op, cnn.loss, cnn.accuracy], feed_dict) logger.info("epoch/step {}/{}: loss {:5.4f}, acc {:5.4f}".format(epoch+1, step, loss, accuracy)) train_summary_writer.add_summary(summaries, step) def validation_step(writer=None): """Evaluates model on a validation set""" total_step = 0 total_loss = 0 total_accuracy = 0 total_recall = 0 total_precision = 0 total_f1 = 0 total_auc = 0 for x_batch_front, x_batch_behind, y_batch in batch_loader.gen_dev_batch(): feed_dict = { cnn.input_x_en: x_batch_front, cnn.input_x_zh: x_batch_behind, cnn.input_y: y_batch, cnn.dropout_keep_prob: 1.0, cnn.is_training: False } step, summaries, loss, accuracy, recall, precision, f1, auc = sess.run( [cnn.global_step, validation_summary_op, cnn.loss, cnn.accuracy, cnn.recall, cnn.precision, cnn.F1, cnn.AUC], feed_dict) total_step += 1 total_loss += loss total_accuracy += accuracy total_recall += recall total_precision += precision total_f1 += f1 # total_auc += auc def get_div(a, b): if b == 0: return 0 return a*1.0/b avg_loss = get_div(total_loss, total_step) avg_accuracy = get_div(total_accuracy, total_step) avg_recall = get_div(total_recall, total_step) avg_precision = get_div(total_precision, total_step) avg_f1 = get_div(total_f1, total_step) # avg_auc = get_div(total_auc, total_step) # logger.info("total_step {0}: avg_loss {1:g}, avg_acc {2:g}, avg_recall {3:g}, avg_precision {4:g}, avg_f1 {5:g}, avg_AUC {6}" # .format(total_step, avg_loss, avg_accuracy, avg_recall, avg_precision, avg_f1, avg_auc)) logger.info("total_step {0}: avg_loss {1:g}, avg_acc {2:g}, avg_recall {3:g}, avg_precision {4:g}, avg_f1 {5:g}" .format(total_step, avg_loss, avg_accuracy, avg_recall, avg_precision, avg_f1)) avg_summaries = tf.Summary() loss_val = avg_summaries.value.add() loss_val.tag = "loss" loss_val.simple_value = avg_loss accuracy_val = avg_summaries.value.add() accuracy_val.tag = "accuracy" accuracy_val.simple_value = avg_accuracy if writer: writer.add_summary(avg_summaries, step) # Training loop. For each batch... for epoch in range(20): for en_batch, zh_batch, y_batch in batch_loader.gen_batch(): train_step(en_batch, zh_batch, y_batch, epoch) current_step = tf.train.global_step(sess, cnn.global_step) if current_step % FLAGS.checkpoint_every == 0: checkpoint_prefix = os.path.join(checkpoint_dir, "model") path = saver.save(sess, checkpoint_prefix, global_step=current_step) logger.info("✔︎ Saved model checkpoint to {0}\n".format(path)) if current_step % FLAGS.evaluate_every == 0: logger.info("\nEvaluation:") validation_step(writer=validation_summary_writer) logger.info("✔︎ Done.")
def train(self, dataset_list, config): """ Args: dataset_list (<StockDataSet>) config (tf.app.flags.FLAGS) """ assert len(dataset_list) > 0 self.merged_sum = tf.summary.merge_all() # Set up the logs folder self.writer = tf.summary.FileWriter( os.path.join("./logs", self.model_name)) self.writer.add_graph(self.sess.graph) if self.use_embed: # Set up embedding visualization # Format: tensorflow/tensorboard/plugins/projector/projector_config.proto projector_config = projector.ProjectorConfig() # You can add multiple embeddings. Here we add only one. added_embed = projector_config.embeddings.add() added_embed.tensor_name = self.embed_matrix.name # Link this tensor to its metadata file (e.g. labels). shutil.copyfile(os.path.join(self.logs_dir, "metadata.tsv"), os.path.join(self.model_logs_dir, "metadata.tsv")) added_embed.metadata_path = "metadata.tsv" # The next line writes a projector_config.pbtxt in the LOG_DIR. TensorBoard will # read this file during startup. projector.visualize_embeddings(self.writer, projector_config) tf.global_variables_initializer().run() # Merged test data of different stocks. merged_test_X = [] merged_test_y = [] merged_test_labels = [] for label_, d_ in enumerate(dataset_list): merged_test_X += list(d_.test_X) merged_test_y += list(d_.test_y) merged_test_labels += [[label_]] * len(d_.test_X) merged_test_X = np.array(merged_test_X) merged_test_y = np.array(merged_test_y) merged_test_labels = np.array(merged_test_labels) print "len(merged_test_X) =", len(merged_test_X) print "len(merged_test_y) =", len(merged_test_y) print "len(merged_test_labels) =", len(merged_test_labels) test_data_feed = { self.learning_rate: 0.0, self.inputs: merged_test_X, self.targets: merged_test_y, self.symbols: merged_test_labels, } global_step = 0 num_batches = sum(len(d_.train_X) for d_ in dataset_list) // config.batch_size random.seed(time.time()) # Select samples for plotting. sample_labels = range(min(config.sample_size, len(dataset_list))) sample_indices = {} for l in sample_labels: sym = dataset_list[l].stock_sym target_indices = np.array([ i for i, sym_label in enumerate(merged_test_labels) if sym_label[0] == l ]) sample_indices[sym] = target_indices print sample_indices print "Start training for stocks:", [d.stock_sym for d in dataset_list] for epoch in xrange(config.max_epoch): epoch_step = 0 learning_rate = config.init_learning_rate * ( config.learning_rate_decay**max( float(epoch + 1 - config.init_epoch), 0.0)) for label_, d_ in enumerate(dataset_list): for batch_X, batch_y in d_.generate_one_epoch( config.batch_size): global_step += 1 epoch_step += 1 batch_labels = np.array([[label_]] * len(batch_X)) train_data_feed = { self.learning_rate: learning_rate, self.inputs: batch_X, self.targets: batch_y, self.symbols: batch_labels, } train_loss, _, train_merged_sum = self.sess.run( [self.loss, self.optim, self.merged_sum], train_data_feed) self.writer.add_summary(train_merged_sum, global_step=global_step) if np.mod(global_step, len(dataset_list) * 100 / config.input_size) == 1: test_loss, test_pred = self.sess.run( [self.loss, self.pred], test_data_feed) print "Step:%d [Epoch:%d] [Learning rate: %.6f] train_loss:%.6f test_loss:%.6f" % ( global_step, epoch, learning_rate, train_loss, test_loss) # Plot samples for sample_sym, indices in sample_indices.iteritems(): image_path = os.path.join( self.model_plots_dir, "{}_epoch{:02d}_step{:04d}.png".format( sample_sym, epoch, epoch_step)) sample_preds = test_pred[indices] sample_truth = merged_test_y[indices] self.plot_samples(sample_preds, sample_truth, image_path, stock_sym=sample_sym) self.save(global_step) final_pred, final_loss = self.sess.run([self.pred, self.loss], test_data_feed) # Save the final model self.save(global_step) return final_pred
def create_tensorboard_visualizations(model, loc, labels=None, write_metadata=True, export_tsv_embeddings=True): """Export embeddings to Tensorboard. This function exports embeddings to disk in a format used by `TensorBoard <https://www.tensorflow.org/tensorboard>`_ and `TensorBoard Embedding Projector <https://projector.tensorflow.org>`_. The function exports: * A number of checkpoint and graph embedding files in the provided location that will allow you to visualize embeddings using Tensorboard. This is generally for use with a `local Tensorboard instance <https://www.tensorflow.org/tensorboard/r1/overview>`_. * a tab-separated file of embeddings ``embeddings_projector.tsv``. This is generally used to visualize embeddings by uploading to `TensorBoard Embedding Projector <https://projector.tensorflow.org>`_. * embeddings metadata (i.e. the embeddings labels from the original knowledge graph), saved to ``metadata.tsv``. Such file can be used in TensorBoard or uploaded to TensorBoard Embedding Projector. The content of ``loc`` will look like: :: tensorboard_files/ ├── checkpoint ├── embeddings_projector.tsv ├── graph_embedding.ckpt.data-00000-of-00001 ├── graph_embedding.ckpt.index ├── graph_embedding.ckpt.meta ├── metadata.tsv └── projector_config.pbtxt .. Note :: A TensorBoard guide is available at `this address <https://www.tensorflow.org/tensorboard/r1/overview>`_. .. Note :: Uploading ``embeddings_projector.tsv`` and ``metadata.tsv`` to `TensorBoard Embedding Projector <https://projector.tensorflow.org>`_ will give a result similar to the picture below: .. image:: ../img/embeddings_projector.png Examples -------- >>> import numpy as np >>> from ampligraph.latent_features import TransE >>> from ampligraph.utils import create_tensorboard_visualizations >>> >>> X = np.array([['a', 'y', 'b'], >>> ['b', 'y', 'a'], >>> ['a', 'y', 'c'], >>> ['c', 'y', 'a'], >>> ['a', 'y', 'd'], >>> ['c', 'y', 'd'], >>> ['b', 'y', 'c'], >>> ['f', 'y', 'e']]) >>> >>> model = TransE(batches_count=1, seed=555, epochs=20, k=10, loss='pairwise', >>> loss_params={'margin':5}) >>> model.fit(X) >>> >>> create_tensorboard_visualizations(model, 'tensorboard_files') Parameters ---------- model: EmbeddingModel A trained neural knowledge graph embedding model, the model must be an instance of TransE, DistMult, ComplEx, or HolE. loc: string Directory where the files are written. labels: pd.DataFrame Label(s) for each embedding point in the Tensorboard visualization. Default behaviour is to use the embeddings labels included in the model. export_tsv_embeddings: bool (Default: True If True, will generate a tab-separated file of embeddings at the given path. This is generally used to visualize embeddings by uploading to `TensorBoard Embedding Projector <https://projector.tensorflow.org>`_. write_metadata: bool (Default: True) If True will write a file named 'metadata.tsv' in the same directory as path. """ # Create loc if it doesn't exist if not os.path.exists(loc): logger.debug('Creating Tensorboard visualization directory: %s' % loc) os.mkdir(loc) if not model.is_fitted: raise ValueError('Cannot write embeddings if model is not fitted.') # If no label data supplied, use model ent_to_idx keys as labels if labels is None: logger.info( 'Using model entity dictionary to create Tensorboard metadata.tsv') labels = list(model.ent_to_idx.keys()) else: if len(labels) != len(model.ent_to_idx): raise ValueError( 'Label data rows must equal number of embeddings.') if write_metadata: logger.debug('Writing metadata.tsv to: %s' % loc) write_metadata_tsv(loc, labels) if export_tsv_embeddings: tsv_filename = "embeddings_projector.tsv" logger.info('Writing embeddings tsv to: %s' % os.path.join(loc, tsv_filename)) np.savetxt(os.path.join(loc, tsv_filename), model.trained_model_params[0], delimiter='\t') checkpoint_path = os.path.join(loc, 'graph_embedding.ckpt') # Create embeddings Variable embedding_var = tf.Variable(model.trained_model_params[0], name='graph_embedding') with tf.Session() as sess: saver = tf.train.Saver([embedding_var]) sess.run(embedding_var.initializer) saver.save(sess, checkpoint_path) config = projector.ProjectorConfig() # One can add multiple embeddings. embedding = config.embeddings.add() embedding.tensor_name = embedding_var.name # Link this tensor to its metadata file (e.g. labels). embedding.metadata_path = 'metadata.tsv' # Saves a config file that TensorBoard will read during startup. projector.visualize_embeddings(tf.summary.FileWriter(loc), config)
loss_summary = tf.scalar_summary("loss", cnn.loss) acc_summary = tf.scalar_summary("accuracy", cnn.accuracy) prec_summary = tf.scalar_summary("precision", cnn.precision) recl_summary = tf.scalar_summary("recall", cnn.recall) # Train Summaries train_summary_op = tf.merge_summary([ loss_summary, acc_summary, prec_summary, recl_summary, grad_summaries_merged ]) train_summary_dir = out_dir #os.path.join(out_dir, "summaries", "train") train_summary_writer = tf.train.SummaryWriter(train_summary_dir, sess.graph) config_pro = projector.ProjectorConfig() embedding = config_pro.embeddings.add() embedding.tensor_name = cnn.embedding.name embedding.metadata_path = os.path.join(out_dir, 'vocab_raw') projector.visualize_embeddings(train_summary_writer, config_pro) # Dev summaries dev_summary_op = tf.merge_summary([loss_summary, acc_summary]) dev_summary_dir = os.path.join(out_dir, "summaries", "dev") dev_summary_writer = tf.train.SummaryWriter(dev_summary_dir, sess.graph) # Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it checkpoint_dir = out_dir #os.path.abspath(os.path.join(out_dir, "checkpoints")) checkpoint_prefix = os.path.join(checkpoint_dir, "model") if not os.path.exists(checkpoint_dir):
def word2vec_basic(log_dir): """Example of building, training and visualizing a word2vec model.""" # Create the directory for TensorBoard variables if there is not. if not os.path.exists(log_dir): os.makedirs(log_dir) # Step 1: Download the data. url = 'http://mattmahoney.net/dc/' # pylint: disable=redefined-outer-name def maybe_download(filename, expected_bytes): """Download a file if not present, and make sure it's the right size.""" local_filename = os.path.join(gettempdir(), filename) if not os.path.exists(local_filename): local_filename, _ = urllib.request.urlretrieve( url + filename, local_filename) statinfo = os.stat(local_filename) if statinfo.st_size == expected_bytes: print('Found and verified', filename) else: print(statinfo.st_size) raise Exception('Failed to verify ' + local_filename + '. Can you get to it with a browser?') return local_filename #filename = maybe_download('text8.zip', 31344016) filename = "/Users/cherry/MachineLearning/tf-scaffold/mnist_datasets/text8.zip" # Read the data into a list of strings. def read_data(filename): """Extract the first file enclosed in a zip file as a list of words.""" with zipfile.ZipFile(filename) as f: data = tf.compat.as_str(f.read(f.namelist()[0])).split() return data vocabulary = read_data(filename) print('Data size', len(vocabulary)) # Step 2: Build the dictionary and replace rare words with UNK token. vocabulary_size = 50000 def build_dataset(words, n_words): """Process raw inputs into a dataset.""" count = [['UNK', -1]] count.extend(collections.Counter(words).most_common(n_words - 1)) dictionary = {} for word, _ in count: dictionary[word] = len(dictionary) data = [] unk_count = 0 for word in words: index = dictionary.get(word, 0) if index == 0: # dictionary['UNK'] unk_count += 1 data.append(index) count[0][1] = unk_count reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys())) return data, count, dictionary, reversed_dictionary # Filling 4 global variables: # data - list of codes (integers from 0 to vocabulary_size-1). # This is the original text but words are replaced by their codes # count - map of words(strings) to count of occurrences # dictionary - map of words(strings) to their codes(integers) # reverse_dictionary - maps codes(integers) to words(strings) data, count, unused_dictionary, reverse_dictionary = build_dataset( vocabulary, vocabulary_size) del vocabulary # Hint to reduce memory. print('Most common words (+UNK)', count[:5]) print('Sample data', data[:10], [reverse_dictionary[i] for i in data[:10]]) # Step 3: Function to generate a training batch for the skip-gram model. def generate_batch(batch_size, num_skips, skip_window): global data_index assert batch_size % num_skips == 0 assert num_skips <= 2 * skip_window batch = np.ndarray(shape=(batch_size), dtype=np.int32) labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32) span = 2 * skip_window + 1 # [ skip_window target skip_window ] buffer = collections.deque(maxlen=span) # pylint: disable=redefined-builtin if data_index + span > len(data): data_index = 0 buffer.extend(data[data_index:data_index + span]) data_index += span for i in range(batch_size // num_skips): context_words = [w for w in range(span) if w != skip_window] words_to_use = random.sample(context_words, num_skips) for j, context_word in enumerate(words_to_use): batch[i * num_skips + j] = buffer[skip_window] labels[i * num_skips + j, 0] = buffer[context_word] if data_index == len(data): buffer.extend(data[0:span]) data_index = span else: buffer.append(data[data_index]) data_index += 1 # Backtrack a little bit to avoid skipping words in the end of a batch data_index = (data_index + len(data) - span) % len(data) return batch, labels batch, labels = generate_batch(batch_size=8, num_skips=2, skip_window=1) for i in range(8): print(batch[i], reverse_dictionary[batch[i]], '->', labels[i, 0], reverse_dictionary[labels[i, 0]]) # Step 4: Build and train a skip-gram model. batch_size = 128 embedding_size = 128 # Dimension of the embedding vector. skip_window = 1 # How many words to consider left and right. num_skips = 2 # How many times to reuse an input to generate a label. num_sampled = 64 # Number of negative examples to sample. # We pick a random validation set to sample nearest neighbors. Here we limit # the validation samples to the words that have a low numeric ID, which by # construction are also the most frequent. These 3 variables are used only for # displaying model accuracy, they don't affect calculation. valid_size = 16 # Random set of words to evaluate similarity on. valid_window = 100 # Only pick dev samples in the head of the distribution. valid_examples = np.random.choice(valid_window, valid_size, replace=False) graph = tf.Graph() with graph.as_default(): # Input data. with tf.name_scope('inputs'): train_inputs = tf.placeholder(tf.int32, shape=[batch_size]) train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1]) valid_dataset = tf.constant(valid_examples, dtype=tf.int32) # Ops and variables pinned to the CPU because of missing GPU implementation with tf.device('/cpu:0'): # Look up embeddings for inputs. with tf.name_scope('embeddings'): embeddings = tf.Variable( tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0)) embed = tf.nn.embedding_lookup(embeddings, train_inputs) # Construct the variables for the NCE loss with tf.name_scope('weights'): nce_weights = tf.Variable( tf.truncated_normal([vocabulary_size, embedding_size], stddev=1.0 / math.sqrt(embedding_size))) with tf.name_scope('biases'): nce_biases = tf.Variable(tf.zeros([vocabulary_size])) # Compute the average NCE loss for the batch. # tf.nce_loss automatically draws a new sample of the negative labels each # time we evaluate the loss. # Explanation of the meaning of NCE loss: # http://mccormickml.com/2016/04/19/word2vec-tutorial-the-skip-gram-model/ with tf.name_scope('loss'): loss = tf.reduce_mean( tf.nn.nce_loss(weights=nce_weights, biases=nce_biases, labels=train_labels, inputs=embed, num_sampled=num_sampled, num_classes=vocabulary_size)) # Add the loss value as a scalar to summary. tf.summary.scalar('loss', loss) # Construct the SGD optimizer using a learning rate of 1.0. with tf.name_scope('optimizer'): optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss) # Compute the cosine similarity between minibatch examples and all # embeddings. norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keepdims=True)) normalized_embeddings = embeddings / norm valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset) similarity = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True) # Merge all summaries. merged = tf.summary.merge_all() # Add variable initializer. init = tf.global_variables_initializer() # Create a saver. saver = tf.train.Saver() # Step 5: Begin training. num_steps = 100001 with tf.Session(graph=graph) as session: # Open a writer to write summaries. writer = tf.summary.FileWriter(log_dir, session.graph) # We must initialize all variables before we use them. init.run() print('Initialized') average_loss = 0 for step in xrange(num_steps): batch_inputs, batch_labels = generate_batch( batch_size, num_skips, skip_window) feed_dict = { train_inputs: batch_inputs, train_labels: batch_labels } # Define metadata variable. run_metadata = tf.RunMetadata() # We perform one update step by evaluating the optimizer op (including it # in the list of returned values for session.run() # Also, evaluate the merged op to get all summaries from the returned # "summary" variable. Feed metadata variable to session for visualizing # the graph in TensorBoard. _, summary, loss_val = session.run([optimizer, merged, loss], feed_dict=feed_dict, run_metadata=run_metadata) average_loss += loss_val # Add returned summaries to writer in each step. writer.add_summary(summary, step) # Add metadata to visualize the graph for the last run. if step == (num_steps - 1): writer.add_run_metadata(run_metadata, 'step%d' % step) if step % 2000 == 0: if step > 0: average_loss /= 2000 # The average loss is an estimate of the loss over the last 2000 # batches. print('Average loss at step ', step, ': ', average_loss) average_loss = 0 # Note that this is expensive (~20% slowdown if computed every 500 steps) if step % 10000 == 0: sim = similarity.eval() for i in xrange(valid_size): valid_word = reverse_dictionary[valid_examples[i]] top_k = 8 # number of nearest neighbors nearest = (-sim[i, :]).argsort()[1:top_k + 1] log_str = 'Nearest to %s:' % valid_word for k in xrange(top_k): close_word = reverse_dictionary[nearest[k]] log_str = '%s %s,' % (log_str, close_word) print(log_str) final_embeddings = normalized_embeddings.eval() # Write corresponding labels for the embeddings. with open(log_dir + '/metadata.tsv', 'w') as f: for i in xrange(vocabulary_size): f.write(reverse_dictionary[i] + '\n') # Save the model for checkpoints. saver.save(session, os.path.join(log_dir, 'model.ckpt')) # Create a configuration for visualizing embeddings with the labels in # TensorBoard. config = projector.ProjectorConfig() embedding_conf = config.embeddings.add() embedding_conf.tensor_name = embeddings.name embedding_conf.metadata_path = os.path.join(log_dir, 'metadata.tsv') projector.visualize_embeddings(writer, config) writer.close() # Step 6: Visualize the embeddings. # pylint: disable=missing-docstring # Function to draw visualization of distance between embeddings. def plot_with_labels(low_dim_embs, labels, filename): assert low_dim_embs.shape[0] >= len( labels), 'More labels than embeddings' plt.figure(figsize=(18, 18)) # in inches for i, label in enumerate(labels): x, y = low_dim_embs[i, :] plt.scatter(x, y) plt.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points', ha='right', va='bottom') plt.savefig(filename) try: # pylint: disable=g-import-not-at-top from sklearn.manifold import TSNE import matplotlib.pyplot as plt tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000, method='exact') plot_only = 500 low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only, :]) labels = [reverse_dictionary[i] for i in xrange(plot_only)] plot_with_labels(low_dim_embs, labels, os.path.join(gettempdir(), 'tsne.png')) except ImportError as ex: print( 'Please install sklearn, matplotlib, and scipy to show embeddings.' ) print(ex)
def main(): cfg = TrainConfig().parse() print(cfg.name) result_dir = os.path.join( cfg.result_root, cfg.name + '_' + datetime.strftime(datetime.now(), '%Y%m%d-%H%M%S')) if not os.path.isdir(result_dir): os.makedirs(result_dir) utils.write_configure_to_file(cfg, result_dir) np.random.seed(seed=cfg.seed) # prepare dataset feat_train = np.load('/mnt/work/CUB_200_2011/data/feat_train.npy') val_feats = np.load('/mnt/work/CUB_200_2011/data/feat_test.npy') label_train = np.load('/mnt/work/CUB_200_2011/data/label_train.npy') label_train -= 1 # make labels start from 0 val_labels = np.load('/mnt/work/CUB_200_2011/data/label_test.npy') class_idx_dict = {} for i, l in enumerate(label_train): l = int(l) if l not in class_idx_dict: class_idx_dict[l] = [i] else: class_idx_dict[l].append(i) C = len(list(class_idx_dict.keys())) val_triplet_idx = select_triplets_random(val_labels, 1000) # generate metadata.tsv for visualize embedding with open(os.path.join(result_dir, 'metadata_val.tsv'), 'w') as fout: for l in val_labels: fout.write('{}\n'.format(int(l))) # construct the graph with tf.Graph().as_default(): tf.set_random_seed(cfg.seed) global_step = tf.Variable(0, trainable=False) lr_ph = tf.placeholder(tf.float32, name='learning_rate') # load backbone model model_emb = networks.CUBLayer(n_input=1024, n_output=cfg.emb_dim) #model_emb = networks.OutputLayer(n_input=1024, n_output=cfg.emb_dim) # get the embedding input_ph = tf.placeholder(tf.float32, shape=[None, 1024]) dropout_ph = tf.placeholder(tf.float32, shape=[]) model_emb.forward(input_ph, dropout_ph) if cfg.normalized: embedding = tf.nn.l2_normalize(model_emb.logits, axis=-1, epsilon=1e-10) else: embedding = model_emb.logits # variable for visualizing the embeddings emb_var = tf.Variable([0.0], name='embeddings') set_emb = tf.assign(emb_var, embedding, validate_shape=False) # calculated for monitoring all-pair embedding distance # diffs = utils.all_diffs_tf(embedding, embedding) # all_dist = utils.cdist_tf(diffs) # tf.summary.histogram('embedding_dists', all_dist) # split embedding into anchor, positive and negative and calculate triplet loss anchor, positive, negative = tf.unstack( tf.reshape(embedding, [-1, 3, cfg.emb_dim]), 3, 1) metric_loss = networks.triplet_loss(anchor, positive, negative, cfg.alpha) regularization_loss = tf.reduce_sum( tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)) total_loss = metric_loss + regularization_loss * cfg.lambda_l2 tf.summary.scalar('learning_rate', lr_ph) train_op = utils.optimize(total_loss, global_step, cfg.optimizer, lr_ph, tf.global_variables()) saver = tf.train.Saver(max_to_keep=10) summary_op = tf.summary.merge_all() # Start running the graph if cfg.gpu: os.environ['CUDA_VISIBLE_DEVICES'] = cfg.gpu gpu_options = tf.GPUOptions(allow_growth=True) sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) summary_writer = tf.summary.FileWriter(result_dir, sess.graph) with sess.as_default(): sess.run(tf.global_variables_initializer()) ################## Training loop ################## for epoch in range(cfg.max_epochs): # learning rate schedule, reference: "In defense of Triplet Loss" if epoch < cfg.static_epochs: learning_rate = cfg.learning_rate else: learning_rate = cfg.learning_rate * \ 0.001**((epoch-cfg.static_epochs)/(cfg.max_epochs-cfg.static_epochs)) # sample images class_in_batch = set() idx_batch = np.array([], dtype=np.int32) while len(idx_batch) < cfg.batch_size: sampled_class = np.random.choice( list(class_idx_dict.keys())) if not sampled_class in class_in_batch: class_in_batch.add(sampled_class) subsample_size = np.random.choice(range(5, 11)) subsample = np.random.permutation( class_idx_dict[sampled_class])[:subsample_size] idx_batch = np.append(idx_batch, subsample) idx_batch = idx_batch[:cfg.batch_size] feat_batch = feat_train[idx_batch] lab_batch = label_train[idx_batch] emb = sess.run(embedding, feed_dict={ input_ph: feat_batch, dropout_ph: 1.0 }) # get distance for all pairs all_diff = utils.all_diffs(emb, emb) triplet_input_idx, active_count = select_triplets_facenet( lab_batch, utils.cdist(all_diff, metric=cfg.metric), cfg.triplet_per_batch, cfg.alpha, num_negative=cfg.num_negative) if triplet_input_idx is not None: triplet_input = feat_batch[triplet_input_idx] # perform training on the selected triplets err, _, step, summ = sess.run( [total_loss, train_op, global_step, summary_op], feed_dict={ input_ph: triplet_input, dropout_ph: cfg.keep_prob, lr_ph: learning_rate }) print ("%s\tEpoch: %d\tImages num: %d\tTriplet num: %d\tLoss %.4f" % \ (cfg.name, epoch+1, feat_batch.shape[0], triplet_input.shape[0]//3, err)) summary = tf.Summary(value=[ tf.Summary.Value(tag="train_loss", simple_value=err), tf.Summary.Value(tag="active_count", simple_value=active_count), tf.Summary.Value(tag="images_num", simple_value=feat_batch.shape[0]), tf.Summary.Value(tag="triplet_num", simple_value=triplet_input.shape[0] // 3) ]) summary_writer.add_summary(summary, step) summary_writer.add_summary(summ, step) # validation on val_set if (epoch + 1) % 100 == 0: print("Evaluating on validation set...") val_err = sess.run(total_loss, feed_dict={ input_ph: val_feats[val_triplet_idx], dropout_ph: 1.0 }) summary = tf.Summary(value=[ tf.Summary.Value(tag="Valiation loss", simple_value=val_err), ]) print("Epoch: [%d]\tloss: %.4f" % (epoch + 1, val_err)) if (epoch + 1) % 1000 == 0: val_embeddings, _ = sess.run([embedding, set_emb], feed_dict={ input_ph: val_feats, dropout_ph: 1.0 }) mAP, mPrec, recall = utils.evaluate_simple( val_embeddings, val_labels) summary = tf.Summary(value=[ tf.Summary.Value(tag="Valiation mAP", simple_value=mAP), tf.Summary.Value(tag="Validation Recall@1", simple_value=recall), tf.Summary.Value(tag="Validation [email protected]", simple_value=mPrec) ]) print("Epoch: [%d]\tmAP: %.4f\trecall: %.4f" % (epoch + 1, mAP, recall)) # config for embedding visualization config = projector.ProjectorConfig() visual_embedding = config.embeddings.add() visual_embedding.tensor_name = emb_var.name visual_embedding.metadata_path = os.path.join( result_dir, 'metadata_val.tsv') projector.visualize_embeddings(summary_writer, config) summary_writer.add_summary(summary, step) # save model saver.save(sess, os.path.join(result_dir, cfg.name + '.ckpt'), global_step=step)
def train_model(self, sentences): sentences, words_to_indices, indices_to_words, words = flatten_and_build_indices( sentences) # Global position within sentences array sentence_index = 0 vocabulary_size = len( set(words)) # Number of unique words in our vocabulary logging.logger.debug("number of training sentences: " + str(len(sentences))) logging.logger.debug("words: " + str(len(words))) logging.logger.debug("vocabulary size: " + str(vocabulary_size)) graph = tf.Graph() with graph.as_default(): with tf.name_scope('inputs'): # Placeholders are structures for feeding input values train_inputs = tf.placeholder(tf.int32, shape=[self.batch_size]) train_labels = tf.placeholder(tf.int32, shape=[self.batch_size, 1]) # Ops and variables pinned to the CPU because of missing GPU implementation with tf.device('/cpu:0'): # Define embedding matrix variable # Variables are the parameters of the model that are being optimized with tf.name_scope('embeddings'): embeddings = tf.Variable(tf.random_uniform( [vocabulary_size, self.embedding_size], -1.0, 1.0), name="embeddings") # Take an input vector of integer indices, # and “look up” these indices in the supplied embeddings tensor. embed = tf.nn.embedding_lookup(embeddings, train_inputs) # Construct the variables for the NCE loss with tf.name_scope('weights'): nce_weights = tf.Variable(tf.truncated_normal( [vocabulary_size, self.embedding_size], stddev=1.0 / math.sqrt(self.embedding_size)), name="weights") with tf.name_scope('biases'): nce_biases = tf.Variable(tf.zeros([vocabulary_size]), name="biases") # Compute the average NCE loss for the batch. # tf.nce_loss automatically draws a new sample of the negative labels each # time we evaluate the loss. with tf.name_scope('loss'): loss = tf.reduce_mean( tf.nn.nce_loss(weights=nce_weights, biases=nce_biases, labels=train_labels, inputs=embed, num_sampled=self.num_sampled, num_classes=vocabulary_size)) # Add the loss value as a scalar to summary. tf.summary.scalar('loss', loss) # Construct the SGD optimizer using a learning rate of 1.0. with tf.name_scope('optimizer'): optimizer = tf.train.GradientDescentOptimizer(1.0).minimize( loss) # Compute the cosine similarity between minibatch examples and all embeddings. norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keepdims=True), name="norm") # normalized_embeddings = embeddings / norm # Merge all summaries. merged = tf.summary.merge_all() # Add variable initializer. init = tf.global_variables_initializer() # Add saver # Save only latest model saver = tf.train.Saver(max_to_keep=1, save_relative_paths=True) # BEGIN TRAINING logging.logger.info("training word2vec model using " + str(len(sentences)) + " samples") logging.init_ticker(total_steps=self.num_steps, desc=self.model_name + " - training word2vec model") with tf.Session(graph=graph) as session: # Open a writer to write summaries. writer = tf.summary.FileWriter(self.log_dir, session.graph) # We must initialize all variables before we use them. init.run() logging.logger.debug('Initialized all variables') logging.logger.debug(norm) average_loss = 0 average_historical_loss = list() step = 0 while step < self.num_steps: logging.tick() batch_inputs, batch_labels, sentence_index = generate_batch( self.batch_size, self.skip_window, sentences, sentence_index) feed_dict = { train_inputs: batch_inputs, train_labels: batch_labels } # Define metadata variable. run_metadata = tf.RunMetadata() # We perform one update step by evaluating the optimizer op (including it in the list of returned values for session.run()) # Also, evaluate the merged op to get all summaries from the returned "summary" variable. # Feed metadata variable to session for visualizing the graph in TensorBoard. _, summary, loss_val = session.run([optimizer, merged, loss], feed_dict=feed_dict, run_metadata=run_metadata) average_loss += loss_val # Add returned summaries to writer in each step. writer.add_summary(summary, step) # Add metadata to visualize the graph for the last run. if step == (self.num_steps - 1): writer.add_run_metadata(run_metadata, 'step%d' % step) if step % 1000 == 0: if step > 0: average_loss /= 1000 average_historical_loss.append(average_loss) # The average loss is an estimate of the loss over the last 1000 steps. logging.logger.info('average loss at step ' + str(step) + ': ' + str(average_loss)) average_loss = 0 # Check if historical loss is showing signs of improvement if len(average_historical_loss) >= 10: if np.std(average_historical_loss[-10:]) < 1: logging.logger.info( "loss seems to have stabilized, stopping training process" ) step = self.num_steps - 1 if step % self.save_every == 0: saver.save(session, self.meta_graph_dir) step = step + 1 # Save used embeddings together with the model with open(self.words_to_indices_filename, "w") as f: json.dump(words_to_indices, f) with open(self.indices_to_words_filename, "w") as f: json.dump(indices_to_words, f) # Write corresponding labels for the embeddings. with open(self.metadata_filename, 'w') as f: for i in range(vocabulary_size): f.write(indices_to_words[i] + '\n') # Create a configuration for visualizing embeddings with the labels in TensorBoard. config = projector.ProjectorConfig() embedding_conf = config.embeddings.add() embedding_conf.tensor_name = embeddings.name embedding_conf.metadata_path = self.metadata_filename projector.visualize_embeddings(writer, config) writer.close()
def trainer_initial(self): graph = tf.Graph() with graph.as_default(): # logging self.logger = tf.summary.FileWriter(self.log_dir) with tf.name_scope("embedding"): batch_inputs = tf.placeholder(tf.int64, shape=([ None, ])) batch_labels = tf.placeholder(tf.int64, shape=([None, 1])) graph_embeddings = tf.Variable(tf.random_uniform( [self.num_words, self.embedding_size], -0.5 / self.embedding_size, 0.5 / self.embedding_size), name='word_embedding') batch_graph_embeddings = tf.nn.embedding_lookup( graph_embeddings, batch_inputs) #hiddeb layer weights = tf.Variable( tf.truncated_normal( [self.num_words, self.embedding_size], stddev=1.0 / math.sqrt(self.embedding_size))) #output layer wt biases = tf.Variable(tf.zeros( self.num_words)) #output layer biases #negative sampling part loss = tf.reduce_mean( tf.nn.nce_loss( weights=weights, biases=biases, labels=batch_labels, inputs=batch_graph_embeddings, num_sampled=self.neg_sampling, num_classes=self.num_words, sampled_values=tf.nn.fixed_unigram_candidate_sampler( true_classes=batch_labels, num_true=1, num_sampled=self.neg_sampling, unique=True, range_max=self.num_words, distortion=0.75, unigrams=self.unigrams ) #word_id_freq_map_as_list is the # frequency of each word in vocabulary )) norm = tf.sqrt( tf.reduce_mean(tf.square(graph_embeddings), 1, keep_dims=True)) normalized_embeddings = graph_embeddings / norm # summary tf.summary.histogram("weights", weights) tf.summary.histogram("biases", biases) tf.summary.scalar("loss", loss) config = projector.ProjectorConfig() emb = config.embeddings.add() emb.tensor_name = normalized_embeddings.name emb.metadata_path = os.path.join(self.log_dir, 'vocab.tsv') projector.visualize_embeddings(self.logger, config) with tf.name_scope('descent'): global_step = tf.Variable(0, trainable=False) learning_rate = tf.train.exponential_decay( self.learning_rate, global_step, 100000, 0.96, staircase=True) #linear decay over time learning_rate = tf.maximum( learning_rate, 0.001 ) #cannot go below 0.001 to ensure at least a minimal learning optimizer = tf.train.GradientDescentOptimizer( learning_rate).minimize(loss, global_step=global_step) self.logger.add_graph(graph) return graph, batch_inputs, batch_labels, normalized_embeddings, loss, optimizer
def main(argv=None): maybe_download() graph = load_graph() basedir = os.path.dirname(__file__) # ensure log directory exists logs_path = os.path.join(basedir, LOG_DIR) if not os.path.exists(logs_path): os.makedirs(logs_path) with tf.Session(graph=graph) as sess: pool3 = sess.graph.get_tensor_by_name('pool_3:0') jpeg_data = tf.placeholder(tf.string) thumbnail = tf.cast( tf.image.resize_images(tf.image.decode_jpeg(jpeg_data, channels=3), [100, 100]), tf.uint8) outputs = [] images = [] # Create metadata metadata_path = os.path.join(basedir, LOG_DIR, 'metadata.tsv') metadata = open(metadata_path, 'w') metadata.write("Name\tLabels\n") for folder_name in os.listdir(IMAGE_DIR): for file_name in os.listdir(IMAGE_DIR + '/' + folder_name): if not file_name.endswith('.jpg'): continue print('process %s...' % file_name) with open( os.path.join(basedir, IMAGE_DIR + '/' + folder_name, file_name), 'rb') as f: data = f.read() results = sess.run([pool3, thumbnail], { 'DecodeJpeg/contents:0': data, jpeg_data: data }) outputs.append(results[0]) images.append(results[1]) metadata.write('{}\t{}\n'.format(file_name, folder_name)) metadata.close() embedding_var = tf.Variable(tf.stack([tf.squeeze(x) for x in outputs], axis=0), trainable=False, name='embed') # prepare projector config config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = embedding_var.name summary_writer = tf.summary.FileWriter(os.path.join(basedir, LOG_DIR)) # link metadata embedding.metadata_path = metadata_path # write to sprite image file image_path = os.path.join(basedir, LOG_DIR, 'sprite.jpg') size = int(math.sqrt(len(images))) + 1 while len(images) < size * size: images.append(np.zeros((100, 100, 3), dtype=np.uint8)) rows = [] for i in range(size): rows.append(tf.concat(images[i * size:(i + 1) * size], 1)) jpeg = tf.image.encode_jpeg(tf.concat(rows, 0)) with open(image_path, 'wb') as f: f.write(sess.run(jpeg)) embedding.sprite.image_path = image_path embedding.sprite.single_image_dim.extend([100, 100]) # save embedding_var projector.visualize_embeddings(summary_writer, config) sess.run(tf.variables_initializer([embedding_var])) saver = tf.train.Saver() saver.save(sess, os.path.join(basedir, LOG_DIR, 'model.ckpt'))
def create_embedding(): print('***** Config *****') print('***** Building Point {}...'.format(MODEL_NAME)) print('** num_frames: {}'.format(cfg.num_frames)) print('** num_classes: {}'.format(cfg.num_classes)) print('** batch_size: {}'.format(cfg.batch_size)) print('** epoch: {}'.format(cfg.epoch)) print('** init_learning_rate: {}'.format(cfg.init_learning_rate)) print('** decay_step: {}'.format(cfg.decay_step)) print('** decay_rate: {}'.format(cfg.decay_rate)) print('** weight_decay: {}'.format(cfg.weight_decay)) with tf.Graph().as_default(): anchor, labels = placeholder_inputs(cfg.batch_size, cfg.num_frames) # input_negative, labels = placeholder_inputs( # cfg.batch_size, cfg.num_frames) # input_positive, labels = placeholder_inputs( # cfg.batch_size, cfg.num_frames) is_training_pl = tf.placeholder(tf.bool, shape=()) keep_prob = tf.placeholder(tf.float32) global_step = tf.Variable(0, dtype=tf.int64) bn_decay = True # tf.summary.scalar('bn_decay', bn_decay) # Get model and loss anchor_embed, pred = build_graph(anchor, is_training_pl, keep_prob, weight_decay=cfg.weight_decay, bn_decay=bn_decay, reuse_layers=False) print("\nplaceholders loaded...") # %% restore a previous model sess = tf.InteractiveSession() # load_model_path = LOGDIR+'/model_epoch_{}'.format(cfg.load_model_epoch) # load_model_path = '/media/tjosh/vault/MSRAction3D/trained_models/logdir_multitask_lowlr_simple_ff_5_96/model_epoch_200' load_model_path = '/media/tjosh/vault/MSRAction3D/trained_models/logdir_multitask_lowlr_128_simple_ff_5_96/model_epoch_65' saver = tf.train.Saver() saver.restore(sess, load_model_path) print("\nModel restored...", load_model_path) # Plot Variable Histogram t_vars = tf.trainable_variables() # Count number of trainable parameters num_params = np.sum([np.prod(v.get_shape().as_list()) for v in t_vars]) print( '************ The Number of Trainable Parameters: {} ************'. format(num_params)) num_g_params = np.sum( [np.prod(v.get_shape().as_list()) for v in tf.global_variables()]) print('************ The Number of Global Parameters: {} ************'. format(num_g_params)) print("\nEmbedding session initialized...\n") # class_list = [2, 4, 5, 6, 7, 9, 10, 11, 12, 13, 14, 16, 17, 19, 20] class_list = [1, 3, 8, 15, 18] # load datasets: test_dataset = np.load( '/media/tjosh/vault/MSRAction3D/one_shot_test_for_known.npy') # '/media/tjosh/vault/MSRAction3D/one_shot_train.npy') # '/media/tjosh/vault/MSRAction3D/one_shot_test_for_unknown.npy') test_data_gen = NewTripletGenerator(test_dataset, classes=class_list, batch_size=cfg.batch_size) current_data, current_label = next(test_data_gen.generator) embedding_var = tf.Variable(tf.zeros( (cfg.batch_size, anchor_embed.get_shape()[1].value)), name=VISUALIZE_NAME) embedding_assign = embedding_var.assign(anchor_embed) summary_writer = tf.summary.FileWriter(EMBED_LOG_DIR) # create embedding projector config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = embedding_var.name # Specify where you find the metadata embedding.metadata_path = 'metadata.tsv' # # Specify where you find the sprite (we dont need this for our case) # embedding.sprite.image_path = path_for_mnist_sprites #'mnistdigits.png' # embedding.sprite.single_image_dim.extend([28,28]) # Say that you want to visualise the embeddings projector.visualize_embeddings(summary_writer, config) # run session to evaluate the embedding tensor current_data = np.array(current_data) embedding_feed_dict = { anchor: current_data[:, 0, :, :], keep_prob: 1.0, is_training_pl: False } sess.run(embedding_assign, feed_dict=embedding_feed_dict) # save the data and checkpoint new_saver = tf.train.Saver() new_saver.save(sess, "embed/model.ckpt", 1) print("logdir: ", "embed/model.ckpt") # save the metadata with open(METADATA_PATH, 'w') as f: f.write("Index\tLabel\n") for index, label in enumerate(current_label): f.write("%d\t%d\n" % (index, label[0]))
def main(_): # Create training directories now = datetime.datetime.now() train_dir_name = now.strftime('alexnet_%Y%m%d_%H%M%S') train_dir = os.path.join(FLAGS.train_root_dir, train_dir_name) checkpoint_dir = os.path.join(train_dir, 'checkpoint') tensorboard_dir = os.path.join(train_dir, 'tensorboard') tensorboard_train_dir = os.path.join(tensorboard_dir, 'train') tensorboard_val_dir = os.path.join(tensorboard_dir, 'val') if not os.path.isdir(FLAGS.train_root_dir): os.mkdir(FLAGS.train_root_dir) if not os.path.isdir(train_dir): os.mkdir(train_dir) if not os.path.isdir(checkpoint_dir): os.mkdir(checkpoint_dir) if not os.path.isdir(tensorboard_dir): os.mkdir(tensorboard_dir) if not os.path.isdir(tensorboard_train_dir): os.mkdir(tensorboard_train_dir) if not os.path.isdir(tensorboard_val_dir): os.mkdir(tensorboard_val_dir) # Write flags to txt flags_file_path = os.path.join(train_dir, 'flags.txt') flags_file = open(flags_file_path, 'w') flags_file.write('learning_rate={}\n'.format(FLAGS.learning_rate)) flags_file.write('dropout_keep_prob={}\n'.format(FLAGS.dropout_keep_prob)) flags_file.write('num_epochs={}\n'.format(FLAGS.num_epochs)) flags_file.write('batch_size={}\n'.format(FLAGS.batch_size)) flags_file.write('train_layers={}\n'.format(FLAGS.train_layers)) flags_file.write('multi_scale={}\n'.format(FLAGS.multi_scale)) flags_file.write('train_root_dir={}\n'.format(FLAGS.train_root_dir)) flags_file.write('log_step={}\n'.format(FLAGS.log_step)) flags_file.close() adlamb = tf.placeholder(tf.float32, name='adlamb') sntglamb = tf.placeholder(tf.float32, name='sntglamb') decay_learning_rate = tf.placeholder(tf.float32) dropout_keep_prob = tf.placeholder(tf.float32) is_training = tf.placeholder(tf.bool) weightlamb = tf.placeholder(tf.float32) # Model train_layers = FLAGS.train_layers.split(',') model = LeNetModel(num_classes=NUM_CLASSES, image_size=32, is_training=is_training, dropout_keep_prob=dropout_keep_prob) # Placeholders if FLAGS.source == "svhn": x_s = tf.placeholder(tf.float32, [None, 32, 32, 3], name='x') elif FLAGS.source == "mnist": x_s = tf.placeholder(tf.float32, [None, 28, 28, 1], name='x') else: x_s = tf.placeholder(tf.float32, [None, 16, 16, 1], name='x') if FLAGS.target == "mnist": x_t = tf.placeholder(tf.float32, [None, 28, 28, 1], name='xt') else: x_t = tf.placeholder(tf.float32, [None, 16, 16, 1], name='xt') x = preprocessing(x_s, model) xt = preprocessing(x_t, model) tf.summary.image('Source Images', x) tf.summary.image('Target Images', xt) print('x_s ', x_s.get_shape()) print('x ', x.get_shape()) print('x_t ', x_t.get_shape()) print('xt ', xt.get_shape()) y = tf.placeholder(tf.float32, [None, NUM_CLASSES], name='y') yt = tf.placeholder(tf.float32, [None, NUM_CLASSES], name='yt') print('y ', y.get_shape()) y_predict, loss = model.loss(x, y) # Training accuracy of the model source_correct_pred = tf.equal(tf.argmax(y_predict, 1), tf.argmax(y, 1)) source_correct = tf.reduce_sum(tf.cast(source_correct_pred, tf.float32)) source_accuracy = tf.reduce_mean(tf.cast(source_correct_pred, tf.float32)) G_loss, D_loss = model.adloss(x, xt, y, yt, FLAGS.LAMBDA, weightlamb) # Testing accuracy of the model with tf.variable_scope('reuse_inference') as scope: scope.reuse_variables() target_feature_test = model.g(xt, training=False) with tf.variable_scope('reuse_inference') as scope: scope.reuse_variables() target_pred_test = model.f(target_feature_test, training=False) correct_pred = tf.equal(tf.argmax(target_pred_test, 1), tf.argmax(yt, 1)) correct = tf.reduce_sum(tf.cast(correct_pred, tf.float32)) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) update_op = model.optimize(decay_learning_rate, train_layers, adlamb, sntglamb, FLAGS.unbalance, FLAGS.revgrad) D_op = model.adoptimize(decay_learning_rate, train_layers) optimizer = tf.group(update_op, D_op) train_writer = tf.summary.FileWriter(tensorboard_dir) train_writer.add_graph(tf.get_default_graph()) config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = model.feature.name embedding.metadata_path = 'domain.csv' projector.visualize_embeddings(train_writer, config) tf.summary.scalar('G_loss', model.G_loss) tf.summary.scalar('D_loss', model.D_loss) tf.summary.scalar('C_loss', model.loss) tf.summary.scalar('Training Accuracy', source_accuracy) tf.summary.scalar('Testing Accuracy', accuracy) merged = tf.summary.merge_all() print( '============================GLOBAL TRAINABLE VARIABLES ============================' ) print(tf.trainable_variables()) #print('============================GLOBAL VARIABLES ======================================') #print tf.global_variables() with tf.Session() as sess: sess.run(tf.global_variables_initializer()) saver = tf.train.Saver() train_writer.add_graph(sess.graph) print("{} Start training...".format(datetime.datetime.now())) gd = 0 step = 1 max_acc = 0 times = -time.time() error_list = [] for epoch in range(40000): # Start training gd += 1 lamb = adaptation_factor(gd * 1.0 / 40000) if FLAGS.unbalance == 1.: lamb2 = math.exp(-(1 - min( (gd - 5000) * 1.0 / 10000, 1.)) * 10) if gd >= 5000 else 0. if FLAGS.source == "mnist" and FLAGS.target == "usps": lamb2 = math.exp(-(1 - min((gd - 2000) * 1.0 / 5000, 1.)) * 10) if gd >= 2000 else 0. else: lamb2 = math.exp(-(1 - min( (gd - 2000) * 1.0 / 2000, 1.)) * 10) if gd >= 2000 else 0. #rate=decay(FLAGS.learning_rate,gd,MAX_STEP) rate = FLAGS.learning_rate batch_xs, batch_ys = TRAIN.next_batch(FLAGS.batch_size) Tbatch_xs, Tbatch_ys = VALID.next_batch(FLAGS.batch_size) summary, _, closs, gloss, dloss, sn_loss, chose_rate_ = sess.run( [ merged, optimizer, model.loss, model.G_loss, model.D_loss, model.sntg_loss, model.chose_rate ], feed_dict={ x_s: batch_xs, x_t: Tbatch_xs, decay_learning_rate: rate, adlamb: lamb, y: batch_ys, yt: Tbatch_ys, sntglamb: lamb2 }) train_writer.add_summary(summary, gd) step += 1 if gd % 250 == 0: epoch = gd / (72357 / 100) print( 'Epoch {} time {}s Step {} lambda {:.4f} lamb2 {:.4f} rate {:.4f} C_loss {:.4f} G_loss {:.4f} D_loss {:.4f} SNTG_loss {:.4f} chose_rate {:.4f}' .format(epoch, times + time.time(), gd, lamb, lamb2, rate, closs, gloss, dloss, sn_loss, chose_rate_)) test_acc = 0. test_count = 0 tt_embs = [] tt_y = [] for _ in range(int((len(TEST.labels)) / 100)): batch_tx, batch_ty = TEST.next_batch(100) #print TEST.pointer,' ',TEST.shuffle acc, t_embs = sess.run([correct, model.feature], feed_dict={ x_t: batch_tx, yt: batch_ty }) tt_embs.append(t_embs) tt_y.append(batch_ty) test_acc += acc test_count += 100 test_acc /= test_count max_acc = max(max_acc, test_acc) print("Validation Accuracy = {:.4f} Max_Accuracy = {:.4f}". format(test_acc, max_acc)) error_list.append(test_acc) if gd % 5000 == 0 and False: tt_embs_s = [] tt_y_s = [] TRAIN.reset_pointer() for _ in range((len(TRAIN.labels)) / 100): batch_tx, batch_ty = TRAIN.next_batch(100) #print TEST.pointer,' ',TEST.shuffle t_embs = sess.run(model.source_feature, feed_dict={ x_s: batch_tx, y: batch_ty }) tt_embs_s.append(t_embs) tt_y_s.append(batch_ty) #np.savez("features_{}.npz".format(FLAGS.tag), x1=np.vstack(tt_embs_s), y1=np.argmax(np.vstack(tt_y_s), axis=1), x2=np.vstack(tt_embs), y2=np.argmax(np.vstack(tt_y), axis=1)) if True: if NUM_CLASSES == 10: test_h_s = np.vstack(tt_embs_s)[:5000] y_test_s = np.ones((test_h_s.shape[0], )) * 8 test_h = np.concatenate( [np.vstack(tt_embs)[:5000], test_h_s], 0) y_test = np.argmax(np.vstack(tt_y)[:5000], axis=1) y_test[y_test == 8] = 10 y_test = np.concatenate([y_test, y_test_s], 0) z_dev_2D = TSNE().fit_transform(test_h) scatter(data=z_dev_2D, label=y_test, dir="./embedding", file_name='s{}_{}_epoch{:03d}.png'.format( FLAGS.source, FLAGS.target, gd / 5000)) else: test_h_s = np.vstack(tt_embs_s) y_test_s = np.argmax(np.vstack(tt_y_s), axis=1) y_test_s[y_test_s == 0] = 2 y_test_s[y_test_s == 1] = 3 test_h = np.concatenate( [np.vstack(tt_embs), test_h_s], 0) y_test = np.argmax(np.vstack(tt_y), axis=1) y_test = np.concatenate([y_test, y_test_s], 0) z_dev_2D = TSNE().fit_transform(test_h) scatter( data=z_dev_2D, label=y_test, dir="./embedding", file_name='cattsne_manifold_epoch{:03d}.png'. format(gd / 5000)) if gd % 5000 == 0 and gd > 0: print() #error_list times = -time.time()
def buildDict(self, train_raw): def summ(x, y): return x+y allwords = reduce( summ, [reduce(summ, h) + q + a for h, q, a in train_raw]) vocab = collections.Counter(allwords) vocab_sort = sorted(vocab, key=vocab.get, reverse=True) # print vocabulary to file with open(self.fvocab, 'w') as f: for word in vocab_sort: print(f, '\t'.join([word, str(vocab[word])])) print ('===============================================') print ('written vocabulary to ', self.fvocab) self.vocab_size = self.vocab_size if self.vocab_size != - \ 1 else len(vocab_sort) + 2 vocab = sorted( zip(vocab_sort[0: self.vocab_size - 2], range(1, self.vocab_size - 1))) filename = maybe_download('text8.zip', 31344016) vocabulary = read_data(filename) print('Data size is here', len(vocabulary)) data, count, dictionary, reverse_dictionary = build_dataset( vocabulary, len(vocabulary)) # del vocab # Hint to reduce memory. # print('Most common words (+UNK)', count[:5]) # print('Sample data', data[:10], [ # reverse_dictionary[i] for i in data[:10]]) # print("DICTIONARY", dictionary) # print("REVERSED DICTIONARY", reverse_dictionary) batch, labels = generate_batch( data, batch_size=8, num_skips=2, skip_window=1) for i in range(8): print(batch[i], reverse_dictionary[batch[i]], '->', labels[i, 0], reverse_dictionary[labels[i, 0]]) batch_size = 128 embedding_size = 128 # Dimension of the embedding vector. skip_window = 1 # How many words to consider left and right. num_skips = 2 # How many times to reuse an input to generate a label. num_sampled = 64 # Number of negative examples to sample. # We pick a random validation set to sample nearest neighbors. Here we limit the # validation samples to the words that have a low numeric ID, which by # construction are also the most frequent. These 3 variables are used only for # displaying model accuracy, they don't affect calculation. valid_size = 16 # Random set of words to evaluate similarity on. valid_window = 100 # Only pick dev samples in the head of the distribution. valid_examples = np.random.choice(valid_window, valid_size, replace=False) graph = tf.Graph() with graph.as_default(): # Input data. with tf.name_scope('inputs'): train_inputs = tf.placeholder(tf.int32, shape=[batch_size]) train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1]) valid_dataset = tf.constant(valid_examples, dtype=tf.int32) # Ops and variables pinned to the CPU because of missing GPU implementation with tf.device('/cpu:0'): # Look up embeddings for inputs. with tf.name_scope('embeddings'): embeddings = tf.Variable( tf.random_uniform([self.vocab_size, embedding_size], -1.0, 1.0)) embed = tf.nn.embedding_lookup(embeddings, train_inputs) # Construct the variables for the NCE loss with tf.name_scope('weights'): nce_weights = tf.Variable( tf.truncated_normal( [self.vocab_size, embedding_size], stddev=1.0 / math.sqrt(embedding_size))) with tf.name_scope('biases'): nce_biases = tf.Variable(tf.zeros([self.vocab_size])) # Compute the average NCE loss for the batch. # tf.nce_loss automatically draws a new sample of the negative labels each # time we evaluate the loss. # Explanation of the meaning of NCE loss: # http://mccormickml.com/2016/04/19/word2vec-tutorial-the-skip-gram-model/ with tf.name_scope('loss'): loss = tf.reduce_mean( tf.nn.nce_loss( weights=nce_weights, biases=nce_biases, labels=train_labels, inputs=embed, num_sampled=num_sampled, num_classes=self.vocab_size)) # Add the loss value as a scalar to summary. tf.summary.scalar('loss', loss) # Construct the SGD optimizer using a learning rate of 1.0. with tf.name_scope('optimizer'): optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss) # Compute the cosine similarity between minibatch examples and all embeddings. norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keepdims=True)) normalized_embeddings = embeddings / norm valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset) similarity = tf.matmul( valid_embeddings, normalized_embeddings, transpose_b=True) # Merge all summaries. merged = tf.summary.merge_all() # Add variable initializer. init = tf.global_variables_initializer() # Create a saver. saver = tf.train.Saver() # Step 5: Begin training. num_steps = 100001 with tf.Session(graph=graph) as session: # Open a writer to write summaries. writer = tf.summary.FileWriter(FLAGS.log_dir, session.graph) # We must initialize all variables before we use them. init.run() # print('Initialized') average_loss = 0 for step in xrange(num_steps): filename = maybe_download('text8.zip', 31344016) vocabulary = read_data(filename) print('Data size is here', len(vocabulary)) data, count, dictionary, reverse_dictionary = build_dataset( vocabulary, len(vocabulary)) batch_inputs, batch_labels = generate_batch(data, batch_size, num_skips, skip_window) feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels} # Define metadata variable. run_metadata = tf.RunMetadata() # print('run_metadata', feed_dict); # We perform one update step by evaluating the optimizer op (including it # in the list of returned values for session.run() # Also, evaluate the merged op to get all summaries from the returned "summary" variable. # Feed metadata variable to session for visualizing the graph in TensorBoard. _, summary, loss_val = session.run( [optimizer, merged, loss], feed_dict=feed_dict, run_metadata=run_metadata) average_loss += loss_val # Add returned summaries to writer in each step. writer.add_summary(summary, step) # Add metadata to visualize the graph for the last run. if step == (num_steps - 1): writer.add_run_metadata(run_metadata, 'step%d' % step) if step % 2000 == 0: if step > 0: average_loss /= 2000 # The average loss is an estimate of the loss over the last 2000 batches. print('Average loss at step ', step, ': ', average_loss) average_loss = 0 # Note that this is expensive (~20% slowdown if computed every 500 steps) if step % 10000 == 0: sim = similarity.eval() for i in xrange(valid_size): valid_word = reverse_dictionary[valid_examples[i]] top_k = 8 # number of nearest neighbors nearest = (-sim[i, :]).argsort()[1:top_k + 1] log_str = 'Nearest to %s:' % valid_word for k in xrange(top_k): close_word = reverse_dictionary[nearest[k]] log_str = '%s %s,' % (log_str, close_word) print(log_str) final_embeddings = normalized_embeddings.eval() # Write corresponding labels for the embeddings. with open(FLAGS.log_dir + '/metadata.tsv', 'w') as f: for i in xrange(self.vocab_size): f.write(reverse_dictionary[i] + '\n') # Save the model for checkpoints. saver.save(session, os.path.join(FLAGS.log_dir, 'model.ckpt')) # Create a configuration for visualizing embeddings with the labels in TensorBoard. config = projector.ProjectorConfig() embedding_conf = config.embeddings.add() embedding_conf.tensor_name = embeddings.name embedding_conf.metadata_path = os.path.join(FLAGS.log_dir, 'metadata.tsv') projector.visualize_embeddings(writer, config) writer.close() # add <unk> and <nil> to vocabulary vocab.append(('<nil>', 0)) vocab.append(('<unk>', self.vocab_size - 1)) assert self.vocab_size == len(vocab) print ('vocabulary size:', self.vocab_size) return dict(vocab)
def restore(checkpoint_file=MODEL_FILE): print("restoring session") with tf.Session() as session: restore_saver = tf.train.import_meta_graph(MODEL_FILE + ".meta") restore_saver.restore(session, tf.train.latest_checkpoint(MODEL_DIR)) graph = tf.get_default_graph() labels = graph.get_tensor_by_name("output:0") inputs = graph.get_tensor_by_name("input:0") in_embedding_var = tf.Variable(tf.zeros([batch_size, feature_size]), name="in_embedding") in_embedding_op = in_embedding_var.assign(inputs) l1 = graph.get_tensor_by_name("dnn/hiddenlayer_0/hiddenlayer_0/Relu:0") l1_embedding_var = tf.Variable(tf.zeros([batch_size, 1024]), name="l1_embedding") l1_embedding_op = l1_embedding_var.assign(l1) l2 = graph.get_tensor_by_name("dnn/hiddenlayer_1/hiddenlayer_1/Relu:0") l2_embedding_var = tf.Variable(tf.zeros([batch_size, 1024]), name="l2_embedding") l2_embedding_op = l2_embedding_var.assign(l2) l3 = graph.get_tensor_by_name("dnn/hiddenlayer_2/hiddenlayer_2/Relu:0") l3_embedding_var = tf.Variable(tf.zeros([batch_size, 1024]), name="l3_embedding") l3_embedding_op = l3_embedding_var.assign(l3) session.run( tf.variables_initializer([ in_embedding_var, l1_embedding_var, l2_embedding_var, l3_embedding_var ])) [Train_index, Val_index, Test_index] = restore_index(WORKINDEX_FILE) print('building filtered batch set') # whitelist follows layout of target # white_list = [[],[KLINKER],[],[],[],[],[],[],[],[b'a', b'A:', b'A+', b'A', b'A~']] white_list = [] batch = get_filtered_batch(Test_index, batch_size, white_listed=white_list) test_features, test_targets, test_labels, test_spectra = batch test_labels = add_prediction(test_features, test_labels) print('running tensorflow with emmbeddings') session.run([ in_embedding_op, l1_embedding_op, l2_embedding_op, l3_embedding_op ], { inputs: test_features, labels: test_targets }) saver = tf.train.Saver() saver.save(session, EMBEDDING_FILE, 2) graph_new = tf.get_default_graph() create_metafile(test_labels, EMBEDDING_LABEL_FILE) print("creating embedding projector") # Format: tensorflow/tensorboard/plugins/projector/projector_config.proto config = projector.ProjectorConfig() embedding_in = config.embeddings.add() embedding_in.tensor_name = in_embedding_var.name embedding_in.metadata_path = EMBEDDING_LABEL_FILE embedding_l1 = config.embeddings.add() embedding_l1.tensor_name = l1_embedding_var.name embedding_l1.metadata_path = EMBEDDING_LABEL_FILE embedding_l2 = config.embeddings.add() embedding_l2.tensor_name = l2_embedding_var.name embedding_l2.metadata_path = EMBEDDING_LABEL_FILE embedding_l3 = config.embeddings.add() embedding_l3.tensor_name = l3_embedding_var.name embedding_l3.metadata_path = EMBEDDING_LABEL_FILE if USE_SPECTRO: plot_width, plot_height = sprite.create_sprite( test_spectra, SPRITE_FILE) embedding_in.sprite.image_path = SPRITE_FILE embedding_in.sprite.single_image_dim.extend( [plot_width, plot_height]) embedding_l1.sprite.image_path = SPRITE_FILE embedding_l1.sprite.single_image_dim.extend( [plot_width, plot_height]) embedding_l2.sprite.image_path = SPRITE_FILE embedding_l2.sprite.single_image_dim.extend( [plot_width, plot_height]) embedding_l3.sprite.image_path = SPRITE_FILE embedding_l3.sprite.single_image_dim.extend( [plot_width, plot_height]) writer = tf.summary.FileWriter(EMBEDDING_DIR) writer.add_graph(graph_new) # The next line writes a projector_config.pbtxt in the LOG_DIR. TensorBoard will # read this file during startup. projector.visualize_embeddings(writer, config)
def run_training(netname, hidden_units, data_sets, epoch_iterations, initial_learning_rate, num_epochs_per_decay, learning_rate_decay_factor): """Train neuralnet for a number of steps.""" #data_sets = input_data.read_data_sets(FLAGS.input_data_dir, FLAGS.fake_data) epoch_length = data_sets.train.num_examples # Tell TensorFlow that the model will be built into the default Graph. with tf.Graph().as_default(): # Generate placeholders for the images and labels. images_placeholder, labels_placeholder = placeholder_inputs( FLAGS.batch_size) # Build a Graph that computes predictions from the inference model. if not CIFAR_MODE: logits = neuralnet.inference(images_placeholder, hidden_units) else: logits = neuralnet.CIFAR_inference(images_placeholder, hidden_units[2]) # Add to the Graph the Ops for loss calculation. loss = neuralnet.loss(logits, labels_placeholder) # Add to the Graph the Ops that calculate and apply gradients. train_op, learning_rate = neuralnet.training( loss, epoch_length, initial_learning_rate, num_epochs_per_decay, learning_rate_decay_factor) # Add the Op to compare the logits to the labels during evaluation. eval_correct = neuralnet.evaluation(logits, labels_placeholder) # Embedding N = epoch_length EMB = np.zeros((N, neuralnet.IMAGE_PIXELS), dtype='float32') for i in range(N): for j in range(neuralnet.IMAGE_PIXELS): EMB[i][j] = data_sets.train.images[i][j] # The embedding variable, which needs to be stored # Note this must a Variable not a Tensor! embedding_var = tf.Variable(EMB, name='Embedding_%s' % (netname)) # Build the summary Tensor based on the TF collection of Summaries. summary = tf.summary.merge_all() # Create a session for running Ops on the Graph. sess = tf.Session() best_pres = 0 last_pres = 0 # Add the variable initializer Op. init = tf.global_variables_initializer() sess.run(init) # Instantiate a SummaryWriter to output summaries and the Graph. summary_writer = tf.summary.FileWriter(FLAGS.log_dir, sess.graph) config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = embedding_var.name names = [ '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12' ] metadata_file = open( os.path.join(FLAGS.log_dir, 'metadata_%s.tsv' % (netname)), 'w') metadata_file.write('Name\tClass\n') for i in range(N): metadata_file.write('%06d\t%s\n' % (i, names[data_sets.train.labels[i]])) metadata_file.close() # Comment out if you don't have metadata embedding.metadata_path = os.path.join(FLAGS.log_dir, 'metadata_%s.tsv' % (netname)) projector.visualize_embeddings(summary_writer, config) saver = tf.train.Saver() saver.save(sess, os.path.join(FLAGS.log_dir, 'model%s.ckpt' % (netname)), 1) #run_testing(0, logits, images_placeholder, sess) accuracy_array = np.zeros([epoch_iterations]) test_ind = 0 for step in range(epoch_length * num_epochs_per_decay * epoch_iterations): start_time = time.time() # Fill a feed dictionary with the actual set of images and labels # for this particular training step. feed_dict = fill_feed_dict(data_sets.train, images_placeholder, labels_placeholder) # Run one step of the model. The return values are the activations # from the `train_op` (which is discarded) and the `loss` Op. To # inspect the values of your Ops or variables, you may include them # in the list passed to sess.run() and the value tensors will be # returned in the tuple from the call. _, loss_value = sess.run([train_op, loss], feed_dict=feed_dict) duration = time.time() - start_time # Write the summaries and print an overview fairly often. if step % 100 == 0: # Print status to stdout. lr_value = sess.run(learning_rate) #print('[%s] Step %d: loss = %.8f (%.3f sec) lr = %.6f bp = %0.04f lp = %0.04f' % (netname, step, loss_value, duration, lr_value, best_pres, last_pres)) # Update the events file. summary_str = sess.run(summary, feed_dict=feed_dict) summary_writer.add_summary(summary_str, step) summary_writer.flush() # Save a checkpoint and evaluate the model periodically. if (step + 1) % (epoch_length * num_epochs_per_decay) == 0 or ( step + 1 ) == epoch_length * num_epochs_per_decay * epoch_iterations: print( '[%s] CHECKPOINT Step %d: loss = %.8f (%.3f sec) lr = %.6f bp = %0.04f lp = %0.04f' % (netname, step, loss_value, duration, lr_value, best_pres, last_pres)) checkpoint_file = os.path.join(FLAGS.log_dir, 'model%s.ckpt' % (netname)) saver.save(sess, checkpoint_file, global_step=step) # Evaluate against the training set. print('Training Data Eval:') do_eval_accuracy(sess, eval_correct, images_placeholder, labels_placeholder, data_sets.train) confmatr = do_eval_confmatr(sess, logits, hidden_units[-1], images_placeholder, labels_placeholder, data_sets.train) prec_rec = do_eval_prec_rec(confmatr, hidden_units[-1]) f1score = do_eval_f1(prec_rec, hidden_units[-1]) # Evaluate against the test set. print('Test Data Eval (accuracy):') accuracy_array[test_ind] = do_eval_accuracy( sess, eval_correct, images_placeholder, labels_placeholder, data_sets.test) print('Test Data Eval (confmatr):') confmatr = do_eval_confmatr(sess, logits, hidden_units[-1], images_placeholder, labels_placeholder, data_sets.test) prec_rec = do_eval_prec_rec(confmatr, hidden_units[-1]) f1score = do_eval_f1(prec_rec, hidden_units[-1]) print("Test #%d ended" % (test_ind)) test_ind += 1 print("Accuracy array: ") print(accuracy_array) checkpoint_file = os.path.join(FLAGS.log_dir, 'model%s_last.ckpt' % (netname)) saver.save(sess, checkpoint_file) return {'logits': logits, 'images_ph': images_placeholder, 'sess': sess}
def main(_): df_train = pd.read_csv(os.getenv('PREPARED_TRAINING')) df_valid = pd.read_csv(os.getenv('PREPARED_VALIDATING')) df_test = pd.read_csv(os.getenv('PREPARED_TESTING')) feature_cols = list(df_train.columns[:-1]) target_col = df_train.columns[-1] X_train = df_train[feature_cols].values y_train = df_train[target_col].values X_valid = df_valid[feature_cols].values y_valid = df_valid[target_col].values X_test = df_test[feature_cols].values dimensions = (5, 5) single = 5 sprites = [None] * 2 sprites[0] = [[1, 1, 1, 1, 1], [1, 1, 1, 1, 1], [1, 0, 0, 0, 1], [1, 1, 1, 1, 1], [1, 1, 1, 1, 1]] sprites[1] = [[1, 1, 1, 1, 1], [1, 1, 0, 1, 1], [1, 1, 0, 1, 1], [1, 1, 0, 1, 1], [1, 1, 1, 1, 1]] sprites[0] = Image.fromarray(np.uint8(sprites[0]) * 0xFF) sprites[1] = Image.fromarray(np.uint8(sprites[1]) * 0xFF) count = X_train.shape[0] size = int(math.ceil(math.sqrt(count))) image = Image.new('1', (size * single, size * single)) logdir = os.path.join(os.getenv('STORING'), 'logs', 'adversarial_{}'.format(int(time.time()))) os.makedirs(logdir, exist_ok=True) handle = open(os.path.join(logdir, 'metadata.tsv'), 'wb') for i in range(count): location = ((i % size) * single, (i // size) * single) label = int(y_train[i]) image.paste(sprites[label], location) handle.write(b'1\n' if label == 1 else b'0\n') handle.close() image.save(os.path.join(logdir, 'sprites.png')) num_features = len(feature_cols) features = tf.placeholder(tf.float32, shape=[None, num_features], name='features') targets = tf.placeholder(tf.int32, shape=[None], name='targets') with tf.name_scope('training'): with tf.variable_scope('adversarial'): train_model = Model(num_features, features, targets, is_training=True) with tf.name_scope('evaluation'): with tf.variable_scope('adversarial', reuse=True): test_model = Model(num_features, features, targets, is_training=False) summary_op = tf.summary.merge_all() supervisor = tf.train.Supervisor(logdir=logdir, summary_op=None) with supervisor.managed_session() as sess: print('Training model with {} parameters...'.format( train_model.num_parameters)) optimize_d, optimize_g = True, True with tqdm(total=FLAGS.num_epochs) as pbar: for epoch in range(FLAGS.num_epochs): summary_writer = tf.summary.FileWriter(logdir, sess.graph) X_train_epoch, y_train_epoch = shuffle(X_train, y_train) losses_d, losses_g = [], [] if optimize_d: _, loss_d = sess.run([ train_model.train_step_d, train_model.loss_d, ], feed_dict={ features: X_train_epoch, targets: y_train_epoch, }) else: loss_d = sess.run(train_model.loss_d, feed_dict={ features: X_train_epoch, targets: y_train_epoch, }) if optimize_g: _, loss_g = sess.run([ train_model.train_step_g, train_model.loss_g, ], feed_dict={ features: X_train_epoch, targets: y_train_epoch, }) else: loss_g = sess.run(train_model.loss_g, feed_dict={ features: X_train_epoch, targets: y_train_epoch, }) losses_d.append(loss_d) losses_g.append(loss_g) loss_train_d = np.mean(losses_d) loss_train_g = np.mean(losses_g) summary_str = sess.run(summary_op, feed_dict={ features: X_valid, targets: y_valid, }) optimize_d = epoch % 2 == 0 optimize_g = True if not optimize_d and not optimize_g: optimize_d = True optimize_g = True summary_writer.add_summary(summary_str, epoch) summary_writer.flush() pbar.set_description( '[{}] loss_train_d ({}): {:.8f}, loss_train_g ({}): {:.8f}' .format(epoch, optimize_d, loss_train_d, optimize_g, loss_train_g)) pbar.update() summary_writer.add_graph(sess.graph) loss_valid_d, loss_valid_g, summary_str = sess.run([ test_model.loss_d, test_model.loss_g, summary_op, ], feed_dict={ features: X_valid, targets: y_valid, }) print('Validation loss (d): {:.8f}, loss (g): {:.8f}'.format( loss_valid_d, loss_valid_g)) z_train = sess.run(test_model.z, feed_dict={features: X_train}) z_valid = sess.run(test_model.z, feed_dict={features: X_valid}) z_test = sess.run(test_model.z, feed_dict={features: X_test}) summary_writer.flush() summary_writer.close() np.savez(os.path.join(os.getenv('STORING'), 'adversarial.npz'), z_train=z_train, z_valid=z_valid, z_test=z_test) tf.reset_default_graph() tf.Graph().as_default() embedding_variable = tf.Variable(z_train, name='adversarial_embedding') summary_writer = tf.summary.FileWriter(logdir) config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = embedding_variable.name embedding.metadata_path = os.path.join(logdir, 'metadata.tsv') embedding.sprite.image_path = os.path.join(logdir, 'sprites.png') embedding.sprite.single_image_dim.extend(dimensions) projector.visualize_embeddings(summary_writer, config) session = tf.InteractiveSession() session.run(tf.global_variables_initializer()) saver = tf.train.Saver() saver.save(session, os.path.join(logdir, 'model.ckpt'), 0)
def word2vec_basic(log_dir): """Example of building, training and visualizing a word2vec model.""" # Create the directory for TensorBoard variables if there is not. if not os.path.exists(log_dir): os.makedirs(log_dir) # Step 1: Analysis the words frequency. # Add all filenames def add_all_files(): global need_to_handle_dirs files = [ os.path.join(d, file) for d in need_to_handle_dirs for file in os.listdir(d) ] file_queue = tf.train.string_input_producer(files) reader = tf.TextLineReader() return reader.read(file_queue) def traverse_all_files(handler, param=None): key, value = add_all_files() key_map = dict() with tf.Session() as sess: # Start populating the filename queue. coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) running = True while running: kama, vamp = sess.run([key, value]) key_map[kama] = 1 if key_map.get( kama) is None else key_map[kama] + 1 if key_map[kama] > 1: running = False else: handler(key_map[kama], tf.compat.as_str(vamp.decode("utf-8")).split(), sess, param) coord.request_stop() coord.join(threads) def count_vocabulary(_, words, sess, param): for word in words: global vocabulary_count_table vocabulary_count_table[word] = 1 if vocabulary_count_table.get(word) is None \ else vocabulary_count_table[word] + 1 count_map_file = os.path.join(out_put_dir, "./count_map.txt") frequency_file = os.path.join(out_put_dir, "./frequency.txt") global all_retrain global append_new_data_source global vocabulary_count_table if all_retrain: # delete all cache file: if os.path.exists(count_map_file): os.remove(count_map_file) if os.path.exists(frequency_file): os.remove(frequency_file) append_new_data_source = False if not os.path.exists(frequency_file) or append_new_data_source: if not os.path.exists(count_map_file): traverse_all_files(count_vocabulary) # 没有count_map文件的话一定不是追加模式 append_new_data_source = False with open(count_map_file, "w", encoding="utf-8") as f: for (k, v) in vocabulary_count_table.items(): f.write(k + " " + str(v)) f.write("\n") vocabulary_count_table = {} with open(count_map_file, "r", encoding="utf-8") as f: line = f.readline() while line: if line != "": strs = line.split(" ") vocabulary_count_table[strs[0]] = int(strs[1]) line = f.readline() # 如果是追加模式 则需要遍历所有新增的文件然后 更新词频统计vocabulary_count_table if append_new_data_source: traverse_all_files(count_vocabulary) with open(count_map_file, "w", encoding="utf-8") as f: for (k, v) in vocabulary_count_table.items(): f.write(k + " " + str(v)) f.write("\n") print("append " + str(need_to_handle_dirs) + "finished.") print("vocabulary_count_table loaded.") # # # Read the data into a list of strings. # def read_data(filename): # """Extract the first file enclosed in a zip file as a list of words.""" # with zipfile.ZipFile(filename) as f: # data = tf.compat.as_str(f.read(f.namelist()[0])).split() # return data # # vocabulary = read_data(filename) # print('Data size', len(vocabulary)) # # # Step 2: Build the dictionary and replace rare words with UNK token. vocabulary_size = 200000 # def build_dataset(words, n_words): """Process raw inputs into a dataset.""" count = [['UNK', -1]] count.extend(collections.Counter(words).most_common(n_words - 1)) dictionary = dict() for word, _ in count: dictionary[word] = len(dictionary) # data = list() # unk_count = 0 # for word in words: # index = dictionary.get(word, 0) # if index == 0: # dictionary['UNK'] # unk_count += 1 # data.append(index) # count[0][1] = unk_count reversed_dictionary = dict( zip(dictionary.values(), dictionary.keys())) global vocabulary_size vocabulary_size = len(dictionary) return count, dictionary, reversed_dictionary # Filling 4 global variables: # data - list of codes (integers from 0 to vocabulary_size-1). # This is the original text but words are replaced by their codes # count - map of words(strings) to count of occurrences # dictionary - map of words(strings) to their codes(integers) # reverse_dictionary - maps codes(integers) to words(strings) count, unused_dictionary, reverse_dictionary = build_dataset( vocabulary_count_table, vocabulary_size) del vocabulary_count_table # Hint to reduce memory. with open(frequency_file, "w", encoding="utf-8") as f: for (k, v) in reverse_dictionary.items(): f.write(str(k) + " " + str(v)) f.write("\n") del count, unused_dictionary, reverse_dictionary with open(frequency_file, "r", encoding="utf-8") as f: line = f.readline() if line.endswith("\n"): line = line[:-1] global frequency global frequency_reverse while line: if "" is not line: line_tuple = line.split(" ") frequency[int(line_tuple[0])] = line_tuple[1] frequency_reverse[line_tuple[1]] = int(line_tuple[0]) line = f.readline() line = line[:-1] # print(frequency_reverse) print("frequency loaded.") print("start training.") def generate_batch(batch_size, num_skips, skip_window, data): global data_index assert batch_size % num_skips == 0 assert num_skips <= 2 * skip_window batch = np.ndarray(shape=(batch_size), dtype=np.int32) labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32) span = 2 * skip_window + 1 # [ skip_window target skip_window ] buffer = collections.deque(maxlen=span) # pylint: disable=redefined-builtin if data_index + span > len(data): data_index = 0 return None, None buffer.extend(data[data_index:data_index + span]) data_index += span for i in range(batch_size // num_skips): context_words = [w for w in range(span) if w != skip_window] words_to_use = random.sample(context_words, num_skips) for j, context_word in enumerate(words_to_use): batch[i * num_skips + j] = buffer[skip_window] labels[i * num_skips + j, 0] = buffer[context_word] if data_index == len(data): buffer.extend(data[0:span]) # data_index = span else: buffer.append(data[data_index]) data_index += 1 # Backtrack a little bit to avoid skipping words in the end of a batch # data_index = (data_index + len(data) - span) % len(data) return batch, labels def train_each_doc(filename, words_sequence, sess, param): words_sequence_filtered = [ 'UNK' if frequency_reverse.get(w) is None else w for w in words_sequence ] word_indexs_sequence = [ frequency_reverse.get(w) for w in words_sequence_filtered ] batch, labels = generate_batch(batch_size, num_skips, skip_window, word_indexs_sequence) while batch is not None: # print([frequency[w] + ":"+frequency[e[0]] for w, e in zip(batch, labels)]) feed_dict = {train_inputs: batch, train_labels: labels} # Define metadata variable. global run_metadata run_metadata = tf.RunMetadata() # We perform one update step by evaluating the optimizer op (including it # in the list of returned values for session.run() # Also, evaluate the merged op to get all summaries from the returned # "summary" variable. Feed metadata variable to session for visualizing # the graph in TensorBoard. _, summary, loss_val = session.run([optimizer, merged, loss], feed_dict=feed_dict, run_metadata=run_metadata) global average_loss, step average_loss += loss_val # Add returned summaries to writer in each step. writer.add_summary(summary, step) step = step + 1 if step % 2000 == 0: if step > 0: average_loss /= 2000 # The average loss is an estimate of the loss over the last 2000 # batches. print('Average loss at step ', step, ': ', average_loss) if average_loss > 300: print("warning") average_loss = 0 # Note that this is expensive (~20% slowdown if computed every 500 steps) if step % 10000 == 0: similarity = param[0] sim = session.run(similarity) for i in xrange(valid_size): valid_word = frequency[valid_examples[i]] top_k = 8 # number of nearest neighbors nearest = (-sim[i, :]).argsort()[1:top_k + 1] log_str = 'Nearest to %s:' % valid_word for k in xrange(top_k): close_word = frequency[nearest[k]] log_str = '%s %s,' % (log_str, close_word) print(log_str) batch, labels = generate_batch(batch_size, num_skips, skip_window, word_indexs_sequence) pass # We pick a random validation set to sample nearest neighbors. Here we limit # the validation samples to the words that have a low numeric ID, which by graph = tf.Graph() with graph.as_default(): # Input data. with tf.name_scope('inputs'): train_inputs = tf.placeholder(tf.int32, shape=[batch_size]) train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1]) valid_dataset = tf.constant(valid_examples, dtype=tf.int32) # Ops and variables pinned to the CPU because of missing GPU implementation with tf.device('/cpu:0'): # Look up embeddings for inputs. with tf.name_scope('embeddings'): embeddings = tf.Variable( tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0)) embed = tf.nn.embedding_lookup(embeddings, train_inputs) # Construct the variables for the NCE loss with tf.name_scope('weights'): nce_weights = tf.Variable( tf.truncated_normal([vocabulary_size, embedding_size], stddev=1.0 / math.sqrt(embedding_size))) with tf.name_scope('biases'): nce_biases = tf.Variable(tf.zeros([vocabulary_size])) # Compute the average NCE loss for the batch. # tf.nce_loss automatically draws a new sample of the negative labels each # time we evaluate the loss. # Explanation of the meaning of NCE loss: # http://mccormickml.com/2016/04/19/word2vec-tutorial-the-skip-gram-model/ with tf.name_scope('loss'): loss = tf.reduce_mean( tf.nn.nce_loss(weights=nce_weights, biases=nce_biases, labels=train_labels, inputs=embed, num_sampled=num_sampled, num_classes=vocabulary_size)) # Add the loss value as a scalar to summary. tf.summary.scalar('loss', loss) # Construct the SGD optimizer using a learning rate of 1.0. with tf.name_scope('optimizer'): optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss) # Compute the cosine similarity between minibatch examples and all # embeddings. norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keepdims=True)) normalized_embeddings = embeddings / norm valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset) similarity = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True) # Merge all summaries. merged = tf.summary.merge_all() # Add variable initializer. init = tf.global_variables_initializer() # Create a saver. saver = tf.train.Saver() # Step 5: Begin training. # num_steps = 1000001 with tf.Session(graph=graph) as session: # Open a writer to write summaries. writer = tf.summary.FileWriter(log_dir, session.graph) # We must initialize all variables before we use them. init.run() print('Initialized') # for step in xrange(num_steps): traverse_all_files(train_each_doc, [similarity, session]) global run_metadata, step writer.add_run_metadata(run_metadata, 'step%d' % step) final_embeddings = normalized_embeddings.eval() # Write corresponding labels for the embeddings. with open(log_dir + '/metadata.tsv', 'w', encoding='utf-8') as f: for i in xrange(vocabulary_size): f.write(frequency[i] + '\n' + str(final_embeddings[i]) + '\n') # Save the model for checkpoints. saver.save(session, os.path.join(log_dir, 'model.ckpt')) # Create a configuration for visualizing embeddings with the labels in # TensorBoard. config = projector.ProjectorConfig() embedding_conf = config.embeddings.add() embedding_conf.tensor_name = embeddings.name embedding_conf.metadata_path = os.path.join(log_dir, 'metadata.tsv') projector.visualize_embeddings(writer, config) writer.close() # Step 6: Visualize the embeddings. # pylint: disable=missing-docstring # Function to draw visualization of distance between embeddings. def plot_with_labels(low_dim_embs, labels, filename): assert low_dim_embs.shape[0] >= len( labels), 'More labels than embeddings' plt.rcParams['font.sans-serif'] = ['SimHei'] # 用来正常显示中文标签 plt.rcParams['axes.unicode_minus'] = False plt.figure(figsize=(18, 18)) # in inches for i, label in enumerate(labels): x, y = low_dim_embs[i, :] plt.scatter(x, y) plt.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points', ha='right', va='bottom') plt.savefig(filename) try: # pylint: disable=g-import-not-at-top from sklearn.manifold import TSNE import matplotlib.pyplot as plt tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000, method='exact') plot_only = 500 low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only, :]) labels = [frequency[i] for i in xrange(plot_only)] plot_with_labels(low_dim_embs, labels, os.path.join('./', 'tsne.png')) except ImportError as ex: print( 'Please install sklearn, matplotlib, and scipy to show embeddings.' ) print(ex)
def main(): cfg = TrainConfig().parse() print(cfg.name) result_dir = os.path.join( cfg.result_root, cfg.name + '_' + datetime.strftime(datetime.now(), '%Y%m%d-%H%M%S')) if not os.path.isdir(result_dir): os.makedirs(result_dir) utils.write_configure_to_file(cfg, result_dir) np.random.seed(seed=cfg.seed) # prepare dataset train_session = cfg.train_session train_set = prepare_multimodal_dataset(cfg.feature_root, train_session, cfg.feat, cfg.label_root) if cfg.task == "supervised": # fully supervised task train_set = train_set[:cfg.label_num] batch_per_epoch = len(train_set) // cfg.sess_per_batch labeled_session = train_session[:cfg.label_num] val_session = cfg.val_session val_set = prepare_multimodal_dataset(cfg.feature_root, val_session, cfg.feat, cfg.label_root) # construct the graph with tf.Graph().as_default(): tf.set_random_seed(cfg.seed) global_step = tf.Variable(0, trainable=False) lr_ph = tf.placeholder(tf.float32, name='learning_rate') ####################### Load models here ######################## sensors_emb_dim = 32 segment_emb_dim = 32 with tf.variable_scope("modality_core"): # load backbone model if cfg.network == "convtsn": model_emb = networks.ConvTSN(n_seg=cfg.num_seg, emb_dim=cfg.emb_dim) elif cfg.network == "convrtsn": model_emb = networks.ConvRTSN(n_seg=cfg.num_seg, emb_dim=cfg.emb_dim) elif cfg.network == "convbirtsn": model_emb = networks.ConvBiRTSN(n_seg=cfg.num_seg, emb_dim=cfg.emb_dim) else: raise NotImplementedError input_ph = tf.placeholder( tf.float32, shape=[None, cfg.num_seg, None, None, None]) dropout_ph = tf.placeholder(tf.float32, shape=[]) model_emb.forward(input_ph, dropout_ph) # for lstm has variable scope with tf.variable_scope("modality_sensors"): model_emb_sensors = networks.RTSN(n_seg=cfg.num_seg, emb_dim=sensors_emb_dim) model_pairsim_sensors = networks.PDDM(n_input=sensors_emb_dim) input_sensors_ph = tf.placeholder(tf.float32, shape=[None, cfg.num_seg, 8]) model_emb_sensors.forward(input_sensors_ph, dropout_ph) var_list = {} for v in tf.global_variables(): if v.op.name.startswith("modality_sensors"): var_list[v.op.name.replace("modality_sensors/", "")] = v restore_saver_sensors = tf.train.Saver(var_list) with tf.variable_scope("modality_segment"): model_emb_segment = networks.RTSN(n_seg=cfg.num_seg, emb_dim=segment_emb_dim, n_input=357) model_pairsim_segment = networks.PDDM(n_input=segment_emb_dim) input_segment_ph = tf.placeholder(tf.float32, shape=[None, cfg.num_seg, 357]) model_emb_segment.forward(input_segment_ph, dropout_ph) var_list = {} for v in tf.global_variables(): if v.op.name.startswith("modality_segment"): var_list[v.op.name.replace("modality_segment/", "")] = v restore_saver_segment = tf.train.Saver(var_list) ############################# Forward Pass ############################# # Core branch if cfg.normalized: embedding = tf.nn.l2_normalize(model_emb.hidden, axis=-1, epsilon=1e-10) else: embedding = model_emb.hidden # get the number of multimodal triplets (x3) mul_num_ph = tf.placeholder(tf.int32, shape=[]) # variable for visualizing the embeddings emb_var = tf.Variable([0.0], name='embeddings') set_emb = tf.assign(emb_var, embedding, validate_shape=False) # calculated for monitoring all-pair embedding distance diffs = utils.all_diffs_tf(embedding, embedding) all_dist = utils.cdist_tf(diffs) tf.summary.histogram('embedding_dists', all_dist) # split embedding into anchor, positive and negative and calculate triplet loss anchor, positive, negative = tf.unstack( tf.reshape(embedding[:(tf.shape(embedding)[0] - mul_num_ph)], [-1, 3, cfg.emb_dim]), 3, 1) anchor_mul, positive_mul, negative_mul = tf.unstack( tf.reshape(embedding[-mul_num_ph:], [-1, 3, cfg.emb_dim]), 3, 1) # Sensors branch emb_sensors = model_emb_sensors.hidden A_sensors, B_sensors, C_sensors = tf.unstack( tf.reshape(emb_sensors, [-1, 3, sensors_emb_dim]), 3, 1) model_pairsim_sensors.forward(tf.stack([A_sensors, B_sensors], axis=1)) pddm_AB_sensors = model_pairsim_sensors.prob[:, 1] model_pairsim_sensors.forward(tf.stack([A_sensors, C_sensors], axis=1)) pddm_AC_sensors = model_pairsim_sensors.prob[:, 1] # Segment branch emb_segment = model_emb_segment.hidden A_segment, B_segment, C_segment = tf.unstack( tf.reshape(emb_segment, [-1, 3, segment_emb_dim]), 3, 1) model_pairsim_segment.forward(tf.stack([A_segment, B_segment], axis=1)) pddm_AB_segment = model_pairsim_segment.prob[:, 1] model_pairsim_segment.forward(tf.stack([A_segment, C_segment], axis=1)) pddm_AC_segment = model_pairsim_segment.prob[:, 1] # fuse prob from all modalities prob_AB = 0.5 * (pddm_AB_sensors + pddm_AB_segment) prob_AC = 0.5 * (pddm_AC_sensors + pddm_AC_segment) ############################# Calculate loss ############################# # triplet loss for labeled inputs metric_loss1 = networks.triplet_loss(anchor, positive, negative, cfg.alpha) # weighted triplet loss for multimodal inputs # if cfg.weighted: # metric_loss2, _ = networks.weighted_triplet_loss(anchor_mul, positive_mul, negative_mul, prob_AB, prob_AC, cfg.alpha) # else: metric_loss2 = networks.triplet_loss(anchor_mul, positive_mul, negative_mul, cfg.alpha) # whether to apply joint optimization if cfg.no_joint: unimodal_var_list = [ v for v in tf.global_variables() if v.op.name.startswith("modality_core") ] train_var_list = unimodal_var_list else: multimodal_var_list = [ v for v in tf.global_variables() if not (v.op.name.startswith("modality_sensors/RTSN") or v.op.name.startswith("modality_segment/RTSN")) ] train_var_list = multimodal_var_list regularization_loss = tf.reduce_sum( tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)) total_loss = tf.cond( tf.greater(mul_num_ph, 0), lambda: tf.cond( tf.equal(mul_num_ph, tf.shape(embedding)[0]), lambda: metric_loss2 * cfg. lambda_multimodal + regularization_loss * cfg.lambda_l2, lambda: metric_loss1 + metric_loss2 * cfg.lambda_multimodal + regularization_loss * cfg.lambda_l2), lambda: metric_loss1 + regularization_loss * cfg.lambda_l2) tf.summary.scalar('learning_rate', lr_ph) train_op = utils.optimize(total_loss, global_step, cfg.optimizer, lr_ph, train_var_list) saver = tf.train.Saver(max_to_keep=10) summary_op = tf.summary.merge_all( ) # not logging histogram of variables because it will cause problem when only unimodal_train_op is called summ_prob_AB = tf.summary.histogram('Prob_AB_histogram', prob_AB) summ_prob_AC = tf.summary.histogram('Prob_AC_histogram', prob_AC) # summ_weights = tf.summary.histogram('Weights_histogram', weights) ######################################################################### # session iterator for session sampling feat_paths_ph = tf.placeholder(tf.string, shape=[None, cfg.sess_per_batch]) feat2_paths_ph = tf.placeholder(tf.string, shape=[None, cfg.sess_per_batch]) feat3_paths_ph = tf.placeholder(tf.string, shape=[None, cfg.sess_per_batch]) label_paths_ph = tf.placeholder(tf.string, shape=[None, cfg.sess_per_batch]) train_data = multimodal_session_generator( feat_paths_ph, feat2_paths_ph, feat3_paths_ph, label_paths_ph, sess_per_batch=cfg.sess_per_batch, num_threads=2, shuffled=False, preprocess_func=[ model_emb.prepare_input, model_emb_sensors.prepare_input, model_emb_segment.prepare_input ]) train_sess_iterator = train_data.make_initializable_iterator() next_train = train_sess_iterator.get_next() # prepare validation data val_sess = [] val_feats = [] val_feats2 = [] val_feats3 = [] val_labels = [] val_boundaries = [] for session in val_set: session_id = os.path.basename(session[1]).split('_')[0] eve_batch, lab_batch, boundary = load_data_and_label( session[0], session[-1], model_emb.prepare_input_test ) # use prepare_input_test for testing time val_feats.append(eve_batch) val_labels.append(lab_batch) val_sess.extend([session_id] * eve_batch.shape[0]) val_boundaries.extend(boundary) eve2_batch, _, _ = load_data_and_label( session[1], session[-1], model_emb_sensors.prepare_input_test) val_feats2.append(eve2_batch) eve3_batch, _, _ = load_data_and_label( session[2], session[-1], model_emb_segment.prepare_input_test) val_feats3.append(eve3_batch) val_feats = np.concatenate(val_feats, axis=0) val_feats2 = np.concatenate(val_feats2, axis=0) val_feats3 = np.concatenate(val_feats3, axis=0) val_labels = np.concatenate(val_labels, axis=0) print("Shape of val_feats: ", val_feats.shape) # generate metadata.tsv for visualize embedding with open(os.path.join(result_dir, 'metadata_val.tsv'), 'w') as fout: fout.write('id\tlabel\tsession_id\tstart\tend\n') for i in range(len(val_sess)): fout.write('{0}\t{1}\t{2}\t{3}\t{4}\n'.format( i, val_labels[i, 0], val_sess[i], val_boundaries[i][0], val_boundaries[i][1])) # compute pairwise embedding distance for each class on validation set # FIXME: don't use np.max # dist_dict = {} # for i in range(1, np.max(val_labels)+1): # temp_feat = val_feats[np.where(val_labels==i)[0]] # dist_dict[i] = [np.mean(utils.cdist(utils.all_diffs(temp_feat, temp_feat), # metric=cfg.metric))] # pdb.set_trace() ######################################################################### # Start running the graph if cfg.gpu: os.environ['CUDA_VISIBLE_DEVICES'] = cfg.gpu gpu_options = tf.GPUOptions(allow_growth=True) sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) summary_writer = tf.summary.FileWriter(result_dir, sess.graph) with sess.as_default(): sess.run(tf.global_variables_initializer()) # load pretrain model, if needed if cfg.model_path: print("Restoring pretrained model: %s" % cfg.model_path) saver.restore(sess, cfg.model_path) print("Restoring sensors model: %s" % cfg.sensors_path) restore_saver_sensors.restore(sess, cfg.sensors_path) print("Restoring segment model: %s" % cfg.segment_path) restore_saver_segment.restore(sess, cfg.segment_path) ################## Training loop ################## epoch = -1 while epoch < cfg.max_epochs - 1: step = sess.run(global_step, feed_dict=None) epoch = step // batch_per_epoch # learning rate schedule, reference: "In defense of Triplet Loss" if epoch < cfg.static_epochs: learning_rate = cfg.learning_rate else: learning_rate = cfg.learning_rate * \ 0.01**((epoch-cfg.static_epochs)/(cfg.max_epochs-cfg.static_epochs)) # prepare data for this epoch random.shuffle(train_set) paths = list(zip(*[iter(train_set)] * cfg.sess_per_batch)) feat_paths = [[p[0] for p in path] for path in paths] feat2_paths = [[p[1] for p in path] for path in paths] feat3_paths = [[p[2] for p in path] for path in paths] label_paths = [[p[-1] for p in path] for path in paths] sess.run(train_sess_iterator.initializer, feed_dict={ feat_paths_ph: feat_paths, feat2_paths_ph: feat2_paths, feat3_paths_ph: feat3_paths, label_paths_ph: label_paths }) # for each epoch batch_count = 1 while True: try: ##################### Data loading ######################## start_time = time.time() eve, eve_sensors, eve_segment, lab, batch_sess = sess.run( next_train) # for memory concern, 1000 events are used in maximum if eve.shape[0] > cfg.event_per_batch: idx = np.random.permutation( eve.shape[0])[:cfg.event_per_batch] eve = eve[idx] eve_sensors = eve_sensors[idx] eve_segment = eve_segment[idx] lab = lab[idx] batch_sess = batch_sess[idx] load_time = time.time() - start_time ##################### Triplet selection ##################### start_time = time.time() # Get the embeddings of all events eve_embedding = np.zeros((eve.shape[0], cfg.emb_dim), dtype='float32') for start, end in zip( range(0, eve.shape[0], cfg.batch_size), range(cfg.batch_size, eve.shape[0] + cfg.batch_size, cfg.batch_size)): end = min(end, eve.shape[0]) emb = sess.run(embedding, feed_dict={ input_ph: eve[start:end], dropout_ph: 1.0 }) eve_embedding[start:end] = np.copy(emb) # sample triplets within sampled sessions all_diff = utils.all_diffs(eve_embedding, eve_embedding) triplet_input_idx, active_count = utils.select_triplets_facenet( lab, utils.cdist(all_diff, metric=cfg.metric), cfg.triplet_per_batch, cfg.alpha, num_negative=cfg.num_negative) if len(triplet_input_idx) == 0: continue triplet_count = len(triplet_input_idx) // 3 multimodal_count = 0 if epoch >= cfg.multimodal_epochs: # Get the similarity of all events sim_prob = np.zeros((eve.shape[0], eve.shape[0]), dtype='float32') * np.nan comb = list( itertools.combinations(range(eve.shape[0]), 2)) for start, end in zip( range(0, len(comb), cfg.batch_size), range(cfg.batch_size, len(comb) + cfg.batch_size, cfg.batch_size)): end = min(end, len(comb)) comb_idx = [] for c in comb[start:end]: comb_idx.extend([c[0], c[1], c[1]]) sim = sess.run(prob_AB, feed_dict={ input_sensors_ph: eve_sensors[comb_idx], input_segment_ph: eve_segment[comb_idx], dropout_ph: 1.0 }) for i in range(sim.shape[0]): sim_prob[comb[start + i][0], comb[start + i][1]] = sim[i] sim_prob[comb[start + i][1], comb[start + i][0]] = sim[i] # sample triplets from similarity prediction # maximum number not exceed the number of triplet_input from facenet selection # hard samples mining triplet_input_idx, triplet_count, multimodal_count1 = select_triplets_mul_hard( triplet_input_idx, lab, sim_prob, cfg.triplet_per_batch, 3, 0.8, 0.2) sensors_input = eve_sensors[ triplet_input_idx[-(3 * multimodal_count1):]] segment_input = eve_segment[ triplet_input_idx[-(3 * multimodal_count1):]] # add up all multimodal triplets multimodal_count = multimodal_count1 print(triplet_count, multimodal_count) triplet_input = eve[triplet_input_idx] select_time = time.time() - start_time if len(triplet_input.shape) > 5: # debugging pdb.set_trace() ##################### Start training ######################## # supervised initialization if multimodal_count == 0: if triplet_count == 0: continue err, metric_err1, _, step, summ = sess.run( [ total_loss, metric_loss1, train_op, global_step, summary_op ], feed_dict={ input_ph: triplet_input, dropout_ph: cfg.keep_prob, mul_num_ph: 0, lr_ph: learning_rate }) metric_err2 = 0 else: err, metric_err1, metric_err2, _, step, summ, s_AB, s_AC = sess.run( [ total_loss, metric_loss1, metric_loss2, train_op, global_step, summary_op, summ_prob_AB, summ_prob_AC ], feed_dict={ input_ph: triplet_input, input_sensors_ph: sensors_input, input_segment_ph: segment_input, mul_num_ph: multimodal_count * 3, dropout_ph: cfg.keep_prob, lr_ph: learning_rate }) summary_writer.add_summary(s_AB, step) summary_writer.add_summary(s_AC, step) print ("%s\tEpoch: [%d][%d/%d]\tEvent num: %d\tTriplet num: %d\tLoad time: %.3f\tSelect time: %.3f\tLoss %.4f" % \ (cfg.name, epoch+1, batch_count, batch_per_epoch, eve.shape[0], triplet_count+multimodal_count, load_time, select_time, err)) summary = tf.Summary(value=[ tf.Summary.Value(tag="train_loss", simple_value=err), tf.Summary.Value(tag="active_count", simple_value=active_count), tf.Summary.Value(tag="triplet_count", simple_value=triplet_count), tf.Summary.Value(tag="multimodal_count", simple_value=multimodal_count), tf.Summary.Value(tag="metric_loss1", simple_value=metric_err1), tf.Summary.Value(tag="metric_loss2", simple_value=metric_err2) ]) summary_writer.add_summary(summary, step) summary_writer.add_summary(summ, step) batch_count += 1 except tf.errors.OutOfRangeError: print("Epoch %d done!" % (epoch + 1)) break # validation on val_set print("Evaluating on validation set...") val_embeddings, _ = sess.run([embedding, set_emb], feed_dict={ input_ph: val_feats, dropout_ph: 1.0 }) mAP, mPrec, recall = utils.evaluate_simple( val_embeddings, val_labels) summary = tf.Summary(value=[ tf.Summary.Value(tag="Valiation mAP", simple_value=mAP), tf.Summary.Value(tag="Validation Recall@1", simple_value=recall), tf.Summary.Value(tag="Validation [email protected]", simple_value=mPrec) ]) summary_writer.add_summary(summary, step) print("Epoch: [%d]\tmAP: %.4f\tmPrec: %.4f" % (epoch + 1, mAP, mPrec)) # config for embedding visualization config = projector.ProjectorConfig() visual_embedding = config.embeddings.add() visual_embedding.tensor_name = emb_var.name visual_embedding.metadata_path = os.path.join( result_dir, 'metadata_val.tsv') projector.visualize_embeddings(summary_writer, config) # save model saver.save(sess, os.path.join(result_dir, cfg.name + '.ckpt'), global_step=step)
def train_model(model, batch_gen, num_train_steps, weights_fld): saver = tf.train.Saver( ) # defaults to saving all variables - in this case embed_matrix, nce_weight, nce_bias initial_step = 0 utils.make_dir('checkpoints') with tf.Session() as sess: sess.run(tf.global_variables_initializer()) ckpt = tf.train.get_checkpoint_state( os.path.dirname('checkpoints/checkpoint')) # if that checkpoint exists, restore from checkpoint if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) else: total_loss = 0.0 # we use this to calculate late average loss in the last SKIP_STEP steps writer = tf.summary.FileWriter( 'improved_graph/lr' + str(LEARNING_RATE), sess.graph) initial_step = model.global_step.eval() for index in range(initial_step, initial_step + num_train_steps): centers, targets = next(batch_gen) feed_dict = { model.center_words: centers, model.target_words: targets } loss_batch, _, summary = sess.run( [model.loss, model.optimizer, model.summary_op], feed_dict=feed_dict) writer.add_summary(summary, global_step=index) total_loss += loss_batch if (index + 1) % SKIP_STEP == 0: print('Average loss at step {}: {:5.1f}'.format( index, total_loss / SKIP_STEP)) total_loss = 0.0 saver.save(sess, 'checkpoints/skip-gram', index) #################### # code to visualize the embeddings. uncomment the below to visualize embeddings # run "'tensorboard --logdir='processed'" to see the embeddings final_embed_matrix = sess.run(model.embed_matrix) # it has to variable. constants don't work here. you can't reuse model.embed_matrix embedding_var = tf.Variable(final_embed_matrix[:1000], name='embedding') sess.run(embedding_var.initializer) config = projector.ProjectorConfig() summary_writer = tf.summary.FileWriter('processed') # add embedding to the config file embedding = config.embeddings.add() embedding.tensor_name = embedding_var.name # link this tensor to its metadata file, in this case the first 500 words of vocab # embedding.metadata_path = 'processed/vocab_1000.tsv' embedding.metadata_path = 'vocab_1000.tsv' # saves a configuration file that TensorBoard will read during startup. projector.visualize_embeddings(summary_writer, config) saver_embed = tf.train.Saver([embedding_var]) saver_embed.save(sess, 'processed/model3.ckpt', 1)
def set_model(self, model): self.model = model if K.backend() == 'tensorflow': self.sess = K.get_session() if self.histogram_freq and self.merged is None: for layer in self.model.layers: for weight in layer.weights: mapped_weight_name = weight.name.replace(':', '_') tf.summary.histogram(mapped_weight_name, weight) if self.write_grads: grads = model.optimizer.get_gradients( model.total_loss, weight) def is_indexed_slices(grad): return type(grad).__name__ == 'IndexedSlices' grads = [ grad.values if is_indexed_slices(grad) else grad for grad in grads ] tf.summary.histogram( '{}_grad'.format(mapped_weight_name), grads) if self.write_images: w_img = tf.squeeze(weight) shape = K.int_shape(w_img) if len(shape) == 2: # dense layer kernel case if shape[0] > shape[1]: w_img = tf.transpose(w_img) shape = K.int_shape(w_img) w_img = tf.reshape(w_img, [1, shape[0], shape[1], 1]) elif len(shape) == 3: # convnet case if K.image_data_format() == 'channels_last': # switch to channels_first to display # every kernel as a separate image w_img = tf.transpose(w_img, perm=[2, 0, 1]) shape = K.int_shape(w_img) w_img = tf.reshape( w_img, [shape[0], shape[1], shape[2], 1]) elif len(shape) == 1: # bias case w_img = tf.reshape(w_img, [1, shape[0], 1, 1]) else: # not possible to handle 3D convnets etc. continue shape = K.int_shape(w_img) assert len(shape) == 4 and shape[-1] in [1, 3, 4] tf.summary.image(mapped_weight_name, w_img) if hasattr(layer, 'output'): tf.summary.histogram('{}_out'.format(layer.name), layer.output) self.merged = tf.summary.merge_all() if self.write_graph: self.writer = tf.summary.FileWriter(self.log_dir, self.sess.graph) else: self.writer = tf.summary.FileWriter(self.log_dir) if self.embeddings_freq: embeddings_layer_names = self.embeddings_layer_names if not embeddings_layer_names: embeddings_layer_names = [ layer.name for layer in self.model.layers if type(layer).__name__ == 'Embedding' ] embeddings = { layer.name: layer.weights[0] for layer in self.model.layers if layer.name in embeddings_layer_names } self.saver = tf.train.Saver(list(embeddings.values())) embeddings_metadata = {} if not isinstance(self.embeddings_metadata, str): embeddings_metadata = self.embeddings_metadata else: embeddings_metadata = { layer_name: self.embeddings_metadata for layer_name in embeddings.keys() } config = projector.ProjectorConfig() self.embeddings_ckpt_path = os.path.join(self.log_dir, 'keras_embedding.ckpt') for layer_name, tensor in embeddings.items(): embedding = config.embeddings.add() embedding.tensor_name = tensor.name if layer_name in embeddings_metadata: embedding.metadata_path = embeddings_metadata[layer_name] projector.visualize_embeddings(self.writer, config)
def fit(self): parameters = self.parameters conf_parameters = self.conf_parameters dataset_filepaths = self.dataset_filepaths dataset = self.dataset dataset_brat_folders = self.dataset_brat_folders sess = self.sess model = self.model transition_params_trained = self.transition_params_trained stats_graph_folder, experiment_timestamp = self._create_stats_graph_folder( parameters) # Initialize and save execution details start_time = time.time() results = {} results['epoch'] = {} results['execution_details'] = {} results['execution_details']['train_start'] = start_time results['execution_details']['time_stamp'] = experiment_timestamp results['execution_details']['early_stop'] = False results['execution_details']['keyboard_interrupt'] = False results['execution_details']['num_epochs'] = 0 results['model_options'] = copy.copy(parameters) model_folder = os.path.join(stats_graph_folder, 'model') utils.create_folder_if_not_exists(model_folder) with open(os.path.join(model_folder, 'parameters.ini'), 'w') as parameters_file: conf_parameters.write(parameters_file) pickle.dump(dataset, open(os.path.join(model_folder, 'dataset.pickle'), 'wb')) tensorboard_log_folder = os.path.join(stats_graph_folder, 'tensorboard_logs') utils.create_folder_if_not_exists(tensorboard_log_folder) tensorboard_log_folders = {} for dataset_type in dataset_filepaths.keys(): tensorboard_log_folders[dataset_type] = os.path.join( stats_graph_folder, 'tensorboard_logs', dataset_type) utils.create_folder_if_not_exists( tensorboard_log_folders[dataset_type]) # Instantiate the writers for TensorBoard writers = {} for dataset_type in dataset_filepaths.keys(): writers[dataset_type] = tf.summary.FileWriter( tensorboard_log_folders[dataset_type], graph=sess.graph) embedding_writer = tf.summary.FileWriter( model_folder ) # embedding_writer has to write in model_folder, otherwise TensorBoard won't be able to view embeddings embeddings_projector_config = projector.ProjectorConfig() tensorboard_token_embeddings = embeddings_projector_config.embeddings.add( ) tensorboard_token_embeddings.tensor_name = model.token_embedding_weights.name token_list_file_path = os.path.join(model_folder, 'tensorboard_metadata_tokens.tsv') tensorboard_token_embeddings.metadata_path = os.path.relpath( token_list_file_path, '..') tensorboard_character_embeddings = embeddings_projector_config.embeddings.add( ) tensorboard_character_embeddings.tensor_name = model.character_embedding_weights.name character_list_file_path = os.path.join( model_folder, 'tensorboard_metadata_characters.tsv') tensorboard_character_embeddings.metadata_path = os.path.relpath( character_list_file_path, '..') projector.visualize_embeddings(embedding_writer, embeddings_projector_config) # Write metadata for TensorBoard embeddings token_list_file = codecs.open(token_list_file_path, 'w', 'UTF-8') for token_index in range(dataset.vocabulary_size): token_list_file.write('{0}\n'.format( dataset.index_to_token[token_index])) token_list_file.close() character_list_file = codecs.open(character_list_file_path, 'w', 'UTF-8') for character_index in range(dataset.alphabet_size): if character_index == dataset.PADDING_CHARACTER_INDEX: character_list_file.write('PADDING\n') else: character_list_file.write('{0}\n'.format( dataset.index_to_character[character_index])) character_list_file.close() # Start training + evaluation loop. Each iteration corresponds to 1 epoch. bad_counter = 0 # number of epochs with no improvement on the validation test in terms of F1-score previous_best_valid_f1_score = 0 epoch_number = -1 try: while True: step = 0 epoch_number += 1 print('\nStarting epoch {0}'.format(epoch_number)) epoch_start_time = time.time() if epoch_number != 0: # Train model: loop over all sequences of training set with shuffling sequence_numbers = list( range(len(dataset.token_indices['train']))) random.shuffle(sequence_numbers) for sequence_number in sequence_numbers: transition_params_trained = train.train_step( sess, dataset, sequence_number, model, parameters) step += 1 if step % 10 == 0: print('Training {0:.2f}% done'.format( step / len(sequence_numbers) * 100), end='\r', flush=True) epoch_elapsed_training_time = time.time() - epoch_start_time print('Training completed in {0:.2f} seconds'.format( epoch_elapsed_training_time), flush=True) y_pred, y_true, output_filepaths = train.predict_labels( sess, model, transition_params_trained, parameters, dataset, epoch_number, stats_graph_folder, dataset_filepaths) # Evaluate model: save and plot results evaluate.evaluate_model(results, dataset, y_pred, y_true, stats_graph_folder, epoch_number, epoch_start_time, output_filepaths, parameters) if parameters['use_pretrained_model'] and not parameters[ 'train_model']: conll_to_brat.output_brat(output_filepaths, dataset_brat_folders, stats_graph_folder) break # Save model model.saver.save( sess, os.path.join(model_folder, 'model_{0:05d}.ckpt'.format(epoch_number))) # Save TensorBoard logs summary = sess.run(model.summary_op, feed_dict=None) writers['train'].add_summary(summary, epoch_number) writers['train'].flush() utils.copytree(writers['train'].get_logdir(), model_folder) # Early stop valid_f1_score = results['epoch'][epoch_number][0]['valid'][ 'f1_score']['micro'] if valid_f1_score > previous_best_valid_f1_score: bad_counter = 0 previous_best_valid_f1_score = valid_f1_score conll_to_brat.output_brat(output_filepaths, dataset_brat_folders, stats_graph_folder, overwrite=True) self.transition_params_trained = transition_params_trained else: bad_counter += 1 print( "The last {0} epochs have not shown improvements on the validation set." .format(bad_counter)) if bad_counter >= parameters['patience']: print('Early Stop!') results['execution_details']['early_stop'] = True break if epoch_number >= parameters['maximum_number_of_epochs']: break except KeyboardInterrupt: results['execution_details']['keyboard_interrupt'] = True print('Training interrupted') print('Finishing the experiment') end_time = time.time() results['execution_details']['train_duration'] = end_time - start_time results['execution_details']['train_end'] = end_time evaluate.save_results(results, stats_graph_folder) for dataset_type in dataset_filepaths.keys(): writers[dataset_type].close()
def train(): train_data = load_file("train.csv") validation_data = load_file("validation.csv") m = model.JointEmbeddingModelForBinaryClassification(conf.embedded_word_size, conf.init_stddev) checkpoint_base_path = conf.log_path + "/" + conf.run_name + "/checkpoint" vocab_size = utils.get_vocab_size(conf.data_dir) with tf.Session() as sess: model_params = m.graph( conf.minibatch_size, vocab_size, conf.word_vector_size, conf.conv_size, conf.conv_stride, conf.conv_features, conf.weights_reg_scale, conf.activity_reg_scale, conf.embedding_reg_scale ) config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = model_params.word_vectors.name embedding.metadata_path = os.path.join(conf.data_dir, "vocab.txt") summary_op = tf.merge_all_summaries() optimiser = tf.train.AdamOptimizer(conf.learning_rate) train_op = optimiser.minimize(model_params.loss, var_list=tf.trainable_variables()) if not os.path.exists(conf.log_path): os.makedirs(conf.log_path) writer = tf.train.SummaryWriter(conf.log_path + "/" + conf.run_name, sess.graph) saver = tf.train.Saver() projector.visualize_embeddings(writer, config) latest_checkpoint = tf.train.latest_checkpoint(conf.log_path) if conf.reuse_checkpoints and latest_checkpoint is not None: print("Restoring checkpoint...: " + latest_checkpoint) saver.restore(sess, latest_checkpoint) starting_iteration = int(latest_checkpoint.split('-')[-1]) + 1 else: print("Initialising new model...") sess.run(tf.initialize_all_variables()) starting_iteration = 0 for i in range(starting_iteration, conf.iterations + 1): X1, X2, Y, M1, M2, S1, S2, _ = get_random_datapoints(train_data) data = {model_params.wordset_1: X1, model_params.wordset_2: X2, model_params.probs: Y, model_params.wordset_1_masks: M1, model_params.wordset_2_masks: M2, model_params.wordset_1_lengths: S1, model_params.wordset_2_lengths: S2} _, summary_value, alpha, loss_value = sess.run([train_op, summary_op, model_params.alpha, model_params.loss], feed_dict=data) writer.add_summary(summary_value, i) writer.flush() if i % conf.report_frequency == 0: print("Iteration #{}, Loss: {}, α: {}.".format(i, loss_value, alpha)) if i % conf.checkpoint_frequency == 0: checkpoint_path = saver.save(sess, checkpoint_base_path, global_step=i) print("Checkpointed: {}.".format(checkpoint_path)) if i % conf.validation_frequency == 0: X1, X2, Y, M1, M2, S1, S2, _ = get_random_datapoints(validation_data) data = {model_params.wordset_1: X1, model_params.wordset_2: X2, model_params.probs: Y, model_params.wordset_1_masks: M1, model_params.wordset_2_masks: M2, model_params.wordset_1_lengths: S1, model_params.wordset_2_lengths: S2} accuracy_value = sess.run(model_params.accuracy, feed_dict=data) print("Iteration #{}, Validation-set accuracy: {}.".format(i, accuracy_value)) if not os.path.exists(conf.save_path): os.makedirs(conf.save_path) saver.save(sess, conf.save_path + "/last_checkpoint")
def __init__(self, filename, batch_size=2000, doc_embed_dim=0, wrd_embed_dim=64, wrd_size_max=10000, loss_type='sampled_softmax_loss', optimizer_type='Adagrad', learning_rate=1.0, n_neg_samples=5, eval_words=None): self.docs, self.doc2id, self.id2doc, self.wrd2id, self.id2wrd, self.cat2id = build_dataset( filename, wrd_size_max) self.doc_size = len(self.doc2id) self.wrd_size = len(self.wrd2id) # bind params to class self.batch_size = batch_size self.doc_embed_dim = doc_embed_dim self.wrd_embed_dim = wrd_embed_dim self.loss_type = loss_type self.optimizer_type = optimizer_type self.learning_rate = learning_rate self.n_neg_samples = n_neg_samples self.eval_examples = [self.wrd2id[wrd] for wrd in eval_words] if eval_words \ else np.random.choice(self.wrd_size, size=10, replace=False) self._init_graph() self.sess = tf.Session(graph=self.graph) self.step = 0 self.epoch = 0 self.doc_idx = 0 # fetch training batch print('doc size {}, word size {}, doc dim {}, word dim {}'.format( self.doc_size, self.wrd_size, self.doc_embed_dim, self.wrd_embed_dim)) print('Sample doc: doc id {}, word id {}\n words {}'.format( self.id2doc[0], self.docs[0], [self.id2wrd[wrd] for wrd in self.docs[0]])) # embedding projector if not os.path.exists(FLAGS.checkpoint_dir): try: os.makedirs(FLAGS.checkpoint_dir) except Exception as e: raise e meta_path = os.path.join(FLAGS.checkpoint_dir, 'metadata.tsv') with open(meta_path, 'w') as f: f.write('word\tlabel\n') for idx in xrange(self.wrd_size): f.write('{}\t{}\n'.format(self.id2wrd[idx], self.id2wrd[idx].split('#')[0])) from tensorflow.contrib.tensorboard.plugins import projector # Use the same LOG_DIR where you stored your checkpoint. summary_writer = tf.summary.FileWriter(FLAGS.checkpoint_dir) # Format: tensorflow/contrib/tensorboard/plugins/projector/projector_config.proto config = projector.ProjectorConfig() # You can add multiple embeddings. Here we add only one. embedding = config.embeddings.add() embedding.tensor_name = self.normalized.name embedding.metadata_path = meta_path embedding2 = config.embeddings.add() embedding2.tensor_name = self.wrd_embeddings.name embedding2.metadata_path = meta_path # Saves a configuration file that TensorBoard will read during startup. projector.visualize_embeddings(summary_writer, config)
def set_model(self, model): self.model = model self.sess = K.get_session() if self.histogram_freq and self.merged is None: for layer in self.model.layers: for weight in layer.weights: tf.summary.histogram(weight.name, weight) if self.write_images: w_img = tf.squeeze(weight) shape = w_img.get_shape() if len(shape) > 1 and shape[0] > shape[1]: w_img = tf.transpose(w_img) if len(shape) == 1: w_img = tf.expand_dims(w_img, 0) w_img = tf.expand_dims(tf.expand_dims(w_img, 0), -1) tf.summary.image(weight.name, w_img) if hasattr(layer, 'output'): tf.summary.histogram('{}_out'.format(layer.name), layer.output) self.merged = tf.summary.merge_all() if self.write_graph: self.writer = tf.summary.FileWriter(self.log_dir, self.sess.graph) else: self.writer = tf.summary.FileWriter(self.log_dir) if self.embeddings_freq: self.saver = tf.train.Saver() embeddings_layer_names = self.embeddings_layer_names if not embeddings_layer_names: embeddings_layer_names = [ layer.name for layer in self.model.layers if type(layer).__name__ == 'Embedding' ] embeddings = { layer.name: layer.weights[0] for layer in self.model.layers if layer.name in embeddings_layer_names } embeddings_metadata = {} if not isinstance(self.embeddings_metadata, str): embeddings_metadata = self.embeddings_metadata else: embeddings_metadata = { layer_name: self.embeddings_metadata for layer_name in embeddings.keys() } config = projector.ProjectorConfig() self.embeddings_logs = [] for layer_name, tensor in embeddings.items(): embedding = config.embeddings.add() embedding.tensor_name = tensor.name self.embeddings_logs.append( os.path.join(self.log_dir, layer_name + '.ckpt')) if layer_name in embeddings_metadata: embedding.metadata_path = embeddings_metadata[layer_name] projector.visualize_embeddings(self.writer, config)
def train_skip_gram(V, data_folder, data_folders, dataset_size, reverse_dictionary, param, valid_examples, log_dir, vocab_metada_file, embeddings_pickle, ckpt_saver_file, ckpt_saver_file_init, ckpt_saver_file_final, restore_variables): """ Train embeddings (Skip-Gram model) :param V: vocabulary size :param data_folder: string containing the path to the parent directory of raw data sub-folders :param data_folders: list of sub-folders containing pre-processed LLVM IR code :param dataset_size: number of data pairs in total in the training data set :param reverse_dictionary: [keys=statement index, values=statement] :param param: parameters of the inst2vec training :param valid_examples: statements to be used as validation examples (list of indices) :param log_dir: logging directory for Tensorboard output :param vocab_metada_file: vocabulary metadata file for Tensorboard :param embeddings_pickle: file in which to pickle embeddings :param ckpt_saver_file: checkpoint saver file (intermediate states of training) :param ckpt_saver_file_init: checkpoint saver file (initial state of training) :param ckpt_saver_file_final: checkpoint saver file (final state of training) :param restore_variables: boolean: whether to restore variables from a previous training :return: embeddings matrix """ #################################################################################################################### # Extract parameters from dictionary "param" N = param['embedding_size'] mini_batch_size = param['mini_batch_size'] num_sampled = param['num_sampled'] num_epochs = param['num_epochs'] learning_rate = param['learning_rate'] l2_reg_scale = param['beta'] freq_print_loss = param['freq_print_loss'] step_print_neighbors = param['step_print_neighbors'] context_width = param['context_width'] #################################################################################################################### # Set up for analogies #analogies, analogy_types, n_questions_total, n_questions_relevant = i2v_eval.load_analogies(data_folder) folder_evaluation = embeddings_pickle.replace('.p', '') + 'eval' if not os.path.exists(folder_evaluation): os.makedirs(folder_evaluation) analogy_evaluation_file = os.path.join(folder_evaluation, "analogy_results") config = None options = None metadata = None if FLAGS.profile: options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) metadata = tf.RunMetadata() if FLAGS.xla: config = tf.ConfigProto() config.graph_options.optimizer_options.global_jit_level = tf.OptimizerOptions.ON_1 #################################################################################################################### # Read data using Tensorflow's data API data_files = get_data_pair_files(data_folders, context_width) print('\ttraining with data from files:', data_files) with tf.name_scope("Reader") as scope: random.shuffle(data_files) dataset_raw = tf.data.FixedLengthRecordDataset( filenames=data_files, record_bytes=8) # <TFRecordDataset shapes: (), types: tf.string> dataset = dataset_raw.map(record_parser) dataset = dataset.shuffle(int(1e5)) dataset_batched = dataset.apply( tf.contrib.data.batch_and_drop_remainder(mini_batch_size)) dataset_batched = dataset_batched.prefetch(int(100000000)) iterator = dataset_batched.make_initializable_iterator() saveable_iterator = tf.contrib.data.make_saveable_from_iterator( iterator) next_batch = iterator.get_next( ) # Tensor("Shape:0", shape=(2,), dtype=int32) #################################################################################################################### # Tensorflow computational graph # Placeholders for inputs with tf.name_scope("Input_Data") as scope: train_inputs = next_batch[:, 0] train_labels = tf.reshape(next_batch[:, 1], shape=[mini_batch_size, 1], name="training_labels") # (input) Embedding matrix with tf.name_scope("Input_Layer") as scope: W_in = tf.Variable(tf.random_uniform([V, N], -1.0, 1.0), name="input-embeddings") # Look up the vector representing each source word in the batch (fetches rows of the embedding matrix) h = tf.nn.embedding_lookup(W_in, train_inputs, name="input_embedding_vectors") # Normalized embedding matrix with tf.name_scope("Embeddings_Normalized") as scope: normalized_embeddings = tf.nn.l2_normalize( W_in, name="embeddings_normalized") # (output) Embedding matrix ("output weights") with tf.name_scope("Output_Layer") as scope: if FLAGS.softmax: W_out = tf.Variable(tf.truncated_normal([N, V], stddev=1.0 / math.sqrt(N)), name="output_embeddings") else: W_out = tf.Variable(tf.truncated_normal([V, N], stddev=1.0 / math.sqrt(N)), name="output_embeddings") # Biases between hidden layer and output layer b_out = tf.Variable(tf.zeros([V]), name="nce_bias") # Optimization with tf.name_scope("Optimization_Block") as scope: # Loss function if FLAGS.softmax: logits = tf.layers.dense(inputs=h, units=V) onehot = tf.one_hot(train_labels, V) loss_tensor = tf.nn.softmax_cross_entropy_with_logits_v2( labels=onehot, logits=logits) else: loss_tensor = tf.nn.nce_loss(weights=W_out, biases=b_out, labels=train_labels, inputs=h, num_sampled=num_sampled, num_classes=V) train_loss = tf.reduce_mean(loss_tensor, name="nce_loss") # Regularization (optional) if l2_reg_scale > 0: tf.add_to_collection(tf.GraphKeys.REGULARIZATION_LOSSES, W_in) tf.add_to_collection(tf.GraphKeys.REGULARIZATION_LOSSES, W_out) regularizer = tf.contrib.layers.l2_regularizer(l2_reg_scale) reg_variables = tf.get_collection( tf.GraphKeys.REGULARIZATION_LOSSES) reg_term = tf.contrib.layers.apply_regularization( regularizer, reg_variables) loss = train_loss + reg_term else: loss = train_loss # Optimizer if FLAGS.optimizer == 'adam': optimizer = tf.train.AdamOptimizer( learning_rate=learning_rate).minimize(loss) elif FLAGS.optimizer == 'nadam': optimizer = tf.contrib.opt.NadamOptimizer( learning_rate=learning_rate).minimize(loss) elif FLAGS.optimizer == 'momentum': global_train_step = tf.Variable(0, trainable=False, dtype=tf.int32, name="global_step") # Passing global_step to minimize() will increment it at each step. optimizer = (tf.train.MomentumOptimizer( learning_rate, 0.95).minimize(loss, global_step=global_train_step)) else: raise ValueError('Unrecognized optimizer ' + FLAGS.optimizer) if FLAGS.optimizer != 'momentum': global_train_step = tf.Variable(0, trainable=False, dtype=tf.int32, name="global_step") #################################################################################################################### # Validation block with tf.name_scope("Validation_Block") as scope: valid_dataset = tf.constant(valid_examples, dtype=tf.int32, name="validation_data_size") valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset) cosine_similarity = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True) #################################################################################################################### # Summaries with tf.name_scope("Summaries") as scope: tf.summary.histogram("input_embeddings", W_in) tf.summary.histogram("input_embeddings_normalized", normalized_embeddings) tf.summary.histogram("output_embeddings", W_out) tf.summary.scalar("nce_loss", loss) analogy_score_tensor = tf.Variable(0, trainable=False, dtype=tf.int32, name="analogy_score") tf.summary.scalar("analogy_score", analogy_score_tensor) #################################################################################################################### # Misc. restore_completed = False init = tf.global_variables_initializer() # variables initializer summary_op = tf.summary.merge_all() # merge summaries into one operation #################################################################################################################### # Training with tf.Session(config=config) as sess: # Add TensorBoard components writer = tf.summary.FileWriter(log_dir) # create summary writer writer.add_graph(sess.graph) gvars = [ gvar for gvar in tf.global_variables() if 'analogy_score' not in gvar.name ] saver = tf.train.Saver(gvars, max_to_keep=5) # create checkpoint saver config = projector.ProjectorConfig() # create projector config embedding = config.embeddings.add() # add embeddings visualizer embedding.tensor_name = W_in.name embedding.metadata_path = vocab_metada_file # link metadata projector.visualize_embeddings( writer, config) # add writer and config to projector # Set up variables if restore_variables: # restore variables from disk restore_file = tf.train.latest_checkpoint(log_dir) assert restore_file is not None, "No restore file found in folder " + log_dir assert os.path.exists(restore_file + ".index"), \ "Trying to restore Tensorflow session from non-existing file: " + restore_file + ".index" init.run() saver.restore(sess, restore_file) print("\tVariables restored from file", ckpt_saver_file, "in TensorFlow ") else: # save the computational graph to file and initialize variables graph_saver = tf.train.Saver(allow_empty=True) init.run() graph_saver.save(sess, ckpt_saver_file_init, global_step=0, write_meta_graph=True) tf.add_to_collection(tf.GraphKeys.SAVEABLE_OBJECTS, saveable_iterator) print("\tVariables initialized in TensorFlow") # Compute the necessary number of steps for this epoch as well as how often to print the avg loss num_steps = int(math.ceil(dataset_size / mini_batch_size)) step_print_loss = int(math.ceil(num_steps / freq_print_loss)) print('\tPrinting loss every ', step_print_loss, 'steps, i.e.', freq_print_loss, 'times per epoch') ################################################################################################################ # Epoch loop epoch = 0 global_step = 0 while epoch < int(num_epochs): print('\n\tStarting epoch ', epoch) sess.run(iterator.initializer) # initialize iterator # If restoring a previous training session, set the right training epoch if restore_variables and not restore_completed: epoch = int( math.floor(global_train_step.eval() / (dataset_size / mini_batch_size))) global_step = global_train_step.eval() print('Starting from epoch', epoch) ############################################################################################################ # Loop over steps (mini batches) inside of epoch step = 0 avg_loss = 0 while True: try: # Print average loss every x steps if step_print_loss > 0 and step % int( step_print_loss) == 0: # update step with logging # If restoring a previous training session, set the right training epoch if restore_variables and not restore_completed: restore_completed = True # Write global step if FLAGS.optimizer != 'momentum': global_train_step.assign(global_step).eval() # Perform an update # print('\tStarting local step {:>6}'.format(step)) # un-comment for debugging [_, loss_val, train_loss_val, global_step] = sess.run( [optimizer, loss, train_loss, global_train_step], options=options, run_metadata=metadata) assert not np.isnan( loss_val), "Loss at step " + str(step) + " is nan" assert not np.isinf( loss_val), "Loss at step " + str(step) + " is inf" avg_loss += loss_val if step > 0: avg_loss /= step_print_loss ''' analogy_score = i2v_eval.evaluate_analogies(W_in.eval(), reverse_dictionary, analogies, analogy_types, analogy_evaluation_file, session=sess, print=i2v_eval.nop) total_analogy_score = sum([a[0] for a in analogy_score]) analogy_score_tensor.assign(total_analogy_score).eval() # for tf.summary ''' [summary, W_in_val] = sess.run([summary_op, W_in]) if FLAGS.savebest is not None: filelist = [f for f in os.listdir(FLAGS.savebest)] scorelist = [ int(s.split('-')[1]) for s in filelist ] if len(scorelist ) == 0 or total_analogy_score > sorted( scorelist)[-1]: i2v_utils.safe_pickle( W_in_val, FLAGS.savebest + '/' + 'score-' + str(total_analogy_score) + '-w.p') # Display average loss ''' print('{} Avg. loss at epoch {:>6,d}, step {:>12,d} of {:>12,d}, global step {:>15} : {:>12.3f}, analogies: {})'.format( str(datetime.now()), epoch, step, num_steps, global_step, avg_loss, str(analogy_score))) ''' print( '{} Avg. loss at epoch {:>6,d}, step {:>12,d} of {:>12,d}, global step {:>15} : {:>12.3f})' .format(str(datetime.now()), epoch, step, num_steps, global_step, avg_loss)) avg_loss = 0 # Pickle intermediate embeddings i2v_utils.safe_pickle(W_in_val, embeddings_pickle) # Write to TensorBoard saver.save(sess, ckpt_saver_file, global_step=global_step, write_meta_graph=False) writer.add_summary(summary, global_step=global_step) if FLAGS.profile: fetched_timeline = timeline.Timeline( metadata.step_stats) chrome_trace = fetched_timeline.generate_chrome_trace_format( ) with open('timeline_step_%d.json' % step, 'w') as f: f.write(chrome_trace) if step > 0 and FLAGS.extreme: sys.exit(22) else: # ordinary update step [_, loss_val] = sess.run([optimizer, loss]) avg_loss += loss_val # Compute and print nearest neighbors every x steps if step_print_neighbors > 0 and step % int( step_print_neighbors) == 0: print_neighbors(op=cosine_similarity, examples=valid_examples, top_k=6, reverse_dictionary=reverse_dictionary) # Update loop index (steps in epoch) step += 1 global_step += 1 except tf.errors.OutOfRangeError: # We reached the end of the epoch print('\n\t Writing embeddings to file ', embeddings_pickle) i2v_utils.safe_pickle([W_in.eval()], embeddings_pickle) # WEIRD! epoch += 1 # update loop index (epochs) break # from this inner loop ################################################################################################################ # End of training: # Print the nearest neighbors at the end of the run if step_print_neighbors == -1: print_neighbors(op=cosine_similarity, examples=valid_examples, top_k=6, reverse_dictionary=reverse_dictionary) # Save state of training and close the TensorBoard summary writer save_path = saver.save(sess, ckpt_saver_file_final, global_step) writer.add_summary(summary, global_step) writer.close() return W_in.eval()
def train(conf): if conf.dataset_name not in NUM_DATASET_MAP: num_dataset, num_classes = dataset.make_tfrecord( conf.dataset_name, conf.dataset_dir, conf.train_fraction, conf.num_channel, conf.num_dataset_parallel) if num_dataset is None: metadata = json.load( open(os.path.join(conf.dataset_dir, "metadata"))) NUM_DATASET_MAP[conf.dataset_name] = [ metadata["num_train"], metadata["num_validation"], metadata["num_classes"], conf.num_channel ] else: NUM_DATASET_MAP[conf.dataset_name] = [ num_dataset * conf.train_fraction, num_dataset * (1 - conf.train_fraction), num_classes, conf.num_channel ] num_channel = NUM_DATASET_MAP[conf.dataset_name][3] num_classes = NUM_DATASET_MAP[conf.dataset_name][2] is_training = tf.placeholder(tf.bool, shape=(), name="is_training") if conf.model_name[:6] == "nasnet": model_f = model_factory.get_network_fn(conf.model_name, num_classes, weight_decay=conf.weight_decay, is_training=True) else: model_f = model_factory.get_network_fn(conf.model_name, num_classes, weight_decay=conf.weight_decay, is_training=is_training) model_image_size = conf.model_image_size or model_f.default_image_size def pre_process(example_proto, training): features = { "image/encoded": tf.FixedLenFeature((), tf.string, default_value=""), "image/class/label": tf.FixedLenFeature((), tf.int64, default_value=0), 'image/height': tf.FixedLenFeature((), tf.int64, default_value=0), 'image/width': tf.FixedLenFeature((), tf.int64, default_value=0) } parsed_features = tf.parse_single_example(example_proto, features) if conf.preprocessing_name: image_preprocessing_fn = preprocessing_factory.get_preprocessing( conf.preprocessing_name, is_training=training) image = tf.image.decode_image(parsed_features["image/encoded"], num_channel) image = tf.clip_by_value( image_preprocessing_fn(image, model_image_size, model_image_size), -1, 1.0) else: image = tf.clip_by_value( tf.image.per_image_standardization( tf.image.resize_images( tf.image.decode_jpeg(parsed_features["image/encoded"], num_channel), [model_image_size, model_image_size])), -1., 1.0) if len(parsed_features["image/class/label"].get_shape()) == 0: label = tf.one_hot(parsed_features["image/class/label"], num_classes) else: label = parsed_features["image/class/label"] return image, label def train_dataset_map(example_proto): return pre_process(example_proto, True) def test_dataset_map(example_proto): return pre_process(example_proto, False) def get_model(): model_name = conf.model_name inputs = tf.placeholder( tf.float32, shape=[None, model_image_size, model_image_size, num_channel], name="inputs") labels = tf.placeholder(tf.float32, shape=[None, num_classes], name="labels") global_step = tf.Variable(0, trainable=False) learning_rate = optimizer.configure_learning_rate( NUM_DATASET_MAP[conf.dataset_name][0], global_step, conf) # learning_rate = tf.placeholder(tf.float32, shape=(), name="learning_rate") conf.num_channel = num_channel conf.num_classes = num_classes if model_name in ["deconv", "ed", "deconv_conv"]: logits, gen_x, gen_x_ = model_f(inputs, model_conf=conf) class_loss_op = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(labels=labels, logits=logits)) gen_loss_op = tf.log( tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(labels=gen_x_, logits=gen_x))) loss_op = tf.add(class_loss_op, gen_loss_op) ops = [class_loss_op, loss_op, gen_loss_op] ops_key = ["class_loss_op", "loss_op", "gen_loss_op"] else: if model_name == "conv": logits = model_f(inputs, model_conf=conf) else: logits, end_points = model_f(inputs) if model_name == "resnet": logits = tf.reshape(logits, [-1, num_classes]) loss_op = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(labels=labels, logits=logits)) ops = [loss_op] ops_key = ["loss_op"] if conf.use_regularizer: weights = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) regularizer = 0 for weight in weights: regularizer += tf.nn.l2_loss(weight) regularizer *= conf.weight_decay loss_op += regularizer tf.summary.scalar('loss', loss_op) opt = optimizer.configure_optimizer(learning_rate, conf) train_op = opt.minimize(loss_op, global_step=global_step) accuracy_op = tf.reduce_mean( tf.cast(tf.equal(tf.argmax(logits, 1), tf.argmax(labels, 1)), tf.float32)) ops.append(tf.argmax(logits, 1)) ops_key.append("predict_idx") tf.summary.scalar('accuracy', accuracy_op) merged = tf.summary.merge_all() return inputs, labels, train_op, accuracy_op, merged, ops, ops_key, logits, end_points if not os.path.exists(conf.dataset_dir): conf.dataset_dir = os.path.join("/home/data", conf.dataset_name) train_filenames = glob.glob( os.path.join(conf.dataset_dir, conf.dataset_name + ("_%s*tfrecord" % conf.train_name))) test_filenames = glob.glob( os.path.join(conf.dataset_dir, conf.dataset_name + ("_%s*tfrecord" % conf.test_name))) inputs, labels, train_op, accuracy_op, merged, ops, ops_key, logits, end_points = get_model( ) config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) summary_dir = os.path.join(conf.log_dir, "summary") train_writer = tf.summary.FileWriter(summary_dir + '/train', sess.graph) test_writer = tf.summary.FileWriter(summary_dir + '/test') sess.run(tf.global_variables_initializer()) saver = tf.train.Saver() if conf.restore_model_path and len( glob.glob(conf.restore_model_path + ".data-00000-of-00001")) > 0: print("restore!!") saver.restore(sess, conf.restore_model_path) train_iterator = tf.data.TFRecordDataset(train_filenames).map( train_dataset_map, conf.num_dataset_parallel).shuffle( buffer_size=conf.shuffle_buffer).batch( conf.batch_size).make_initializable_iterator() train_next = train_iterator.get_next() test_iterator = tf.data.TFRecordDataset(test_filenames).map( test_dataset_map, conf.num_dataset_parallel).batch( conf.batch_size).make_initializable_iterator() test_next = test_iterator.get_next() num_train = NUM_DATASET_MAP[conf.dataset_name][0] // conf.batch_size num_test = NUM_DATASET_MAP[conf.dataset_name][1] // conf.batch_size if conf.vis_epoch is not None: config = projector.ProjectorConfig() vis_dir = os.path.join(conf.log_dir, "embedding") total_dataset = None total_labels = None total_activations = None heatmap_imgs = {} bb_imgs = {} for epoch in range(conf.epoch): train_step = 0 if conf.vis_epoch is not None and total_dataset is not None: total_dataset = None total_labels = None total_activations = None if conf.train: sess.run(train_iterator.initializer) total_train_accuracy = .0 inner_train_step = 0 while True: try: batch_xs, batch_ys = sess.run(train_next) results = sess.run([ train_op, merged, accuracy_op, ] + ops, feed_dict={ inputs: batch_xs, labels: batch_ys, is_training: True }) total_train_accuracy += results[2] now = datetime.now().strftime('%Y/%m/%d %H:%M:%S') if train_step % conf.summary_interval == 0: ops_results = " ".join( list( map(lambda x: str(x), list(zip(ops_key, results[3:]))))) print( ("[%s TRAIN %d epoch, %d / %d step] accuracy: %f" % (now, epoch, train_step, num_train, results[2])) + ops_results) train_writer.add_summary( results[1], train_step + epoch * num_train) train_step += 1 inner_train_step += 1 except tf.errors.OutOfRangeError: break if inner_train_step > 0: print("Avg Train Accuracy : %f" % (float(total_train_accuracy) / inner_train_step)) if epoch % conf.num_save_interval == 0: saver.save(sess, conf.log_dir + "/model_epoch_%d.ckpt" % epoch) if conf.eval: total_accuracy = 0 test_step = 0 sess.run(test_iterator.initializer) while True: try: test_xs, test_ys = sess.run(test_next) results = sess.run([merged, accuracy_op, logits] + ops, feed_dict={ inputs: test_xs, labels: test_ys, is_training: False }) total_accuracy += results[1] now = datetime.now().strftime('%Y/%m/%d %H:%M:%S') ops_results = " ".join( list( map(lambda x: str(x), list(zip(ops_key, results[3:]))))) print(("[%s TEST %d epoch, %d /%d step] accuracy: %f" % (now, epoch, test_step, num_test, results[1])) + ops_results + "labels", test_ys.argmax(1)) test_writer.add_summary( results[0], test_step + (train_step + epoch * num_train)) test_step += 1 if conf.vis_epoch is not None and epoch % conf.vis_epoch == 0: if conf.num_vis_steps >= test_step: if conf.use_predict_of_test_for_embed_vis: predict_y = np.zeros( (conf.batch_size, num_classes)) predict_y[np.arange(conf.batch_size), results[-1]] = 1 tmp_labels = predict_y else: tmp_labels = test_ys if total_dataset is None: total_dataset = test_xs total_labels = tmp_labels total_activations = results[2] else: total_dataset = np.append(test_xs, total_dataset, axis=0) total_labels = np.append(tmp_labels, total_labels, axis=0) total_activations = np.append( results[2], total_activations, axis=0) ### Create CAM image if end_points and conf.num_cam: grad_cam_plus_plus = GradCamPlusPlus( end_points[ model_f.default_logit_layer_name], end_points[ model_f.default_last_conv_layer_name], inputs, is_training) cam_imgs, class_indices = grad_cam_plus_plus.create_cam_imgs( sess, test_xs, results[2]) for i in range(conf.num_cam): box_img = np.copy(test_xs[i]) # for j in range(GradCamPlusPlus.TOP3): ### Overlay heatmap heapmap = grad_cam_plus_plus.convert_cam_2_heatmap( cam_imgs[i][0]) overlay_img = grad_cam_plus_plus.overlay_heatmap( test_xs[i], heapmap) if test_ys[i].argmax( ) == results[2][i].argmax(): key = "label_%dd" % test_ys[i].argmax() else: key = "fail_label_%d_pred_%d" % ( test_ys[i].argmax(), results[2][i].argmax()) if key not in heatmap_imgs: heatmap_imgs[key] = [] bb_imgs[key] = [] if len(test_xs[i].shape ) != 3 or test_xs[i].shape[2] != 3: test = cv2.cvtColor( test_xs[i], cv2.COLOR_GRAY2BGR)[..., ::-1] else: test = test_xs[i] heatmap_imgs[key].append( overlay_img[..., ::-1]) heatmap_imgs[key].append(test) ### Boxing box_img = grad_cam_plus_plus.draw_rectangle( box_img, cam_imgs[i][0], [255, 0, 0]) bb_imgs[key].append(box_img) except tf.errors.OutOfRangeError: break if conf.vis_epoch is not None and epoch % conf.vis_epoch == 0: for key in heatmap_imgs: write_summary(test_writer, "heatmap_epoch_%d_%s" % (epoch, key), heatmap_imgs[key], sess) write_summary(test_writer, "bb_epoch_%d_%s" % (epoch, key), bb_imgs[key], sess) heatmap_imgs = {} bb_imgs = {} if test_step > 0: print("Avg Accuracy : %f" % (float(total_accuracy) / test_step)) if conf.vis_epoch is not None and epoch % conf.vis_epoch == 0: # vis_dir = os.path.join(conf.log_dir, "embed_vis_%d" % epoch) visualizer.add_embedding( config, sess=sess, embedding_list=[total_activations], embedding_path=vis_dir, image_size=model_image_size, channel=num_channel, labels=total_labels, prefix="epoch" + str(epoch)) if not conf.train: break ### Write summary # write_summary(test_writer, summary_names, result_imgs, sess) if conf.vis_epoch is not None and conf.eval and total_dataset is not None: visualizer.write_embedding(config, sess, total_dataset, embedding_path=vis_dir, image_size=model_image_size, channel=num_channel, labels=total_labels) sess.close()
def save_tensorboard(self, dirpath=None): if not self.trained: raise Exception('Train `Word2Vec` first.') os.makedirs(dirpath, exist_ok=True) weights = self.embed.wv.vectors idx2words = self.embed.wv.index2word vocab_size = weights.shape[0] embedding_dim = weights.shape[1] with open(os.path.join(dirpath, "metadata.tsv"), 'w') as f: f.writelines("\n".join(idx2words)) tf.reset_default_graph() W = tf.Variable( tf.constant(0., shape=[vocab_size, embedding_dim]), trainable=False, name="W", ) embedding_placeholder = tf.placeholder( tf.float32, [vocab_size, embedding_dim], ) embedding_init = W.assign(embedding_placeholder) writer = tf.summary.FileWriter( dirpath, graph=tf.get_default_graph(), ) saver = tf.train.Saver() # tf.contrib.tensorboard.plugins.projector.projector_config.proto config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = W.name embedding.metadata_path = os.path.join( # filepath, "metadata.tsv", ) # Saves a configuration file that TensorBoard will read during startup. projector.visualize_embeddings(writer, config) with tf.Session() as sess: sess.run( embedding_init, feed_dict={embedding_placeholder: weights}, ) save_path = saver.save( sess, os.path.join( dirpath, "tf-model.cpkt", ), ) print(f"'Projector Saved: '{save_path}'") return save_path
def build_graph(): vocabulary_size = FLAGS.vocabulary_size or int( redis_client.zcard(FLAGS.redis_key_vocabulary)) valid_examples = np.random.choice(100, FLAGS.valid_size, replace=False) graph = tf.Graph() with graph.as_default(): with tf.name_scope('Inputs'): # input data train_inputs = tf.placeholder(tf.int32, shape=[FLAGS.batch_size]) train_labels = tf.placeholder(tf.int32, shape=[FLAGS.batch_size, 1]) valid_dataset = tf.constant(valid_examples) with tf.device('/cpu:0'): with tf.name_scope('embedings'): # Look up embeddings for inputs. embeddings = tf.Variable( tf.random_uniform([vocabulary_size, FLAGS.embedding_size], -1.0, 1.0)) embeded = tf.nn.embedding_lookup(embeddings, train_inputs) with tf.name_scope('weight'): # Construct the variables for the NCE loss nce_weights = tf.Variable( tf.truncated_normal( [vocabulary_size, FLAGS.embedding_size], stddev=1.0 / np.sqrt(FLAGS.embedding_size))) with tf.name_scope('biases'): nce_biases = tf.Variable(tf.zeros([vocabulary_size])) # Compute the average NCE loss for the batch. # tf.nce_loss automatically draws a new sample of the negative labels each # time we evaluate the loss. # TODO num_sampled? loss = tf.reduce_mean( tf.nn.nce_loss(weights=nce_weights, biases=nce_biases, labels=train_labels, inputs=embeded, num_sampled=64, num_classes=vocabulary_size)) # Add the loss value as a scalar to summary. tf.summary.scalar('loss', loss) with tf.name_scope('optimizer'): optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss) # Compute the cosine similarity between minibatch examples and all embeddings. norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True)) normalized_embeddings = embeddings / norm valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset) # 余弦相似性 # (valid_size, embeddings_size) * (vocabulary_size, embeddings_size)^T similarity = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True) # Merge all summaries. merged = tf.summary.merge_all() # Add variable initializer. init = tf.global_variables_initializer() # Create a saver. saver = tf.train.Saver() with tf.Session(graph=graph) as session: writer = tf.summary.FileWriter(FLAGS.log_dir, session.graph) init.run() print('Initialized') target_index = 0 average_loss = 0 for step in range(FLAGS.num_steps): batch_train, labels_train, target_index = generate_train_batch( target_index, FLAGS.window_width, FLAGS.batch_size, FLAGS.num_skips, FLAGS.redis_key_index) feed_dict = {train_inputs: batch_train, train_labels: labels_train} # Define metadata variable. run_metadata = tf.RunMetadata() _, loss_val, summary = session.run([optimizer, loss, merged], feed_dict=feed_dict) average_loss += loss_val # Add returned summaries to writer in each step. writer.add_summary(summary, step) # Add metadata to visualize the graph for the last run. if step == (FLAGS.num_steps - 1): writer.add_run_metadata(run_metadata, 'step%d' % step) if step % 2000 == 0: if step > 0: average_loss /= 2000 print('Average loss at step ', step, ': ', average_loss) average_loss = 0 # 查看相似的词组 if step % 10000 == 0: sim = similarity.eval() for i in range(FLAGS.valid_size): valid_word = \ redis_client.zrevrange(FLAGS.redis_key_vocabulary, valid_examples[i], valid_examples[i])[ 0] top_k = 8 # 最相似的8个 nearest = (-sim[i, :]).argsort()[1:top_k + 1] log_str = 'Nearest {}:'.format(valid_word.decode('utf8')) for k in range(top_k): close_word = redis_client.zrevrange( FLAGS.redis_key_vocabulary, nearest[k], nearest[k])[0] log_str = '{} {},'.format(log_str, close_word.decode('utf8')) print(log_str) final_embeddings = normalized_embeddings.eval() # Write corresponding labels for the embeddings. with open(FLAGS.log_dir + '/metadata.tsv', 'w') as f: for i in range(vocabulary_size): f.write( redis_client.zrevrange(FLAGS.redis_key_vocabulary, i, i) [0].decode('utf8') + '\n') # Save the model for checkpoints. saver.save(session, os.path.join(FLAGS.log_dir, 'model.ckpt')) # Create a configuration for visualizing embeddings with the labels in TensorBoard. config = projector.ProjectorConfig() embedding_conf = config.embeddings.add() embedding_conf.tensor_name = embeddings.name embedding_conf.metadata_path = os.path.join('metadata.tsv') projector.visualize_embeddings(writer, config) plot_samples(final_embeddings, vocabulary_size)
def set_model(self, model): self.model = model if K.backend() == 'tensorflow': self.sess = K.get_session() if self.histogram_freq and self.merged is None: for layer in self.model.layers: for weight in layer.weights: mapped_weight_name = weight.name.replace(':', '_') tf.summary.histogram(mapped_weight_name, weight) if self.write_grads: grads = model.optimizer.get_gradients( model.total_loss, weight) def is_indexed_slices(grad): return type(grad).__name__ == 'IndexedSlices' grads = [ grad.values if is_indexed_slices(grad) else grad for grad in grads ] tf.summary.histogram( '{}_grad'.format(mapped_weight_name), grads) if self.write_images: w_img = tf.squeeze(weight) shape = K.int_shape(w_img) if len(shape) == 2: # dense layer kernel case if shape[0] > shape[1]: w_img = tf.transpose(w_img) shape = K.int_shape(w_img) w_img = tf.reshape(w_img, [1, shape[0], shape[1], 1]) elif len(shape) == 3: # convnet case if K.image_data_format() == 'channels_last': # switch to channels_first to display # every kernel as a separate image w_img = tf.transpose(w_img, perm=[2, 0, 1]) shape = K.int_shape(w_img) w_img = tf.reshape( w_img, [shape[0], shape[1], shape[2], 1]) elif len(shape) == 1: # bias case w_img = tf.reshape(w_img, [1, shape[0], 1, 1]) else: # not possible to handle 3D convnets etc. continue shape = K.int_shape(w_img) assert len(shape) == 4 and shape[-1] in [1, 3, 4] tf.summary.image(mapped_weight_name, w_img) if hasattr(layer, 'output'): if isinstance(layer.output, list): for i, output in enumerate(layer.output): tf.summary.histogram( '{}_out_{}'.format(layer.name, i), output) else: tf.summary.histogram('{}_out'.format(layer.name), layer.output) self.merged = tf.summary.merge_all() if self.write_graph: self.writer = tf.summary.FileWriter(self.log_dir, self.sess.graph) else: self.writer = tf.summary.FileWriter(self.log_dir) if self.embeddings_freq and self.embeddings_data is not None: self.embeddings_data = standardize_input_data( self.embeddings_data, model.input_names) embeddings_layer_names = self.embeddings_layer_names if not embeddings_layer_names: embeddings_layer_names = [ layer.name for layer in self.model.layers if type(layer).__name__ == 'Embedding' ] self.assign_embeddings = [] embeddings_vars = {} self.batch_id = batch_id = tf.placeholder(tf.int32) self.step = step = tf.placeholder(tf.int32) for layer in self.model.layers: if layer.name in embeddings_layer_names: embedding_input = self.model.get_layer(layer.name).output embedding_size = np.prod(embedding_input.shape[1:]) embedding_input = tf.reshape(embedding_input, (step, int(embedding_size))) shape = (self.embeddings_data[0].shape[0], int(embedding_size)) embedding = tf.Variable(tf.zeros(shape), name=layer.name + '_embedding') embeddings_vars[layer.name] = embedding batch = tf.assign(embedding[batch_id:batch_id + step], embedding_input) self.assign_embeddings.append(batch) self.saver = tf.train.Saver(list(embeddings_vars.values())) embeddings_metadata = {} if not isinstance(self.embeddings_metadata, str): embeddings_metadata = self.embeddings_metadata else: embeddings_metadata = { layer_name: self.embeddings_metadata for layer_name in embeddings_vars.keys() } config = projector.ProjectorConfig() for layer_name, tensor in embeddings_vars.items(): embedding = config.embeddings.add() embedding.tensor_name = tensor.name if layer_name in embeddings_metadata: embedding.metadata_path = embeddings_metadata[layer_name] projector.visualize_embeddings(self.writer, config)
#产生metdata文件 if tf.gfile.Exists(DIR + 'projector\\projector\\metadata.tsv'): tf.gfile.DeleteRecursively(DIR + 'projector\\progector\\metadata.tsv') with open(DIR + 'projector\\progector\\metadata.tsv', 'w') as f: labels = sess.run(tf.argmax(mnist.test.labels[:], 1)) for i in range(image_num): f.write(str(labels[i]) + '\n') #合并所有的summary merged = tf.summary.merge_all() projector_writer = tf.summary.FileWriter(DIR + 'projector/progector/', sess.graph) saver = tf.train.Saver() config = projector.ProjectorConfig() embed = config.embeddings.add() embed.metadata_path = DIR + 'projector/progector/metadata.tsv' embed.sprite.image_path = DIR + 'projector/data/mnist_10k_sprite.png' embed.sprite.single_image_dim.extend([28, 28]) projector.visualize_embeddings(projector_writer, config) for i in range(max_steps): #每个批次100个样本 batch_xs, batch_ys = mnist.train.next_batch(100) run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) run_metadata = tf.RunMetadata() summary, _ = sess.run([merged, train_step], feed_dict={ x: batch_xs, y: batch_ys
def train(FLAGS, graph=None, init=None, optimizer=None, merged=None, loss=None, similarity=None, reverse_dictionary=None, normalized_embeddings=None, saver=None, embeddings=None, vocabulary_size=None, valid_size=None, valid_examples=None, data=None, train_inputs=None, train_labels=None): # We pick a random validation set to sample nearest neighbors. Here we limit the # validation samples to the words that have a low numeric ID, which by # construction are also the most frequent. These 3 variables are used only for # displaying model accuracy, they don't affect calculation. global i, final_embeddings num_steps = 100001 with tf.Session(graph=graph) as session: # Open a writer to write summaries. writer = tf.summary.FileWriter(FLAGS.log_dir, session.graph) # We must initialize all variables before we use them. init.run() print('Initialized') average_loss = 0 for step in xrange(num_steps): batch_inputs, batch_labels = generate_batch( batch_size, num_skips, skip_window, data) feed_dict = { train_inputs: batch_inputs, train_labels: batch_labels } # Define metadata variable. run_metadata = tf.RunMetadata() # We perform one update step by evaluating the optimizer op (including it # in the list of returned values for session.run() # Also, evaluate the merged op to get all summaries from the returned "summary" variable. # Feed metadata variable to session for visualizing the graph in TensorBoard. _, summary, loss_val = session.run([optimizer, merged, loss], feed_dict=feed_dict, run_metadata=run_metadata) average_loss += loss_val # Add returned summaries to writer in each step. writer.add_summary(summary, step) # Add metadata to visualize the graph for the last run. if step == (num_steps - 1): writer.add_run_metadata(run_metadata, 'step%d' % step) if step % 2000 == 0: if step > 0: average_loss /= 2000 # The average loss is an estimate of the loss over the last 2000 batches. print('Average loss at step ', step, ': ', average_loss) average_loss = 0 # Note that this is expensive (~20% slowdown if computed every 500 steps) if step % 10000 == 0: sim = similarity.eval() for i in xrange(valid_size): valid_word = reverse_dictionary[valid_examples[i]] top_k = 8 # number of nearest neighbors nearest = (-sim[i, :]).argsort()[1:top_k + 1] log_str = 'Nearest to %s:' % valid_word for k in xrange(top_k): close_word = reverse_dictionary[nearest[k]] log_str = '%s %s,' % (log_str, close_word) print(log_str) final_embeddings = normalized_embeddings.eval() # Write corresponding labels for the embeddings. with open(FLAGS.log_dir + '/metadata.tsv', 'w') as f: for i in xrange(vocabulary_size): f.write(reverse_dictionary[i] + '\n') # Save the model for checkpoints. saver.save(session, os.path.join(FLAGS.log_dir, 'model.ckpt')) # Create a configuration for visualizing embeddings with the labels in TensorBoard. config = projector.ProjectorConfig() embedding_conf = config.embeddings.add() embedding_conf.tensor_name = embeddings.name embedding_conf.metadata_path = os.path.join(FLAGS.log_dir, 'metadata.tsv') projector.visualize_embeddings(writer, config) writer.close() return final_embeddings