def plot_to_projector( x, feature_vector, y, class_names, log_dir="default_log_dir", meta_file="metadata.tsv", ): assert x.ndim == 4 # (BATCH, H, W, C) if os.path.isdir(log_dir): shutil.rmtree(log_dir) # Create a new clean fresh folder :) os.mkdir(log_dir) SPRITES_FILE = os.path.join(log_dir, "sprites.png") sprite = create_sprite(x) cv2.imwrite(SPRITES_FILE, sprite) # Generate label names labels = [class_names[y[i]] for i in range(int(y.shape[0]))] with open(os.path.join(log_dir, meta_file), "w") as f: for label in labels: f.write("{}\n".format(label)) if feature_vector.ndim != 2: print("NOTE: Feature vector is not of form (BATCH, FEATURES)" " reshaping to try and get it to this form!") feature_vector = tf.reshape(feature_vector, [feature_vector.shape[0], -1]) feature_vector = tf.Variable(feature_vector) checkpoint = tf.train.Checkpoint(embedding=feature_vector) checkpoint.save(os.path.join(log_dir, "embeddings.ckpt")) # Set up config config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = "embedding/.ATTRIBUTES/VARIABLE_VALUE" embedding.metadata_path = meta_file embedding.sprite.image_path = "sprites.png" embedding.sprite.single_image_dim.extend((x.shape[1], x.shape[2])) projector.visualize_embeddings(log_dir, config)
def plot_with_tensorboard( logdir: Path, sentences: List[str], clusters: Union[List, np.ndarray], embeddings: Dict[str, np.ndarray], ): """ Saves Tensorboard embeddings to projector. Args: logdir (Path): Directory where to save. sentences (List[str]): Sentences for metadata. embeddings (Dict[str, np.ndarray]): Embeddings to plot. """ # Set up a logs directory, so Tensorboard knows where to look for files if not logdir.exists(): logdir.mkdir(exist_ok=True, parents=True) # Save Labels separately on a line-by-line manner. with open(str(logdir / 'metadata.tsv'), "w") as f: f.write('Index\tSentence\tCluster\n') for i, (sentence, cluster) in enumerate(zip(sentences, clusters)): f.write(f"{i}\t{sentence}\t{cluster}\n") # Set up config config = projector.ProjectorConfig() variables = {} for name, embedding in embeddings.items(): # Save the weights we want to analyse as a variable. Note that the first # value represents any unknown word, which is not in the metadata, so # we will remove that value. variable = tf.Variable(embedding, name=name) # Create a checkpoint from embedding, the filename and key are # name of the tensor. variables = {**variables, name: variable} embedding = config.embeddings.add() # The name of the tensor will be suffixed by `/.ATTRIBUTES/VARIABLE_VALUE` embedding.tensor_name = f"{name}/.ATTRIBUTES/VARIABLE_VALUE" embedding.metadata_path = "metadata.tsv" checkpoint = tf.train.Checkpoint(**variables) checkpoint.save(str(logdir / f"embeddings.ckpt")) projector.visualize_embeddings(str(logdir), config)
def export_movies_embeddings(self, path: pathlib, movie_to_title: dict): # Creates folders if don't exist if not path.exists(): path.mkdir(parents=True) # Creates metadata for the embedding projector with open(path / 'metadata.tsv', "w") as f: f.writelines([f'{title}\n' for title in movie_to_title.values()]) # Creates checkpoint storing variable with embeddings matching the metadata info indexes = self.movie_to_index[tf.constant(list(movie_to_title.keys()))] embeddings = tf.gather(self.movies_embeddings, indices=indexes, axis=0) checkpoint = tf.train.Checkpoint(embeddings=tf.Variable(embeddings)) checkpoint.save(str(path / "embeddings.ckpt")) # Set up projector config config = projector.ProjectorConfig() embedding = config.embeddings.add() # The name of the tensor will be suffixed by `/.ATTRIBUTES/VARIABLE_VALUE` embedding.tensor_name = "embeddings/.ATTRIBUTES/VARIABLE_VALUE" embedding.metadata_path = 'metadata.tsv' projector.visualize_embeddings(str(path), config)
def main(): model = tf.keras.models.load_model(MODEL_PATH) # with open(os.path.join(LOG_DIR, 'metadata.tsv'), "w") as f: # for subwords in encoder.subwords: # f.write("{}\n".format(subwords)) # # Fill in the rest of the labels with "unknown" # for unknown in range(1, encoder.vocab_size - len(encoder.subwords)): # f.write("unknown #{}\n".format(unknown)) weights = tf.Variable(model.embed.get_weights()[0][1:]) # Create a checkpoint from embedding, the filename and key are # name of the tensor. checkpoint = tf.train.Checkpoint(embedding=weights) checkpoint.save(os.path.join(LOG_DIR, "embedding.ckpt")) config = projector.ProjectorConfig() embedding = config.embeddings.add() # The name of the tensor will be suffixed by `/.ATTRIBUTES/VARIABLE_VALUE` embedding.tensor_name = "embedding/.ATTRIBUTES/VARIABLE_VALUE" # embedding.metadata_path = 'metadata.tsv' projector.visualize_embeddings(LOG_DIR, config)
def testVisualizeEmbeddings(self): # Create a dummy configuration. config = projector.ProjectorConfig() config.model_checkpoint_path = 'test' emb1 = config.embeddings.add() emb1.tensor_name = 'tensor1' emb1.metadata_path = 'metadata1' # Call the API method to save the configuration to a temporary dir. temp_dir = self.get_temp_dir() self.addCleanup(shutil.rmtree, temp_dir) writer = tf.summary.FileWriter(temp_dir) projector.visualize_embeddings(writer, config) # Read the configurations from disk and make sure it matches the original. with tf.gfile.GFile(os.path.join(temp_dir, 'projector_config.pbtxt')) as f: config2 = projector.ProjectorConfig() text_format.Parse(f.read(), config2) self.assertEqual(config, config2)
def create_config(self, with_sprite=True): """ Create a congfig files that defines image tensor name, path to metadata file, path to the sprite image, and the size of individual image whithin the sprite image. Parameters ---------- with_sprite : bool, optional If to save sprite or not, by default True """ config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = f'{self.data_name}/.ATTRIBUTES/VARIABLE_VALUE' embedding.metadata_path = f'metadata_{self.data_name}.tsv' if with_sprite: embedding.sprite.image_path = f'sprite_{self.data_name}.png' embedding.sprite.single_image_dim.extend( [self.image_width, self.image_height]) projector.visualize_embeddings(self.log_dir, config)
def _configure_embeddings(self): """Configure the Projector for embeddings. Implementation from tensorflow codebase, but supports multiple models """ try: # noinspection PyPackageRequirements from tensorboard.plugins import projector except ImportError: raise ImportError( 'Failed to import TensorBoard. Please make sure that ' 'TensorBoard integration is complete."') config = projector.ProjectorConfig() for model_name, model in self.network.model.items(): for layer in model.layers: if isinstance(layer, embeddings.Embedding): embedding = config.embeddings.add() embedding.tensor_name = layer.embeddings.name if self.embeddings_metadata is not None: if isinstance(self.embeddings_metadata, str): embedding.metadata_path = self.embeddings_metadata else: if layer.name in embedding.metadata_path: embedding.metadata_path = self.embeddings_metadata.pop( layer.name) if self.embeddings_metadata: raise ValueError( 'Unrecognized `Embedding` layer names passed to ' '`keras.callbacks.TensorBoard` `embeddings_metadata` ' 'argument: ' + str(self.embeddings_metadata)) class DummyWriter(object): """Dummy writer to conform to `Projector` API.""" def __init__(self, logdir): self.logdir = logdir def get_logdir(self): return self.logdir writer = DummyWriter(self.train_log_dir) projector.visualize_embeddings(writer, config)
def create_config_file(path_logs, tensor_filename, features, image_sprite_filename, metadata_filename, image_sprite_size): with open(tensor_filename, 'w') as fw: csv_writer = csv.writer(fw, delimiter='\t') csv_writer.writerows(features) config = projector.ProjectorConfig() # One can add multiple embeddings. embedding = config.embeddings.add() embedding.tensor_path = tensor_filename # Link this tensor to its metadata file (e.g. labels). embedding.metadata_path = metadata_filename # Comment out if you don't want sprites embedding.sprite.image_path = image_sprite_filename embedding.sprite.single_image_dim.extend( [image_sprite_size[0], image_sprite_size[1]]) # Saves a config file that TensorBoard will read during startup. projector.visualize_embeddings(path_logs, config)
def visualize_data_frame(self, features: embeddings.LabelledFeatures) -> None: print("Exporting tensorboard logs to: {}".format(self._output_path)) features = _sample_if_needed(features) path_metadata = self._resolved_path(FILENAME_METADATA) path_features = self._resolved_path(FILENAME_FEATURES) _write_labels(features.labels, path_metadata) _save_embedding_as_checkpoint( self._maybe_project(features.features), path_features ) projector_config = _create_projector_config( path_metadata, self._maybe_create_sprite(features.image_paths) ) projector.visualize_embeddings(self._output_path, projector_config)
def write_embedding(log_dir): """Writes embedding data and projector configuration to the logdir.""" metadata_filename = "metadata.tsv" tensor_filename = "tensor.tsv" labels = ANIMALS.strip().splitlines() labels_to_tensors = {label: tensor_for_label(label) for label in labels} os.makedirs(log_dir, exist_ok=True) with open(os.path.join(log_dir, metadata_filename), "w") as f: for label in labels_to_tensors: f.write("{}\n".format(label)) with open(os.path.join(log_dir, tensor_filename), "w") as f: for tensor in labels_to_tensors.values(): f.write("{}\n".format("\t".join(str(x) for x in tensor))) config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.metadata_path = metadata_filename embedding.tensor_path = tensor_filename projector.visualize_embeddings(log_dir, config)
def setup_sample_training_data(self, log_dir, writer): # get some samples segs = [self.datasets.train.sample_segment() for _ in range(1000)] sample_X = [] sample_y = [] for segment in segs: data = self.datasets.train.fetch_segment(segment, augment=False) sample_X.append(data) sample_y.append(self.labels.index(segment.label)) X = np.asarray(sample_X, dtype=np.float32) y = np.asarray(sample_y, dtype=np.int32) data = (X, y, segs) sprite_path = os.path.join(log_dir, "examples.png") meta_path = os.path.join(log_dir, "examples.tsv") config = projector.ProjectorConfig() for var_name in ["sample_logits", "sample_hidden"]: embedding = config.embeddings.add() embedding.tensor_name = var_name embedding.metadata_path = meta_path embedding.sprite.image_path = sprite_path embedding.sprite.single_image_dim.extend([48, 48]) projector.visualize_embeddings(writer, config) # save tsv file containing labels" with open(meta_path, "w") as f: f.write("Index\tLabel\tSource\n") for index, segment in enumerate(segs): f.write("{}\t{}\t{}\n".format(index, segment.label, segment.clip_id)) # save out image previews to_vis = X[:, self.training_segment_frames // 2, 0] sprite_image = self._create_sprite_image(to_vis) plt.imsave(sprite_path, sprite_image, cmap="gray") return data
def save(df: pd.DataFrame, data_out: str): """Saves artifacts for tensorboard embeddings projector""" df.to_csv(f'{data_out}metadata.tsv', sep='\t', columns=['label', 'title'], index=False, header=True) embeddings = df['embedding'] embeddings = embeddings.tolist() embeddings = np.array(embeddings) embeddings_tensor = tf.Variable(embeddings, name='embedding') checkpoint = tf.train.Checkpoint(embedding=embeddings_tensor) checkpoint.save(os.path.join(data_out, 'embedding.ckpt')) config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = "embedding/.ATTRIBUTES/VARIABLE_VALUE" embedding.metadata_path = 'metadata.tsv' projector.visualize_embeddings(data_out, config)
def _configure_embeddings(self): """Configure the Projector for embeddings.""" # TODO(omalleyt): Add integration tests. from tensorflow.python.keras.layers import embeddings try: from tensorboard.plugins import projector except ImportError: raise ImportError( 'Failed to import TensorBoard. Please make sure that ' 'TensorBoard integration is complete."') config = projector.ProjectorConfig() for layer in self.model.layers: if isinstance(layer, embeddings.Embedding): embedding = config.embeddings.add() embedding.tensor_name = layer.embeddings.name if self.embeddings_metadata is not None: if isinstance(self.embeddings_metadata, str): embedding.metadata_path = self.embeddings_metadata else: if layer.name in embedding.metadata_path: embedding.metadata_path = self.embeddings_metadata.pop( layer.name) if self.embeddings_metadata: raise ValueError( 'Unrecognized `Embedding` layer names passed to ' '`keras.callbacks.TensorBoard` `embeddings_metadata` ' 'argument: ' + str(self.embeddings_metadata.keys())) class DummyWriter(object): """Dummy writer to conform to `Projector` API.""" def __init__(self, logdir): self.logdir = logdir def get_logdir(self): return self.logdir writer = DummyWriter(self.log_dir) projector.visualize_embeddings(writer, config)
def visualize(model, output_path): meta_file = "test/w2x_metadata.tsv" placeholder = np.zeros((len(model.wv.index2word), 50)) with open(os.path.join(output_path, meta_file), 'wb') as file_metadata: for i, word in enumerate(model.wv.index2word): placeholder[i] = model[word] # temporary solution for https://github.com/tensorflow/tensorflow/issues/9094 if word == '': print( "Emply Line, should replecaed by any thing else, or will cause a bug of tensorboard" ) file_metadata.write( "{0}".format('<Empty Line>').encode('utf-8') + b'\n') else: file_metadata.write("{0}".format(word).encode('utf-8') + b'\n') # define the model without training sess = tf.InteractiveSession() embedding = tf.Variable(placeholder, trainable=False, name='w2x_metadata') tf.global_variables_initializer().run() tf.disable_eager_execution() saver = tf.train.Saver() writer = tf.summary.FileWriter(output_path, sess.graph) # adding into projector config = projector.ProjectorConfig() embed = config.embeddings.add() embed.tensor_name = 'w2x_metadata' embed.metadata_path = meta_file # Specify the width and height of a single thumbnail. projector.visualize_embeddings(writer, config) saver.save(sess=sess, save_path=os.path.join(output_path, 'w2x_metadata.ckpt')) print( 'Run `tensorboard --logdir={0}` to run visualize result on tensorboard' .format(output_path))
def project(word_vecs, id2word): log_dir = 'logs' if not os.path.exists(log_dir): os.mkdir(log_dir) with open(os.path.join(log_dir, 'metadata.tsv'), "w") as f: f.write("{}\t{}\n".format("index", "word")) for i, word in id2word.items(): f.write("{}\t{}\n".format(i, word)) weights = tf.Variable(word_vecs) checkpoint = tf.train.Checkpoint(embedding=weights) checkpoint.save(os.path.join(log_dir, "embedding.ckpt")) # Set up config config = projector.ProjectorConfig() embedding = config.embeddings.add() # The name of the tensor will be suffixed by `/.ATTRIBUTES/VARIABLE_VALUE` embedding.tensor_name = "embedding/.ATTRIBUTES/VARIABLE_VALUE" embedding.metadata_path = 'metadata.tsv' projector.visualize_embeddings(log_dir, config)
def visualize_embeddings(images, embeddings, output_dir, thumbnail_size=(32, 32)): """ Args: images: embeddings: output_dir: thumbnail_size: Returns: """ summary_writer = tf.summary.FileWriter(output_dir) sprite_path = os.path.abspath(os.path.join(output_dir, 'sprite.png')) metadata_path = os.path.abspath(os.path.join(output_dir, 'metadata.csv')) embeddings_path = os.path.join(output_dir, 'embeddings.ckpt') embedding_var = tf.Variable(embeddings, name='embeddings') sprite = images_to_sprite(images) imsave(os.path.join(output_dir, 'sprite.png'), sprite) with tf.Session() as sess: sess.run(embedding_var.initializer) config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = embedding_var.name embedding.metadata_path = metadata_path embedding.sprite.image_path = sprite_path embedding.sprite.single_image_dim.extend(thumbnail_size) projector.visualize_embeddings(summary_writer, config) saver = tf.train.Saver([embedding_var]) saver.save(sess, embeddings_path, 1)
def write_embeddings(self, Wv, name="WordVectors"): """Write embedding matrix to the right place. Args: Wv: (numpy.ndarray) |V| x d matrix of word embeddings """ #with tf.Graph().as_default(), tf.Session() as session: with tf.Graph().as_default(), tf.compat.v1.Session() as session: ## # Feed embeddings to tf, and save. embedding_var = tf.Variable(Wv, name=name, dtype=tf.float32) session.run(tf.compat.v1.global_variables_initializer()) saver = tf.compat.v1.train.Saver() saver.save(session, self.CHECKPOINT_FILE, 0) ## # Save metadata summary_writer = tf.compat.v1.summary.FileWriter(self.LOGDIR) config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = embedding_var.name embedding.metadata_path = self.VOCAB_FILE_BASE projector.visualize_embeddings(summary_writer, config) msg = "Saved {s0:d} x {s1:d} embedding matrix '{name}'" msg += " to LOGDIR='{logdir}'" print( msg.format(s0=Wv.shape[0], s1=Wv.shape[1], name=name, logdir=self.LOGDIR)) print("To view, run:") print("\n tensorboard --logdir=\"{logdir}\"\n".format( logdir=self.LOGDIR)) print("and navigate to the \"Embeddings\" tab in the web interface.")
def visualization(self, num_visualize, visual_fld): """ run "'tensorboard --logdir='visualization'" to see the embeddings """ # create the list of num_variable most common words to visualize word2vec_utils.most_common_words(visual_fld, num_visualize) saver = tf.train.Saver() with tf.Session() as sess: sess.run(tf.global_variables_initializer()) ckpt = tf.train.get_checkpoint_state( os.path.dirname('checkpoints/checkpoint')) # if that checkpoint exists, restore from checkpoint if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) final_embed_matrix = sess.run(self.embed_matrix) # you have to store embeddings in a new variable embedding_var = tf.Variable(final_embed_matrix[:num_visualize], name='embedding') sess.run(embedding_var.initializer) config = projector.ProjectorConfig() summary_writer = tf.summary.FileWriter(visual_fld) # add embedding to the config file embedding = config.embeddings.add() embedding.tensor_name = embedding_var.name # link this tensor to its metadata file, in this case the first NUM_VISUALIZE words of vocab embedding.metadata_path = 'vocab_' + str(num_visualize) + '.tsv' # saves a configuration file that TensorBoard will read during startup. projector.visualize_embeddings(summary_writer, config) saver_embed = tf.train.Saver([embedding_var]) saver_embed.save(sess, os.path.join(visual_fld, 'model.ckpt'), 1)
def visualize_embeddings(images, embeddings, output_dir, thumbnail_size=(32, 32)): """ Args: images: embeddings: output_dir: thumbnail_size: Returns: """ summary_writer = tf.summary.FileWriter(output_dir) sprite_path = os.path.abspath(os.path.join(output_dir, 'sprite.png')) metadata_path = os.path.abspath(os.path.join(output_dir, 'metadata.csv')) embeddings_path = os.path.join(output_dir, 'embeddings.ckpt') embedding_var = tf.Variable(embeddings, name='embeddings') sprite = images_to_sprite(images) imsave(os.path.join(output_dir, 'sprite.png'), sprite) with tf.Session() as sess: sess.run(embedding_var.initializer) config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = embedding_var.name embedding.metadata_path = metadata_path embedding.sprite.image_path = sprite_path embedding.sprite.single_image_dim.extend(thumbnail_size) projector.visualize_embeddings(summary_writer, config) saver = tf.train.Saver([embedding_var]) saver.save(sess, embeddings_path, 1)
def embedding(vlist, rlist, metaPath, spSize): vs = [] for i in range(0, len(vlist)): v = tf.Variable(rlist[i], name=vlist[i].name.split('/')[0]) # x/relu vs.append(v) with tf.Session() as sess: tf.variables_initializer(vs).run() # assign to vs saver = tf.train.Saver(vs) saver.save(sess, './log/model.ckpt', 0) # only contain vs # get writer and config summary_writer = tf.summary.FileWriter('./log/') config = projector.ProjectorConfig() # set config for v in vs: embed = config.embeddings.add() embed.tensor_name = v.name embed.metadata_path = metaPath + 'meta.tsv' embed.sprite.image_path = metaPath + 'meta.png' embed.sprite.single_image_dim.extend(spSize) # write projector.visualize_embeddings(summary_writer, config)
def tensorboard_view(v_data, words=None): df = pd.DataFrame.from_records(data=v_data) tf_data = tf.Variable( df.values.transpose()) if words is None else tf.Variable( df[words].values.transpose()) ## Running Tensorlow Session with tf.Session() as sess: saver = tf.train.Saver([tf_data]) sess.run(tf_data.initializer) saver.save(sess, os.path.join(LOG_DIR, 'tf_data.ckpt')) config = projector.ProjectorConfig() # One can add multiple embeddings. embedding = config.embeddings.add() embedding.tensor_name = tf_data.name # Specify where you find the sprite (we will create this later) path_for_mnist_sprites = os.path.join(LOG_DIR, 'mnistdigits.png') embedding.sprite.image_path = path_for_mnist_sprites embedding.sprite.single_image_dim.extend([28, 28]) # Link this tensor to its metadata(Labels) file path_for_metadata = os.path.join(LOG_DIR, 'metadata.tsv') with open(path_for_metadata, 'w+') as metadata_file: _words = v_data.keys() if words is None else words for word in _words: metadata_file.write(f'{word}\n') embedding.metadata_path = path_for_metadata # Saves a config file that TensorBoard will read during startup. tf.compat.v1.disable_eager_execution() projector.visualize_embeddings( tf.compat.v1.summary.FileWriter(LOG_DIR), config) print(f'create logs on {LOG_DIR}')
def visualize_embeddings(model_path, dataset, log_dir): model = tf.keras.models.load_model(model_path) labels = [] embeddings = [] for n, (x, y) in enumerate(dataset): pred_y = model(x).numpy() for m in pred_y: embeddings.append(n) for m in y: labels.append(m) # Set up a logs directory, so Tensorboard knows where to look for files if not os.path.exists(log_dir): os.makedirs(log_dir) # Save Labels separately on a line-by-line manner. with open(os.path.join(log_dir, 'metadata.tsv'), "w") as f: for l in labels: f.write("{}\n".format(l)) # Save the weights we want to analyse as a variable. Note that the first # value represents any unknown word, which is not in the metadata, so # we will remove that value. embedding_array = np.array(embeddings) embedding = tf.Variable(embedding_array, name='embedding') # Create a checkpoint from embedding, the filename and key are # name of the tensor. checkpoint = tf.train.Checkpoint(embedding=embedding) checkpoint.save(os.path.join(log_dir, "embedding.ckpt")) # Set up config config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = "embedding/.ATTRIBUTES/VARIABLE_VALUE" embedding.metadata_path = 'metadata.tsv' projector.visualize_embeddings(log_dir, config)
def train_cnn(): """Training CNN model.""" # Load sentences, labels, and training parameters logger.info("✔︎ Loading data...") logger.info("✔︎ Training data processing...") train_data = dh.load_data_and_labels(FLAGS.training_data_file, FLAGS.embedding_dim) logger.info("✔︎ Validation data processing...") validation_data = dh.load_data_and_labels(FLAGS.validation_data_file, FLAGS.embedding_dim) logger.info("Recommended padding Sequence length is: {0}".format(FLAGS.pad_seq_len)) logger.info("✔︎ Training data padding...") x_train_front, x_train_behind, y_train = dh.pad_data(train_data, FLAGS.pad_seq_len) logger.info("✔︎ Validation data padding...") x_validation_front, x_validation_behind, y_validation = dh.pad_data(validation_data, FLAGS.pad_seq_len) # Build vocabulary VOCAB_SIZE = dh.load_vocab_size(FLAGS.embedding_dim) pretrained_word2vec_matrix = dh.load_word2vec_matrix(VOCAB_SIZE, FLAGS.embedding_dim) # Build a graph and cnn object with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) session_conf.gpu_options.allow_growth = FLAGS.gpu_options_allow_growth sess = tf.Session(config=session_conf) with sess.as_default(): cnn = TextCNN( sequence_length=FLAGS.pad_seq_len, num_classes=y_train.shape[1], vocab_size=VOCAB_SIZE, fc_hidden_size=FLAGS.fc_hidden_size, embedding_size=FLAGS.embedding_dim, embedding_type=FLAGS.embedding_type, filter_sizes=list(map(int, FLAGS.filter_sizes.split(','))), num_filters=FLAGS.num_filters, l2_reg_lambda=FLAGS.l2_reg_lambda, pretrained_embedding=pretrained_word2vec_matrix) # Define training procedure with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)): learning_rate = tf.train.exponential_decay(learning_rate=FLAGS.learning_rate, global_step=cnn.global_step, decay_steps=FLAGS.decay_steps, decay_rate=FLAGS.decay_rate, staircase=True) optimizer = tf.train.AdamOptimizer(learning_rate) grads, vars = zip(*optimizer.compute_gradients(cnn.loss)) grads, _ = tf.clip_by_global_norm(grads, clip_norm=FLAGS.norm_ratio) train_op = optimizer.apply_gradients(zip(grads, vars), global_step=cnn.global_step, name="train_op") # Keep track of gradient values and sparsity (optional) grad_summaries = [] for g, v in zip(grads, vars): if g is not None: grad_hist_summary = tf.summary.histogram("{0}/grad/hist".format(v.name), g) sparsity_summary = tf.summary.scalar("{0}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g)) grad_summaries.append(grad_hist_summary) grad_summaries.append(sparsity_summary) grad_summaries_merged = tf.summary.merge(grad_summaries) # Output directory for models and summaries if FLAGS.train_or_restore == 'R': MODEL = input("☛ Please input the checkpoints model you want to restore, " "it should be like(1490175368): ") # The model you want to restore while not (MODEL.isdigit() and len(MODEL) == 10): MODEL = input("✘ The format of your input is illegal, please re-input: ") logger.info("✔︎ The format of your input is legal, now loading to next step...") out_dir = os.path.abspath(os.path.join(os.path.curdir, "runs", MODEL)) logger.info("✔︎ Writing to {0}\n".format(out_dir)) else: timestamp = str(int(time.time())) out_dir = os.path.abspath(os.path.join(os.path.curdir, "runs", timestamp)) logger.info("✔︎ Writing to {0}\n".format(out_dir)) checkpoint_dir = os.path.abspath(os.path.join(out_dir, "checkpoints")) best_checkpoint_dir = os.path.abspath(os.path.join(out_dir, "bestcheckpoints")) # Summaries for loss and accuracy loss_summary = tf.summary.scalar("loss", cnn.loss) acc_summary = tf.summary.scalar("accuracy", cnn.accuracy) # Train summaries train_summary_op = tf.summary.merge([loss_summary, acc_summary, grad_summaries_merged]) train_summary_dir = os.path.join(out_dir, "summaries", "train") train_summary_writer = tf.summary.FileWriter(train_summary_dir, sess.graph) # Validation summaries validation_summary_op = tf.summary.merge([loss_summary, acc_summary]) validation_summary_dir = os.path.join(out_dir, "summaries", "validation") validation_summary_writer = tf.summary.FileWriter(validation_summary_dir, sess.graph) saver = tf.train.Saver(tf.global_variables(), max_to_keep=FLAGS.num_checkpoints) best_saver = cm.BestCheckpointSaver(save_dir=best_checkpoint_dir, num_to_keep=3, maximize=True) if FLAGS.train_or_restore == 'R': # Load cnn model logger.info("✔︎ Loading model...") checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir) logger.info(checkpoint_file) # Load the saved meta graph and restore variables saver = tf.train.import_meta_graph("{0}.meta".format(checkpoint_file)) saver.restore(sess, checkpoint_file) else: if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) # Embedding visualization config config = projector.ProjectorConfig() embedding_conf = config.embeddings.add() embedding_conf.tensor_name = "embedding" embedding_conf.metadata_path = FLAGS.metadata_file projector.visualize_embeddings(train_summary_writer, config) projector.visualize_embeddings(validation_summary_writer, config) # Save the embedding visualization saver.save(sess, os.path.join(out_dir, "embedding", "embedding.ckpt")) current_step = sess.run(cnn.global_step) def train_step(x_batch_front, x_batch_behind, y_batch): """A single training step""" feed_dict = { cnn.input_x_front: x_batch_front, cnn.input_x_behind: x_batch_behind, cnn.input_y: y_batch, cnn.dropout_keep_prob: FLAGS.dropout_keep_prob, cnn.is_training: True } _, step, summaries, loss, accuracy = sess.run( [train_op, cnn.global_step, train_summary_op, cnn.loss, cnn.accuracy], feed_dict) logger.info("step {0}: loss {1:g}, acc {2:g}".format(step, loss, accuracy)) train_summary_writer.add_summary(summaries, step) def validation_step(x_batch_front, x_batch_behind, y_batch, writer=None): """Evaluates model on a validation set""" feed_dict = { cnn.input_x_front: x_batch_front, cnn.input_x_behind: x_batch_behind, cnn.input_y: y_batch, cnn.dropout_keep_prob: 1.0, cnn.is_training: False } step, summaries, loss, accuracy, recall, precision, f1, auc = sess.run( [cnn.global_step, validation_summary_op, cnn.loss, cnn.accuracy, cnn.recall, cnn.precision, cnn.F1, cnn.AUC], feed_dict) logger.info("step {0}: loss {1:g}, acc {2:g}, recall {3:g}, precision {4:g}, f1 {5:g}, AUC {6}" .format(step, loss, accuracy, recall, precision, f1, auc)) if writer: writer.add_summary(summaries, step) return accuracy # Generate batches batches = dh.batch_iter( list(zip(x_train_front, x_train_behind, y_train)), FLAGS.batch_size, FLAGS.num_epochs) num_batches_per_epoch = int((len(x_train_front) - 1) / FLAGS.batch_size) + 1 # Training loop. For each batch... for batch in batches: x_batch_front, x_batch_behind, y_batch = zip(*batch) train_step(x_batch_front, x_batch_behind, y_batch) current_step = tf.train.global_step(sess, cnn.global_step) if current_step % FLAGS.evaluate_every == 0: logger.info("\nEvaluation:") accuracy = validation_step(x_validation_front, x_validation_behind, y_validation, writer=validation_summary_writer) best_saver.handle(accuracy, sess, current_step) if current_step % FLAGS.checkpoint_every == 0: checkpoint_prefix = os.path.join(checkpoint_dir, "model") path = saver.save(sess, checkpoint_prefix, global_step=current_step) logger.info("✔︎ Saved model checkpoint to {0}\n".format(path)) if current_step % num_batches_per_epoch == 0: current_epoch = current_step // num_batches_per_epoch logger.info("✔︎ Epoch {0} has finished!".format(current_epoch)) logger.info("✔︎ Done.")
def train_rcnn(): """Training RCNN model.""" # Load sentences, labels, and training parameters logger.info("✔︎ Loading data...") logger.info("✔︎ Training data processing...") train_data = dh.load_data_and_labels(FLAGS.training_data_file, FLAGS.num_classes, FLAGS.embedding_dim, data_aug_flag=False) logger.info("✔︎ Validation data processing...") val_data = dh.load_data_and_labels(FLAGS.validation_data_file, FLAGS.num_classes, FLAGS.embedding_dim, data_aug_flag=False) logger.info("Recommended padding Sequence length is: {0}".format(FLAGS.pad_seq_len)) logger.info("✔︎ Training data padding...") x_train, y_train = dh.pad_data(train_data, FLAGS.pad_seq_len) logger.info("✔︎ Validation data padding...") x_val, y_val = dh.pad_data(val_data, FLAGS.pad_seq_len) # Build vocabulary VOCAB_SIZE = dh.load_vocab_size(FLAGS.embedding_dim) pretrained_word2vec_matrix = dh.load_word2vec_matrix(VOCAB_SIZE, FLAGS.embedding_dim) # Build a graph and rcnn object with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) session_conf.gpu_options.allow_growth = FLAGS.gpu_options_allow_growth sess = tf.Session(config=session_conf) with sess.as_default(): rcnn = TextRCNN( sequence_length=FLAGS.pad_seq_len, num_classes=FLAGS.num_classes, vocab_size=VOCAB_SIZE, lstm_hidden_size=FLAGS.lstm_hidden_size, fc_hidden_size=FLAGS.fc_hidden_size, embedding_size=FLAGS.embedding_dim, embedding_type=FLAGS.embedding_type, filter_sizes=list(map(int, FLAGS.filter_sizes.split(','))), num_filters=FLAGS.num_filters, l2_reg_lambda=FLAGS.l2_reg_lambda, pretrained_embedding=pretrained_word2vec_matrix) # Define training procedure with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)): learning_rate = tf.train.exponential_decay(learning_rate=FLAGS.learning_rate, global_step=rcnn.global_step, decay_steps=FLAGS.decay_steps, decay_rate=FLAGS.decay_rate, staircase=True) optimizer = tf.train.AdamOptimizer(learning_rate) grads, vars = zip(*optimizer.compute_gradients(rcnn.loss)) grads, _ = tf.clip_by_global_norm(grads, clip_norm=FLAGS.norm_ratio) train_op = optimizer.apply_gradients(zip(grads, vars), global_step=rcnn.global_step, name="train_op") # Keep track of gradient values and sparsity (optional) grad_summaries = [] for g, v in zip(grads, vars): if g is not None: grad_hist_summary = tf.summary.histogram("{0}/grad/hist".format(v.name), g) sparsity_summary = tf.summary.scalar("{0}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g)) grad_summaries.append(grad_hist_summary) grad_summaries.append(sparsity_summary) grad_summaries_merged = tf.summary.merge(grad_summaries) # Output directory for models and summaries if FLAGS.train_or_restore == 'R': MODEL = input("☛ Please input the checkpoints model you want to restore, " "it should be like(1490175368): ") # The model you want to restore while not (MODEL.isdigit() and len(MODEL) == 10): MODEL = input("✘ The format of your input is illegal, please re-input: ") logger.info("✔︎ The format of your input is legal, now loading to next step...") out_dir = os.path.abspath(os.path.join(os.path.curdir, "runs", MODEL)) logger.info("✔︎ Writing to {0}\n".format(out_dir)) else: timestamp = str(int(time.time())) out_dir = os.path.abspath(os.path.join(os.path.curdir, "runs", timestamp)) logger.info("✔︎ Writing to {0}\n".format(out_dir)) checkpoint_dir = os.path.abspath(os.path.join(out_dir, "checkpoints")) best_checkpoint_dir = os.path.abspath(os.path.join(out_dir, "bestcheckpoints")) # Summaries for loss loss_summary = tf.summary.scalar("loss", rcnn.loss) # Train summaries train_summary_op = tf.summary.merge([loss_summary, grad_summaries_merged]) train_summary_dir = os.path.join(out_dir, "summaries", "train") train_summary_writer = tf.summary.FileWriter(train_summary_dir, sess.graph) # Validation summaries validation_summary_op = tf.summary.merge([loss_summary]) validation_summary_dir = os.path.join(out_dir, "summaries", "validation") validation_summary_writer = tf.summary.FileWriter(validation_summary_dir, sess.graph) saver = tf.train.Saver(tf.global_variables(), max_to_keep=FLAGS.num_checkpoints) best_saver = cm.BestCheckpointSaver(save_dir=best_checkpoint_dir, num_to_keep=3, maximize=True) if FLAGS.train_or_restore == 'R': # Load rcnn model logger.info("✔︎ Loading model...") checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir) logger.info(checkpoint_file) # Load the saved meta graph and restore variables saver = tf.train.import_meta_graph("{0}.meta".format(checkpoint_file)) saver.restore(sess, checkpoint_file) else: if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) # Embedding visualization config config = projector.ProjectorConfig() embedding_conf = config.embeddings.add() embedding_conf.tensor_name = "embedding" embedding_conf.metadata_path = FLAGS.metadata_file projector.visualize_embeddings(train_summary_writer, config) projector.visualize_embeddings(validation_summary_writer, config) # Save the embedding visualization saver.save(sess, os.path.join(out_dir, "embedding", "embedding.ckpt")) current_step = sess.run(rcnn.global_step) def train_step(x_batch, y_batch): """A single training step""" feed_dict = { rcnn.input_x: x_batch, rcnn.input_y: y_batch, rcnn.dropout_keep_prob: FLAGS.dropout_keep_prob, rcnn.is_training: True } _, step, summaries, loss = sess.run( [train_op, rcnn.global_step, train_summary_op, rcnn.loss], feed_dict) logger.info("step {0}: loss {1:g}".format(step, loss)) train_summary_writer.add_summary(summaries, step) def validation_step(x_val, y_val, writer=None): """Evaluates model on a validation set""" batches_validation = dh.batch_iter(list(zip(x_val, y_val)), FLAGS.batch_size, 1) # Predict classes by threshold or topk ('ts': threshold; 'tk': topk) eval_counter, eval_loss = 0, 0.0 eval_pre_tk = [0.0] * FLAGS.top_num eval_rec_tk = [0.0] * FLAGS.top_num eval_F_tk = [0.0] * FLAGS.top_num true_onehot_labels = [] predicted_onehot_scores = [] predicted_onehot_labels_ts = [] predicted_onehot_labels_tk = [[] for _ in range(FLAGS.top_num)] for batch_validation in batches_validation: x_batch_val, y_batch_val = zip(*batch_validation) feed_dict = { rcnn.input_x: x_batch_val, rcnn.input_y: y_batch_val, rcnn.dropout_keep_prob: 1.0, rcnn.is_training: False } step, summaries, scores, cur_loss = sess.run( [rcnn.global_step, validation_summary_op, rcnn.scores, rcnn.loss], feed_dict) # Prepare for calculating metrics for i in y_batch_val: true_onehot_labels.append(i) for j in scores: predicted_onehot_scores.append(j) # Predict by threshold batch_predicted_onehot_labels_ts = \ dh.get_onehot_label_threshold(scores=scores, threshold=FLAGS.threshold) for k in batch_predicted_onehot_labels_ts: predicted_onehot_labels_ts.append(k) # Predict by topK for top_num in range(FLAGS.top_num): batch_predicted_onehot_labels_tk = dh.get_onehot_label_topk(scores=scores, top_num=top_num+1) for i in batch_predicted_onehot_labels_tk: predicted_onehot_labels_tk[top_num].append(i) eval_loss = eval_loss + cur_loss eval_counter = eval_counter + 1 if writer: writer.add_summary(summaries, step) eval_loss = float(eval_loss / eval_counter) # Calculate Precision & Recall & F1 (threshold & topK) eval_pre_ts = precision_score(y_true=np.array(true_onehot_labels), y_pred=np.array(predicted_onehot_labels_ts), average='micro') eval_rec_ts = recall_score(y_true=np.array(true_onehot_labels), y_pred=np.array(predicted_onehot_labels_ts), average='micro') eval_F_ts = f1_score(y_true=np.array(true_onehot_labels), y_pred=np.array(predicted_onehot_labels_ts), average='micro') for top_num in range(FLAGS.top_num): eval_pre_tk[top_num] = precision_score(y_true=np.array(true_onehot_labels), y_pred=np.array(predicted_onehot_labels_tk[top_num]), average='micro') eval_rec_tk[top_num] = recall_score(y_true=np.array(true_onehot_labels), y_pred=np.array(predicted_onehot_labels_tk[top_num]), average='micro') eval_F_tk[top_num] = f1_score(y_true=np.array(true_onehot_labels), y_pred=np.array(predicted_onehot_labels_tk[top_num]), average='micro') # Calculate the average AUC eval_auc = roc_auc_score(y_true=np.array(true_onehot_labels), y_score=np.array(predicted_onehot_scores), average='micro') # Calculate the average PR eval_prc = average_precision_score(y_true=np.array(true_onehot_labels), y_score=np.array(predicted_onehot_scores), average='micro') return eval_loss, eval_auc, eval_prc, eval_rec_ts, eval_pre_ts, eval_F_ts, \ eval_rec_tk, eval_pre_tk, eval_F_tk # Generate batches batches_train = dh.batch_iter( list(zip(x_train, y_train)), FLAGS.batch_size, FLAGS.num_epochs) num_batches_per_epoch = int((len(x_train) - 1) / FLAGS.batch_size) + 1 # Training loop. For each batch... for batch_train in batches_train: x_batch_train, y_batch_train = zip(*batch_train) train_step(x_batch_train, y_batch_train) current_step = tf.train.global_step(sess, rcnn.global_step) if current_step % FLAGS.evaluate_every == 0: logger.info("\nEvaluation:") eval_loss, eval_auc, eval_prc, \ eval_rec_ts, eval_pre_ts, eval_F_ts, eval_rec_tk, eval_pre_tk, eval_F_tk = \ validation_step(x_val, y_val, writer=validation_summary_writer) logger.info("All Validation set: Loss {0:g} | AUC {1:g} | AUPRC {2:g}" .format(eval_loss, eval_auc, eval_prc)) # Predict by threshold logger.info("☛ Predict by threshold: Precision {0:g}, Recall {1:g}, F {2:g}" .format(eval_pre_ts, eval_rec_ts, eval_F_ts)) # Predict by topK logger.info("☛ Predict by topK:") for top_num in range(FLAGS.top_num): logger.info("Top{0}: Precision {1:g}, Recall {2:g}, F {3:g}" .format(top_num+1, eval_pre_tk[top_num], eval_rec_tk[top_num], eval_F_tk[top_num])) best_saver.handle(eval_prc, sess, current_step) if current_step % FLAGS.checkpoint_every == 0: checkpoint_prefix = os.path.join(checkpoint_dir, "model") path = saver.save(sess, checkpoint_prefix, global_step=current_step) logger.info("✔︎ Saved model checkpoint to {0}\n".format(path)) if current_step % num_batches_per_epoch == 0: current_epoch = current_step // num_batches_per_epoch logger.info("✔︎ Epoch {0} has finished!".format(current_epoch)) logger.info("✔︎ Done.")
def train_han(): """Training HAN model.""" # Print parameters used for the model dh.tab_printer(args, logger) # Load sentences, labels, and training parameters logger.info("Loading data...") logger.info("Data processing...") train_data = dh.load_data_and_labels(args.train_file, args.num_classes, args.word2vec_file, data_aug_flag=False) val_data = dh.load_data_and_labels(args.validation_file, args.num_classes, args.word2vec_file, data_aug_flag=False) logger.info("Data padding...") x_train, y_train = dh.pad_data(train_data, args.pad_seq_len) x_val, y_val = dh.pad_data(val_data, args.pad_seq_len) # Build vocabulary VOCAB_SIZE, EMBEDDING_SIZE, pretrained_word2vec_matrix = dh.load_word2vec_matrix(args.word2vec_file) # Build a graph and han object with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=args.allow_soft_placement, log_device_placement=args.log_device_placement) session_conf.gpu_options.allow_growth = args.gpu_options_allow_growth sess = tf.Session(config=session_conf) with sess.as_default(): han = TextHAN( sequence_length=args.pad_seq_len, vocab_size=VOCAB_SIZE, embedding_type=args.embedding_type, embedding_size=EMBEDDING_SIZE, lstm_hidden_size=args.lstm_dim, fc_hidden_size=args.fc_dim, num_classes=args.num_classes, l2_reg_lambda=args.l2_lambda, pretrained_embedding=pretrained_word2vec_matrix) # Define training procedure with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)): learning_rate = tf.train.exponential_decay(learning_rate=args.learning_rate, global_step=han.global_step, decay_steps=args.decay_steps, decay_rate=args.decay_rate, staircase=True) optimizer = tf.train.AdamOptimizer(learning_rate) grads, vars = zip(*optimizer.compute_gradients(han.loss)) grads, _ = tf.clip_by_global_norm(grads, clip_norm=args.norm_ratio) train_op = optimizer.apply_gradients(zip(grads, vars), global_step=han.global_step, name="train_op") # Keep track of gradient values and sparsity (optional) grad_summaries = [] for g, v in zip(grads, vars): if g is not None: grad_hist_summary = tf.summary.histogram("{0}/grad/hist".format(v.name), g) sparsity_summary = tf.summary.scalar("{0}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g)) grad_summaries.append(grad_hist_summary) grad_summaries.append(sparsity_summary) grad_summaries_merged = tf.summary.merge(grad_summaries) # Output directory for models and summaries out_dir = dh.get_out_dir(OPTION, logger) checkpoint_dir = os.path.abspath(os.path.join(out_dir, "checkpoints")) best_checkpoint_dir = os.path.abspath(os.path.join(out_dir, "bestcheckpoints")) # Summaries for loss loss_summary = tf.summary.scalar("loss", han.loss) # Train summaries train_summary_op = tf.summary.merge([loss_summary, grad_summaries_merged]) train_summary_dir = os.path.join(out_dir, "summaries", "train") train_summary_writer = tf.summary.FileWriter(train_summary_dir, sess.graph) # Validation summaries validation_summary_op = tf.summary.merge([loss_summary]) validation_summary_dir = os.path.join(out_dir, "summaries", "validation") validation_summary_writer = tf.summary.FileWriter(validation_summary_dir, sess.graph) saver = tf.train.Saver(tf.global_variables(), max_to_keep=args.num_checkpoints) best_saver = cm.BestCheckpointSaver(save_dir=best_checkpoint_dir, num_to_keep=3, maximize=True) if OPTION == 'R': # Load han model logger.info("Loading model...") checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir) logger.info(checkpoint_file) # Load the saved meta graph and restore variables saver = tf.train.import_meta_graph("{0}.meta".format(checkpoint_file)) saver.restore(sess, checkpoint_file) if OPTION == 'T': if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) # Embedding visualization config config = projector.ProjectorConfig() embedding_conf = config.embeddings.add() embedding_conf.tensor_name = "embedding" embedding_conf.metadata_path = args.metadata_file projector.visualize_embeddings(train_summary_writer, config) projector.visualize_embeddings(validation_summary_writer, config) # Save the embedding visualization saver.save(sess, os.path.join(out_dir, "embedding", "embedding.ckpt")) current_step = sess.run(han.global_step) def train_step(x_batch, y_batch): """A single training step""" feed_dict = { han.input_x: x_batch, han.input_y: y_batch, han.dropout_keep_prob: args.dropout_rate, han.is_training: True } _, step, summaries, loss = sess.run( [train_op, han.global_step, train_summary_op, han.loss], feed_dict) logger.info("step {0}: loss {1:g}".format(step, loss)) train_summary_writer.add_summary(summaries, step) def validation_step(x_val, y_val, writer=None): """Evaluates model on a validation set""" batches_validation = dh.batch_iter(list(zip(x_val, y_val)), args.batch_size, 1) # Predict classes by threshold or topk ('ts': threshold; 'tk': topk) eval_counter, eval_loss = 0, 0.0 eval_pre_tk = [0.0] * args.topK eval_rec_tk = [0.0] * args.topK eval_F1_tk = [0.0] * args.topK true_onehot_labels = [] predicted_onehot_scores = [] predicted_onehot_labels_ts = [] predicted_onehot_labels_tk = [[] for _ in range(args.topK)] for batch_validation in batches_validation: x_batch_val, y_batch_val = zip(*batch_validation) feed_dict = { han.input_x: x_batch_val, han.input_y: y_batch_val, han.dropout_keep_prob: 1.0, han.is_training: False } step, summaries, scores, cur_loss = sess.run( [han.global_step, validation_summary_op, han.scores, han.loss], feed_dict) # Prepare for calculating metrics for i in y_batch_val: true_onehot_labels.append(i) for j in scores: predicted_onehot_scores.append(j) # Predict by threshold batch_predicted_onehot_labels_ts = \ dh.get_onehot_label_threshold(scores=scores, threshold=args.threshold) for k in batch_predicted_onehot_labels_ts: predicted_onehot_labels_ts.append(k) # Predict by topK for top_num in range(args.topK): batch_predicted_onehot_labels_tk = dh.get_onehot_label_topk(scores=scores, top_num=top_num+1) for i in batch_predicted_onehot_labels_tk: predicted_onehot_labels_tk[top_num].append(i) eval_loss = eval_loss + cur_loss eval_counter = eval_counter + 1 if writer: writer.add_summary(summaries, step) eval_loss = float(eval_loss / eval_counter) # Calculate Precision & Recall & F1 eval_pre_ts = precision_score(y_true=np.array(true_onehot_labels), y_pred=np.array(predicted_onehot_labels_ts), average='micro') eval_rec_ts = recall_score(y_true=np.array(true_onehot_labels), y_pred=np.array(predicted_onehot_labels_ts), average='micro') eval_F1_ts = f1_score(y_true=np.array(true_onehot_labels), y_pred=np.array(predicted_onehot_labels_ts), average='micro') for top_num in range(args.topK): eval_pre_tk[top_num] = precision_score(y_true=np.array(true_onehot_labels), y_pred=np.array(predicted_onehot_labels_tk[top_num]), average='micro') eval_rec_tk[top_num] = recall_score(y_true=np.array(true_onehot_labels), y_pred=np.array(predicted_onehot_labels_tk[top_num]), average='micro') eval_F1_tk[top_num] = f1_score(y_true=np.array(true_onehot_labels), y_pred=np.array(predicted_onehot_labels_tk[top_num]), average='micro') # Calculate the average AUC eval_auc = roc_auc_score(y_true=np.array(true_onehot_labels), y_score=np.array(predicted_onehot_scores), average='micro') # Calculate the average PR eval_prc = average_precision_score(y_true=np.array(true_onehot_labels), y_score=np.array(predicted_onehot_scores), average='micro') return eval_loss, eval_auc, eval_prc, eval_pre_ts, eval_rec_ts, eval_F1_ts, \ eval_pre_tk, eval_rec_tk, eval_F1_tk # Generate batches batches_train = dh.batch_iter( list(zip(x_train, y_train)), args.batch_size, args.epochs) num_batches_per_epoch = int((len(x_train) - 1) / args.batch_size) + 1 # Training loop. For each batch... for batch_train in batches_train: x_batch_train, y_batch_train = zip(*batch_train) train_step(x_batch_train, y_batch_train) current_step = tf.train.global_step(sess, han.global_step) if current_step % args.evaluate_steps == 0: logger.info("\nEvaluation:") eval_loss, eval_auc, eval_prc, \ eval_pre_ts, eval_rec_ts, eval_F1_ts, eval_pre_tk, eval_rec_tk, eval_F1_tk = \ validation_step(x_val, y_val, writer=validation_summary_writer) logger.info("All Validation set: Loss {0:g} | AUC {1:g} | AUPRC {2:g}" .format(eval_loss, eval_auc, eval_prc)) # Predict by threshold logger.info("Predict by threshold: Precision {0:g}, Recall {1:g}, F1 {2:g}" .format(eval_pre_ts, eval_rec_ts, eval_F1_ts)) # Predict by topK logger.info("Predict by topK:") for top_num in range(args.topK): logger.info("Top{0}: Precision {1:g}, Recall {2:g}, F1 {3:g}" .format(top_num+1, eval_pre_tk[top_num], eval_rec_tk[top_num], eval_F1_tk[top_num])) best_saver.handle(eval_prc, sess, current_step) if current_step % args.checkpoint_steps == 0: checkpoint_prefix = os.path.join(checkpoint_dir, "model") path = saver.save(sess, checkpoint_prefix, global_step=current_step) logger.info("Saved model checkpoint to {0}\n".format(path)) if current_step % num_batches_per_epoch == 0: current_epoch = current_step // num_batches_per_epoch logger.info("Epoch {0} has finished!".format(current_epoch)) logger.info("All Done.")
def train_mann(): """Training MANN model.""" # Load sentences, labels, and training parameters logger.info('✔︎ Loading data...') logger.info('✔︎ Training data processing...') train_data = dh.load_data_and_labels(FLAGS.training_data_file, FLAGS.num_classes, FLAGS.embedding_dim) logger.info('✔︎ Validation data processing...') validation_data = \ dh.load_data_and_labels(FLAGS.validation_data_file, FLAGS.num_classes, FLAGS.embedding_dim) logger.info('Recommended padding Sequence length is: {0}'.format( FLAGS.pad_seq_len)) logger.info('✔︎ Training data padding...') x_train, y_train = dh.pad_data(train_data, FLAGS.pad_seq_len) logger.info('✔︎ Validation data padding...') x_validation, y_validation = dh.pad_data(validation_data, FLAGS.pad_seq_len) # Build vocabulary VOCAB_SIZE = dh.load_vocab_size(FLAGS.embedding_dim) pretrained_word2vec_matrix = dh.load_word2vec_matrix( VOCAB_SIZE, FLAGS.embedding_dim) # Build a graph and mann object with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) session_conf.gpu_options.allow_growth = FLAGS.gpu_options_allow_growth sess = tf.Session(config=session_conf) with sess.as_default(): mann = TextMANN(sequence_length=FLAGS.pad_seq_len, num_classes=FLAGS.num_classes, batch_size=FLAGS.batch_size, vocab_size=VOCAB_SIZE, lstm_hidden_size=FLAGS.lstm_hidden_size, fc_hidden_size=FLAGS.fc_hidden_size, embedding_size=FLAGS.embedding_dim, embedding_type=FLAGS.embedding_type, l2_reg_lambda=FLAGS.l2_reg_lambda, pretrained_embedding=pretrained_word2vec_matrix) # Define training procedure with tf.control_dependencies( tf.get_collection(tf.GraphKeys.UPDATE_OPS)): learning_rate = tf.train.exponential_decay( learning_rate=FLAGS.learning_rate, global_step=mann.global_step, decay_steps=FLAGS.decay_steps, decay_rate=FLAGS.decay_rate, staircase=True) optimizer = tf.train.AdamOptimizer(learning_rate) grads, vars = zip(*optimizer.compute_gradients(mann.loss)) grads, _ = tf.clip_by_global_norm(grads, clip_norm=FLAGS.norm_ratio) train_op = optimizer.apply_gradients( zip(grads, vars), global_step=mann.global_step, name="train_op") # Keep track of gradient values and sparsity (optional) grad_summaries = [] for g, v in zip(grads, vars): if g is not None: grad_hist_summary = tf.summary.histogram( "{0}/grad/hist".format(v.name), g) sparsity_summary = tf.summary.scalar( "{0}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g)) grad_summaries.append(grad_hist_summary) grad_summaries.append(sparsity_summary) grad_summaries_merged = tf.summary.merge(grad_summaries) # Output directory for models and summaries if FLAGS.train_or_restore == 'R': MODEL = input( "☛ Please input the checkpoints model you want to restore, " "it should be like(1490175368): " ) # The model you want to restore while not (MODEL.isdigit() and len(MODEL) == 10): MODEL = input( '✘ The format of your input is illegal, please re-input: ' ) logger.info( '✔︎ The format of your input is legal, now loading to next step...' ) checkpoint_dir = 'runs/' + MODEL + '/checkpoints/' out_dir = os.path.abspath( os.path.join(os.path.curdir, "runs", MODEL)) logger.info("✔︎ Writing to {0}\n".format(out_dir)) else: timestamp = str(int(time.time())) out_dir = os.path.abspath( os.path.join(os.path.curdir, "runs", timestamp)) logger.info("✔︎ Writing to {0}\n".format(out_dir)) # Summaries for loss and accuracy loss_summary = tf.summary.scalar("loss", mann.loss) # Train summaries train_summary_op = tf.summary.merge( [loss_summary, grad_summaries_merged]) train_summary_dir = os.path.join(out_dir, "summaries", "train") train_summary_writer = tf.summary.FileWriter( train_summary_dir, sess.graph) # Validation summaries validation_summary_op = tf.summary.merge([loss_summary]) validation_summary_dir = os.path.join(out_dir, "summaries", "validation") validation_summary_writer = tf.summary.FileWriter( validation_summary_dir, sess.graph) saver = tf.train.Saver(tf.global_variables(), max_to_keep=FLAGS.num_checkpoints) if FLAGS.train_or_restore == 'R': # Load mann model logger.info("✔ Loading model...") checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir) logger.info(checkpoint_file) # Load the saved meta graph and restore variables saver = tf.train.import_meta_graph( "{0}.meta".format(checkpoint_file)) saver.restore(sess, checkpoint_file) else: checkpoint_dir = os.path.abspath( os.path.join(out_dir, "checkpoints")) if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) # Embedding visualization config config = projector.ProjectorConfig() embedding_conf = config.embeddings.add() embedding_conf.tensor_name = 'embedding' embedding_conf.metadata_path = FLAGS.metadata_file projector.visualize_embeddings(train_summary_writer, config) projector.visualize_embeddings(validation_summary_writer, config) # Save the embedding visualization saver.save( sess, os.path.join(out_dir, 'embedding', 'embedding.ckpt')) current_step = sess.run(mann.global_step) def train_step(x_batch, y_batch): """A single training step""" feed_dict = { mann.input_x: x_batch, mann.input_y: y_batch, mann.dropout_keep_prob: FLAGS.dropout_keep_prob, mann.is_training: True } _, step, summaries, loss = sess.run( [train_op, mann.global_step, train_summary_op, mann.loss], feed_dict) logger.info("step {0}: loss {1:g}".format(step, loss)) train_summary_writer.add_summary(summaries, step) def validation_step(x_validation, y_validation, writer=None): """Evaluates model on a validation set""" batches_validation = dh.batch_iter( list(zip(x_validation, y_validation)), FLAGS.batch_size, FLAGS.num_epochs) # Predict classes by threshold or topk ('ts': threshold; 'tk': topk) eval_counter, eval_loss, eval_rec_ts, eval_acc_ts, eval_F_ts = 0, 0.0, 0.0, 0.0, 0.0 eval_rec_tk = [0.0] * FLAGS.top_num eval_acc_tk = [0.0] * FLAGS.top_num eval_F_tk = [0.0] * FLAGS.top_num for batch_validation in batches_validation: x_batch_validation, y_batch_validation = zip( *batch_validation) feed_dict = { mann.input_x: x_batch_validation, mann.input_y: y_batch_validation, mann.dropout_keep_prob: 1.0, mann.is_training: False } step, summaries, scores, cur_loss = sess.run([ mann.global_step, validation_summary_op, mann.scores, mann.loss ], feed_dict) # Predict by threshold predicted_labels_threshold, predicted_values_threshold = \ dh.get_label_using_scores_by_threshold(scores=scores, threshold=FLAGS.threshold) cur_rec_ts, cur_acc_ts, cur_F_ts = 0.0, 0.0, 0.0 for index, predicted_label_threshold in enumerate( predicted_labels_threshold): rec_inc_ts, acc_inc_ts, F_inc_ts = dh.cal_metric( predicted_label_threshold, y_batch_validation[index]) cur_rec_ts, cur_acc_ts, cur_F_ts = cur_rec_ts + rec_inc_ts, \ cur_acc_ts + acc_inc_ts, \ cur_F_ts + F_inc_ts cur_rec_ts = cur_rec_ts / len(y_batch_validation) cur_acc_ts = cur_acc_ts / len(y_batch_validation) cur_F_ts = cur_F_ts / len(y_batch_validation) eval_rec_ts, eval_acc_ts, eval_F_ts = eval_rec_ts + cur_rec_ts, \ eval_acc_ts + cur_acc_ts, \ eval_F_ts + cur_F_ts # Predict by topK topK_predicted_labels = [] for top_num in range(FLAGS.top_num): predicted_labels_topk, predicted_values_topk = \ dh.get_label_using_scores_by_topk(scores=scores, top_num=top_num+1) topK_predicted_labels.append(predicted_labels_topk) cur_rec_tk = [0.0] * FLAGS.top_num cur_acc_tk = [0.0] * FLAGS.top_num cur_F_tk = [0.0] * FLAGS.top_num for top_num, predicted_labels_topK in enumerate( topK_predicted_labels): for index, predicted_label_topK in enumerate( predicted_labels_topK): rec_inc_tk, acc_inc_tk, F_inc_tk = dh.cal_metric( predicted_label_topK, y_batch_validation[index]) cur_rec_tk[top_num], cur_acc_tk[top_num], cur_F_tk[top_num] = \ cur_rec_tk[top_num] + rec_inc_tk, \ cur_acc_tk[top_num] + acc_inc_tk, \ cur_F_tk[top_num] + F_inc_tk cur_rec_tk[top_num] = cur_rec_tk[top_num] / len( y_batch_validation) cur_acc_tk[top_num] = cur_acc_tk[top_num] / len( y_batch_validation) cur_F_tk[top_num] = cur_F_tk[top_num] / len( y_batch_validation) eval_rec_tk[top_num], eval_acc_tk[top_num], eval_F_tk[top_num] = \ eval_rec_tk[top_num] + cur_rec_tk[top_num], \ eval_acc_tk[top_num] + cur_acc_tk[top_num], \ eval_F_tk[top_num] + cur_F_tk[top_num] eval_loss = eval_loss + cur_loss eval_counter = eval_counter + 1 logger.info("✔︎ validation batch {0}: loss {1:g}".format( eval_counter, cur_loss)) logger.info( "︎☛ Predict by threshold: recall {0:g}, accuracy {1:g}, F {2:g}" .format(cur_rec_ts, cur_acc_ts, cur_F_ts)) logger.info("︎☛ Predict by topK:") for top_num in range(FLAGS.top_num): logger.info( "Top{0}: recall {1:g}, accuracy {2:g}, F {3:g}". format(top_num + 1, cur_rec_tk[top_num], cur_acc_tk[top_num], cur_F_tk[top_num])) if writer: writer.add_summary(summaries, step) eval_loss = float(eval_loss / eval_counter) eval_rec_ts = float(eval_rec_ts / eval_counter) eval_acc_ts = float(eval_acc_ts / eval_counter) eval_F_ts = float(eval_F_ts / eval_counter) for top_num in range(FLAGS.top_num): eval_rec_tk[top_num] = float(eval_rec_tk[top_num] / eval_counter) eval_acc_tk[top_num] = float(eval_acc_tk[top_num] / eval_counter) eval_F_tk[top_num] = float(eval_F_tk[top_num] / eval_counter) return eval_loss, eval_rec_ts, eval_acc_ts, eval_F_ts, eval_rec_tk, eval_acc_tk, eval_F_tk # Generate batches batches_train = dh.batch_iter(list(zip(x_train, y_train)), FLAGS.batch_size, FLAGS.num_epochs) num_batches_per_epoch = int( (len(x_train) - 1) / FLAGS.batch_size) + 1 # Training loop. For each batch... for batch_train in batches_train: x_batch_train, y_batch_train = zip(*batch_train) train_step(x_batch_train, y_batch_train) current_step = tf.train.global_step(sess, mann.global_step) if current_step % FLAGS.evaluate_every == 0: logger.info("\nEvaluation:") eval_loss, eval_rec_ts, eval_acc_ts, eval_F_ts, eval_rec_tk, eval_acc_tk, eval_F_tk = \ validation_step(x_validation, y_validation, writer=validation_summary_writer) logger.info( "All Validation set: Loss {0:g}".format(eval_loss)) # Predict by threshold logger.info( "︎☛ Predict by threshold: Recall {0:g}, Accuracy {1:g}, F {2:g}" .format(eval_rec_ts, eval_acc_ts, eval_F_ts)) # Predict by topK logger.info("︎☛ Predict by topK:") for top_num in range(FLAGS.top_num): logger.info( "Top{0}: Recall {1:g}, Accuracy {2:g}, F {3:g}". format(top_num + 1, eval_rec_tk[top_num], eval_acc_tk[top_num], eval_F_tk[top_num])) if current_step % FLAGS.checkpoint_every == 0: checkpoint_prefix = os.path.join(checkpoint_dir, "model") path = saver.save(sess, checkpoint_prefix, global_step=current_step) logger.info( "✔︎ Saved model checkpoint to {0}\n".format(path)) if current_step % num_batches_per_epoch == 0: current_epoch = current_step // num_batches_per_epoch logger.info( "✔︎ Epoch {0} has finished!".format(current_epoch)) logger.info("✔︎ Done.")
def train_sann(): """Training RNN model.""" # Print parameters used for the model dh.tab_printer(args, logger) # Load word2vec model word2idx, embedding_matrix = dh.load_word2vec_matrix(args.word2vec_file) # Load sentences, labels, and training parameters logger.info("Loading data...") logger.info("Data processing...") train_data = dh.load_data_and_labels(args, args.train_file, word2idx) val_data = dh.load_data_and_labels(args, args.validation_file, word2idx) # Build a graph and sann object with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=args.allow_soft_placement, log_device_placement=args.log_device_placement) session_conf.gpu_options.allow_growth = args.gpu_options_allow_growth sess = tf.Session(config=session_conf) with sess.as_default(): sann = TextSANN(sequence_length=args.pad_seq_len, vocab_size=len(word2idx), embedding_type=args.embedding_type, embedding_size=args.embedding_dim, lstm_hidden_size=args.lstm_dim, attention_unit_size=args.attention_dim, attention_hops_size=args.attention_hops_dim, fc_hidden_size=args.fc_dim, num_classes=args.num_classes, l2_reg_lambda=args.l2_lambda, pretrained_embedding=embedding_matrix) # Define training procedure with tf.control_dependencies( tf.get_collection(tf.GraphKeys.UPDATE_OPS)): learning_rate = tf.train.exponential_decay( learning_rate=args.learning_rate, global_step=sann.global_step, decay_steps=args.decay_steps, decay_rate=args.decay_rate, staircase=True) optimizer = tf.train.AdamOptimizer(learning_rate) grads, vars = zip(*optimizer.compute_gradients(sann.loss)) grads, _ = tf.clip_by_global_norm(grads, clip_norm=args.norm_ratio) train_op = optimizer.apply_gradients( zip(grads, vars), global_step=sann.global_step, name="train_op") # Keep track of gradient values and sparsity (optional) grad_summaries = [] for g, v in zip(grads, vars): if g is not None: grad_hist_summary = tf.summary.histogram( "{0}/grad/hist".format(v.name), g) sparsity_summary = tf.summary.scalar( "{0}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g)) grad_summaries.append(grad_hist_summary) grad_summaries.append(sparsity_summary) grad_summaries_merged = tf.summary.merge(grad_summaries) # Output directory for models and summaries out_dir = dh.get_out_dir(OPTION, logger) checkpoint_dir = os.path.abspath( os.path.join(out_dir, "checkpoints")) best_checkpoint_dir = os.path.abspath( os.path.join(out_dir, "bestcheckpoints")) # Summaries for loss loss_summary = tf.summary.scalar("loss", sann.loss) # Train summaries train_summary_op = tf.summary.merge( [loss_summary, grad_summaries_merged]) train_summary_dir = os.path.join(out_dir, "summaries", "train") train_summary_writer = tf.summary.FileWriter( train_summary_dir, sess.graph) # Validation summaries validation_summary_op = tf.summary.merge([loss_summary]) validation_summary_dir = os.path.join(out_dir, "summaries", "validation") validation_summary_writer = tf.summary.FileWriter( validation_summary_dir, sess.graph) saver = tf.train.Saver(tf.global_variables(), max_to_keep=args.num_checkpoints) best_saver = cm.BestCheckpointSaver(save_dir=best_checkpoint_dir, num_to_keep=3, maximize=True) if OPTION == 'R': # Load sann model logger.info("Loading model...") checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir) logger.info(checkpoint_file) # Load the saved meta graph and restore variables saver = tf.train.import_meta_graph( "{0}.meta".format(checkpoint_file)) saver.restore(sess, checkpoint_file) if OPTION == 'T': if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) # Embedding visualization config config = projector.ProjectorConfig() embedding_conf = config.embeddings.add() embedding_conf.tensor_name = "embedding" embedding_conf.metadata_path = args.metadata_file projector.visualize_embeddings(train_summary_writer, config) projector.visualize_embeddings(validation_summary_writer, config) # Save the embedding visualization saver.save( sess, os.path.join(out_dir, "embedding", "embedding.ckpt")) current_step = sess.run(sann.global_step) def train_step(batch_data): """A single training step.""" x_f, x_b, y_onehot = zip(*batch_data) feed_dict = { sann.input_x_front: x_f, sann.input_x_behind: x_b, sann.input_y: y_onehot, sann.dropout_keep_prob: args.dropout_rate, sann.is_training: True } _, step, summaries, loss = sess.run( [train_op, sann.global_step, train_summary_op, sann.loss], feed_dict) logger.info("step {0}: loss {1:g}".format(step, loss)) train_summary_writer.add_summary(summaries, step) def validation_step(val_loader, writer=None): """Evaluates model on a validation set.""" batches_validation = dh.batch_iter( list(create_input_data(val_loader)), args.batch_size, 1) eval_counter, eval_loss = 0, 0.0 true_labels = [] predicted_scores = [] predicted_labels = [] for batch_validation in batches_validation: x_f, x_b, y_onehot = zip(*batch_validation) feed_dict = { sann.input_x_front: x_f, sann.input_x_behind: x_b, sann.input_y: y_onehot, sann.dropout_keep_prob: 1.0, sann.is_training: False } step, summaries, predictions, cur_loss = sess.run([ sann.global_step, validation_summary_op, sann.topKPreds, sann.loss ], feed_dict) # Prepare for calculating metrics for i in y_onehot: true_labels.append(np.argmax(i)) for j in predictions[0]: predicted_scores.append(j[0]) for k in predictions[1]: predicted_labels.append(k[0]) eval_loss = eval_loss + cur_loss eval_counter = eval_counter + 1 if writer: writer.add_summary(summaries, step) eval_loss = float(eval_loss / eval_counter) # Calculate Precision & Recall & F1 eval_acc = accuracy_score(y_true=np.array(true_labels), y_pred=np.array(predicted_labels)) eval_pre = precision_score(y_true=np.array(true_labels), y_pred=np.array(predicted_labels), average='micro') eval_rec = recall_score(y_true=np.array(true_labels), y_pred=np.array(predicted_labels), average='micro') eval_F1 = f1_score(y_true=np.array(true_labels), y_pred=np.array(predicted_labels), average='micro') # Calculate the average AUC eval_auc = roc_auc_score(y_true=np.array(true_labels), y_score=np.array(predicted_scores), average='micro') return eval_loss, eval_acc, eval_pre, eval_rec, eval_F1, eval_auc # Generate batches batches_train = dh.batch_iter(list(create_input_data(train_data)), args.batch_size, args.epochs) num_batches_per_epoch = int( (len(train_data['f_pad_seqs']) - 1) / args.batch_size) + 1 # Training loop. For each batch... for batch_train in batches_train: train_step(batch_train) current_step = tf.train.global_step(sess, sann.global_step) if current_step % args.evaluate_steps == 0: logger.info("\nEvaluation:") eval_loss, eval_acc, eval_pre, eval_rec, eval_F1, eval_auc = \ validation_step(val_data, writer=validation_summary_writer) logger.info( "All Validation set: Loss {0:g} | Acc {1:g} | Precision {2:g} | " "Recall {3:g} | F1 {4:g} | AUC {5:g}".format( eval_loss, eval_acc, eval_pre, eval_rec, eval_F1, eval_auc)) best_saver.handle(eval_acc, sess, current_step) if current_step % args.checkpoint_steps == 0: checkpoint_prefix = os.path.join(checkpoint_dir, "model") path = saver.save(sess, checkpoint_prefix, global_step=current_step) logger.info("Saved model checkpoint to {0}\n".format(path)) if current_step % num_batches_per_epoch == 0: current_epoch = current_step // num_batches_per_epoch logger.info( "Epoch {0} has finished!".format(current_epoch)) logger.info("All Done.")
checkpoint.save(os.path.join(embedding_path, "embedding.ckpt")) # Set up config config = projector.ProjectorConfig() embedding = config.embeddings.add() # The name of the tensor will be suffixed by `/.ATTRIBUTES/VARIABLE_VALUE` embedding.tensor_name = "embedding/.ATTRIBUTES/VARIABLE_VALUE" # Link this tensor to its metadata file (e.g. labels). embedding.metadata_path = 'metadata.tsv' # Comment out if you don't want sprites embedding.sprite.image_path = 'sprite.png' embedding.sprite.single_image_dim.extend( [img_data.shape[1], img_data.shape[1]]) # Saves a config file that TensorBoard will read during startup. projector.visualize_embeddings(embedding_path, config) #%% ''' # Use the iris dataset to illustrate PCA: import pandas as pd url = "https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data" # load dataset into Pandas DataFrame df = pd.read_csv(url, names=['sepal length','sepal width','petal length','petal width','target']) df.head() from sklearn.preprocessing import StandardScaler variables = ['sepal length','sepal width','petal length','petal width'] x = df.loc[:, variables].values y = df.loc[:,['target']].values x = StandardScaler().fit_transform(x)
# Report only errors by default if not args.verbose: os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" # Generate the embeddings for the projector tf.summary.create_file_writer(args.output_dir) with open(args.input_embeddings, "r") as embedding_file: _, dim = map(int, embedding_file.readline().split()) embeddings = np.zeros([args.elements, dim], np.float32) with open(os.path.join(args.output_dir, "metadata.tsv"), "w") as metadata_file: for i, line in zip(range(args.elements), embedding_file): form, *embedding = line.split() print(form, file=metadata_file) embeddings[i] = list(map(float, embedding)) # Save the variable embeddings = tf.Variable(embeddings, tf.float32) checkpoint = tf.train.Checkpoint(embeddings=embeddings) checkpoint.save(os.path.join(args.output_dir, "embeddings.ckpt")) # Set up the projector config config = projector.ProjectorConfig() embeddings = config.embeddings.add() # The name of the tensor will be suffixed by `/.ATTRIBUTES/VARIABLE_VALUE` embeddings.tensor_name = "embeddings/.ATTRIBUTES/VARIABLE_VALUE" embeddings.metadata_path = "metadata.tsv" projector.visualize_embeddings(args.output_dir, config)
PATH = os.getcwd() LOG_DIR = PATH + '/log-1/' metadata = os.path.join(LOG_DIR, 'metadata.tsv') Voices = tf.Variable(o.reshape((len(o), -1)), name='Voices') #def save_metadata(file): with open(metadata, 'w') as metadata_file: for row in range(702): c = emotion[y_train[row]] metadata_file.write('{}\n'.format(c)) with tf.compat.v1.Session() as sess: saver = tf.compat.v1.train.Saver([Voices]) sess.run(Voices.initializer) saver.save(sess, os.path.join(LOG_DIR, 'model.ckpt')) config = projector.ProjectorConfig() # One can add multiple embeddings. embedding = config.embeddings.add() embedding.tensor_name = Voices.name # Link this tensor to its metadata file (e.g. labels). embedding.metadata_path = metadata # Saves a config file that TensorBoard will read during startup. projector.visualize_embeddings(tf.compat.v1.summary.FileWriter(LOG_DIR), config) #get_ipython().system('tensorboard --logdir=./log-1/')
def build_model(self): with tf.variable_scope('Placeholder'): self.nodes_placeholder = tf.placeholder(tf.int32, (None, ), name='nodes_placeholder') self.seqlen_placeholder = tf.placeholder(tf.int32, (None, ), name='seqlen_placeholder') self.neighborhood_placeholder = tf.placeholder( tf.int32, (None, self.args.sampling_size), name='neighborhood_placeholder') self.label_placeholder = tf.placeholder(tf.float32, (None, ), name='label_placeholder') self.data = network.next_batch(self.graph, self.degree_max, sampling=True, sampling_size=self.args.sampling_size) with tf.variable_scope('Embeddings'): self.embeddings = tf.get_variable( 'embeddings', [len(self.graph), self.args.embedding_size], initializer=tf.constant_initializer( utils.init_embedding(self.degree, self.degree_max, self.args.embedding_size))) with tf.variable_scope('LSTM'): cell = tf.contrib.rnn.DropoutWrapper( #tf.contrib.rnn.BasicLSTMCell(num_units=self.args.embedding_size), tf.contrib.rnn.LayerNormBasicLSTMCell( num_units=self.args.embedding_size, layer_norm=False), input_keep_prob=1.0, output_keep_prob=1.0) _, states = tf.nn.dynamic_rnn( cell, tf.nn.embedding_lookup(self.embeddings, self.neighborhood_placeholder), dtype=tf.float32, sequence_length=self.seqlen_placeholder) self.lstm_output = states.h with tf.variable_scope('Guilded'): self.predict_info = tf.squeeze( tf.layers.dense(self.lstm_output, units=1, activation=utils.selu)) with tf.variable_scope('Loss'): self.structure_loss = tf.losses.mean_squared_error( tf.nn.embedding_lookup(self.embeddings, self.nodes_placeholder), self.lstm_output) self.guilded_loss = tf.reduce_mean( tf.abs(tf.subtract(self.predict_info, self.label_placeholder))) self.orth_loss = tf.losses.mean_squared_error( tf.matmul(self.embeddings, self.embeddings, transpose_a=True), tf.eye(self.args.embedding_size)) self.total_loss = self.structure_loss + self.args.alpha * self.orth_loss + self.args.lamb * self.guilded_loss with tf.variable_scope('Optimizer'): #self.optimizer = tf.train.AdamOptimizer(self.args.learning_rate) self.optimizer = tf.train.RMSPropOptimizer(self.args.learning_rate) tvars = tf.trainable_variables() grads, self.global_norm = tf.clip_by_global_norm( tf.gradients(self.total_loss, tvars), self.args.grad_clip) self.train_op = self.optimizer.apply_gradients(zip(grads, tvars)) with tf.variable_scope('Summary'): tf.summary.scalar("orth_loss", self.orth_loss) tf.summary.scalar("guilded_loss", self.guilded_loss) tf.summary.scalar("structure_loss", self.structure_loss) tf.summary.scalar("total_loss", self.total_loss) tf.summary.scalar("globol_norm", self.global_norm) for (grad, var) in zip(grads, tvars): if grad is not None: tf.summary.histogram('grad/{}'.format(var.name), grad) tf.summary.histogram('weight/{}'.format(var.name), var) log_dir = os.path.join(self.save_path, 'logs') if os.path.exists(log_dir): shutil.rmtree(log_dir) self.summary_writer = tf.summary.FileWriter( log_dir, self.sess.graph) config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = self.embeddings.name embedding.metadata_path = os.path.join( os.path.join(self.args.save_path, 'data', 'index.tsv')) projector.visualize_embeddings(self.summary_writer, config) self.merged_summary = tf.summary.merge_all() self.saver = tf.train.Saver() self.sess.run(tf.global_variables_initializer())
def set_model(self, model): """Sets Keras model and creates summary ops.""" self.model = model self._init_writer(model) # histogram summaries only enabled in graph mode if not context.executing_eagerly(): self._make_histogram_ops(model) self.merged = tf_summary.merge_all() # If both embedding_freq and embeddings_data are available, we will # visualize embeddings. if self.embeddings_freq and self.embeddings_data is not None: # Avoid circular dependency. from tensorflow.python.keras.engine import training_utils # pylint: disable=g-import-not-at-top self.embeddings_data = training_utils.standardize_input_data( self.embeddings_data, model.input_names) # If embedding_layer_names are not provided, get all of the embedding # layers from the model. embeddings_layer_names = self.embeddings_layer_names if not embeddings_layer_names: embeddings_layer_names = [ layer.name for layer in self.model.layers if type(layer).__name__ == 'Embedding' ] self.assign_embeddings = [] embeddings_vars = {} self.batch_id = batch_id = array_ops.placeholder(dtypes.int32) self.step = step = array_ops.placeholder(dtypes.int32) for layer in self.model.layers: if layer.name in embeddings_layer_names: embedding_input = self.model.get_layer(layer.name).output embedding_size = np.prod(embedding_input.shape[1:]) embedding_input = array_ops.reshape(embedding_input, (step, int(embedding_size))) shape = (self.embeddings_data[0].shape[0], int(embedding_size)) embedding = variables.Variable( array_ops.zeros(shape), name=layer.name + '_embedding') embeddings_vars[layer.name] = embedding batch = state_ops.assign(embedding[batch_id:batch_id + step], embedding_input) self.assign_embeddings.append(batch) self.saver = saver.Saver(list(embeddings_vars.values())) # Create embeddings_metadata dictionary if isinstance(self.embeddings_metadata, str): embeddings_metadata = { layer_name: self.embeddings_metadata for layer_name in embeddings_vars.keys() } else: # If embedding_metadata is already a dictionary embeddings_metadata = self.embeddings_metadata try: from tensorboard.plugins import projector except ImportError: raise ImportError('Failed to import TensorBoard. Please make sure that ' 'TensorBoard integration is complete."') # TODO(psv): Add integration tests to test embedding visualization # with TensorBoard callback. We are unable to write a unit test for this # because TensorBoard dependency assumes TensorFlow package is installed. config = projector.ProjectorConfig() for layer_name, tensor in embeddings_vars.items(): embedding = config.embeddings.add() embedding.tensor_name = tensor.name if (embeddings_metadata is not None and layer_name in embeddings_metadata): embedding.metadata_path = embeddings_metadata[layer_name] projector.visualize_embeddings(self.writer, config)
def set_model(self, model): """Sets Keras model and creates summary ops.""" self.model = model self.sess = K.get_session() # only make histogram summary op if it hasn't already been made if self.histogram_freq and self.merged is None: for layer in self.model.layers: for weight in layer.weights: mapped_weight_name = weight.name.replace(':', '_') tf_summary.histogram(mapped_weight_name, weight) if self.write_images: w_img = array_ops.squeeze(weight) shape = K.int_shape(w_img) if len(shape) == 2: # dense layer kernel case if shape[0] > shape[1]: w_img = array_ops.transpose(w_img) shape = K.int_shape(w_img) w_img = array_ops.reshape(w_img, [1, shape[0], shape[1], 1]) elif len(shape) == 3: # convnet case if K.image_data_format() == 'channels_last': # switch to channels_first to display # every kernel as a separate image w_img = array_ops.transpose(w_img, perm=[2, 0, 1]) shape = K.int_shape(w_img) w_img = array_ops.reshape(w_img, [shape[0], shape[1], shape[2], 1]) elif len(shape) == 1: # bias case w_img = array_ops.reshape(w_img, [1, shape[0], 1, 1]) else: # not possible to handle 3D convnets etc. continue shape = K.int_shape(w_img) assert len(shape) == 4 and shape[-1] in [1, 3, 4] tf_summary.image(mapped_weight_name, w_img) if self.write_grads: for weight in layer.trainable_weights: mapped_weight_name = weight.name.replace(':', '_') grads = model.optimizer.get_gradients(model.total_loss, weight) def is_indexed_slices(grad): return type(grad).__name__ == 'IndexedSlices' grads = [grad.values if is_indexed_slices(grad) else grad for grad in grads] tf_summary.histogram('{}_grad'.format(mapped_weight_name), grads) if hasattr(layer, 'output'): if isinstance(layer.output, list): for i, output in enumerate(layer.output): tf_summary.histogram('{}_out_{}'.format(layer.name, i), output) else: tf_summary.histogram('{}_out'.format(layer.name), layer.output) self.merged = tf_summary.merge_all() if self.write_graph: self.writer = self._writer_class(self.log_dir, self.sess.graph) else: self.writer = self._writer_class(self.log_dir) # If both embedding_freq and embeddings_data are available, we will # visualize embeddings. if self.embeddings_freq and self.embeddings_data is not None: self.embeddings_data = standardize_input_data(self.embeddings_data, model.input_names) # If embedding_layer_names are not provided, get all of the embedding # layers from the model. embeddings_layer_names = self.embeddings_layer_names if not embeddings_layer_names: embeddings_layer_names = [ layer.name for layer in self.model.layers if type(layer).__name__ == 'Embedding' ] self.assign_embeddings = [] embeddings_vars = {} self.batch_id = batch_id = array_ops.placeholder(dtypes.int32) self.step = step = array_ops.placeholder(dtypes.int32) for layer in self.model.layers: if layer.name in embeddings_layer_names: embedding_input = self.model.get_layer(layer.name).output embedding_size = np.prod(embedding_input.shape[1:]) embedding_input = array_ops.reshape(embedding_input, (step, int(embedding_size))) shape = (self.embeddings_data[0].shape[0], int(embedding_size)) embedding = variables.Variable( array_ops.zeros(shape), name=layer.name + '_embedding') embeddings_vars[layer.name] = embedding batch = state_ops.assign(embedding[batch_id:batch_id + step], embedding_input) self.assign_embeddings.append(batch) self.saver = saver.Saver(list(embeddings_vars.values())) # Create embeddings_metadata dictionary if isinstance(self.embeddings_metadata, str): embeddings_metadata = { layer_name: self.embeddings_metadata for layer_name in embeddings_vars.keys() } else: # If embedding_metadata is already a dictionary embeddings_metadata = self.embeddings_metadata try: from tensorboard.plugins import projector except ImportError: raise ImportError('Failed to import TensorBoard. Please make sure that ' 'TensorBoard integration is complete."') # TODO(psv): Add integration tests to test embedding visualization # with TensorBoard callback. We are unable to write a unit test for this # because TensorBoard dependency assumes TensorFlow package is installed. config = projector.ProjectorConfig() for layer_name, tensor in embeddings_vars.items(): embedding = config.embeddings.add() embedding.tensor_name = tensor.name if (embeddings_metadata is not None and layer_name in embeddings_metadata): embedding.metadata_path = embeddings_metadata[layer_name] projector.visualize_embeddings(self.writer, config)