def visualize_char(model, path="/home/aegis/igor/LM4paper/tests/textchar.txt", ): chars = open(path, 'r').read().splitlines() embedding = np.empty(shape=(len(chars), model.hps.emb_char_size), dtype=np.float32) for i, char in enumerate(chars): embedding[i] = model.get_char_embedding(char) print(embedding) print(embedding.shape) logdir = "/data/visualog/char/" metadata = os.path.join(logdir, "metadata.tsv") with open(metadata, "w") as metadata_file: for c in chars: metadata_file.write("%s\n" % c) tf.reset_default_graph() with tf.Session() as sess: X = tf.Variable([0.0], name='embedding') place = tf.placeholder(tf.float32, shape=embedding.shape) set_x = tf.assign(X, place, validate_shape=False) sess.run(tf.global_variables_initializer()) sess.run(set_x, feed_dict={place: embedding}) saver = tf.train.Saver([X]) saver.save(sess, os.path.join(logdir, 'char.ckpt')) config = projector.ProjectorConfig() # One can add multiple embeddings. embedding = config.embeddings.add() embedding.tensor_name = X.name # Link this tensor to its metadata file (e.g. labels). embedding.metadata_path = metadata # Saves a config file that TensorBoard will read during startup. projector.visualize_embeddings(tf.summary.FileWriter(logdir), config)
def generate_embeddings(): # Import data mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True, fake_data=FLAGS.fake_data) sess = tf.InteractiveSession() # Input set for Embedded TensorBoard visualization # Performed with cpu to conserve memory and processing power with tf.device("/cpu:0"): embedding = tf.Variable(tf.stack(mnist.test.images[:FLAGS.max_steps], axis=0), trainable=False, name='embedding') tf.global_variables_initializer().run() saver = tf.train.Saver() writer = tf.summary.FileWriter(FLAGS.log_dir + '/projector', sess.graph) # Add embedding tensorboard visualization. Need tensorflow version # >= 0.12.0RC0 config = projector.ProjectorConfig() embed= config.embeddings.add() embed.tensor_name = 'embedding:0' embed.metadata_path = os.path.join(FLAGS.log_dir + '/projector/metadata.tsv') embed.sprite.image_path = os.path.join(FLAGS.data_dir + '/mnist_10k_sprite.png') # Specify the width and height of a single thumbnail. embed.sprite.single_image_dim.extend([28, 28]) projector.visualize_embeddings(writer, config) saver.save(sess, os.path.join( FLAGS.log_dir, 'projector/a_model.ckpt'), global_step=FLAGS.max_steps)
def set_model(self, model): if self.embeddings_freq: self.saver = tf.train.Saver() embeddings_layer_names = self.embeddings_layer_names elayers = find_embedding_layers(model.layers) if not embeddings_layer_names: embeddings_layer_names = [layer.name for layer in elayers] embeddings = {layer.name: layer.weights[0] for layer in elayers if layer.name in embeddings_layer_names} embeddings_metadata = {} if not isinstance(self.embeddings_metadata, str): embeddings_metadata = self.embeddings_metadata else: embeddings_metadata = {layer_name: self.embeddings_metadata for layer_name in embeddings.keys()} config = projector.ProjectorConfig() self.embeddings_logs = [] for layer_name, tensor in embeddings.items(): embedding = config.embeddings.add() embedding.tensor_name = tensor.name self.embeddings_logs.append(os.path.join(self.log_dir, layer_name + '.ckpt')) if layer_name in embeddings_metadata: embedding.metadata_path = embeddings_metadata[layer_name] projector.visualize_embeddings(self.writer, config)
def visualize(self, visual_fld, num_visualize): """ run "'tensorboard --logdir='visualization'" to see the embeddings """ # create the list of num_variable most common words to visualize word2vec_utils.most_common_words(visual_fld, num_visualize) saver = tf.train.Saver() with tf.Session() as sess: sess.run(tf.global_variables_initializer()) ckpt = tf.train.get_checkpoint_state(os.path.dirname('checkpoints/checkpoint')) # if that checkpoint exists, restore from checkpoint if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) final_embed_matrix = sess.run(self.embed_matrix) # you have to store embeddings in a new variable embedding_var = tf.Variable(final_embed_matrix[:num_visualize], name='embedding') sess.run(embedding_var.initializer) config = projector.ProjectorConfig() summary_writer = tf.summary.FileWriter(visual_fld) # add embedding to the config file embedding = config.embeddings.add() embedding.tensor_name = embedding_var.name # link this tensor to its metadata file, in this case the first NUM_VISUALIZE words of vocab embedding.metadata_path = 'vocab_' + str(num_visualize) + '.tsv' # saves a configuration file that TensorBoard will read during startup. projector.visualize_embeddings(summary_writer, config) saver_embed = tf.train.Saver([embedding_var]) saver_embed.save(sess, os.path.join(visual_fld, 'model.ckpt'), 1)
def test(self, step, number=400): # 256 self.batch_size session = sess = self.session config = projector.ProjectorConfig() if visualize_cluster: embedding = config.embeddings.add() # You can add multiple embeddings. Here just one. embedding.tensor_name = self.last_layer.name # last_dense # embedding.tensor_path # embedding.tensor_shape embedding.sprite.image_path = PATH_TO_SPRITE_IMAGE # help(embedding.sprite) embedding.sprite.single_image_dim.extend([width, hight]) # if mnist thumbnail # embedding.single_image_dim.extend([28, 28]) # if mnist thumbnail # Link this tensor to its metadata file (e.g. labels). embedding.metadata_path = os.path.join(LOG_DIR, 'metadata.tsv') # Saves a configuration file that TensorBoard will read during startup. projector.visualize_embeddings(self.summary_writer, config) run_metadata = tf.RunMetadata() run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) # Calculate accuracy for 256 mnist test images test_images, test_labels = self.next_batch(number, session, test=True) feed_dict = {self.x: test_images, self.y: test_labels, self.keep_prob: 1., self.train_phase: False} # accuracy,summary= self.session.run([self.accuracy, self.summaries], feed_dict=feed_dict) accuracy, summary = session.run([self.accuracy, self.summaries], feed_dict, run_options, run_metadata) print('\t' * 3 + "Test Accuracy: ", accuracy) self.summary_writer.add_run_metadata(run_metadata, 'step #%03d' % step) self.summary_writer.add_summary(summary, global_step=step)
def write_embeddings(self, Wv, name="WordVectors"): """Write embedding matrix to the right place. Args: Wv: (numpy.ndarray) |V| x d matrix of word embeddings """ with tf.Graph().as_default(), tf.Session() as session: ## # Feed embeddings to tf, and save. embedding_var = tf.Variable(Wv, name=name, dtype=tf.float32) session.run(tf.global_variables_initializer()) saver = tf.train.Saver() saver.save(session, self.CHECKPOINT_FILE, 0) ## # Save metadata summary_writer = tf.summary.FileWriter(self.LOGDIR) config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = embedding_var.name embedding.metadata_path = self.VOCAB_FILE_BASE projector.visualize_embeddings(summary_writer, config) msg = "Saved {s0:d} x {s1:d} embedding matrix '{name}'" msg += " to LOGDIR='{logdir}'" print(msg.format(s0=Wv.shape[0], s1=Wv.shape[1], name=name, logdir=self.LOGDIR)) print("To view, run:") print("\n tensorboard --logdir=\"{logdir}\"\n".format(logdir=self.LOGDIR)) print("and navigate to the \"Embeddings\" tab in the web interface.")
def save_embeddings(model: adagram.VectorModel, output: Path, words: List[str]): labels = [] senses = [] for word in words: for sense, _ in model.word_sense_probs(word): labels.append('{} #{}'.format(word, sense)) v = model.sense_vector(word, sense) senses.append(v / np.linalg.norm(v)) output.mkdir(exist_ok=True) labels_path = output.joinpath('labels.tsv') labels_path.write_text('\n'.join(labels)) senses = np.array(senses) with tf.Session() as session: embedding_var = tf.Variable(senses, trainable=False, name='senses') session.run(tf.global_variables_initializer()) saver = tf.train.Saver() saver.save(session, str(output.joinpath('model.ckpt'))) summary_writer = tf.train.SummaryWriter(str(output)) config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = embedding_var.name embedding.metadata_path = str(labels_path) projector.visualize_embeddings(summary_writer, config)
def save_tb_embeddings(embeddings_filename): f = open(embeddings_filename, 'rb') embeddings = pickle.load(f) images = embeddings['images'] zs = embeddings['zs'] # overwrite Tensorboard log dir if necessary if os.path.exists(TB_DIR): shutil.rmtree(TB_DIR) os.makedirs(TB_DIR) # create grid image img_width, img_height = save_sprite_image(images) with tf.device('cpu:0'): # create embedding var embedding_var = tf.Variable(initial_value=zs) # save projector config summary_writer = tf.summary.FileWriter(TB_DIR) config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = embedding_var.name embedding.sprite.image_path = SPRITE_IMAGE_FILENAME embedding.sprite.single_image_dim.extend([img_width, img_height]) projector.visualize_embeddings(summary_writer, config) # save embeddings sess = tf.Session() sess.run(embedding_var.initializer) saver = tf.train.Saver([embedding_var]) saver.save(sess, os.path.join(TB_DIR, 'model.ckpt'))
def main(vectors_loc, out_loc, name="spaCy_vectors"): meta_file = "{}.tsv".format(name) out_meta_file = path.join(out_loc, meta_file) print('Loading spaCy vectors model: {}'.format(vectors_loc)) model = spacy.load(vectors_loc) print('Finding lexemes with vectors attached: {}'.format(vectors_loc)) strings_stream = tqdm.tqdm(model.vocab.strings, total=len(model.vocab.strings), leave=False) queries = [w for w in strings_stream if model.vocab.has_vector(w)] vector_count = len(queries) print('Building Tensorboard Projector metadata for ({}) vectors: {}'.format(vector_count, out_meta_file)) # Store vector data in a tensorflow variable tf_vectors_variable = numpy.zeros((vector_count, model.vocab.vectors.shape[1])) # Write a tab-separated file that contains information about the vectors for visualization # # Reference: https://www.tensorflow.org/programmers_guide/embedding#metadata with open(out_meta_file, 'wb') as file_metadata: # Define columns in the first row file_metadata.write("Text\tFrequency\n".encode('utf-8')) # Write out a row for each vector that we add to the tensorflow variable we created vec_index = 0 for text in tqdm.tqdm(queries, total=len(queries), leave=False): # https://github.com/tensorflow/tensorflow/issues/9094 text = '<Space>' if text.lstrip() == '' else text lex = model.vocab[text] # Store vector data and metadata tf_vectors_variable[vec_index] = model.vocab.get_vector(text) file_metadata.write("{}\t{}\n".format(text, math.exp(lex.prob) * vector_count).encode('utf-8')) vec_index += 1 print('Running Tensorflow Session...') sess = tf.InteractiveSession() tf.Variable(tf_vectors_variable, trainable=False, name=name) tf.global_variables_initializer().run() saver = tf.train.Saver() writer = tf.summary.FileWriter(out_loc, sess.graph) # Link the embeddings into the config config = ProjectorConfig() embed = config.embeddings.add() embed.tensor_name = name embed.metadata_path = meta_file # Tell the projector about the configured embeddings and metadata file visualize_embeddings(writer, config) # Save session and print run command to the output print('Saving Tensorboard Session...') saver.save(sess, path.join(out_loc, '{}.ckpt'.format(name))) print('Done. Run `tensorboard --logdir={0}` to view in Tensorboard'.format(out_loc))
def _add_emb_vis(self, embedding_var): """Do setup so that we can view word embedding visualization in Tensorboard, as described here: https://www.tensorflow.org/get_started/embedding_viz Make the vocab metadata file, then make the projector config file pointing to it.""" train_dir = os.path.join(FLAGS.log_root, "train") vocab_metadata_path = os.path.join(train_dir, "vocab_metadata.tsv") self._vocab.write_metadata(vocab_metadata_path) # write metadata file summary_writer = tf.summary.FileWriter(train_dir) config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = embedding_var.name embedding.metadata_path = vocab_metadata_path projector.visualize_embeddings(summary_writer, config)
def testVisualizeEmbeddings(self): # Create a dummy configuration. config = projector_config_pb2.ProjectorConfig() config.model_checkpoint_path = 'test' emb1 = config.embeddings.add() emb1.tensor_name = 'tensor1' emb1.metadata_path = 'metadata1' # Call the API method to save the configuration to a temporary dir. temp_dir = self.get_temp_dir() self.addCleanup(shutil.rmtree, temp_dir) writer = writer_lib.FileWriter(temp_dir) projector.visualize_embeddings(writer, config) # Read the configuratin from disk and make sure it matches the original. with gfile.GFile(os.path.join(temp_dir, 'projector_config.pbtxt')) as f: config2 = projector_config_pb2.ProjectorConfig() text_format.Parse(f.read(), config2) self.assertEqual(config, config2)
def __get_tensorboard_writer(self, path): tensorboard_writer = tf.summary.FileWriter(path, graph=self.graph, filename_suffix=".bot") # set the projector's configuration to add the embedding summary also: conf = projector.ProjectorConfig() embedding_field = conf.embeddings.add() embedding_content_label = conf.embeddings.add() # set the tensors to these embedding matrices embedding_field.tensor_name = self.field_embedding_matrix.name embedding_content_label.tensor_name = self.content_label_embedding_matrix.name # add the metadata paths to these embedding_summaries: embedding_field.metadata_path = os.path.join("..", "Metadata/fields.vocab") embedding_content_label.metadata_path = os.path.join("..", "Metadata/content_labels.vocab") # save the configuration file for this projector.visualize_embeddings(tensorboard_writer, conf) # return the so created tensorboard_writer return tensorboard_writer
def run_latent(model, n_images, data_out_path, sprite=True): tensorboard_path = os.path.join(data_out_path, 'tensorboard') saver = tf.train.Saver() with tf.Session() as session: # Initializer and restoring model. session.run(tf.global_variables_initializer()) check = get_checkpoint(data_out_path) saver.restore(session, check) # Inputs for tensorboard. tf_data = tf.Variable(tf.zeros((n_images, model.z_dim)), name='tf_data') input_sample = tf.placeholder(tf.float32, shape=(n_images, model.z_dim)) set_tf_data = tf.assign(tf_data, input_sample, validate_shape=False) if sprite: # Sample images. gen_samples, sample_z = show_generated(session=session, z_input=model.z_input, z_dim=model.z_dim, output_fake=model.output_gen, n_images=n_images, show=False) # Generate sprite of images. write_sprite_image(filename=os.path.join(data_out_path, 'gen_sprite.png'), data=gen_samples) else: sample_z = np.random.uniform(low=-1., high=1., size=(n_images, model.z_dim)) # Variable for embedding. saver_latent = tf.train.Saver([tf_data]) session.run(set_tf_data, feed_dict={input_sample: sample_z}) saver_latent.save(sess=session, save_path=os.path.join(tensorboard_path, 'tf_data.ckpt')) # Tensorflow embedding. config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = tf_data.name if sprite: embedding.metadata_path = os.path.join(data_out_path, 'metadata.tsv') embedding.sprite.image_path = os.path.join(data_out_path, 'gen_sprite.png') embedding.sprite.single_image_dim.extend([model.image_height, model.image_width]) projector.visualize_embeddings(tf.summary.FileWriter(tensorboard_path), config)
def visualize_embeddings(self, sess, tensor, name): """ Visualises an embedding vector into Tensorboard :param sess: Tensorflow session object :param tensor: The embedding tensor to be visualizd :param name: Name of the tensor """ # make directory if not exist if not tf.os.path.exists(self.save_dir): tf.os.makedirs(self.save_dir) # summary writer summary_writer = tf.summary.FileWriter(self.save_dir, graph=tf.get_default_graph()) # embedding visualizer config = projector.ProjectorConfig() emb = config.embeddings.add() emb.tensor_name = name # tensor emb.metadata_path = tf.os.path.join(self.DEFAULT_META_DATA_DIR, self.meta_file) # metadata file print(tf.os.path.abspath(emb.metadata_path)) projector.visualize_embeddings(summary_writer, config)
def embedding_view(EMB, y, EMB_C, sess, opt): EMB = [(x / np.linalg.norm(x)).tolist() for x in EMB] EMB_C = [(x / np.linalg.norm(x) ).tolist() for x in EMB_C] embedding_var = tf.Variable(EMB + EMB_C, name='Embedding_of_sentence') sess.run(embedding_var.initializer) EB_summary_writer = tf.summary.FileWriter(opt.log_path) config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.metadata_path = os.path.join(opt.log_path, 'metadata.tsv') projector.visualize_embeddings(EB_summary_writer, config) saver = tf.train.Saver([embedding_var]) saver.save(sess, os.path.join(opt.log_path, 'model2.ckpt'), 1) metadata_file = open(os.path.join(opt.log_path, 'metadata.tsv'), 'w') metadata_file.write('ClassID\tClass\n') for i in range(len(y)): metadata_file.write('%06d\t%s\n' % (y[i], opt.class_name[y[i]])) for i in range(opt.num_class): metadata_file.write('%06d\t%s\n' % (i, "class_"+opt.class_name[i])) metadata_file.close() print("embedding created")
def visualize_embeddings(self) -> None: """Insert visualization of embeddings in TensorBoard. Visualize the embeddings of `EmbeddedFactorSequence` objects specified in the `main.visualize_embeddings` config attribute. """ tb_projector = projector.ProjectorConfig() for sequence in self.model.visualize_embeddings: for i, (vocabulary, emb_matrix) in enumerate( zip(sequence.vocabularies, sequence.embedding_matrices)): # TODO when vocabularies will have name parameter, change it path = self.get_path("seq.{}-{}.tsv".format(sequence.name, i)) vocabulary.save_wordlist(path) embedding = tb_projector.embeddings.add() # pylint: disable=unsubscriptable-object embedding.tensor_name = emb_matrix.name embedding.metadata_path = path # pylint: enable=unsubscriptable-object summary_writer = tf.summary.FileWriter(self.model.output) projector.visualize_embeddings(summary_writer, tb_projector)
def _model_fn(inputs, context_indices, mode): if mode == ModeKeys.INFER: sparse_index_tensor = tf.string_split( [tf.read_file(vocab_file)], delimiter='\n' ) index_tensor = tf.squeeze(tf.sparse_to_dense( sparse_index_tensor.indices, [1, vocab_size], sparse_index_tensor.values, default_value='UNK' )) reverse_index = tf.contrib.lookup.HashTable( tf.contrib.lookup.KeyValueTensorInitializer( index_tensor, tf.constant(range(vocab_size), dtype=tf.int64) ), 0 ) target_indices = reverse_index.lookup(inputs) else: target_indices = inputs with tf.device(tf.train.replica_device_setter()): with tf.variable_scope('nce', partitioner=tf.fixed_size_partitioner( num_partitions)): embeddings = tf.get_variable( 'embeddings', shape=[vocab_size, embedding_size], dtype=tf.float32, initializer=tf.random_uniform_initializer(-1.0, 1.0) ) if mode in [ModeKeys.TRAIN, ModeKeys.EVAL]: nce_weights = tf.get_variable( 'nce_weights', shape=[vocab_size, embedding_size], dtype=tf.float32, initializer=tf.truncated_normal_initializer( stddev=1.0 / math.sqrt(embedding_size) ) ) nce_biases = tf.get_variable( 'nce_biases', initializer=tf.zeros_initializer([vocab_size]), dtype=tf.float32 ) tensors, loss, train_op = ({}, None, None) if mode in [ModeKeys.TRAIN, ModeKeys.EVAL]: embedded = tf.nn.embedding_lookup(embeddings, target_indices) loss = tf.reduce_mean(tf.nn.nce_loss( nce_weights, nce_biases, embedded, context_indices, num_sampled, vocab_size )) tf.summary.scalar('loss', loss) tf.summary.scalar('training/hptuning/metric', loss) # Embedding Visualizer embedding_writer = tf.summary.FileWriter(output_path) config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = embeddings.name embedding.metadata_path = vocab_file projector.visualize_embeddings(embedding_writer, config) if mode == ModeKeys.TRAIN: train_op = tf.train.GradientDescentOptimizer( learning_rate ).minimize( loss, global_step=tf.contrib.framework.get_or_create_global_step() ) if mode == ModeKeys.INFER: # Compute the cosine similarity between examples and embeddings. norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True)) normalized_embeddings = embeddings / norm valid_embeddings = tf.nn.embedding_lookup( normalized_embeddings, tf.squeeze(target_indices)) similarity = tf.matmul( valid_embeddings, normalized_embeddings, transpose_b=True) tensors['values'], predictions = tf.nn.top_k( similarity, sorted=True, k=num_sim) index_tensor = tf.concat(0, [tf.constant(['UNK']), index_tensor]) tensors['predictions'] = tf.gather(index_tensor, predictions) return tensors, loss, train_op
def word2vec_basic(log_dir): """Example of building, training and visualizing a word2vec model.""" # Create the directory for TensorBoard variables if there is not. if not os.path.exists(log_dir): os.makedirs(log_dir) # Step 1: Download the data. url = 'http://mattmahoney.net/dc/' # pylint: disable=redefined-outer-name def maybe_download(filename, expected_bytes): """Download a file if not present, and make sure it's the right size.""" local_filename = os.path.join(gettempdir(), filename) if not os.path.exists(local_filename): local_filename, _ = urllib.request.urlretrieve(url + filename, local_filename) statinfo = os.stat(local_filename) if statinfo.st_size == expected_bytes: print('Found and verified', filename) else: print(statinfo.st_size) raise Exception('Failed to verify ' + local_filename + '. Can you get to it with a browser?') return local_filename filename = maybe_download('text8.zip', 31344016) # Read the data into a list of strings. def read_data(filename): """Extract the first file enclosed in a zip file as a list of words.""" with zipfile.ZipFile(filename) as f: data = tf.compat.as_str(f.read(f.namelist()[0])).split() return data vocabulary = read_data(filename) print('Data size', len(vocabulary)) # Step 2: Build the dictionary and replace rare words with UNK token. vocabulary_size = 50000 def build_dataset(words, n_words): """Process raw inputs into a dataset.""" count = [['UNK', -1]] count.extend(collections.Counter(words).most_common(n_words - 1)) dictionary = {} for word, _ in count: dictionary[word] = len(dictionary) data = [] unk_count = 0 for word in words: index = dictionary.get(word, 0) if index == 0: # dictionary['UNK'] unk_count += 1 data.append(index) count[0][1] = unk_count reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys())) return data, count, dictionary, reversed_dictionary # Filling 4 global variables: # data - list of codes (integers from 0 to vocabulary_size-1). # This is the original text but words are replaced by their codes # count - map of words(strings) to count of occurrences # dictionary - map of words(strings) to their codes(integers) # reverse_dictionary - maps codes(integers) to words(strings) data, count, unused_dictionary, reverse_dictionary = build_dataset( vocabulary, vocabulary_size) del vocabulary # Hint to reduce memory. print('Most common words (+UNK)', count[:5]) print('Sample data', data[:10], [reverse_dictionary[i] for i in data[:10]]) # Step 3: Function to generate a training batch for the skip-gram model. def generate_batch(batch_size, num_skips, skip_window): global data_index assert batch_size % num_skips == 0 assert num_skips <= 2 * skip_window batch = np.ndarray(shape=(batch_size), dtype=np.int32) labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32) span = 2 * skip_window + 1 # [ skip_window target skip_window ] buffer = collections.deque(maxlen=span) # pylint: disable=redefined-builtin if data_index + span > len(data): data_index = 0 buffer.extend(data[data_index:data_index + span]) data_index += span for i in range(batch_size // num_skips): context_words = [w for w in range(span) if w != skip_window] words_to_use = random.sample(context_words, num_skips) for j, context_word in enumerate(words_to_use): batch[i * num_skips + j] = buffer[skip_window] labels[i * num_skips + j, 0] = buffer[context_word] if data_index == len(data): buffer.extend(data[0:span]) data_index = span else: buffer.append(data[data_index]) data_index += 1 # Backtrack a little bit to avoid skipping words in the end of a batch data_index = (data_index + len(data) - span) % len(data) return batch, labels batch, labels = generate_batch(batch_size=8, num_skips=2, skip_window=1) for i in range(8): print(batch[i], reverse_dictionary[batch[i]], '->', labels[i, 0], reverse_dictionary[labels[i, 0]]) # Step 4: Build and train a skip-gram model. batch_size = 128 embedding_size = 128 # Dimension of the embedding vector. skip_window = 1 # How many words to consider left and right. num_skips = 2 # How many times to reuse an input to generate a label. num_sampled = 64 # Number of negative examples to sample. # We pick a random validation set to sample nearest neighbors. Here we limit # the validation samples to the words that have a low numeric ID, which by # construction are also the most frequent. These 3 variables are used only for # displaying model accuracy, they don't affect calculation. valid_size = 16 # Random set of words to evaluate similarity on. valid_window = 100 # Only pick dev samples in the head of the distribution. valid_examples = np.random.choice(valid_window, valid_size, replace=False) graph = tf.Graph() with graph.as_default(): # Input data. with tf.name_scope('inputs'): train_inputs = tf.placeholder(tf.int32, shape=[batch_size]) train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1]) valid_dataset = tf.constant(valid_examples, dtype=tf.int32) # Ops and variables pinned to the CPU because of missing GPU implementation with tf.device('/cpu:0'): # Look up embeddings for inputs. with tf.name_scope('embeddings'): embeddings = tf.Variable( tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0)) embed = tf.nn.embedding_lookup(embeddings, train_inputs) # Construct the variables for the NCE loss with tf.name_scope('weights'): nce_weights = tf.Variable( tf.truncated_normal([vocabulary_size, embedding_size], stddev=1.0 / math.sqrt(embedding_size))) with tf.name_scope('biases'): nce_biases = tf.Variable(tf.zeros([vocabulary_size])) # Compute the average NCE loss for the batch. # tf.nce_loss automatically draws a new sample of the negative labels each # time we evaluate the loss. # Explanation of the meaning of NCE loss: # http://mccormickml.com/2016/04/19/word2vec-tutorial-the-skip-gram-model/ with tf.name_scope('loss'): loss = tf.reduce_mean( tf.nn.nce_loss( weights=nce_weights, biases=nce_biases, labels=train_labels, inputs=embed, num_sampled=num_sampled, num_classes=vocabulary_size)) # Add the loss value as a scalar to summary. tf.summary.scalar('loss', loss) # Construct the SGD optimizer using a learning rate of 1.0. with tf.name_scope('optimizer'): optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss) # Compute the cosine similarity between minibatch examples and all # embeddings. norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keepdims=True)) normalized_embeddings = embeddings / norm valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset) similarity = tf.matmul( valid_embeddings, normalized_embeddings, transpose_b=True) # Merge all summaries. merged = tf.summary.merge_all() # Add variable initializer. init = tf.global_variables_initializer() # Create a saver. saver = tf.train.Saver() # Step 5: Begin training. num_steps = 100001 with tf.Session(graph=graph) as session: # Open a writer to write summaries. writer = tf.summary.FileWriter(log_dir, session.graph) # We must initialize all variables before we use them. init.run() print('Initialized') average_loss = 0 for step in xrange(num_steps): batch_inputs, batch_labels = generate_batch(batch_size, num_skips, skip_window) feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels} # Define metadata variable. run_metadata = tf.RunMetadata() # We perform one update step by evaluating the optimizer op (including it # in the list of returned values for session.run() # Also, evaluate the merged op to get all summaries from the returned # "summary" variable. Feed metadata variable to session for visualizing # the graph in TensorBoard. _, summary, loss_val = session.run([optimizer, merged, loss], feed_dict=feed_dict, run_metadata=run_metadata) average_loss += loss_val # Add returned summaries to writer in each step. writer.add_summary(summary, step) # Add metadata to visualize the graph for the last run. if step == (num_steps - 1): writer.add_run_metadata(run_metadata, 'step%d' % step) if step % 2000 == 0: if step > 0: average_loss /= 2000 # The average loss is an estimate of the loss over the last 2000 # batches. print('Average loss at step ', step, ': ', average_loss) average_loss = 0 # Note that this is expensive (~20% slowdown if computed every 500 steps) if step % 10000 == 0: sim = similarity.eval() for i in xrange(valid_size): valid_word = reverse_dictionary[valid_examples[i]] top_k = 8 # number of nearest neighbors nearest = (-sim[i, :]).argsort()[1:top_k + 1] log_str = 'Nearest to %s:' % valid_word for k in xrange(top_k): close_word = reverse_dictionary[nearest[k]] log_str = '%s %s,' % (log_str, close_word) print(log_str) final_embeddings = normalized_embeddings.eval() # Write corresponding labels for the embeddings. with open(log_dir + '/metadata.tsv', 'w') as f: for i in xrange(vocabulary_size): f.write(reverse_dictionary[i] + '\n') # Save the model for checkpoints. saver.save(session, os.path.join(log_dir, 'model.ckpt')) # Create a configuration for visualizing embeddings with the labels in # TensorBoard. config = projector.ProjectorConfig() embedding_conf = config.embeddings.add() embedding_conf.tensor_name = embeddings.name embedding_conf.metadata_path = os.path.join(log_dir, 'metadata.tsv') projector.visualize_embeddings(writer, config) writer.close() # Step 6: Visualize the embeddings. # pylint: disable=missing-docstring # Function to draw visualization of distance between embeddings. def plot_with_labels(low_dim_embs, labels, filename): assert low_dim_embs.shape[0] >= len(labels), 'More labels than embeddings' plt.figure(figsize=(18, 18)) # in inches for i, label in enumerate(labels): x, y = low_dim_embs[i, :] plt.scatter(x, y) plt.annotate( label, xy=(x, y), xytext=(5, 2), textcoords='offset points', ha='right', va='bottom') plt.savefig(filename) try: # pylint: disable=g-import-not-at-top from sklearn.manifold import TSNE import matplotlib.pyplot as plt tsne = TSNE( perplexity=30, n_components=2, init='pca', n_iter=5000, method='exact') plot_only = 500 low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only, :]) labels = [reverse_dictionary[i] for i in xrange(plot_only)] plot_with_labels(low_dim_embs, labels, os.path.join(gettempdir(), 'tsne.png')) except ImportError as ex: print('Please install sklearn, matplotlib, and scipy to show embeddings.') print(ex)
train_summary_writer = tf.summary.FileWriter('/tmp/train', session.graph) test_images, test_labels = mnist.test.images, mnist.test.labels for batch_no in range(total_batches): image_batch = mnist.train.next_batch(100) _, merged_summary = session.run([optimiser, merged_summary_operation], feed_dict={ x_input: image_batch[0], y_input: image_batch[1], dropout_bool: True }) train_summary_writer.add_summary(merged_summary, batch_no) work_dir = '' # change path metadata_path = '/tmp/train/metadata.tsv' with open(metadata_path, 'w') as metadata_file: for i in range(no_embedding_data): metadata_file.write('{}\n'.format( np.nonzero(mnist.test.labels[::1])[1:][0][i])) from tensorflow.contrib.tensorboard.plugins import projector projector_config = projector.ProjectorConfig() embedding_projection = projector_config.embeddings.add() embedding_projection.tensor_name = embedding_variable.name embedding_projection.metadata_path = metadata_path embedding_projection.sprite.image_path = os.path.join(work_dir + '/mnist_10k_sprite.png') embedding_projection.sprite.single_image_dim.extend([28, 28]) projector.visualize_embeddings(train_summary_writer, projector_config) tf.train.Saver().save(session, '/tmp/train/model.ckpt', global_step=1)
log_dir = args.log_dir if args.log_dir is None: log_dir = os.path.join(os.getcwd(), "log_dir") # Load data data_frame = pd.read_csv(args.vector, index_col=False, header=None, sep=' ') metadata = args.labels # Generating PCA and pca = PCA(n_components=50, random_state=123, svd_solver='auto') df_pca = pd.DataFrame(pca.fit_transform(data_frame)) df_pca = df_pca.values # Start tensorflow variable setup tf_data = tf.Variable(df_pca) # Start TF session with tf.Session() as sess: saver = tf.train.Saver([tf_data]) sess.run(tf_data.initializer) saver.save(sess, os.path.join(log_dir, 'tf_data.ckpt')) config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = tf_data.name # Link this tensor to its metadata(Labels) file embedding.metadata_path = metadata # Saves a config file that TensorBoard will read during startup. projector.visualize_embeddings(tf.summary.FileWriter(log_dir), config)
def create_embedding(self, batch, img_shape, labels=None, invert_colors=True): """ This will eventually be called inside some convenient training routine that will be exposed to the user. This creates the logs and variables necessary to visualize the latency space with the embedding visualizer in tensorboard. batch - (array) Input to the network. Typically of shape (batch_size, [input_dimensions]) img_shape - (array like) Can be an array, tuple or list containing the dimensions of one image in (height, width) format. For example, if using MNIST img_shape might be equal to (28,28) or [28,28] labels - (array) One dimensional array containing labels. The element in the ith index of 'labels' is the associated label for ith training element in batch """ EMBED_VAR_NAME = "Latent_Space" EMBED_LOG_DIR = self.LOG_DIR SPRITES_PATH = os.path.join(self.LOG_DIR, 'sprites.png') METADATA_PATH = os.path.join(self.LOG_DIR, 'metadata.tsv') img_h, img_w = img_shape with tf.name_scope('Create_Embedding'): # ----- Create the embedding variable to be visualized ----- # Paths to sprites and metadata must be relative to the location of # the projector config file which is determined by the line below embedding_writer = tf.summary.FileWriter(EMBED_LOG_DIR) latent_var = self.sess.run(self.z, feed_dict={self.network_input: batch}) embedding_var = tf.Variable(latent_var, trainable=False, name=EMBED_VAR_NAME) # Initialize the newly created embedding variable init_embedding_var_op = tf.variables_initializer([embedding_var]) self.sess.run(init_embedding_var_op) # Create a projector configuartion object config = projector.ProjectorConfig() # Create an embedding embedding = config.embeddings.add() # This is where the name is important again. Specify # which variable to embed embedding.tensor_name = embedding_var.name # Specify where you find the metadata embedding.metadata_path = METADATA_PATH #'metadata.tsv' # Specify where you find the sprite (we will create this later) embedding.sprite.image_path = SPRITES_PATH #'mnistdigits.png' embedding.sprite.single_image_dim.extend([img_h, img_w]) # Say that you want to visualise the embeddings projector.visualize_embeddings(embedding_writer, config) # ----- Construct the Sprites for visualization ----- images = np.reshape(batch, (-1, img_h, img_w)) # Maybe invert greyscale... MNIST looks prettier in tensorboard if invert_colors: images = 1 - images batch_size = images.shape[0] n_plots = int(np.ceil(np.sqrt(batch_size))) spriteimage = np.ones((img_h * n_plots, img_w * n_plots)) for i in range(n_plots): for j in range(n_plots): this_filter = i * n_plots + j if this_filter < images.shape[0]: this_img = images[this_filter] spriteimage[i * img_h:(i + 1) * img_h, j * img_w:(j + 1) * img_w] = this_img plt.imsave(SPRITES_PATH, spriteimage, cmap='gray') # ----- Create the metadata file for visualization ----- with open(METADATA_PATH, 'w') as f: if labels is None: for index in range(batch_size): f.write("%d\n" % (index)) else: f.write("Index\tLabel\n") for index, label in enumerate(labels): f.write("%d\t%d\n" % (index, label)) embed_saver = tf.train.Saver() embed_saver.save(self.sess, os.path.join(EMBED_LOG_DIR, "embed.ckpt"), self.CHECKPOINT_COUNTER) self.CHECKPOINT_COUNTER += 1
import pickle from tensorflow.contrib.tensorboard.plugins import projector LOG_DIR = 'logs' metadata ='./data/metadata.tsv' embed = pickle.load(open('./data/embedding.p','rb')) vocabs = pickle.load(open('./data/vocab.p','rb')) embeds = tf.Variable(embed, name='embeds') with open(metadata, 'w', encoding='utf-8') as metadata_file: metadata_file.write('Name\tPOS\n') for row in vocabs: metadata_file.write('%s\t%s\n' % (row[0],row[1])) with tf.Session() as sess: saver = tf.train.Saver([embeds]) sess.run(embeds.initializer) saver.save(sess, os.path.join(LOG_DIR, 'embeds.ckpt')) config = projector.ProjectorConfig() # One can add multiple embeddings. embedding = config.embeddings.add() embedding.tensor_name = embeds.name # Link this tensor to its metadata file (e.g. labels). embedding.metadata_path = metadata # Saves a config file that TensorBoard will read during startup. projector.visualize_embeddings(tf.summary.FileWriter(LOG_DIR), config)
def main(args): # pass the args as params so the model_fn can use # the TPU specific args params = vars(args) if args.use_tpu: # additional configs required for using TPUs tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver( args.tpu) tpu_config = tf.contrib.tpu.TPUConfig( num_shards=8, # using Cloud TPU v2-8 iterations_per_loop=args.save_checkpoints_steps) # use the TPU version of RunConfig config = tf.contrib.tpu.RunConfig( cluster=tpu_cluster_resolver, model_dir=args.model_dir, tpu_config=tpu_config, save_checkpoints_steps=args.save_checkpoints_steps, save_summary_steps=100) # TPUEstimator estimator = tf.contrib.tpu.TPUEstimator( model_fn=model_fn, config=config, params=params, train_batch_size=args.train_batch_size, # Calling TPUEstimator.predict requires setting predict_bath_size. predict_batch_size=PREDICT_BATCH_SIZE, eval_batch_size=32, export_to_tpu=False) else: config = tf.estimator.RunConfig(model_dir=args.model_dir) estimator = tf.estimator.Estimator(model_fn, config=config, params=params) estimator.train(train_input_fn, max_steps=args.max_steps) # After training, apply the learned embedding to the test data and visualize with tensorboard Projector. embeddings = next( estimator.predict(predict_input_fn, yield_single_examples=False))['embeddings'] # Put the embeddings into a variable to be visualized. embedding_var = tf.Variable(embeddings, name='test_embeddings') # Labels do not pass through the estimator.predict call, so we get it separately. _, (_, labels) = tf.keras.datasets.mnist.load_data() labels = labels[:PREDICT_BATCH_SIZE] # Write the metadata file for the projector. metadata_path = os.path.join(estimator.model_dir, 'metadata.tsv') with tf.gfile.GFile(metadata_path, 'w') as f: f.write('index\tlabel\n') for i, label in enumerate(labels): f.write('{}\t{}\n'.format(i, label)) # Configure the projector. projector_config = projector.ProjectorConfig() embedding_config = projector_config.embeddings.add() embedding_config.tensor_name = embedding_var.name # The metadata_path is relative to the summary_writer's log_dir. embedding_config.metadata_path = 'metadata.tsv' summary_writer = tf.summary.FileWriter(estimator.model_dir) projector.visualize_embeddings(summary_writer, projector_config) # Start a session to actually write the embeddings into a new checkpoint. sess = tf.Session() sess.run(tf.global_variables_initializer()) saver = tf.train.Saver() saver.save(sess, os.path.join(estimator.model_dir, 'model.ckpt'), args.max_steps + 1)
def _model_fn(features, labels, mode, params): """A model function for item recommendation. Two Tower Architecture: Builds neural nets for users and items that learn n-dimensional representations of each. The distance between these representations is used to make a prediction for a binary classification. Args: features: a batch of features. labels: a batch of labels or None if predicting. mode: an instance of tf.estimator.ModeKeys. params: a dict of additional params. Returns: A tf.estimator.EstimatorSpec that fully defines the model that will be run by an Estimator. """ tft_output = tft.TFTransformOutput(params["tft_dir"]) tft_features = tft_output.transform_raw_features(features) hparams = params["hparams"] # Prediction op: Return the top k closest items for a given user or item. if mode == tf.estimator.ModeKeys.PREDICT: table = tf.contrib.lookup.index_to_string_table_from_file( tft_output.vocabulary_file_by_name(constants.ITEM_VOCAB_NAME)) item_embedding = _get_embedding_matrix(hparams.embedding_size, tft_output, constants.ITEM_VOCAB_NAME) item_sims, item_top_k = _get_top_k(tft_features, item_embedding, constants.TFT_ITEM_KEY, item_embedding) user_embedding = _get_embedding_matrix(hparams.embedding_size, tft_output, constants.USER_VOCAB_NAME) user_sims, user_top_k = _get_top_k(tft_features, user_embedding, constants.TFT_USER_KEY, item_embedding) predictions = { constants.USER_KEY: tf.identity(features[constants.USER_KEY]), "user_top_k": table.lookup(tf.cast(user_top_k, tf.int64)), "user_sims": user_sims, constants.ITEM_KEY: tf.identity(features[constants.ITEM_KEY]), "item_top_k": table.lookup(tf.cast(item_top_k, tf.int64)), "item_sims": item_sims, } return tf.estimator.EstimatorSpec(mode, predictions=predictions) # Build user and item networks. user_net, user_size = _make_input_layer(tft_features, tft_output, constants.TFT_USER_KEY, constants.USER_VOCAB_NAME, constants.USER_NUMERICAL_FEATURES, constants.USER_CATEGORICAL_FEATURES, constants.USER_CATEGORICAL_VOCABS, hparams.user_embed_mult) item_net, item_size = _make_input_layer(tft_features, tft_output, constants.TFT_ITEM_KEY, constants.ITEM_VOCAB_NAME, constants.ITEM_NUMERICAL_FEATURES, constants.ITEM_CATEGORICAL_FEATURES, constants.ITEM_CATEGORICAL_VOCABS, hparams.item_embed_mult) user_net, item_net = _resize_networks(user_net, user_size, item_net, item_size, hparams.num_layers, hparams.embedding_size) user_norm = tf.nn.l2_normalize(user_net, 1) item_norm = tf.nn.l2_normalize(item_net, 1) preds = tf.abs(tf.reduce_sum(tf.multiply(user_norm, item_norm), axis=1)) loss = tf.losses.mean_squared_error(labels, preds, weights=labels**.5) user_embedding = _update_embedding_matrix( tft_features[constants.TFT_USER_KEY], user_norm, hparams.embedding_size, tft_output, constants.USER_VOCAB_NAME) item_embedding = _update_embedding_matrix( tft_features[constants.TFT_ITEM_KEY], item_norm, hparams.embedding_size, tft_output, constants.ITEM_VOCAB_NAME) # Eval op: Log the recall of the batch and save a sample of user+item shared # embedding space for tensorboard projector. item_sims = tf.matmul(user_norm, item_embedding, transpose_b=True) metrics = {} with tf.name_scope("recall"): for k in constants.EVAL_RECALLS: key = "recall_{0}".format(k) recall = tf.metrics.recall_at_k( tft_features[constants.TFT_TOP_10_KEY], item_sims, k) metrics["recall/{0}".format(key)] = recall tf.summary.scalar(key, recall[1]) tf.summary.merge_all() if mode == tf.estimator.ModeKeys.EVAL: samples, config = _get_projector_data(user_embedding, params["projector_users"], item_embedding, params["projector_items"]) writer = tf.summary.FileWriter(params["model_dir"]) projector.visualize_embeddings(writer, config) with tf.control_dependencies([samples]): loss = tf.identity(loss) return tf.estimator.EstimatorSpec(mode, loss=loss, eval_metric_ops=metrics) # Training op: Update the weights via backpropagation. num_samples = constants.NUM_PROJECTOR_USERS + constants.NUM_PROJECTOR_ITEMS sample = tf.get_variable(constants.PROJECTOR_NAME, [num_samples, hparams.embedding_size]) optimizer = tf.train.AdagradOptimizer(learning_rate=hparams.learning_rate) with tf.control_dependencies([user_embedding, item_embedding, sample]): train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step()) return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)
def fit(self): """ Fit the model. """ parameters = self.parameters conf_parameters = self.conf_parameters dataset_filepaths = self.dataset_filepaths modeldata = self.modeldata dataset_brat_folders = self.dataset_brat_folders sess = self.sess model = self.model transition_params_trained = self.transition_params_trained stats_graph_folder, experiment_timestamp = self._create_stats_graph_folder( parameters) # Initialize and save execution details start_time = time.time() results = {} results['epoch'] = {} results['execution_details'] = {} results['execution_details']['train_start'] = start_time results['execution_details']['time_stamp'] = experiment_timestamp results['execution_details']['early_stop'] = False results['execution_details']['keyboard_interrupt'] = False results['execution_details']['num_epochs'] = 0 results['model_options'] = copy.copy(parameters) model_folder = os.path.join(stats_graph_folder, 'model') utils.create_folder_if_not_exists(model_folder) with open(os.path.join(model_folder, 'parameters.ini'), 'w') as parameters_file: conf_parameters.write(parameters_file) pickle.dump(modeldata, open(os.path.join(model_folder, 'dataset.pickle'), 'wb')) tensorboard_log_folder = os.path.join(stats_graph_folder, 'tensorboard_logs') utils.create_folder_if_not_exists(tensorboard_log_folder) tensorboard_log_folders = {} for dataset_type in dataset_filepaths.keys(): tensorboard_log_folders[dataset_type] = os.path.join( stats_graph_folder, 'tensorboard_logs', dataset_type) utils.create_folder_if_not_exists( tensorboard_log_folders[dataset_type]) # Instantiate the writers for TensorBoard writers = {} for dataset_type in dataset_filepaths.keys(): writers[dataset_type] = tf.summary.FileWriter( tensorboard_log_folders[dataset_type], graph=sess.graph) # embedding_writer has to write in model_folder, otherwise TensorBoard won't be able to view embeddings embedding_writer = tf.summary.FileWriter(model_folder) embeddings_projector_config = projector.ProjectorConfig() tensorboard_token_embeddings = embeddings_projector_config.embeddings.add( ) tensorboard_token_embeddings.tensor_name = model.token_embedding_weights.name token_list_file_path = os.path.join(model_folder, 'tensorboard_metadata_tokens.tsv') tensorboard_token_embeddings.metadata_path = os.path.relpath( token_list_file_path, '.') tensorboard_character_embeddings = embeddings_projector_config.embeddings.add( ) tensorboard_character_embeddings.tensor_name = model.character_embedding_weights.name character_list_file_path = os.path.join( model_folder, 'tensorboard_metadata_characters.tsv') tensorboard_character_embeddings.metadata_path = os.path.relpath( character_list_file_path, '.') projector.visualize_embeddings(embedding_writer, embeddings_projector_config) # Write metadata for TensorBoard embeddings token_list_file = codecs.open(token_list_file_path, 'w', 'UTF-8') for token_index in range(modeldata.vocabulary_size): token_list_file.write('{0}\n'.format( modeldata.index_to_token[token_index])) token_list_file.close() character_list_file = codecs.open(character_list_file_path, 'w', 'UTF-8') for character_index in range(modeldata.alphabet_size): if character_index == modeldata.PADDING_CHARACTER_INDEX: character_list_file.write('PADDING\n') else: character_list_file.write('{0}\n'.format( modeldata.index_to_character[character_index])) character_list_file.close() # Start training + evaluation loop. Each iteration corresponds to 1 epoch. # number of epochs with no improvement on the validation test in terms of F1-score bad_counter = 0 previous_best_valid_f1_score = 0 epoch_number = -1 try: while True: step = 0 epoch_number += 1 print('\nStarting epoch {0}'.format(epoch_number)) epoch_start_time = time.time() if epoch_number != 0: # Train model: loop over all sequences of training set with shuffling sequence_numbers = list( range(len(modeldata.token_indices['train']))) random.shuffle(sequence_numbers) for sequence_number in sequence_numbers: transition_params_trained = train.train_step( sess, modeldata, sequence_number, model, parameters) step += 1 if step % 10 == 0: print('Training {0:.2f}% done'.format( step / len(sequence_numbers) * 100), end='\r', flush=True) epoch_elapsed_training_time = time.time() - epoch_start_time print('Training completed in {0:.2f} seconds'.format( epoch_elapsed_training_time), flush=True) y_pred, y_true, output_filepaths = train.predict_labels( sess, model, transition_params_trained, parameters, modeldata, epoch_number, stats_graph_folder, dataset_filepaths) # Evaluate model: save and plot results evaluate.evaluate_model(results, modeldata, y_pred, y_true, stats_graph_folder, epoch_number, epoch_start_time, output_filepaths, parameters) if parameters['use_pretrained_model'] and not parameters[ 'train_model']: conll_to_brat.output_brat(output_filepaths, dataset_brat_folders, stats_graph_folder) break # Save model model.saver.save( sess, os.path.join(model_folder, 'model_{0:05d}.ckpt'.format(epoch_number))) # Save TensorBoard logs summary = sess.run(model.summary_op, feed_dict=None) writers['train'].add_summary(summary, epoch_number) writers['train'].flush() utils.copytree(writers['train'].get_logdir(), model_folder) # Early stop valid_f1_score = results['epoch'][epoch_number][0]['valid'][ 'f1_score']['micro'] if valid_f1_score > previous_best_valid_f1_score: bad_counter = 0 previous_best_valid_f1_score = valid_f1_score conll_to_brat.output_brat(output_filepaths, dataset_brat_folders, stats_graph_folder, overwrite=True) self.transition_params_trained = transition_params_trained else: bad_counter += 1 print( "The last {0} epochs have not shown improvements on the validation set." .format(bad_counter)) if bad_counter >= parameters['patience']: print('Early Stop!') results['execution_details']['early_stop'] = True break if epoch_number >= parameters['maximum_number_of_epochs']: break except KeyboardInterrupt: results['execution_details']['keyboard_interrupt'] = True print('Training interrupted') print('Finishing the experiment') end_time = time.time() results['execution_details']['train_duration'] = end_time - start_time results['execution_details']['train_end'] = end_time evaluate.save_results(results, stats_graph_folder) for dataset_type in dataset_filepaths.keys(): writers[dataset_type].close()
def run_training(): """Train MNIST for a number of steps.""" # Get the sets of images and labels for training, validation, and # test on MNIST. data_sets = input_data.read_data_sets(FLAGS.input_data_dir, FLAGS.fake_data) # Tell TensorFlow that the model will be built into the default Graph. with tf.Graph().as_default(): # Generate placeholders for the images and labels. images_placeholder, labels_placeholder = placeholder_inputs( FLAGS.batch_size) # Build a Graph that computes predictions from the inference model. logits = mnist.inference(images_placeholder, FLAGS.hidden1, FLAGS.hidden2) # Add to the Graph the Ops for loss calculation. loss = mnist.loss(logits, labels_placeholder) # Add to the Graph the Ops that calculate and apply gradients. train_op = mnist.training(loss, FLAGS.learning_rate) # Add the Op to compare the logits to the labels during evaluation. eval_correct = mnist.evaluation(logits, labels_placeholder) # Build the summary Tensor based on the TF collection of Summaries. summary = tf.summary.merge_all() # Add the variable initializer Op. init = tf.global_variables_initializer() # Create a saver for writing training checkpoints. saver = tf.train.Saver() # Create a session for running Ops on the Graph. sess = tf.Session() # Instantiate a SummaryWriter to output summaries and the Graph. summary_writer = tf.summary.FileWriter(FLAGS.log_dir, sess.graph) # And then after everything is built: # Run the Op to initialize the variables. sess.run(init) # Start the training loop. for step in xrange(FLAGS.max_steps): start_time = time.time() # Fill a feed dictionary with the actual set of images and labels # for this particular training step. feed_dict = fill_feed_dict(data_sets.train, images_placeholder, labels_placeholder) # Run one step of the model. The return values are the activations # from the `train_op` (which is discarded) and the `loss` Op. To # inspect the values of your Ops or variables, you may include them # in the list passed to sess.run() and the value tensors will be # returned in the tuple from the call. _, loss_value = sess.run([train_op, loss], feed_dict=feed_dict) duration = time.time() - start_time # Write the summaries and print an overview fairly often. if step % 100 == 0: # Print status to stdout. print('Step %d: loss = %.2f (%.3f sec)' % (step, loss_value, duration)) # Update the events file. summary_str = sess.run(summary, feed_dict=feed_dict) summary_writer.add_summary(summary_str, step) summary_writer.flush() # Save a checkpoint and evaluate the model periodically. if (step + 1) % 1000 == 0 or (step + 1) == FLAGS.max_steps: checkpoint_file = os.path.join(FLAGS.log_dir, 'model.ckpt') saver.save(sess, checkpoint_file, global_step=step) # Evaluate against the training set. print('Training Data Eval:') do_eval(sess, eval_correct, images_placeholder, labels_placeholder, data_sets.train) # Evaluate against the validation set. print('Validation Data Eval:') do_eval(sess, eval_correct, images_placeholder, labels_placeholder, data_sets.validation) # Evaluate against the test set. print('Test Data Eval:') do_eval(sess, eval_correct, images_placeholder, labels_placeholder, data_sets.test) # Compute embeddings and save them. thumbnail_size = int(np.sqrt(mnist.IMAGE_PIXELS)) for data_set, name in [ (data_sets.train, 'train'), (data_sets.validation, 'validation'), (data_sets.test, 'test')]: output_path = os.path.join(FLAGS.log_dir, 'embed', name) print('Computing %s Embedding' % name) (all_images, all_labels, hidden1_vectors, hidden2_vectors) = do_eval( sess, eval_correct, images_placeholder, labels_placeholder, data_set, True) embed_tensors = [] summary_writer = tf.summary.FileWriter(output_path, sess.graph) config = projector.ProjectorConfig() for layer, embed_vectors in enumerate([hidden1_vectors, hidden2_vectors]): embed_tensor = tf.Variable( np.array(embed_vectors).reshape( len(embed_vectors) * embed_vectors[0].shape[0], -1), name=('%s_layer_%s' % (name, layer))) embed_tensors.append(embed_tensor) sess.run(embed_tensor.initializer) embedding = config.embeddings.add() embedding.tensor_name = embed_tensor.name embedding.metadata_path = os.path.join(output_path, 'labels.tsv') embedding.sprite.image_path = os.path.join(output_path, 'sprite.png') embedding.sprite.single_image_dim.extend( [thumbnail_size, thumbnail_size]) projector.visualize_embeddings(summary_writer, config) result = sess.run(embed_tensors) saver = tf.train.Saver(embed_tensors) saver.save(sess, os.path.join(output_path, 'model.ckpt'), layer) # Make sprite and labels. images = np.array(all_images).reshape( -1, thumbnail_size, thumbnail_size).astype(np.float32) sprite = images_to_sprite(images) scipy.misc.imsave(os.path.join(output_path, 'sprite.png'), sprite) all_labels = np.array(all_labels).flatten() metadata_file = open(os.path.join(output_path, 'labels.tsv'), 'w') metadata_file.write('Name\tClass\n') for ll in xrange(len(all_labels)): metadata_file.write('%06d\t%d\n' % (ll, all_labels[ll])) metadata_file.close()
def train(self, dataset_list, config): """ Args: dataset_list (<StockDataSet>) config (tf.app.flags.FLAGS) """ assert len(dataset_list) > 0 self.merged_sum = tf.summary.merge_all() # Set up the logs folder self.writer = tf.summary.FileWriter( os.path.join("./logs", self.model_name)) self.writer.add_graph(self.sess.graph) if self.use_embed: # Set up embedding visualization # Format: tensorflow/tensorboard/plugins/projector/projector_config.proto projector_config = projector.ProjectorConfig() # You can add multiple embeddings. Here we add only one. added_embed = projector_config.embeddings.add() added_embed.tensor_name = self.embed_matrix.name # Link this tensor to its metadata file (e.g. labels). shutil.copyfile(os.path.join(self.logs_dir, "metadata.tsv"), os.path.join(self.model_logs_dir, "metadata.tsv")) added_embed.metadata_path = "metadata.tsv" # The next line writes a projector_config.pbtxt in the LOG_DIR. TensorBoard will # read this file during startup. projector.visualize_embeddings(self.writer, projector_config) tf.global_variables_initializer().run() # Merged test data of different stocks. merged_test_X = [] merged_test_y = [] merged_test_labels = [] for label_, d_ in enumerate(dataset_list): merged_test_X += list(d_.test_X) merged_test_y += list(d_.test_y) merged_test_labels += [[label_]] * len(d_.test_X) merged_test_X = np.array(merged_test_X) merged_test_y = np.array(merged_test_y) merged_test_labels = np.array(merged_test_labels) print "len(merged_test_X) =", len(merged_test_X) print "len(merged_test_y) =", len(merged_test_y) print "len(merged_test_labels) =", len(merged_test_labels) test_data_feed = { self.learning_rate: 0.0, self.keep_prob: 1.0, self.inputs: merged_test_X, self.targets: merged_test_y, self.symbols: merged_test_labels, } global_step = 0 num_batches = sum(len(d_.train_X) for d_ in dataset_list) // config.batch_size random.seed(time.time()) # Select samples for plotting. sample_labels = range(min(config.sample_size, len(dataset_list))) sample_indices = {} for l in sample_labels: sym = dataset_list[l].stock_sym target_indices = np.array([ i for i, sym_label in enumerate(merged_test_labels) if sym_label[0] == l ]) sample_indices[sym] = target_indices print sample_indices print "Start training for stocks:", [d.stock_sym for d in dataset_list] for epoch in xrange(config.max_epoch): epoch_step = 0 learning_rate = config.init_learning_rate * ( config.learning_rate_decay**max( float(epoch + 1 - config.init_epoch), 0.0)) for label_, d_ in enumerate(dataset_list): for batch_X, batch_y in d_.generate_one_epoch( config.batch_size): global_step += 1 epoch_step += 1 batch_labels = np.array([[label_]] * len(batch_X)) train_data_feed = { self.learning_rate: learning_rate, self.keep_prob: config.keep_prob, self.inputs: batch_X, self.targets: batch_y, self.symbols: batch_labels, } train_loss, _, train_merged_sum = self.sess.run( [self.loss, self.optim, self.merged_sum], train_data_feed) self.writer.add_summary(train_merged_sum, global_step=global_step) if np.mod(global_step, len(dataset_list) * 200 / config.input_size) == 1: test_loss, test_pred = self.sess.run( [self.loss_test, self.pred], test_data_feed) print "Step:%d [Epoch:%d] [Learning rate: %.6f] train_loss:%.6f test_loss:%.6f" % ( global_step, epoch, learning_rate, train_loss, test_loss) # Plot samples for sample_sym, indices in sample_indices.iteritems(): image_path = os.path.join( self.model_plots_dir, "{}_epoch{:02d}_step{:04d}.png".format( sample_sym, epoch, epoch_step)) sample_preds = test_pred[indices] sample_truth = merged_test_y[indices] self.plot_samples(sample_preds, sample_truth, image_path, stock_sym=sample_sym) self.save(global_step) final_pred, final_loss = self.sess.run([self.pred, self.loss], test_data_feed) # Save the final model self.save(global_step) return final_pred
def set_model(self, model): self.model = model self.sess = K.get_session() if self.histogram_freq and self.merged is None: for layer in self.model.layers: for weight in layer.weights: tf.summary.histogram(weight.name, weight) if self.write_images: w_img = tf.squeeze(weight) shape = w_img.get_shape() if len(shape) > 1 and shape[0] > shape[1]: w_img = tf.transpose(w_img) if len(shape) == 1: w_img = tf.expand_dims(w_img, 0) w_img = tf.expand_dims(tf.expand_dims(w_img, 0), -1) tf.summary.image(weight.name, w_img) if hasattr(layer, 'output'): tf.summary.histogram('{}_out'.format(layer.name), layer.output) self.merged = tf.summary.merge_all() if self.write_graph: self.writer = tf.summary.FileWriter(self.log_dir, self.sess.graph) else: self.writer = tf.summary.FileWriter(self.log_dir) if self.embeddings_freq: self.saver = tf.train.Saver() embeddings_layer_names = self.embeddings_layer_names if not embeddings_layer_names: embeddings_layer_names = [layer.name for layer in self.model.layers if type(layer).__name__ == 'Embedding'] embeddings = {layer.name: layer.weights[0] for layer in self.model.layers if layer.name in embeddings_layer_names} embeddings_metadata = {} if not isinstance(self.embeddings_metadata, str): embeddings_metadata = self.embeddings_metadata else: embeddings_metadata = {layer_name: self.embeddings_metadata for layer_name in embeddings.keys()} config = projector.ProjectorConfig() self.embeddings_logs = [] for layer_name, tensor in embeddings.items(): embedding = config.embeddings.add() embedding.tensor_name = tensor.name self.embeddings_logs.append(os.path.join(self.log_dir, layer_name + '.ckpt')) if layer_name in embeddings_metadata: embedding.metadata_path = embeddings_metadata[layer_name] projector.visualize_embeddings(self.writer, config)
def run(self): ''' Runs the model according to the specified settings - If mode = Train: Train a GRU model using the training data - If mode = Val: Load the saved GRU model and evaluate it on the validation fold - If mode = Test: Load the saved GRU model and evaluate it on the blind test set ''' self.is_train = (self.mode == 'Train') if not os.path.exists(self.path): os.mkdir(self.path) # Load the training data with open('train_data.pkl', 'rb') as f: data_sequences = pkl.load(f) with open('train_labels.pkl', 'rb') as f: data_labels = pkl.load(f) dictionary, reverse_dictionary, data_lengths, self.max_seq_len, enc_sequences = build_dictionary( data_sequences) self.dictionary = sorted(dictionary.items(), key=operator.itemgetter(1)) print(self.dictionary) self.vocabulary_size = len(dictionary) self.val_size = len(data_sequences) // self.folds fold = self.mask print('Training fold number %d. Each fold of size %d' % (fold, len(data_sequences) // self.folds)) # Truncates sequences at length 2000 and returns descriptive statistics. # This is done by concatenating the first 1900 and the last 100 amino acids. if self.is_train: self.max_seq_len = 2000 original_lengths = copy(data_lengths) data_sequences = enc_sequences[:, :self.max_seq_len] for i in range(len(data_lengths)): if data_lengths[i] > self.max_seq_len: data_sequences[i] = np.concatenate( (enc_sequences[i, :self.max_seq_len - 100], enc_sequences[i, -100:]), axis=0) data_lengths[i] = self.max_seq_len if self.folds == 1: val_mask = np.array([False]) else: val_mask = np.arange(self.val_size * (fold - 1), self.val_size * (fold)) # Use seed to ensure same randomisation is applied for each fold np.random.seed(4) perm = np.random.permutation(len(data_sequences)) data_labels = np.array(data_labels) data_sequences = data_sequences[perm] data_labels = data_labels[perm] data_lenghts = data_lengths[perm] original_lengths = original_lengths[perm] self.val_data = data_sequences[val_mask] self.val_labels = data_labels[val_mask] self.val_lengths = data_lengths[val_mask] self.val_original_lengths = original_lengths[val_mask] self.train_data = np.delete(data_sequences, val_mask, axis=0) self.train_labels = np.delete(data_labels, val_mask, axis=0) self.train_lengths = np.delete(data_lengths, val_mask, axis=0) self.train_original_lengths = np.delete(original_lengths, val_mask, axis=0) self.train_statistics, self.train_frame = self.summary_stats( self.train_lengths, self.train_labels, 'train') if self.folds == 1: self.val_statistics = np.array([]) self.val_frame = np.array([]) self.val_original_lengths = np.array([]) else: self.val_statistics, self.val_frame = self.summary_stats( self.val_lengths, self.val_labels, 'validation') this_data = [ self.train_data, self.train_labels, self.train_lengths, self.val_data, self.val_labels, self.val_lengths, self.train_statistics, self.train_frame, self.val_statistics, self.val_frame, self.train_original_lengths, self.val_original_lengths ] with open(self.path + 'this_data.pkl', 'wb') as f: pkl.dump(this_data, f) else: with open(self.path + 'this_data.pkl', 'rb') as f: self.train_data, self.train_labels, self.train_lengths, self.val_data, self.val_labels, self.val_lengths, self.train_statistics, self.train_frame, self.val_statistics, self.val_frame, self.train_original_lengths, self.val_original_lengths = pkl.load( f) # Now construct the Tensorflow graph print('\r~~~~~~~ Building model ~~~~~~~\r') # Define placeholders and variables initializer = tf.random_normal_initializer() self.word_embeddings = tf.get_variable( 'embeddings', [self.vocabulary_size, self.embedding_size], tf.float32, initializer=initializer) sequences = tf.placeholder(tf.int32, [None, None], "sequences") sequences_lengths = tf.placeholder(tf.int32, [None], "sequences_lengths") labels = tf.placeholder(tf.int64, [None], "labels") keep_prob_dropout = tf.placeholder(tf.float32, name='dropout') global_step = tf.Variable(0, name='global_step', trainable=False) # Embed and encode sequences sequences_embedded = self.embed_data(sequences) encoded_sequences = self.encoder(sequences_embedded, sequences_lengths, keep_prob_dropout, bidirectional=self.bidirectional) # Take last hidden state of GRU and put them through a nonlinear and a linear FC layer with tf.name_scope('non_linear_layer'): encoded_sentences_BN = self.batch_norm_wrapper( encoded_sequences, self.is_train) non_linear = tf.nn.dropout(tf.nn.relu( tf.contrib.layers.linear(encoded_sentences_BN, 64)), keep_prob=keep_prob_dropout) with tf.name_scope('final_layer'): non_linear_BN = self.batch_norm_wrapper(non_linear, self.is_train) logits = tf.contrib.layers.linear(non_linear_BN, 4) # Compute mean loss on this batch, consisting of cross entropy loss and L2 loss CE_loss = self.get_CE_loss(labels, logits) L2_loss = self.get_L2_loss() loss = CE_loss + L2_loss # Perform training operation learning_rate = tf.train.exponential_decay(self.learning_rate, global_step, 100, 0.96, staircase=True) opt_op = tf.contrib.layers.optimize_loss(loss=loss, global_step=global_step, learning_rate=learning_rate, optimizer='Adam', clip_gradients=2.0, learning_rate_decay_fn=None, summaries=None) # Define scalars for Tensorboard tf.summary.scalar('CE_loss', CE_loss) tf.summary.scalar('L2_loss', L2_loss) tf.summary.scalar('loss', loss) tf.summary.scalar('learning_rate', learning_rate) # Compute accuracy of prediction probs = tf.nn.softmax(logits) with tf.name_scope('accuracy'): pred = tf.argmax(logits, 1) correct_prediction = tf.equal(labels, pred) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) tf.summary.scalar('accuracy', accuracy) # If in training mode: # - shuffle data set before each epoch # - train model using mini batches # - track performance on train and validation set throughout training if self.is_train == True: with tf.Session() as session: train_loss_writer = tf.summary.FileWriter( str(self.path + 'tensorboard/train_loss'), session.graph) train_summary_writer = tf.summary.FileWriter( str(self.path + 'tensorboard/train_summary'), session.graph) val_summary_writer = tf.summary.FileWriter( str(self.path + 'tensorboard/val_summary'), session.graph) # Use the same LOG_DIR where you stored your checkpoint. embedding_writer = tf.summary.FileWriter( str(self.path + 'tensorboard/'), session.graph) config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = self.word_embeddings.name # Link this tensor to its metadata file (e.g. labels). embedding.metadata_path = os.path.join('./metadata.tsv') # Saves a configuration file that TensorBoard will read during startup. projector.visualize_embeddings(embedding_writer, config) merged = tf.summary.merge_all() print('\r~~~~~~~ Initializing variables ~~~~~~~\r') tf.global_variables_initializer().run() start_time = time.time() min_train_loss = np.inf batch_times = [] n = self.train_data.shape[0] print('\r~~~~~~~ Starting training ~~~~~~~\r') try: train_summaryIndex = -1 for epoch in range(self.num_epochs): self.is_train = True epoch_time = time.time() print('----- Epoch', epoch, '-----') print('Shuffling dataset') perm = np.random.permutation(len(self.train_data)) self.train_data_perm = self.train_data[perm] self.train_labels_perm = self.train_labels[perm] self.train_lengths_perm = self.train_lengths[perm] total_loss = 0 for i in range(n // self.batch_size): batch_start = time.time() batch_data = self.train_data_perm[i * self.batch_size: (i + 1) * self.batch_size] batch_lengths = self.train_lengths_perm[ i * self.batch_size:(i + 1) * self.batch_size] batch_labels = self.train_labels_perm[ i * self.batch_size:(i + 1) * self.batch_size] train_dict = { sequences: batch_data, sequences_lengths: batch_lengths, labels: batch_labels, keep_prob_dropout: self.keep_prob_dropout } _, batch_loss, batch_accuracy, batch_summary = session.run( [opt_op, loss, accuracy, merged], feed_dict=train_dict) total_loss += batch_loss batch_times.append(time.time() - batch_start) train_loss_writer.add_summary( batch_summary, i + (n // self.batch_size) * epoch) if i % 10 == 0 and i > 0: # Print loss every 10 batches time_per_epoch = np.mean(batch_times) * ( n // self.batch_size) remaining_time = int(time_per_epoch - time.time() + epoch_time) string_out = '\rEnd of batch ' + str( i) + ' Train loss: ' + str( total_loss / (i * self.batch_size) ) + ' Accuracy: ' + str( batch_accuracy) string_out += ' Elapsed training time : ' + str( int(time.time() - start_time)) + "s, " string_out += str( remaining_time ) + "s remaining for this epoch" string_out += ' (' + str( time_per_epoch * 100 / 60 // 1 / 100) + ' min/epoch)' stdout.write(string_out) # Train accuracy train_dict = { sequences: self.train_data_perm[:1000], sequences_lengths: self.train_lengths_perm[:1000], labels: self.train_labels_perm[:1000], keep_prob_dropout: 1.0 } train_summary, train_loss, train_accuracy = session.run( [merged, loss, accuracy], feed_dict=train_dict) train_summary_writer.add_summary(train_summary, epoch) print('\nEpoch train loss: ', train_loss, 'Epoch train accuracy: ', train_accuracy) # Val accuracy val_dict = { sequences: self.val_data, sequences_lengths: self.val_lengths, labels: self.val_labels, keep_prob_dropout: 1.0 } val_summary, val_loss, val_accuracy = session.run( [merged, loss, accuracy], feed_dict=val_dict) val_summary_writer.add_summary(val_summary, epoch) print('\nEpoch val loss: ', val_loss, 'Epoch val accuracy: ', val_accuracy) self.save_model(session, epoch) saver = tf.train.Saver( write_version=tf.train.SaverDef.V2) saver.save( session, os.path.join(self.path + '/tensorboard/', 'model.ckpt')) except KeyboardInterrupt: save = input('save?') if 'y' in save: self.save_model(session, epoch) # If in validation mode: # - Load saved model and evaluate on validation fold # - Return list containing confusion matrices, and accuracy measures such as FPR and TPR elif self.mode == 'Val': with tf.Session() as session: print('Restoring model...') saver = tf.train.Saver(write_version=tf.train.SaverDef.V2) saver.restore(session, self.path + 'tensorboard/model.ckpt') print('Model restored!') val_dict = { sequences: self.val_data, sequences_lengths: self.val_lengths, labels: self.val_labels, keep_prob_dropout: 1.0 } self.val_pred, self.val_accuracy, self.val_probs = session.run( [pred, accuracy, probs], feed_dict=val_dict) _ = self.summary_stats(self.val_lengths, self.val_labels, 'val') print('\nConfusion matrix (all sequence lengths):') val_confusion_1 = self.confusion( gold=self.val_labels, prediction=self.val_pred, lengths=self.val_original_lengths, min_length=0, max_length=np.inf) print(val_confusion_1) print('\nConfusion matrix (sequence length < 2000):') val_confusion_2 = self.confusion( gold=self.val_labels, prediction=self.val_pred, lengths=self.val_original_lengths, min_length=0, max_length=2000) print(val_confusion_2) print('\nConfusion matrix (sequence length > 2000):') val_confusion_3 = self.confusion( gold=self.val_labels, prediction=self.val_pred, lengths=self.val_original_lengths, min_length=2000, max_length=np.inf) print(val_confusion_3) print('\n Val accuracy:', self.val_accuracy) print( '\n Val accuracy when length <2000:', np.sum((self.val_pred == self.val_labels) * (self.val_original_lengths <= 2000)) / np.sum(self.val_original_lengths <= 2000)) print( '\n Val accuracy when length >2000:', np.sum((self.val_pred == self.val_labels) * (self.val_original_lengths > 2000)) / np.sum(self.val_original_lengths > 2000)) this_sum = np.zeros([3, 5]) this_auc = np.zeros([1, 5]) this_TPR = [] this_FPR = [] total_tp = 0 total_fp = 0 total_fn = 0 total_tn = 0 for i in range(4): tp = np.sum((self.val_labels == i) * (self.val_pred == i)) fp = np.sum((self.val_labels != i) * (self.val_pred == i)) fn = np.sum((self.val_labels == i) * (self.val_pred != i)) tn = np.sum((self.val_labels != i) * (self.val_pred != i)) total_tp += tp total_fp += fp total_fn += fn total_tn += tn prec = tp / (tp + fp) if (tp + fp) > 0 else 0.0 recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0 f1 = 2 * prec * recall / ( prec + recall) if prec * recall > 0 else 0.0 this_sum[:, i] = np.array([prec, recall, f1]) this_auc[:, i] = roc_auc_score(self.val_labels == i, self.val_pred == i) if i < 4: this_FPR.append( roc_curve(self.val_labels == i, self.val_probs[:, i])[0]) this_TPR.append( roc_curve(self.val_labels == i, self.val_probs[:, i])[1]) prec = total_tp / (total_tp + total_fp) if ( total_tp + total_fp) > 0 else 0.0 recall = total_tp / (total_tp + total_fn) if ( total_tp + total_fn) > 0 else 0.0 f1 = 2 * prec * recall / (prec + recall) if prec * recall > 0 else 0.0 this_sum[:, 4] = np.array([prec, recall, f1]) this_sum = np.concatenate((this_sum, this_auc), 0) self.this_sum = pd.DataFrame(this_sum) self.this_sum.index = pd.Index( ['Precision', 'Recall', 'F1', 'AUC']) self.this_sum.columns = pd.Index( ['cyto', 'secreted', 'mito', 'nucleus', 'Total']) print(self.this_sum) if self.is_train == False: return [ val_confusion_1, val_confusion_2, val_confusion_3, self.this_sum, this_FPR, this_TPR ] # If in test model: # - Load saved model and evaluate on test set # - Print predicted probabilities for each protein in the test set elif self.mode == 'Test': with tf.Session() as session: print('Restoring model...') saver = tf.train.Saver(write_version=tf.train.SaverDef.V2) saver.restore(session, self.path + 'model.checkpoint') print('Model restored!') with open('test_data.pkl', 'rb') as f: test_sequences = pkl.load(f) with open('test_labels.pkl', 'rb') as f: test_labels = pkl.load(f) _, _, data_lengths, _, enc_sequences = build_dictionary( test_sequences, vocab=dictionary) test_dict = { sequences: enc_sequences, sequences_lengths: data_lengths, keep_prob_dropout: 1.0 } self.probs, self.pred = session.run([probs, pred], feed_dict=test_dict) result = pd.DataFrame( np.concatenate((self.probs, np.expand_dims(self.pred, 1)), 1)) result.columns = pd.Index( ['cyto', 'secreted', 'mito', 'nucleus', 'prediction']) print(result)
def train(tfrecord_dir_pre, model_pre, logs_folder, pic_size, num, forward, end_epoch, display=100, base_lr=0.001): """placeholder的声明""" input_image = tf.placeholder( tf.float32, shape=[config.BATCH_SIZE, pic_size, pic_size, 3], name='input_image') label = tf.placeholder(tf.float32, shape=[config.BATCH_SIZE], name='label') box_target = tf.placeholder(tf.float32, shape=[config.BATCH_SIZE, 4], name='box_target') mark_target = tf.placeholder(tf.float32, shape=[config.BATCH_SIZE, 10], name='mark_target') """forward op,train_op,lr_op结果是损失""" input_image = image_color_distort(input_image) cls_loss_op, box_loss_op,mark_loss_op, L2_loss_op,accuracy_op =\ forward(pic_size,input_image, label, box_target, mark_target, training = True)#前向传播 """误差权重确认,以及整体误差""" if pic_size == 12: radio_cls_loss = 1.0 radio_box_loss = 0.5 radio_mark_loss = 0.5 elif pic_size == 24: radio_cls_loss = 1.0 radio_box_loss = 0.5 radio_mark_loss = 0.5 elif pic_size == 48: radio_cls_loss = 1.0 radio_box_loss = 0.5 radio_mark_loss = 1.0 total_loss_op = (radio_cls_loss * cls_loss_op + radio_box_loss * box_loss_op + radio_mark_loss * mark_loss_op + L2_loss_op) """对网络训练的操作,以及对学习率改变的操作""" train_op, lr_op = train_lr_op(base_lr, total_loss_op, num) """tensorboard中记录的标量""" tf.summary.scalar('cls_loss', cls_loss_op) tf.summary.scalar('box_loss', box_loss_op) tf.summary.scalar('landmark_loss', mark_loss_op) tf.summary.scalar('cls_accuracy', accuracy_op) tf.summary.scalar('total_loss', total_loss_op) summary_op = tf.summary.merge_all() """tfrecord文件路径指定, 训练数据集从tfrecord读出""" image_batch, label_batch, box_batch, mark_batch = read_tfrecord( tfrecord_dir_pre, config.BATCH_SIZE, pic_size) print('############################################################') """session 初始化""" init = tf.global_variables_initializer() sess = tf.Session() saver = tf.train.Saver(max_to_keep=0) sess.run(init) writer = tf.summary.FileWriter(logs_folder, sess.graph) projector_config = projector.ProjectorConfig() projector.visualize_embeddings(writer, projector_config) """协同管理""" coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) batch_done_count = 0 #网络计算了批量数,每个epoch都重新计数 MAX_STEP = int(num / config.BATCH_SIZE + 1) * end_epoch epoch_count = 0 try: for step in range(1): batch_done_count += 1 if coord.should_stop(): break image_batch_array, label_batch_array,box_batch_array,mark_batch_array = \ sess.run([image_batch, label_batch,box_batch,mark_batch]) #随机旋转 image_batch_array,mark_batch_array = \ random_flip_images(image_batch_array, label_batch_array, mark_batch_array) print(label_batch_array) print(mark_batch_array[:100]) _, _, summary = sess.run( [train_op, lr_op, summary_op], feed_dict={ input_image: image_batch_array, label: label_batch_array, box_target: box_batch_array, mark_target: mark_batch_array }) writer.add_summary(summary, global_step=step) """满100就展示""" if (step + 1) % (display) == 0: cls_loss, box_loss, mark_loss, L2_loss, lr, acc = sess.run( [ cls_loss_op, box_loss_op, mark_loss_op, L2_loss_op, lr_op, accuracy_op ], feed_dict={ input_image: image_batch_array, label: label_batch_array, box_target: box_batch_array, mark_target: mark_batch_array }) total_loss = radio_cls_loss * cls_loss + radio_box_loss * box_loss +\ radio_mark_loss * mark_loss + L2_loss print( "Step:%d/%d, acc:%3f, cls loss:%4f, box_loss:%4f, mark loss:%4f, L2 loss: %4f, Total Loss:%4f, lr:%f" % (step + 1, MAX_STEP, acc, cls_loss, box_loss, mark_loss, L2_loss, total_loss, lr)) """下一轮迭代""" if batch_done_count * config.BATCH_SIZE > num * 2: epoch_count = epoch_count + 1 batch_done_count = 0 path_prefix = saver.save(sess, model_pre, global_step=epoch_count * 2) print('path prefix is :', path_prefix) except tf.errors.OutOfRangeError: print("训练完成!!!") finally: coord.request_stop() writer.close() coord.join(threads) sess.close()
def main(argv=None): #maybe_download() graph = load_graph() #basedir = os.path.dirname(__file__) # ensure log directory exists logs_path = LOG_DIR if not os.path.exists(logs_path): os.makedirs(logs_path) #generate metadata with tf.Session(graph=graph) as sess: pool3 = sess.graph.get_tensor_by_name('pool_3:0') jpeg_data = tf.placeholder(tf.string) outputs = [] # Create metadata metadata_path = os.path.join(LOG_DIR, 'metadata.tsv') metadata = open(metadata_path, 'w') metadata.write("Name\tLabels\n") for folder_name in os.listdir(IMAGE_DIR): for file_name in os.listdir(IMAGE_DIR + '/' + folder_name): if not file_name.endswith('.jpg'): continue print('process %s...' % file_name) with open( os.path.join(IMAGE_DIR + '/' + folder_name, file_name), 'rb') as f: data = f.read() results = sess.run(pool3, { 'DecodeJpeg/contents:0': data, jpeg_data: data }) #print("results", results) outputs.append(results[0]) metadata.write('{}\t{}\n'.format(file_name, folder_name)) metadata.close() embedding_var = tf.Variable(tf.stack([tf.squeeze(x) for x in outputs], axis=0), trainable=False, name='embed') # prepare projector config config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = embedding_var.name summary_writer = tf.summary.FileWriter(LOG_DIR) # link metadata embedding.metadata_path = metadata_path # write to sprite image file img = images_to_sprite(img_data) cv2.imwrite(os.path.join(LOG_DIR, 'sprite_classes.png'), img) #print("img", img) embedding.sprite.image_path = os.path.join(LOG_DIR, 'sprite_classes.png') embedding.sprite.single_image_dim.extend([100, 100]) # save embedding_var projector.visualize_embeddings(summary_writer, config) sess.run(tf.variables_initializer([embedding_var])) saver = tf.train.Saver() saver.save(sess, os.path.join(LOG_DIR, 'model.ckpt'))
def train(data): data = shuffle(data) numSamples = data.shape[0] split = round(0.1 * numSamples) ValidData = data[0:split, :] TestData = data[split:2 * split, :] TrainData = data[2 * split:numSamples, :] dataset = tf.data.Dataset.from_tensor_slices(TrainData) dataset = dataset.batch(BatchSize) iterator = dataset.make_initializable_iterator() next_element = iterator.get_next() ## Sparse autoencoder ae = autoencoder() rho_hat = tf.reduce_mean(ae['latent']) kl = kl_divergence(rho, rho_hat) sparse = beta * tf.reduce_sum(kl) with tf.name_scope("Train"): optimizer = tf.train.AdamOptimizer(LearningRate).minimize(ae['cost'] + sparse) init = tf.global_variables_initializer() sess = tf.Session() sess.run(init) merged_summary = tf.summary.merge_all() writer = tf.summary.FileWriter(LogDir) writer.add_graph(sess.graph) TestWriter = tf.summary.FileWriter(LogDir + "_Test") # +"_Test") TestWriter.add_graph(sess.graph) # Training prev_loss = 500000 epoch_idx = 0 trainingLoss = list() validationLoss = list() # testingLoss = list() for i in range(1, NumEpochs): sess.run(iterator.initializer) loss = 0 num_batch = 0 saver = tf.train.Saver(max_to_keep=5) while True: try: batch_x = sess.run(next_element) _, l = sess.run([optimizer, ae['cost']], feed_dict={ae['x']: batch_x}) loss += l num_batch += 1 except tf.errors.OutOfRangeError: s = sess.run(merged_summary, feed_dict={ae['x']: TrainData}) writer.add_summary(s, i) trainLoss = loss / num_batch print('Epoch = %i\tTraining Loss = %f' % (i, trainLoss)) trainingLoss.append(trainLoss) break if i % 10 == 0: with tf.name_scope("TestingMeasurements"): s, validation_cost = sess.run([merged_summary, ae['cost']], feed_dict={ae['x']: ValidData}) print('Epoch = %i\t*** Validation Loss = %f' % (i, validation_cost)) validationLoss.append(validation_cost) if (validation_cost < prev_loss): # Save best model based on validation loss prev_loss = validation_cost saver.save(sess, os.path.join(LogDir, 'model.ckpt'), global_step=i) epoch_idx = i TestWriter.add_summary(s, i) ## Apply best model to the test set bestmodel = os.path.join(LogDir, 'model.ckpt-' + str(epoch_idx)) print("Best model occured at epoch = ", epoch_idx) print("Best Model path = ", bestmodel) saver.restore(sess, bestmodel) s, test_cost = sess.run([merged_summary, ae['cost']], feed_dict={ae['x']: TestData}) embedded_data = sess.run(ae['latent'], feed_dict={ae['x']: data}) np.savetxt(LogDir + "_trainingLoss.csv", trainingLoss, delimiter=",") np.savetxt(LogDir + "_validationLoss.csv", validationLoss, delimiter=",") csv = open(LogDir + "_testingLoss.csv", "w") csv.write(str(test_cost)) ### Visualize on Tensorboard with tf.device("/cpu:0"): embedding = tf.Variable(tf.stack(embedded_data, axis=0), trainable=False, name='embedding') sess.run(tf.global_variables_initializer()) config = projector.ProjectorConfig() embed = config.embeddings.add() embed.tensor_name = 'embedding:0' embed.metadata_path = os.path.join(LogDir + '/metadata.tsv') projector.visualize_embeddings(writer, config) sess.close() return embedded_data
def main(): # Load configurations and write to config.txt cfg = TrainConfig().parse() print(cfg.name) result_dir = os.path.join( cfg.result_root, cfg.name + '_' + datetime.strftime(datetime.now(), '%Y%m%d-%H%M%S')) if not os.path.isdir(result_dir): os.makedirs(result_dir) utils.write_configure_to_file(cfg, result_dir) np.random.seed(seed=cfg.seed) # prepare dataset train_session = cfg.train_session train_set = prepare_dataset(cfg.feature_root, train_session, cfg.feat, cfg.label_root) train_set = train_set[:cfg.label_num] batch_per_epoch = len(train_set) // cfg.sess_per_batch val_session = cfg.val_session val_set = prepare_dataset(cfg.feature_root, val_session, cfg.feat, cfg.label_root) # construct the graph with tf.Graph().as_default(): tf.set_random_seed(cfg.seed) global_step = tf.Variable(0, trainable=False) lr_ph = tf.placeholder(tf.float32, name='learning_rate') ####################### Define model here ######################## with tf.variable_scope("modality_core"): # load backbone model if cfg.network == "tsn": model_emb = networks.TSN(n_seg=cfg.num_seg, emb_dim=cfg.emb_dim) elif cfg.network == "rtsn": model_emb = networks.RTSN(n_seg=cfg.num_seg, emb_dim=cfg.emb_dim) elif cfg.network == "convtsn": model_emb = networks.ConvTSN(n_seg=cfg.num_seg, emb_dim=cfg.emb_dim) elif cfg.network == "convrtsn": model_emb = networks.ConvRTSN(n_seg=cfg.num_seg, emb_dim=cfg.emb_dim, n_h=cfg.n_h, n_w=cfg.n_w, n_C=cfg.n_C, n_input=cfg.n_input) elif cfg.network == "convbirtsn": model_emb = networks.ConvBiRTSN(n_seg=cfg.num_seg, emb_dim=cfg.emb_dim) else: raise NotImplementedError # get the embedding if cfg.feat == "sensors" or cfg.feat == "segment": input_ph = tf.placeholder(tf.float32, shape=[None, cfg.num_seg, None]) elif cfg.feat == "resnet" or cfg.feat == "segment_down": input_ph = tf.placeholder( tf.float32, shape=[None, cfg.num_seg, None, None, None]) dropout_ph = tf.placeholder(tf.float32, shape=[]) model_emb.forward(input_ph, dropout_ph) if cfg.normalized: embedding = tf.nn.l2_normalize(model_emb.hidden, axis=-1, epsilon=1e-10) else: embedding = model_emb.hidden # split embedding into anchor, positive and negative and calculate triplet loss anchor, positive, negative = tf.unstack( tf.reshape(embedding, [-1, 3, cfg.emb_dim]), 3, 1) metric_loss = networks.triplet_loss(anchor, positive, negative, cfg.alpha) regularization_loss = tf.reduce_sum( tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)) total_loss = metric_loss + regularization_loss * cfg.lambda_l2 tf.summary.scalar('learning_rate', lr_ph) train_op = utils.optimize(total_loss, global_step, cfg.optimizer, lr_ph, tf.global_variables()) ####################### Define data loader ############################ # session iterator for session sampling feat_paths_ph = tf.placeholder(tf.string, shape=[None, cfg.sess_per_batch]) label_paths_ph = tf.placeholder(tf.string, shape=[None, cfg.sess_per_batch]) train_data = session_generator(feat_paths_ph, label_paths_ph, sess_per_batch=cfg.sess_per_batch, num_threads=2, shuffled=False, preprocess_func=model_emb.prepare_input) train_sess_iterator = train_data.make_initializable_iterator() next_train = train_sess_iterator.get_next() # prepare validation data val_sess = [] val_feats = [] val_labels = [] val_boundaries = [] for session in val_set: session_id = os.path.basename(session[1]).split('_')[0] eve_batch, lab_batch, boundary = load_data_and_label( session[0], session[-1], model_emb.prepare_input_test ) # use prepare_input_test for testing time val_feats.append(eve_batch) val_labels.append(lab_batch) val_sess.extend([session_id] * eve_batch.shape[0]) val_boundaries.extend(boundary) val_feats = np.concatenate(val_feats, axis=0) val_labels = np.concatenate(val_labels, axis=0) print("Shape of val_feats: ", val_feats.shape) # generate metadata.tsv for visualize embedding with open(os.path.join(result_dir, 'metadata_val.tsv'), 'w') as fout: fout.write('id\tlabel\tsession_id\tstart\tend\n') for i in range(len(val_sess)): fout.write('{0}\t{1}\t{2}\t{3}\t{4}\n'.format( i, val_labels[i, 0], val_sess[i], val_boundaries[i][0], val_boundaries[i][1])) # Variable for visualizing the embeddings emb_var = tf.Variable(tf.zeros([val_feats.shape[0], cfg.emb_dim]), name='embeddings') set_emb = tf.assign(emb_var, embedding, validate_shape=False) # calculated for monitoring all-pair embedding distance diffs = utils.all_diffs_tf(embedding, embedding) all_dist = utils.cdist_tf(diffs) tf.summary.histogram('embedding_dists', all_dist) summary_op = tf.summary.merge_all() saver = tf.train.Saver(max_to_keep=10) ######################################################################### # Start running the graph if cfg.gpu: os.environ['CUDA_VISIBLE_DEVICES'] = cfg.gpu gpu_options = tf.GPUOptions(allow_growth=True) sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) summary_writer = tf.summary.FileWriter(result_dir, sess.graph) with sess.as_default(): sess.run(tf.global_variables_initializer()) # load pretrain model, if needed if cfg.model_path: print("Restoring pretrained model: %s" % cfg.model_path) saver.restore(sess, cfg.model_path) ################## Training loop ################## epoch = -1 while epoch < cfg.max_epochs - 1: step = sess.run(global_step, feed_dict=None) epoch = step // batch_per_epoch # learning rate schedule, reference: "In defense of Triplet Loss" if epoch < cfg.static_epochs: learning_rate = cfg.learning_rate else: learning_rate = cfg.learning_rate * \ 0.01**((epoch-cfg.static_epochs)/(cfg.max_epochs-cfg.static_epochs)) # prepare data for this epoch random.shuffle(train_set) feat_paths = [path[0] for path in train_set] label_paths = [path[1] for path in train_set] # reshape a list to list of list # interesting hacky code from: https://stackoverflow.com/questions/10124751/convert-a-flat-list-to-list-of-list-in-python feat_paths = list(zip(*[iter(feat_paths)] * cfg.sess_per_batch)) label_paths = list( zip(*[iter(label_paths)] * cfg.sess_per_batch)) sess.run(train_sess_iterator.initializer, feed_dict={ feat_paths_ph: feat_paths, label_paths_ph: label_paths }) # for each epoch batch_count = 1 while True: try: # Hierarchical sampling (same as fast rcnn) start_time_select = time.time() # First, sample sessions for a batch eve, se, lab = sess.run(next_train) # for memory concern, cfg.event_per_batch events are used in maximum if eve.shape[0] > cfg.event_per_batch: idx = np.random.permutation( eve.shape[0])[:cfg.event_per_batch] eve = eve[idx] se = se[idx] lab = lab[idx] select_time1 = time.time() - start_time_select # Get the embeddings of all events eve_embedding = np.zeros((eve.shape[0], cfg.emb_dim), dtype='float32') for start, end in zip( range(0, eve.shape[0], cfg.batch_size), range(cfg.batch_size, eve.shape[0] + cfg.batch_size, cfg.batch_size)): end = min(end, eve.shape[0]) emb = sess.run(embedding, feed_dict={ input_ph: eve[start:end], dropout_ph: 1.0 }) eve_embedding[start:end] = emb # Second, sample triplets within sampled sessions # get distance for all pairs all_diff = utils.all_diffs(eve_embedding, eve_embedding) triplet_selected, active_count = utils.select_triplets_facenet( lab, utils.cdist(all_diff, metric=cfg.metric), cfg.triplet_per_batch, cfg.alpha) select_time2 = time.time( ) - start_time_select - select_time1 # perform training on the selected triplets start_time_train = time.time() triplet_input_idx = [ idx for triplet in triplet_selected for idx in triplet ] triplet_input = eve[triplet_input_idx] err, _, step, summ = sess.run( [total_loss, train_op, global_step, summary_op], feed_dict={ input_ph: triplet_input, dropout_ph: cfg.keep_prob, lr_ph: learning_rate }) train_time = time.time() - start_time_train print ("%s\tEpoch: [%d][%d/%d]\tEvent num: %d\tTriplet num: %d\tSelect_time1: %.3f\tSelect_time2: %.3f\tTrain_time: %.3f\tLoss %.4f" % \ (cfg.name, epoch+1, batch_count, batch_per_epoch, eve.shape[0], triplet_input.shape[0]//3, select_time1, select_time2, train_time, err)) summary = tf.Summary(value=[ tf.Summary.Value(tag="train_loss", simple_value=err), tf.Summary.Value(tag="active_count", simple_value=active_count), tf.Summary.Value( tag="triplet_num", simple_value=triplet_input.shape[0] // 3) ]) summary_writer.add_summary(summary, step) summary_writer.add_summary(summ, step) batch_count += 1 except tf.errors.OutOfRangeError: print("Epoch %d done!" % (epoch + 1)) break # validation on val_set print("Evaluating on validation set...") val_embeddings, _ = sess.run([embedding, set_emb], feed_dict={ input_ph: val_feats, dropout_ph: 1.0 }) mAP, mPrec, recall = utils.evaluate_simple( val_embeddings, val_labels) summary = tf.Summary(value=[ tf.Summary.Value(tag="Valiation mAP", simple_value=mAP), tf.Summary.Value(tag="Validation Recall@1", simple_value=recall), tf.Summary.Value(tag="Validation [email protected]", simple_value=mPrec) ]) summary_writer.add_summary(summary, step) print("Epoch: [%d]\tmAP: %.4f\tmPrec: %.4f" % (epoch + 1, mAP, mPrec)) # config for embedding visualization config = projector.ProjectorConfig() visual_embedding = config.embeddings.add() visual_embedding.tensor_name = emb_var.name visual_embedding.metadata_path = os.path.join( result_dir, 'metadata_val.tsv') projector.visualize_embeddings(summary_writer, config) # save model saver.save(sess, os.path.join(result_dir, cfg.name + '.ckpt'), global_step=step)
def fit_emb(reviews, batch_feeder, config): do_log_save = False do_profiling = False log_save_path = 'log' # separate a validation set use_valid_set = True if use_valid_set: reviews, valid_reviews = separate_valid(reviews, 0.1) # build model graph with tf.device('/gpu:0'): graph = tf.Graph() with graph.as_default(): tf.set_random_seed(27) builder = GraphBuilder() problem_size = {'num_reviews': reviews.shape[0], 'num_items': reviews.shape[1]} inputs, outputs, model_param = builder.construct_model_graph(problem_size, config, init_model=None, training=True) model_vars = [model_param['alpha'], model_param['rho'], model_param['intercept'], model_param['prior_logit']] optimizer = tf.train.AdagradOptimizer(0.1).minimize(outputs['objective'], var_list=model_vars) if config['model'] in ['context_select']: net_vars = builder.infer_net.param_list() net_optimizer = tf.train.AdagradOptimizer(0.1).minimize(outputs['objective'], var_list=net_vars) init = tf.global_variables_initializer() # for visualization vis_conf = projector.ProjectorConfig() embedding = vis_conf.embeddings.add() embedding.tensor_name = model_param['alpha'].name # optimize the model with tf.Session(graph=graph) as session: # initialize all variables init.run() # Merge all the summaries and write them out to /tmp/mnist_logs (by defaul) if do_log_save: merged = tf.summary.merge_all() train_writer = tf.summary.FileWriter(log_save_path, session.graph) projector.visualize_embeddings(train_writer, vis_conf) else: merged = [] if do_profiling: run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) run_metadata = tf.RunMetadata() nprint = config['nprint'] val_accum = np.array([0.0, 0.0]) count_accum = 0 train_logg = np.zeros([int(config['max_iter'] / nprint) + 1, 3]) review_size = reviews.shape[0] for step in xrange(1, config['max_iter'] + 1): rind = np.random.choice(review_size) indices, labels = batch_feeder(reviews[rind]) if indices.shape[0] <= 1: # neglect views with only one entry raise Exception('Row %d of the data has only one non-zero entry.' % rind) feed_dict = {inputs['input_ind']: indices, inputs['input_label']: labels} if config['model'] in ['context_select']: _, net_debugv, summary = session.run((net_optimizer, outputs['debugv'], merged), feed_dict=feed_dict) else: net_debugv = '' _, llh_val, nums, obj_val, debug_val, summary = session.run((optimizer, outputs['llh_sum'], outputs['num_items'], \ outputs['objective'], outputs['debugv'], merged), feed_dict=feed_dict) if do_log_save: train_writer.add_summary(summary, step) # record llh, and objective val_accum = val_accum + np.array([llh_val, obj_val]) count_accum = count_accum + (nums[0] + nums[1]); # print loss every nprint iterations if step % nprint == 0 or np.isnan(llh_val) or np.isinf(llh_val): # do validation valid_llh = 0.0 if use_valid_set: valid_llh = validate(valid_reviews, batch_feeder, session, inputs, outputs) # record the three values ibatch = int(step / nprint) train_logg[ibatch, :] = np.array([val_accum[0] / count_accum, val_accum[1] / nprint, valid_llh]) val_accum[:] = 0.0 # reset the accumulater count_accum = 0 print("iteration[", step, "]: average llh, obj, valid_llh, and debug_val are ", train_logg[ibatch, :], debug_val, net_debugv) #check nan value if np.isnan(llh_val) or np.isinf(llh_val): print('Loss value is ', llh_val, ', and the debug value is ', debug_val) raise Exception('Bad values') model = get_model(model_param, session, config) if do_log_save: tf.train.Saver().save(session, log_save_path, step) # Create the Timeline object, and write it to a json if do_profiling: tl = timeline.Timeline(run_metadata.step_stats) ctf = tl.generate_chrome_trace_format() with open(log_save_path + '/timeline_step%d.json' % (step / nprint), 'w') as f: f.write(ctf) model = get_model(model_param, session, config) return model, train_logg
def main(current_path, args): print(args) # Give a folder path as an argument with '--log_dir' to save # TensorBoard summaries. Default is a log folder in current directory. # Create the directory for TensorBoard variables if there is not. if not os.path.exists(args.log_dir): os.makedirs(args.log_dir) filename = args.file # preprocessing raw data and building vocabulary vocabulary = read_data(filename) print('Data size', len(vocabulary)) # Step 2: Build the dictionary and replace rare words with UNK token # Filling 4 global variables: # data - list of codes (integers from 0 to vocabulary_size-1). # This is the original text but words are replaced by their codes # count - map of words(strings) to count of occurrences # dictionary - map of words(strings) to their codes(integers) # reverse_dictionary - maps codes(integers) to words(strings) data, count, dictionary, reverse_dictionary = build_dataset(vocabulary, 0) del vocabulary # Hint to reduce memory. vocabulary_size = len(dictionary) num_sampled = args.neg_samples # Number of negative examples to sample. train_args = { 'batch_size': args.train_bs, 'embedding_size': args.emb_size, # Dimension of the embedding vector. 'skip_window': args.skip_window, # How many words to consider left and right. 'num_skips': args. num_skips, # How many times to reuse an input to generate a label. 'num_steps': args.epochs, # Number of epochs 'log_dir': args.log_dir } model = Word2Vec(args.train_bs, args.emb_size, vocabulary_size, num_sampled) graph = model._graph embeddings = None with tf.Session(graph=graph) as session: # Open a writer to write summaries. writer = tf.summary.FileWriter(args.log_dir, session.graph) embeddings = train_model(session, model, reverse_dictionary, writer, data, **train_args) #similarity = model.similarity.eval() np.savetxt(args.emb_file, embeddings) # Save the model for checkpoints. model.savemodel.save(session, os.path.join(args.log_dir, 'model.ckpt')) # Create a configuration for visualizing embeddings with the labels in TensorBoard. config = projector.ProjectorConfig() embedding_conf = config.embeddings.add() embedding_conf.tensor_name = model.embeddings.name embedding_conf.metadata_path = os.path.join(args.log_dir, 'metadata.tsv') projector.visualize_embeddings(writer, config) # Closing writer writer.close() print("Plotting items on TSNE plot...") runTSNE(embeddings, reverse_dictionary, args.tsne_img_file)
def main(args): tf.logging.set_verbosity(tf.logging.INFO) embedding_dir_path = args.MODEL_CHPT_DIR_PATH + "-embedding" if not os.path.exists(embedding_dir_path): os.makedirs(embedding_dir_path) json_path = os.path.join(args.MODEL_DIR_PATH, "params.json") if not json_path: tf.logging.error( "No json configuration file found at {}".format(json_path)) params = get_params(json_path) train_config = tf.estimator.RunConfig( tf_random_seed=230, model_dir=args.MODEL_CHPT_DIR_PATH, save_summary_steps=params["save_summary_steps"]) estimator = tf.estimator.Estimator(model_fn, params=params, config=train_config) #### prepare sprite.png and metadata.tsv sprite_image_path = os.path.join(embedding_dir_path, "sprite.png") if not os.path.exists(sprite_image_path): save_sprite_image(embedding_dir_path, args.DATA_DIR_PATH) metadata_path = os.path.join(embedding_dir_path, "metadata.tsv") if not os.path.exists(metadata_path): write_matadata_tsv(embedding_dir_path, args.DATA_DIR_PATH) # Compute embeddings on the test set tf.logging.info("Predicting") predictions = estimator.predict( lambda: test_input_fn(args.DATA_DIR_PATH, params)) embeddings = np.zeros((params["test_size"], params["embedding_size"])) for i, p in enumerate(predictions): embeddings[i] = p["embeddings"] # Visualize test embeddings embedding_var = tf.Variable(embeddings, name="mnist_embedding") summary_writer = tf.summary.FileWriter(embedding_dir_path) embedding_config = projector.ProjectorConfig() embedding = embedding_config.embeddings.add() embedding.tensor_name = embedding_var.name # Specify where you find the metadata embedding.metadata_path = "metadata.tsv" #'metadata.tsv' # Specify where you find the sprite (we will create this later) embedding.sprite.image_path = "sprite.png" embedding.sprite.single_image_dim.extend( [params["image_size"], params["image_size"]]) # Say that you want to visualise the embeddings projector.visualize_embeddings(summary_writer, embedding_config) saver = tf.train.Saver() with tf.Session() as sess: sess.run(embedding_var.initializer) saver.save(sess, os.path.join(embedding_dir_path, "embeddings.ckpt"))
log_path = "tf_writer" model_path = './singleNerualNode' removeFileInDir(log_path) [inputData, labels] = generateDataAndLabels(batchSize, numberOfInputDims) combinedDataAndLabel = np.column_stack((inputData, labels)) np.savetxt('trainDataAndLabel.csv', combinedDataAndLabel, delimiter=",") np.savetxt('label.csv', labels, delimiter=",") embedding_var = tf.Variable(inputData, 'data_embeding') config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = embedding_var.name embedding.metadata_path = 'label.csv' projector.visualize_embeddings(tf.summary.FileWriter(log_path), config) inputTensor = tf.placeholder(tf.float32, [None, numberOfInputDims], name='inputTensor') labelTensor = tf.placeholder(tf.float32, [None, 1], name='LabelTensor') with tf.name_scope('Nerual_Node'): W = tf.Variable(tf.random_normal([numberOfInputDims, 1]), name='weights') tf.summary.histogram('weights', W) b = tf.Variable(tf.zeros([1]), name='biases') tf.summary.histogram('biases', b) a = tf.nn.sigmoid(tf.matmul(inputTensor, W) + b, name='activation') with tf.name_scope('evaluation'): loss = tf.nn.l2_loss(a - labels, name='L2Loss') / batchSize threshold = 0.5
def main(argv=None): classify_image.maybe_download_and_extract() classify_image.create_graph() basedir = os.path.dirname(__file__) with tf.Session() as sess: pool3 = sess.graph.get_tensor_by_name('pool_3:0') jpeg_data = tf.placeholder(tf.string) thumbnail = tf.cast( tf.image.resize_images(tf.image.decode_jpeg(jpeg_data), [100, 100]), tf.uint8) outputs = [] files = [] images = [] for filename in os.listdir('images'): if not filename.endswith('.JPG'): continue print('process %s...' % filename) files.append(filename) with open(os.path.join(basedir, 'images', filename), 'rb') as f: data = f.read() results = sess.run([pool3, thumbnail], { 'DecodeJpeg/contents:0': data, jpeg_data: data }) outputs.append(results[0]) images.append(results[1]) embedding_var = tf.Variable(tf.pack([tf.squeeze(x) for x in outputs], axis=0), trainable=False, name='pool3') # prepare projector config config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = embedding_var.name summary_writer = tf.train.SummaryWriter(os.path.join( basedir, 'logdir')) # link metadata metadata_path = os.path.join(basedir, 'logdir', 'metadata.tsv') with open(metadata_path, 'w') as f: for name in files: f.write('%s\n' % name) embedding.metadata_path = metadata_path # write to sprite image file image_path = os.path.join(basedir, 'logdir', 'sprite.jpg') size = int(math.sqrt(len(images))) + 1 while len(images) < size * size: images.append(np.zeros((100, 100, 3), dtype=np.uint8)) rows = [] for i in range(size): rows.append(tf.concat(1, images[i * size:(i + 1) * size])) jpeg = tf.image.encode_jpeg(tf.concat(0, rows)) with open(image_path, 'wb') as f: f.write(sess.run(jpeg)) embedding.sprite.image_path = image_path embedding.sprite.single_image_dim.extend([100, 100]) # save embedding_var projector.visualize_embeddings(summary_writer, config) sess.run(tf.variables_initializer([embedding_var])) saver = tf.train.Saver([embedding_var]) saver.save(sess, os.path.join(basedir, 'logdir', 'model.ckpt'))
def main(_): # Create training directories now = datetime.datetime.now() train_dir_name = now.strftime('alexnet_%Y%m%d_%H%M%S') train_dir = os.path.join(FLAGS.train_root_dir, train_dir_name) checkpoint_dir = os.path.join(train_dir, 'checkpoint') tensorboard_dir = os.path.join(train_dir, 'tensorboard') tensorboard_train_dir = os.path.join(tensorboard_dir, 'train') tensorboard_val_dir = os.path.join(tensorboard_dir, 'val') if not os.path.isdir(FLAGS.train_root_dir): os.mkdir(FLAGS.train_root_dir) if not os.path.isdir(train_dir): os.mkdir(train_dir) if not os.path.isdir(checkpoint_dir): os.mkdir(checkpoint_dir) if not os.path.isdir(tensorboard_dir): os.mkdir(tensorboard_dir) if not os.path.isdir(tensorboard_train_dir): os.mkdir(tensorboard_train_dir) if not os.path.isdir(tensorboard_val_dir): os.mkdir(tensorboard_val_dir) # Write flags to txt flags_file_path = os.path.join(train_dir, 'flags.txt') flags_file = open(flags_file_path, 'w') flags_file.write('learning_rate={}\n'.format(FLAGS.learning_rate)) flags_file.write('dropout_keep_prob={}\n'.format(FLAGS.dropout_keep_prob)) flags_file.write('num_epochs={}\n'.format(FLAGS.num_epochs)) flags_file.write('batch_size={}\n'.format(FLAGS.batch_size)) flags_file.write('train_layers={}\n'.format(FLAGS.train_layers)) flags_file.write('multi_scale={}\n'.format(FLAGS.multi_scale)) flags_file.write('train_root_dir={}\n'.format(FLAGS.train_root_dir)) flags_file.write('log_step={}\n'.format(FLAGS.log_step)) flags_file.close() adlamb = tf.placeholder(tf.float32, name='adlamb') num_update = tf.placeholder(tf.float32, name='num_update') decay_learning_rate = tf.placeholder(tf.float32) dropout_keep_prob = tf.placeholder(tf.float32) is_training = tf.placeholder(tf.bool) time = tf.placeholder(tf.float32, [1]) # Model train_layers = FLAGS.train_layers.split(',') model = LeNetModel(num_classes=NUM_CLASSES, image_size=28, is_training=is_training, dropout_keep_prob=dropout_keep_prob) # Placeholders x_s = tf.placeholder(tf.float32, [None, 32, 32, 3], name='x') x_t = tf.placeholder(tf.float32, [None, 28, 28, 1], name='xt') x = preprocessing(x_s, model) xt = preprocessing(x_t, model) tf.summary.image('Source Images', x) tf.summary.image('Target Images', xt) print 'x_s ', x_s.get_shape() print 'x ', x.get_shape() print 'x_t ', x_t.get_shape() print 'xt ', xt.get_shape() y = tf.placeholder(tf.float32, [None, NUM_CLASSES], name='y') yt = tf.placeholder(tf.float32, [None, NUM_CLASSES], name='yt') loss = model.loss(x, y) # Training accuracy of the model source_correct_pred = tf.equal(tf.argmax(model.score, 1), tf.argmax(y, 1)) source_correct = tf.reduce_sum(tf.cast(source_correct_pred, tf.float32)) source_accuracy = tf.reduce_mean(tf.cast(source_correct_pred, tf.float32)) G_loss, D_loss, sc, tc = model.adloss(x, xt, y, yt) # Testing accuracy of the model correct_pred = tf.equal(tf.argmax(model.score, 1), tf.argmax(yt, 1)) correct = tf.reduce_sum(tf.cast(correct_pred, tf.float32)) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) update_op = model.optimize(decay_learning_rate, train_layers, adlamb, sc, tc) D_op = model.adoptimize(decay_learning_rate, train_layers) optimizer = tf.group(update_op, D_op) train_writer = tf.summary.FileWriter('./log/tensorboard') train_writer.add_graph(tf.get_default_graph()) config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = model.feature.name embedding.metadata_path = 'domain.csv' projector.visualize_embeddings(train_writer, config) tf.summary.scalar('G_loss', model.G_loss) tf.summary.scalar('D_loss', model.D_loss) tf.summary.scalar('C_loss', model.loss) tf.summary.scalar('SA_loss', model.Semanticloss) tf.summary.scalar('Training Accuracy', source_accuracy) tf.summary.scalar('Testing Accuracy', accuracy) merged = tf.summary.merge_all() print '============================GLOBAL TRAINABLE VARIABLES ============================' print tf.trainable_variables() #print '============================GLOBAL VARIABLES ======================================' #print tf.global_variables() with tf.Session() as sess: sess.run(tf.global_variables_initializer()) saver = tf.train.Saver() #saver.restore(sess,'log/checkpoint') # Load the pretrained weights #model.load_original_weights(sess, skip_layers=train_layers) train_writer.add_graph(sess.graph) # Directly restore (your model should be exactly the same with checkpoint) # saver.restore(sess, "/Users/dgurkaynak/Projects/marvel-training/alexnet64-fc6/model_epoch10.ckpt") print("{} Start training...".format(datetime.datetime.now())) #print("{} Open Tensorboard at --logdir {}".format(datetime.datetime.now(), tensorboard_dir)) gd = 0 step = 1 for epoch in range(300000): # Start training gd += 1 lamb = adaptation_factor(gd * 1.0 / MAX_STEP) #rate=decay(FLAGS.learning_rate,gd,MAX_STEP) power = gd / 10000 rate = FLAGS.learning_rate tt = pow(0.1, power) batch_xs, batch_ys = TRAIN.next_batch(FLAGS.batch_size) Tbatch_xs, Tbatch_ys = VALID.next_batch(FLAGS.batch_size) #print batch_xs.shape #print Tbatch_xs.shape summary, _, closs, gloss, dloss, smloss = sess.run( [ merged, optimizer, model.loss, model.G_loss, model.D_loss, model.Semanticloss ], feed_dict={ x_s: batch_xs, x_t: Tbatch_xs, time: [1.0 * gd], decay_learning_rate: rate, adlamb: lamb, is_training: True, y: batch_ys, dropout_keep_prob: 0.5, yt: Tbatch_ys }) train_writer.add_summary(summary, gd) step += 1 if gd % 250 == 0: epoch = gd / (72357 / 100) print 'lambda: ', lamb print 'rate: ', rate print 'Epoch {5:<10} Step {3:<10} C_loss {0:<10} G_loss {1:<10} D_loss {2:<10} Sem_loss {4:<10}'.format( closs, gloss, dloss, gd, smloss, epoch) print("{} Start validation".format(datetime.datetime.now())) test_acc = 0. test_count = 0 print 'test_iter ', len(TEST.labels) for _ in xrange((len(TEST.labels)) / 5000): batch_tx, batch_ty = TEST.next_batch(5000) #print TEST.pointer,' ',TEST.shuffle acc = sess.run(correct, feed_dict={ x_t: batch_tx, yt: batch_ty, is_training: True, dropout_keep_prob: 1. }) test_acc += acc test_count += 5000 print test_acc, test_count test_acc /= test_count if epoch == 300: return #batch_tx, batch_ty = TEST.next_batch(len(TEST.labels)) #test_acc=sess.run(accuracy,feed_dict={x_t:batch_tx,y:batch_ty,is_training:False,dropout_keep_prob:1.0}) print len(batch_tx) print("{} Validation Accuracy = {:.4f}".format( datetime.datetime.now(), test_acc)) if gd % 10000 == 0 and gd > 0: #saver.save(sess,'./log/mstn2model'+str(gd)+'.ckpt') #print 'tensorboard --logdir ./log/tensorboard' #return pass
def tensorboard_op(): # 载入数据集 mnist = input_data.read_data_sets("./../../../datas/mnist/",one_hot=True) # 运行次数 max_steps = 10001 # 图片数量 image_num = 3000 # 文件路径 DIR = "./../../../datas/logs/share/mnist/fc/" # 定义会话 sess = tf.Session() # 载入图片 embedding = tf.Variable(tf.stack(mnist.test.images[:image_num]), trainable=False, name='embedding') # 参数概要 def variable_summaries(var): with tf.name_scope('summaries'): mean = tf.reduce_mean(var) tf.summary.scalar('mean', mean) # 平均值 with tf.name_scope('stddev'): stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean))) tf.summary.scalar('stddev', stddev) # 标准差 tf.summary.scalar('max', tf.reduce_max(var)) # 最大值 tf.summary.scalar('min', tf.reduce_min(var)) # 最小值 tf.summary.histogram('histogram', var) # 直方图 # 命名空间 with tf.name_scope('input'): # 这里的none表示第一个维度可以是任意的长度 x = tf.placeholder(tf.float32, [None, 784], name='x-input') # 正确的标签 y = tf.placeholder(tf.float32, [None, 10], name='y-input') keep_prob = tf.placeholder(tf.float32) lr = tf.Variable(0.001, dtype=tf.float32) # 显示图片 with tf.name_scope('input_reshape'): image_shaped_input = tf.reshape(x, [-1, 28, 28, 1]) tf.summary.image('input', image_shaped_input, 10) with tf.name_scope('layer1'): # 创建一个简单神经网络 with tf.name_scope('weights1'): W1 = tf.Variable(tf.truncated_normal([784, 500], stddev=0.1), name='W1') variable_summaries(W1) with tf.name_scope('biases1'): b1 = tf.Variable(tf.zeros([500]) + 0.1, name='b1') variable_summaries(b1) with tf.name_scope('wx_plus_b1'): wx_plus_b1 = tf.matmul(x, W1) + b1 with tf.name_scope('tanh1'): L1 = tf.nn.tanh(wx_plus_b1) with tf.name_scope('dropout1'): L1_drop = tf.nn.dropout(L1, keep_prob) with tf.name_scope('layer2'): # 创建一个简单神经网络 with tf.name_scope('weights2'): W2 = tf.Variable(tf.truncated_normal([500, 300], stddev=0.1), name='W2') variable_summaries(W2) with tf.name_scope('biases2'): b2 = tf.Variable(tf.zeros([300]) + 0.1, name='b2') variable_summaries(b2) with tf.name_scope('wx_plus_b2'): wx_plus_b2 = tf.matmul(L1_drop, W2) + b2 with tf.name_scope('tanh2'): L2 = tf.nn.tanh(wx_plus_b2) with tf.name_scope('dropout2'): L2_drop = tf.nn.dropout(L2, keep_prob) with tf.name_scope('layer3'): # 创建一个简单神经网络 with tf.name_scope('weights3'): W3 = tf.Variable(tf.truncated_normal([300, 10], stddev=0.1), name='W3') variable_summaries(W3) with tf.name_scope('biases3'): b3 = tf.Variable(tf.zeros([10]) + 0.1, name='b3') variable_summaries(b3) with tf.name_scope('wx_plus_b3'): wx_plus_b3 = tf.matmul(L2_drop, W3) + b3 with tf.name_scope('softmax'): prediction = tf.nn.softmax(wx_plus_b3) with tf.name_scope('loss'): # 交叉熵代价函数 loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=y, logits=prediction)) tf.summary.scalar('loss', loss) with tf.name_scope('train'): # 使用梯度下降法 train_step = tf.train.GradientDescentOptimizer(0.2).minimize(loss) #train_step = tf.train.AdamOptimizer(lr).minimize(loss) # 初始化变量 sess.run(tf.global_variables_initializer()) with tf.name_scope('accuracy'): with tf.name_scope('correct_prediction'): # 结果存放在一个布尔型列表中 correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(prediction, 1)) # argmax返回一维张量中最大的值所在的位置 with tf.name_scope('accuracy'): # 求准确率 accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) # 把correct_prediction变为float32类型 tf.summary.scalar('accuracy', accuracy) # 产生metadata文件 if tf.gfile.Exists(DIR + 'projector/projector/metadata.tsv'): tf.gfile.DeleteRecursively(DIR + 'projector/projector/metadata.tsv') with open(DIR + 'projector/projector/metadata.tsv', 'w') as f: labels = sess.run(tf.argmax(mnist.test.labels[:], 1)) for i in range(image_num): f.write(str(labels[i]) + '\n') # 合并所有的summary merged = tf.summary.merge_all() projector_writer = tf.summary.FileWriter(DIR + 'projector/projector', sess.graph) saver = tf.train.Saver() config = projector.ProjectorConfig() embed = config.embeddings.add() embed.tensor_name = embedding.name embed.metadata_path = 'metadata.tsv' embed.sprite.image_path = 'mnist_10k_sprite.png' embed.sprite.single_image_dim.extend([28, 28]) projector.visualize_embeddings(projector_writer, config) count = 0 for i in range(max_steps): # 每个批次100个样本 # if i % 20 == 0 and count < 50: # count += 1 # sess.run(tf.assign(lr, 0.001 * (0.95 ** i))) batch_xs, batch_ys = mnist.train.next_batch(100) run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) run_metadata = tf.RunMetadata() summary, _ = sess.run([merged, train_step], feed_dict={x: batch_xs, y: batch_ys,keep_prob:1.0}, options=run_options, run_metadata=run_metadata) projector_writer.add_run_metadata(run_metadata, 'step%03d' % i) projector_writer.add_summary(summary, i) if i % 100 == 0: acc = sess.run(accuracy, feed_dict={x: mnist.test.images, y: mnist.test.labels,keep_prob:1.0}) print("Iter " + str(i) + ", Testing Accuracy= " + str(acc)) saver.save(sess, DIR + 'projector/projector/a_model.ckpt', global_step=max_steps) projector_writer.close() sess.close()
def train_main(args): """ trains model specfied in args. main method for train subcommand. """ # load text with open(args.text_path) as f: text = f.read() logger.info("corpus length: %s.", len(text)) # restore or build model if args.restore: load_path = args.checkpoint_path if args.restore is True else args.restore with open("{}.json".format(args.checkpoint_path)) as f: model_args = json.load(f) logger.info("model restored: %s.", load_path) else: load_path = None model_args = { "batch_size": args.batch_size, "vocab_size": VOCAB_SIZE, "embedding_size": args.embedding_size, "rnn_size": args.rnn_size, "num_layers": args.num_layers, "p_keep": 1 - args.drop_rate, "learning_rate": args.learning_rate, "clip_norm": args.clip_norm } # build train model train_graph = tf.Graph() with train_graph.as_default(): train_model = build_model(**model_args) with tf.Session(graph=train_graph) as train_sess: # restore or initialise model weights if load_path is not None: train_model["saver"].restore(train_sess, load_path) logger.info("model weights restored: %s.", load_path) else: train_sess.run(train_model["init_op"]) # clear checkpoint directory log_dir = make_dirs(args.checkpoint_path, empty=True) # save model with open("{}.json".format(args.checkpoint_path), "w") as f: json.dump(train_model["args"], f, indent=2) checkpoint_path = train_model["saver"].save(train_sess, args.checkpoint_path) logger.info("model saved: %s.", checkpoint_path) # tensorboard logger summary_writer = tf.summary.FileWriter(log_dir, train_sess.graph) # embeddings visualisation config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = "EmbedSequence/embeddings" embedding.metadata_path = os.path.abspath( os.path.join("data", "id2char.tsv")) projector.visualize_embeddings(summary_writer, config) logger.info("tensorboard set up.") # build infer model inference_graph = tf.Graph() with inference_graph.as_default(): inference_model = load_inference_model(args.checkpoint_path) # training start num_batches = (len(text) - 1) // (args.batch_size * args.seq_len) data_iter = batch_generator(encode_text(text), args.batch_size, args.seq_len) fetches = [ train_model["train_op"], train_model["output_state"], train_model["loss"], train_model["summary"] ] state = train_sess.run(train_model["input_state"]) logger.info("start of training.") time_train = time.time() for i in range(args.num_epochs): epoch_losses = np.empty(num_batches) time_epoch = time.time() # training epoch for j in tqdm(range(num_batches), desc="epoch {}/{}".format(i + 1, args.num_epochs)): x, y = next(data_iter) feed_dict = { train_model["X"]: x, train_model["Y"]: y, train_model["input_state"]: state } _, state, loss, summary_log = train_sess.run( fetches, feed_dict) epoch_losses[j] = loss # logs duration_epoch = time.time() - time_epoch logger.info("epoch: %s, duration: %ds, loss: %.6g.", i + 1, duration_epoch, epoch_losses.mean()) # tensorboard logs summary_writer.add_summary(summary_log, i + 1) summary_writer.flush() # checkpoint checkpoint_path = train_model["saver"].save( train_sess, args.checkpoint_path) logger.info("model saved: %s.", checkpoint_path) # generate text seed = generate_seed(text) with tf.Session(graph=inference_graph) as infer_sess: # restore weights inference_model["saver"].restore(infer_sess, checkpoint_path) generate_text(inference_model, infer_sess, seed) # training end duration_train = time.time() - time_train logger.info("end of training, duration: %ds.", duration_train) # generate text seed = generate_seed(text) with tf.Session(graph=inference_graph) as infer_sess: # restore weights inference_model["saver"].restore(infer_sess, checkpoint_path) generate_text(inference_model, infer_sess, seed, 1024, 3) return train_model
def main(_): # Create training directories now = datetime.datetime.now() train_dir_name = now.strftime('alexnet_%Y%m%d_%H%M%S') train_dir = os.path.join(FLAGS.train_root_dir, train_dir_name) checkpoint_dir = os.path.join(train_dir, 'checkpoint') tensorboard_dir = os.path.join(train_dir, 'tensorboard') tensorboard_train_dir = os.path.join(tensorboard_dir, 'train') tensorboard_val_dir = os.path.join(tensorboard_dir, 'val') if not os.path.isdir(FLAGS.train_root_dir): os.mkdir(FLAGS.train_root_dir) if not os.path.isdir(train_dir): os.mkdir(train_dir) if not os.path.isdir(checkpoint_dir): os.mkdir(checkpoint_dir) if not os.path.isdir(tensorboard_dir): os.mkdir(tensorboard_dir) if not os.path.isdir(tensorboard_train_dir): os.mkdir(tensorboard_train_dir) if not os.path.isdir(tensorboard_val_dir): os.mkdir(tensorboard_val_dir) # Write flags to txt flags_file_path = os.path.join(train_dir, 'flags.txt') flags_file = open(flags_file_path, 'w') flags_file.write('learning_rate={}\n'.format(FLAGS.learning_rate)) flags_file.write('dropout_keep_prob={}\n'.format(FLAGS.dropout_keep_prob)) flags_file.write('num_epochs={}\n'.format(FLAGS.num_epochs)) flags_file.write('batch_size={}\n'.format(FLAGS.batch_size)) flags_file.write('train_layers={}\n'.format(FLAGS.train_layers)) flags_file.write('multi_scale={}\n'.format(FLAGS.multi_scale)) flags_file.write('train_root_dir={}\n'.format(FLAGS.train_root_dir)) flags_file.write('log_step={}\n'.format(FLAGS.log_step)) flags_file.close() adlamb=tf.placeholder(tf.float32,name='adlamb') num_update=tf.placeholder(tf.float32,name='num_update') decay_learning_rate=tf.placeholder(tf.float32) dropout_keep_prob = tf.placeholder(tf.float32) is_training=tf.placeholder(tf.bool) time=tf.placeholder(tf.float32,[1]) # Model train_layers = FLAGS.train_layers.split(',') model = LeNetModel(num_classes=NUM_CLASSES, image_size=28,is_training=is_training,dropout_keep_prob=dropout_keep_prob) # Placeholders x_s = tf.placeholder(tf.float32, [None, 32, 32, 3],name='x') x_t = tf.placeholder(tf.float32, [None, 28, 28, 1],name='xt') x=preprocessing(x_s,model) xt=preprocessing(x_t,model) tf.summary.image('Source Images',x) tf.summary.image('Target Images',xt) print 'x_s ',x_s.get_shape() print 'x ',x.get_shape() print 'x_t ',x_t.get_shape() print 'xt ',xt.get_shape() y = tf.placeholder(tf.float32, [None, NUM_CLASSES],name='y') yt = tf.placeholder(tf.float32, [None, NUM_CLASSES],name='yt') loss = model.loss(x, y) # Training accuracy of the model source_correct_pred = tf.equal(tf.argmax(model.score, 1), tf.argmax(y, 1)) source_correct=tf.reduce_sum(tf.cast(source_correct_pred,tf.float32)) source_accuracy = tf.reduce_mean(tf.cast(source_correct_pred, tf.float32)) G_loss,D_loss,sc,tc=model.adloss(x,xt,y,yt) # Testing accuracy of the model correct_pred = tf.equal(tf.argmax(model.score, 1), tf.argmax(yt, 1)) correct=tf.reduce_sum(tf.cast(correct_pred,tf.float32)) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) update_op = model.optimize(decay_learning_rate,train_layers,adlamb,sc,tc) D_op=model.adoptimize(decay_learning_rate,train_layers) optimizer=tf.group(update_op,D_op) train_writer=tf.summary.FileWriter('./log/tensorboard') train_writer.add_graph(tf.get_default_graph()) config=projector.ProjectorConfig() embedding=config.embeddings.add() embedding.tensor_name=model.feature.name embedding.metadata_path='domain.csv' projector.visualize_embeddings(train_writer,config) tf.summary.scalar('G_loss',model.G_loss) tf.summary.scalar('D_loss',model.D_loss) tf.summary.scalar('C_loss',model.loss) tf.summary.scalar('SA_loss',model.Semanticloss) tf.summary.scalar('Training Accuracy',source_accuracy) tf.summary.scalar('Testing Accuracy',accuracy) merged=tf.summary.merge_all() print '============================GLOBAL TRAINABLE VARIABLES ============================' print tf.trainable_variables() #print '============================GLOBAL VARIABLES ======================================' #print tf.global_variables() with tf.Session() as sess: sess.run(tf.global_variables_initializer()) saver=tf.train.Saver() #saver.restore(sess,'log/checkpoint') # Load the pretrained weights #model.load_original_weights(sess, skip_layers=train_layers) train_writer.add_graph(sess.graph) # Directly restore (your model should be exactly the same with checkpoint) # saver.restore(sess, "/Users/dgurkaynak/Projects/marvel-training/alexnet64-fc6/model_epoch10.ckpt") print("{} Start training...".format(datetime.datetime.now())) #print("{} Open Tensorboard at --logdir {}".format(datetime.datetime.now(), tensorboard_dir)) gd=0 step = 1 for epoch in range(300000): # Start training gd+=1 lamb=adaptation_factor(gd*1.0/MAX_STEP) #rate=decay(FLAGS.learning_rate,gd,MAX_STEP) power=gd/10000 rate=FLAGS.learning_rate tt=pow(0.1,power) batch_xs, batch_ys = TRAIN.next_batch(FLAGS.batch_size) Tbatch_xs, Tbatch_ys = VALID.next_batch(FLAGS.batch_size) #print batch_xs.shape #print Tbatch_xs.shape summary,_,closs,gloss,dloss,smloss=sess.run([merged,optimizer,model.loss,model.G_loss,model.D_loss,model.Semanticloss], feed_dict={x_s: batch_xs,x_t: Tbatch_xs,time:[1.0*gd],decay_learning_rate:rate,adlamb:lamb,is_training:True,y: batch_ys,dropout_keep_prob:0.5,yt:Tbatch_ys}) train_writer.add_summary(summary,gd) step += 1 if gd%250==0: epoch=gd/(72357/100) print 'lambda: ',lamb print 'rate: ',rate print 'Epoch {5:<10} Step {3:<10} C_loss {0:<10} G_loss {1:<10} D_loss {2:<10} Sem_loss {4:<10}'.format(closs,gloss,dloss,gd,smloss,epoch) print("{} Start validation".format(datetime.datetime.now())) test_acc = 0. test_count = 0 print 'test_iter ',len(TEST.labels) for _ in xrange((len(TEST.labels))/5000): batch_tx, batch_ty = TEST.next_batch(5000) #print TEST.pointer,' ',TEST.shuffle acc = sess.run(correct, feed_dict={x_t: batch_tx, yt: batch_ty, is_training:True,dropout_keep_prob: 1.}) test_acc += acc test_count += 5000 print test_acc,test_count test_acc /= test_count if epoch==300: return #batch_tx, batch_ty = TEST.next_batch(len(TEST.labels)) #test_acc=sess.run(accuracy,feed_dict={x_t:batch_tx,y:batch_ty,is_training:False,dropout_keep_prob:1.0}) print len(batch_tx) print("{} Validation Accuracy = {:.4f}".format(datetime.datetime.now(), test_acc)) if gd%10000==0 and gd>0: #saver.save(sess,'./log/mstn2model'+str(gd)+'.ckpt') #print 'tensorboard --logdir ./log/tensorboard' #return pass
def word2vec(batch_gen): # Phase1: assemble your graph # Step 1: define the placeholders for input and output with tf.name_scope("data"): center_words = tf.placeholder(tf.int32, shape=[BATCH_SIZE], name='center_words') target_words = tf.placeholder(tf.int32, shape=[BATCH_SIZE, 1], name='target_words') # Step 2: define weights with tf.name_scope("weight"): embed_matrix = tf.Variable(tf.random_uniform([VOCAB_SIZE, EMBED_SIZE], -1.0, 1.0), name='embed_matrix') # Step 3: define inference model with tf.name_scope("loss"): embed = tf.nn.embedding_lookup(embed_matrix, center_words, name='embed') nce_weight = tf.Variable(tf.truncated_normal([VOCAB_SIZE, EMBED_SIZE], stddev=1.0 / (EMBED_SIZE**0.5)), name='nce_weight') nce_bias = tf.Variable(tf.zeros([VOCAB_SIZE]), name='nce_bias') # Step 4: define the loss function loss = tf.reduce_mean(tf.nn.nce_loss(weights=nce_weight, biases=nce_bias, labels=target_words, inputs=embed, num_sampled=NUM_SAMPLED, num_classes=VOCAB_SIZE), name='loss') # Step 5: define the optimizer optimizer = tf.train.GradientDescentOptimizer(LEARNING_RATE).minimize(loss) # Phase2: ececute the computation with tf.Session() as sess: sess.run(tf.global_variables_initializer()) writer = tf.summary.FileWriter('./graphs/', sess.graph) total_loss = 0 for index in range(TRAIN_STEPS): centers, targets = next(batch_gen) batch_loss, _ = sess.run([loss, optimizer], feed_dict={ center_words: centers, target_words: targets }) total_loss += batch_loss if (index + 1) % DISPLAY_STEP == 0: print('Average loss at step {}: {:5.1f}'.format( index, total_loss / DISPLAY_STEP)) total_loss = 0.0 writer.close() # Phase3: visulize # code to visualize the embeddings. uncomment the below to visualize embeddings # run "'tensorboard --logdir='processed'" to see the embeddings final_embed_matrix = sess.run( embed_matrix) #result of sess.run is constant rather than variable # # it has to variable. constants don't work here. you can't reuse model.embed_matrix embedding_var = tf.Variable(final_embed_matrix[:1000], name='embedding') sess.run(embedding_var.initializer) config = projector.ProjectorConfig() summary_writer = tf.summary.FileWriter('processed') # # add embedding to the config file embedding = config.embeddings.add() embedding.tensor_name = embedding_var.name # # link this tensor to its metadata file, in this case the first 500 words of vocab embedding.metadata_path = 'processed/vocab_1000.tsv' # # saves a configuration file that TensorBoard will read during startup. projector.visualize_embeddings(summary_writer, config) saver_embed = tf.train.Saver([embedding_var]) saver_embed.save(sess, 'processed/model3.ckpt', 1)
w2v_samples = np.zeros((args.samples, model.vector_size)) with open('{}/{}_metadata.tsv'.format(args.projector, args.prefix), 'w+') as file_metadata: for i, word in enumerate(model.wv.index2word[:args.samples]): w2v_samples[i] = model[word] file_metadata.write(word + '\n') # define the model without training sess = tf.InteractiveSession() with tf.device("/cpu:0"): embedding = tf.Variable(w2v_samples, trainable=False, name='{}_embedding'.format(args.prefix)) tf.global_variables_initializer().run() saver = tf.train.Saver() writer = tf.summary.FileWriter(args.projector, sess.graph) # adding into projector config = projector.ProjectorConfig() embed = config.embeddings.add() embed.tensor_name = '{}_embedding'.format(args.prefix) embed.metadata_path = './{}_metadata.tsv'.format(args.prefix) # Specify the width and height of a single thumbnail. projector.visualize_embeddings(writer, config) saver.save(sess, '{}/{}_model.ckpt'.format(args.projector, args.prefix), global_step=args.samples) print('Start tensorboard with: \'tensorboard --logdir=\"projector\"\'\n' 'and check http://localhost:6006/#embeddings to view your embedding')
final_embeddings = normalized_embeddings.eval() # Write corresponding labels for the embeddings. with open(FLAGS.log_dir + '/metadata.tsv', 'w') as f: for i in xrange(vocabulary_size): f.write(reverse_dictionary[i] + '\n') # Save the model for checkpoints. saver.save(session, os.path.join(FLAGS.log_dir, 'model.ckpt')) # Create a configuration for visualizing embeddings with the labels in TensorBoard. config = projector.ProjectorConfig() embedding_conf = config.embeddings.add() embedding_conf.tensor_name = embeddings.name embedding_conf.metadata_path = os.path.join(FLAGS.log_dir, 'metadata.tsv') projector.visualize_embeddings(writer, config) writer.close() # Step 6: Visualize the embeddings. # pylint: disable=missing-docstring # Function to draw visualization of distance between embeddings. def plot_with_labels(low_dim_embs, labels, filename): assert low_dim_embs.shape[0] >= len(labels), 'More labels than embeddings' plt.figure(figsize=(18, 18)) # in inches for i, label in enumerate(labels): x, y = low_dim_embs[i, :] plt.scatter(x, y) plt.annotate(
def set_model(self, model): self.model = model if K.backend() == 'tensorflow': self.sess = K.get_session() if self.histogram_freq and self.merged is None: for layer in self.model.layers: for weight in layer.weights: mapped_weight_name = weight.name.replace(':', '_') tf.summary.histogram(mapped_weight_name, weight) if self.write_grads: grads = model.optimizer.get_gradients(model.total_loss, weight) def is_indexed_slices(grad): return type(grad).__name__ == 'IndexedSlices' grads = [ grad.values if is_indexed_slices(grad) else grad for grad in grads] tf.summary.histogram('{}_grad'.format(mapped_weight_name), grads) if self.write_images: w_img = tf.squeeze(weight) shape = K.int_shape(w_img) if len(shape) == 2: # dense layer kernel case if shape[0] > shape[1]: w_img = tf.transpose(w_img) shape = K.int_shape(w_img) w_img = tf.reshape(w_img, [1, shape[0], shape[1], 1]) elif len(shape) == 3: # convnet case if K.image_data_format() == 'channels_last': # switch to channels_first to display # every kernel as a separate image w_img = tf.transpose(w_img, perm=[2, 0, 1]) shape = K.int_shape(w_img) w_img = tf.reshape(w_img, [shape[0], shape[1], shape[2], 1]) elif len(shape) == 1: # bias case w_img = tf.reshape(w_img, [1, shape[0], 1, 1]) else: # not possible to handle 3D convnets etc. continue shape = K.int_shape(w_img) assert len(shape) == 4 and shape[-1] in [1, 3, 4] tf.summary.image(mapped_weight_name, w_img) if hasattr(layer, 'output'): if isinstance(layer.output, list): for i, output in enumerate(layer.output): tf.summary.histogram('{}_out_{}'.format(layer.name, i), output) else: tf.summary.histogram('{}_out'.format(layer.name), layer.output) self.merged = tf.summary.merge_all() if self.write_graph: self.writer = tf.summary.FileWriter(self.log_dir, self.sess.graph) else: self.writer = tf.summary.FileWriter(self.log_dir) if self.embeddings_freq and self.embeddings_data is not None: self.embeddings_data = standardize_input_data(self.embeddings_data, model.input_names) embeddings_layer_names = self.embeddings_layer_names if not embeddings_layer_names: embeddings_layer_names = [layer.name for layer in self.model.layers if type(layer).__name__ == 'Embedding'] self.assign_embeddings = [] embeddings_vars = {} self.batch_id = batch_id = tf.placeholder(tf.int32) self.step = step = tf.placeholder(tf.int32) for layer in self.model.layers: if layer.name in embeddings_layer_names: embedding_input = self.model.get_layer(layer.name).output embedding_size = np.prod(embedding_input.shape[1:]) embedding_input = tf.reshape(embedding_input, (step, int(embedding_size))) shape = (self.embeddings_data[0].shape[0], int(embedding_size)) embedding = tf.Variable(tf.zeros(shape), name=layer.name + '_embedding') embeddings_vars[layer.name] = embedding batch = tf.assign(embedding[batch_id:batch_id + step], embedding_input) self.assign_embeddings.append(batch) self.saver = tf.train.Saver(list(embeddings_vars.values())) embeddings_metadata = {} if not isinstance(self.embeddings_metadata, str): embeddings_metadata = self.embeddings_metadata else: embeddings_metadata = {layer_name: self.embeddings_metadata for layer_name in embeddings_vars.keys()} config = projector.ProjectorConfig() for layer_name, tensor in embeddings_vars.items(): embedding = config.embeddings.add() embedding.tensor_name = tensor.name if layer_name in embeddings_metadata: embedding.metadata_path = embeddings_metadata[layer_name] projector.visualize_embeddings(self.writer, config)
def word2vec_basic(log_dir): """Example of building, training and visualizing a word2vec model.""" # Create the directory for TensorBoard variables if there is not. if not os.path.exists(log_dir): os.makedirs(log_dir) filename = 'dataset.txt' # Read the data into a list of strings. def read_data(filename): dir_path = os.path.dirname(os.path.realpath(__file__)) with open(os.path.join(dir_path, filename)) as f: data = tf.compat.as_str(f.read()).split() return data vocabulary = read_data(filename) print(vocabulary) print('Data size', len(vocabulary)) # sys.exit() # Step 2: Build the dictionary and replace rare words with UNK token. vocabulary_size = 100 def build_dataset(words, n_words): """Process raw inputs into a dataset.""" count = [['UNK', -1]] count.extend(collections.Counter(words).most_common(n_words - 1)) dictionary = {} for word, _ in count: dictionary[word] = len(dictionary) data = [] unk_count = 0 for word in words: index = dictionary.get(word, 0) if index == 0: # dictionary['UNK'] unk_count += 1 data.append(index) count[0][1] = unk_count reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys())) return data, count, dictionary, reversed_dictionary # Filling 4 global variables: # data - list of codes (integers from 0 to vocabulary_size-1). # This is the original text but words are replaced by their codes # count - map of words(strings) to count of occurrences # dictionary - map of words(strings) to their codes(integers) # reverse_dictionary - maps codes(integers) to words(strings) data, count, unused_dictionary, reverse_dictionary = build_dataset( vocabulary, vocabulary_size) del vocabulary # Hint to reduce memory. print('Most common words (+UNK)', count[:5]) print('Sample data', data[:10], [reverse_dictionary[i] for i in data[:10]]) # Step 3: Function to generate a training batch for the skip-gram model. def generate_batch(batch_size, num_skips, skip_window): global data_index assert batch_size % num_skips == 0 assert num_skips <= 2 * skip_window batch = np.ndarray(shape=(batch_size), dtype=np.int32) labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32) span = 2 * skip_window + 1 # [ skip_window target skip_window ] buffer = collections.deque(maxlen=span) # pylint: disable=redefined-builtin if data_index + span > len(data): data_index = 0 buffer.extend(data[data_index:data_index + span]) data_index += span for i in range(batch_size // num_skips): context_words = [w for w in range(span) if w != skip_window] words_to_use = random.sample(context_words, num_skips) for j, context_word in enumerate(words_to_use): batch[i * num_skips + j] = buffer[skip_window] labels[i * num_skips + j, 0] = buffer[context_word] if data_index == len(data): buffer.extend(data[0:span]) data_index = span else: buffer.append(data[data_index]) data_index += 1 # Backtrack a little bit to avoid skipping words in the end of a batch data_index = (data_index + len(data) - span) % len(data) return batch, labels batch, labels = generate_batch(batch_size=8, num_skips=2, skip_window=1) for i in range(8): print(batch[i], reverse_dictionary[batch[i]], '->', labels[i, 0], reverse_dictionary[labels[i, 0]]) # Step 4: Build and train a skip-gram model. batch_size = 64 # 128 embedding_size = 64 # Dimension of the embedding vector. 128 skip_window = 1 # How many words to consider left and right. num_skips = 2 # How many times to reuse an input to generate a label. num_sampled = 64 # Number of negative examples to sample. # We pick a random validation set to sample nearest neighbors. Here we limit # the validation samples to the words that have a low numeric ID, which by # construction are also the most frequent. These 3 variables are used only for # displaying model accuracy, they don't affect calculation. valid_size = 16 # Random set of words to evaluate similarity on. valid_window = 20 # Only pick dev samples in the head of the distribution. 100 valid_examples = np.random.choice(valid_window, valid_size, replace=False) graph = tf.Graph() with graph.as_default(): # Input data. with tf.name_scope('inputs'): train_inputs = tf.placeholder(tf.int32, shape=[batch_size]) train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1]) valid_dataset = tf.constant(valid_examples, dtype=tf.int32) # Ops and variables pinned to the CPU because of missing GPU implementation with tf.device('/cpu:0'): # Look up embeddings for inputs. with tf.name_scope('embeddings'): embeddings = tf.Variable( tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0)) embed = tf.nn.embedding_lookup(embeddings, train_inputs) # Construct the variables for the NCE loss with tf.name_scope('weights'): nce_weights = tf.Variable( tf.truncated_normal([vocabulary_size, embedding_size], stddev=1.0 / math.sqrt(embedding_size))) with tf.name_scope('biases'): nce_biases = tf.Variable(tf.zeros([vocabulary_size])) # Compute the average NCE loss for the batch. # tf.nce_loss automatically draws a new sample of the negative labels each # time we evaluate the loss. # Explanation of the meaning of NCE loss: # http://mccormickml.com/2016/04/19/word2vec-tutorial-the-skip-gram-model/ with tf.name_scope('loss'): loss = tf.reduce_mean( tf.nn.nce_loss(weights=nce_weights, biases=nce_biases, labels=train_labels, inputs=embed, num_sampled=num_sampled, num_classes=vocabulary_size)) # Add the loss value as a scalar to summary. tf.summary.scalar('loss', loss) # Construct the SGD optimizer using a learning rate of 1.0. with tf.name_scope('optimizer'): optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss) # Compute the cosine similarity between minibatch examples and all # embeddings. norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keepdims=True)) normalized_embeddings = embeddings / norm valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset) similarity = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True) # Merge all summaries. merged = tf.summary.merge_all() # Add variable initializer. init = tf.global_variables_initializer() # Create a saver. saver = tf.train.Saver() # Step 5: Begin training. num_steps = 20001 # 100001 with tf.Session(graph=graph) as session: # Open a writer to write summaries. writer = tf.summary.FileWriter(log_dir, session.graph) # We must initialize all variables before we use them. init.run() print('Initialized') average_loss = 0 for step in xrange(num_steps): batch_inputs, batch_labels = generate_batch( batch_size, num_skips, skip_window) feed_dict = { train_inputs: batch_inputs, train_labels: batch_labels } # Define metadata variable. run_metadata = tf.RunMetadata() # We perform one update step by evaluating the optimizer op (including it # in the list of returned values for session.run() # Also, evaluate the merged op to get all summaries from the returned # "summary" variable. Feed metadata variable to session for visualizing # the graph in TensorBoard. _, summary, loss_val = session.run([optimizer, merged, loss], feed_dict=feed_dict, run_metadata=run_metadata) average_loss += loss_val # Add returned summaries to writer in each step. writer.add_summary(summary, step) # Add metadata to visualize the graph for the last run. if step == (num_steps - 1): writer.add_run_metadata(run_metadata, 'step%d' % step) if step % 2000 == 0: if step > 0: average_loss /= 2000 # The average loss is an estimate of the loss over the last 2000 # batches. print('Average loss at step ', step, ': ', average_loss) average_loss = 0 # Note that this is expensive (~20% slowdown if computed every 500 steps) if step % 10000 == 0: sim = similarity.eval() for i in xrange(valid_size): valid_word = reverse_dictionary[valid_examples[i]] top_k = 8 # number of nearest neighbors nearest = (-sim[i, :]).argsort()[1:top_k + 1] log_str = 'Nearest to %s:' % valid_word for k in xrange(top_k): close_word = reverse_dictionary[nearest[k]] log_str = '%s %s,' % (log_str, close_word) print(log_str) final_embeddings = normalized_embeddings.eval() # Write corresponding labels for the embeddings. with open(log_dir + '/metadata.tsv', 'w') as f: for i in xrange(vocabulary_size): f.write(reverse_dictionary[i] + '\n') # Save the model for checkpoints. saver.save(session, os.path.join(log_dir, 'model.ckpt')) # Create a configuration for visualizing embeddings with the labels in # TensorBoard. config = projector.ProjectorConfig() embedding_conf = config.embeddings.add() embedding_conf.tensor_name = embeddings.name embedding_conf.metadata_path = os.path.join(log_dir, 'metadata.tsv') projector.visualize_embeddings(writer, config) writer.close() def save_tsne(low_dim_embs, labels, filename): import json tsne = {} xs = [] ys = [] for i, label in enumerate(labels): x, y = low_dim_embs[i, :] xs.append(x) ys.append(y) for i, label in enumerate(labels): x, y = low_dim_embs[i, :] j = np.interp(x, [min(xs), max(xs)], [0, 700]) k = np.interp(y, [min(ys), max(ys)], [0, 700]) tsne[label] = [float(j), float(k)] with open(filename, 'w', encoding='utf-8') as f: json.dump(tsne, f, ensure_ascii=False, indent=2) # Step 6: Visualize the embeddings. # pylint: disable=missing-docstring # Function to draw visualization of distance between embeddings. def plot_with_labels(low_dim_embs, labels, filename): assert low_dim_embs.shape[0] >= len( labels), 'More labels than embeddings' plt.figure(figsize=(18, 18)) # in inches for i, label in enumerate(labels): x, y = low_dim_embs[i, :] plt.scatter(x, y) plt.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points', ha='right', va='bottom') plt.savefig(filename) try: # pylint: disable=g-import-not-at-top from sklearn.manifold import TSNE import matplotlib.pyplot as plt tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000, method='exact') plot_only = 100 low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only, :]) labels = [reverse_dictionary[i] for i in xrange(plot_only)] dir_path = os.path.dirname(os.path.realpath(__file__)) plot_with_labels(low_dim_embs, labels, os.path.join(dir_path, 'tsne.png')) save_tsne(low_dim_embs, labels, os.path.join(dir_path, 'tsne.json')) except ImportError as ex: print( 'Please install sklearn, matplotlib, and scipy to show embeddings.' ) print(ex)
def main(_): """Runs main labeled eval loop.""" # Parse config dict from yaml config files / command line flags. config = util.ParseConfigsToLuaTable(FLAGS.config_paths, FLAGS.model_params) # Choose an estimator based on training strategy. checkpointdir = FLAGS.checkpointdir checkpoint_path = os.path.join( '%s/model.ckpt-%s' % (checkpointdir, FLAGS.checkpoint_iter)) estimator = get_estimator(config, checkpointdir) # Get records to embed. validation_dir = FLAGS.embedding_records validation_records = util.GetFilesRecursively(validation_dir) sequences_to_data = {} for (view_embeddings, view_raw_image_strings, seqname) in estimator.inference( validation_records, checkpoint_path, config.data.embed_batch_size, num_sequences=FLAGS.num_sequences): sequences_to_data[seqname] = { 'embeddings': view_embeddings, 'images': view_raw_image_strings, } all_embeddings = np.zeros((0, config.embedding_size)) all_ims = [] all_seqnames = [] num_embeddings = FLAGS.num_embed # Concatenate all views from all sequences into a big flat list. for seqname, data in sequences_to_data.iteritems(): embs = data['embeddings'] ims = data['images'] for v in range(config.data.num_views): for (emb, im) in zip(embs[v], ims[v]): all_embeddings = np.append(all_embeddings, [emb], axis=0) all_ims.append(im) all_seqnames.append(seqname) # Choose N indices uniformly from all images. random_indices = range(all_embeddings.shape[0]) random.shuffle(random_indices) viz_indices = random_indices[:num_embeddings] # Extract embs. viz_embs = np.array(all_embeddings[viz_indices]) # Extract and decode ims. viz_ims = list(np.array(all_ims)[viz_indices]) decoded_ims = [] sprite_dim = FLAGS.sprite_dim for i, im in enumerate(viz_ims): if i % 100 == 0: print('Decoding image %d/%d.' % (i, num_embeddings)) nparr_i = np.fromstring(str(im), np.uint8) img_np = cv2.imdecode(nparr_i, 1) img_np = img_np[..., [2, 1, 0]] img_np = imresize(img_np, [sprite_dim, sprite_dim, 3]) decoded_ims.append(img_np) decoded_ims = np.array(decoded_ims) # Extract sequence names. outdir = FLAGS.outdir # The embedding variable, which needs to be stored # Note this must a Variable not a Tensor! embedding_var = tf.Variable(viz_embs, name='viz_embs') with tf.Session() as sess: sess.run(embedding_var.initializer) summary_writer = tf.summary.FileWriter(outdir) config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = embedding_var.name # Comment out if you don't want sprites embedding.sprite.image_path = os.path.join(outdir, 'sprite.png') embedding.sprite.single_image_dim.extend( [decoded_ims.shape[1], decoded_ims.shape[1]]) projector.visualize_embeddings(summary_writer, config) saver = tf.train.Saver([embedding_var]) saver.save(sess, os.path.join(outdir, 'model2.ckpt'), 1) sprite = images_to_sprite(decoded_ims) imsave(os.path.join(outdir, 'sprite.png'), sprite)
def main(_): if tf.gfile.Exists(FLAGS.log_dir): tf.gfile.DeleteRecursively(FLAGS.log_dir) tf.gfile.MakeDirs(FLAGS.log_dir) # Hyper-parameters width, height = 32, 32 size = (width, height) classes = 1721 batch_size = 50 steps = 10000 save_location = "/tmp/tensorflow/kanji_simple/1" # Import data training, t_labels, validation, v_labels = prep.data_from_base( 'train_val_test_data_32') t_labels = onehot_labels(t_labels, classes) v_labels = onehot_labels(v_labels, classes) print('data imported') sess = tf.InteractiveSession() # Create the model with tf.name_scope('input'): x = tf.placeholder(tf.float32, [None, width * height]) y_ = tf.placeholder(tf.float32, [None, classes]) with tf.name_scope('input_reshape'): x_image = tf.reshape(x, [-1, width, height, 1]) tf.summary.image('input', x_image, classes) # setting up the cnn def weight_variable(shape, nme): initial = tf.truncated_normal(shape, stddev=0.1) return tf.Variable(initial, name=nme) def bias_variable(shape, nme): initial = tf.constant(0.1, shape=shape) return tf.Variable(initial, name=nme) def conv2d(x, W): return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME') def max_pool_2x2(x): return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') def variable_summaries(var): with tf.name_scope('summaries'): mean = tf.reduce_mean(var) tf.summary.scalar('mean', mean) tf.summary.scalar('max', tf.reduce_max(var)) tf.summary.scalar('min', tf.reduce_min(var)) tf.summary.histogram('histogram', var) with tf.name_scope('stddev'): stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean))) tf.summary.scalar('stddev', stddev) # adding the first convolutional layer with tf.name_scope('conv_layer1'): with tf.name_scope('weights'): W_conv1 = weight_variable([5, 5, 1, 32], "w1") variable_summaries(W_conv1) with tf.name_scope('biases'): b_conv1 = bias_variable([32], "b1") variable_summaries(b_conv1) with tf.name_scope('activation'): h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1) tf.summary.histogram('activations', h_conv1) # adding the first pooling layer with tf.name_scope('pooling1'): h_pool1 = max_pool_2x2(h_conv1) pool1_img = tf.reshape(h_pool1, [-1, width, height, 1]) tf.summary.image('pool1', pool1_img, classes) # adding the third convolutional layer with tf.name_scope('conv_layer2'): with tf.name_scope('weights'): W_conv3 = weight_variable([5, 5, 32, 64], "w3") variable_summaries(W_conv3) with tf.name_scope('biases'): b_conv3 = bias_variable([64], "b3") variable_summaries(b_conv3) with tf.name_scope('activation'): h_conv3 = tf.nn.relu(conv2d(h_pool1, W_conv3) + b_conv3) tf.summary.histogram('activations', h_conv3) # the second pooling layer with tf.name_scope('pooling2'): h_pool2 = max_pool_2x2(h_conv3) pool2_img = tf.reshape(h_pool2, [-1, width, height, 1]) tf.summary.image('pool2', pool2_img, classes) # adding the fifth convolutional layer with tf.name_scope('conv_layer3'): with tf.name_scope('weights'): W_conv5 = weight_variable([5, 5, 64, 64], "w5") variable_summaries(W_conv5) with tf.name_scope('biases'): b_conv5 = bias_variable([64], "b5") variable_summaries(b_conv5) with tf.name_scope('activation'): h_conv5 = tf.nn.relu(conv2d(h_pool2, W_conv5) + b_conv5) tf.summary.histogram('activations', h_conv5) # the third pooling layer h_pool3 = max_pool_2x2(h_conv5) #adding the final layer with tf.name_scope('final_layer'): with tf.name_scope('weights'): W_fc1 = weight_variable([4 * 4 * 64, 1024], "W_fc1") variable_summaries(W_fc1) with tf.name_scope('biases'): b_fc1 = bias_variable([1024], "b_fc1") variable_summaries(b_fc1) h_pool3_flat = tf.reshape(h_pool3, [-1, 4 * 4 * 64]) with tf.name_scope('Wx_plus_b'): preactivate = tf.matmul(h_pool3_flat, W_fc1) + b_fc1 tf.summary.histogram('pre_activations', preactivate) h_fc1 = tf.nn.relu(preactivate) tf.summary.histogram('activations', h_fc1) # adding the dropout with tf.name_scope('dropout'): keep_prob = tf.placeholder(tf.float32) tf.summary.scalar('dropout_keep_probability', keep_prob) h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob) # adding the readout layer with tf.name_scope('readout_layer'): with tf.name_scope('weights'): W_fc3 = weight_variable([1024, classes], "w_read") variable_summaries(W_fc3) with tf.name_scope('biases'): b_fc3 = bias_variable([classes], "b_read") variable_summaries(b_fc3) with tf.name_scope('Wx_plus_b'): y_conv = tf.matmul(h_fc1_drop, W_fc3) + b_fc3 tf.summary.histogram('activations', y_conv) with tf.name_scope('cross_entropy'): # The raw formulation of cross-entropy, # # tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)), # reduction_indices=[1])) # # can be numerically unstable. # # So here we use tf.nn.softmax_cross_entropy_with_logits on the raw # outputs of 'y', and then average across the batch. diff = tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv) with tf.name_scope('total'): cross_entropy = tf.reduce_mean(diff) tf.summary.scalar('cross_entropy', cross_entropy) with tf.name_scope('train'): train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy) # Test trained model with tf.name_scope('accuracy'): with tf.name_scope('correct_prediction'): correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1)) with tf.name_scope('accuracy'): accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) tf.summary.scalar('accuracy', accuracy) merged = tf.summary.merge_all() train_writer = tf.summary.FileWriter( '/tmp/tensorflow/kanji_simple/1/logs/kanji_with_summaries/train', sess.graph) test_writer = tf.summary.FileWriter( '/tmp/tensorflow/kanji_simple/1/logs/kanji_with_summaries/test') tf.global_variables_initializer().run() saver = tf.train.Saver() # if os.path.exists(os.path.join(save_location)): # saver.restore(sess, save_location + "/model.ckpt") epoch = -1 test_batch = 500 # Train for i in range(steps): a = i * batch_size % len(training) batchx = training[a:a + batch_size] batchy = t_labels[a:a + batch_size] summary, _ = sess.run([merged, train_step], feed_dict={ x: batchx, y_: batchy, keep_prob: 0.5 }) train_writer.add_summary(summary, i) if i % 100 == 0: run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) run_metadata = tf.RunMetadata() summary, _ = sess.run([merged, train_step], feed_dict={ x: batchx, y_: batchy, keep_prob: 0.5 }, options=run_options, run_metadata=run_metadata) train_writer.add_run_metadata(run_metadata, 'step%03d' % i) train_writer.add_summary(summary, i) print('Adding run metadata for', i) summary, acc = sess.run([merged, accuracy], feed_dict={ x: validation, y_: v_labels, keep_prob: 1.0 }) test_writer.add_summary(summary, i) print('Accuracy at step %s: %s' % (i, acc)) save_path = saver.save(sess, save_location + "/model.ckpt", i) save_path = saver.save(sess, save_location + "/model.ckpt", steps) summary, acc = sess.run([merged, accuracy], feed_dict={ x: validation, y_: v_labels, keep_prob: 1.0 }) test_writer.add_summary(summary, i) print('Accuracy at step %s: %s' % (i, acc)) prediction = tf.argmax(y_conv, 1) print( "predictions", prediction.eval(feed_dict={ x: validation, y_: v_labels, keep_prob: 1.0 }, session=sess)) print( "correct predictions", correct_prediction.eval(feed_dict={ x: validation, y_: v_labels, keep_prob: 1.0 }, session=sess)) # Create randomly initialized embedding weights which will be trained. N = classes # Number of items (classes). D = 200 # Dimensionality of the embedding. embedding_var = tf.Variable(tf.random_normal([N, D]), name='image_embedding') # Format: tensorflow/tensorboard/plugins/projector/projector_config.proto config = projector.ProjectorConfig() # You can add multiple embeddings. Here we add only one. embedding = config.embeddings.add() embedding.sprite.image_path = 'home/workspace/kanji_tests/sprites20/master.jpg' # Specify the width and height of a single thumbnail. embedding.sprite.single_image_dim.extend([20, 20]) embedding.tensor_name = embedding_var.name # Link this tensor to its metadata file (e.g. labels). # embedding.metadata_path = os.path.join('sprites20/', 'labels.tsv') # Use the same LOG_DIR where you stored your checkpoint. summary_writer = tf.summary.FileWriter(save_location) # The next line writes a projector_config.pbtxt in the LOG_DIR. TensorBoard will # read this file during startup. projector.visualize_embeddings(summary_writer, config) train_writer.close() test_writer.close()
np.savetxt('Xtest.tsv', X, fmt='%.6e', delimiter='\t') np.savetxt('Ytest.tsv', Y_str, fmt='%s') plt.imsave('zalando-mnist-sprite.png', get_sprite_image(X), cmap='gray') exit() embedding_var = tf.Variable(X, name='mnist_pixels') # Format: tensorflow/tensorboard/plugins/projector/projector_config.proto config = projector.ProjectorConfig() # You can add multiple embeddings. Here we add only one. embedding = config.embeddings.add() embedding.tensor_name = embedding_var.name # Link this tensor to its metadata file (e.g. labels). embedding.metadata_path = VIS_DIR + 'Ytest.tsv' embedding.sprite.image_path = VIS_DIR + 'zalando-mnist-sprite.png' # Specify the width and height of a single thumbnail. embedding.sprite.single_image_dim.extend([28, 28]) # Use the same LOG_DIR where you stored your checkpoint. summary_writer = tf.summary.FileWriter(LOG_DIR + 'visualization') # The next line writes a projector_config.pbtxt in the LOG_DIR. TensorBoard will # read this file during startup. projector.visualize_embeddings(summary_writer, config) sess = tf.Session() sess.run(tf.global_variables_initializer()) saver = tf.train.Saver() saver.save(sess, LOG_DIR + 'visualization/model.ckpt', 0)
data = np.pad(data, padding, mode='constant', constant_values=0) # Tile the individual thumbnails into an image. data = data.reshape((n, n) + data.shape[1:]).transpose((0, 2, 1, 3) + tuple(range(4, data.ndim + 1))) data = data.reshape((n * data.shape[1], n * data.shape[3]) + data.shape[4:]) data = (data * 225).astype(np.uint8) return data #%% sprite = images_to_sprite(img_data) cv2.imwrite(os.path.join(LOG_DIR, 'sprite_24_classes.png'), sprite) #%% modified Prarthana with tf.Session() as sess: saver = tf.train.Saver([features]) sess.run(features.initializer) saver.save(sess, os.path.join(LOG_DIR, 'images_24_classes.ckpt')) config = projector.ProjectorConfig() # One can add multiple embeddings. embedding = config.embeddings.add() embedding.tensor_name = features.name # Link this tensor to its metadata file (e.g. labels). embedding.metadata_path = os.path.join(LOG_DIR, 'metadata_24_classes.tsv') # Comment out if you don't want sprites embedding.sprite.image_path = os.path.join(LOG_DIR, 'sprite_24_classes.png') embedding.sprite.single_image_dim.extend([img_data.shape[1], img_data.shape[1]]) # Saves a config file that TensorBoard will read during startup. projector.visualize_embeddings(tf.summary.FileWriter(LOG_DIR), config)
def __init__(self, d, K, sig, sess, logdir): self.K = K self.sig = sig self.sess = sess self.logdir = logdir with tf.name_scope('model'): # Data Placeholder with tf.name_scope('input'): self.placeholders = tf.placeholder(tf.int32) self.words = self.placeholders # Index Masks with tf.name_scope('context_mask'): self.p_mask = tf.cast( tf.range(d.cs / 2, d.n_minibatch + d.cs / 2), tf.int32) rows = tf.cast( tf.tile(tf.expand_dims(tf.range(0, d.cs / 2), [0]), [d.n_minibatch, 1]), tf.int32) columns = tf.cast( tf.tile(tf.expand_dims(tf.range(0, d.n_minibatch), [1]), [1, d.cs / 2]), tf.int32) self.ctx_mask = tf.concat( [rows + columns, rows + columns + d.cs / 2 + 1], 1) with tf.name_scope('embeddings'): # Embedding vectors self.rho = tf.Variable(tf.random_normal([d.L, self.K]) / self.K, name='rho') # Context vectors self.alpha = tf.Variable(tf.random_normal([d.L, self.K]) / self.K, name='alpha') with tf.name_scope('priors'): prior = Normal(loc=0.0, scale=self.sig) self.log_prior = tf.reduce_sum( prior.log_prob(self.rho) + prior.log_prob(self.alpha)) with tf.name_scope('natural_param'): # Taget and Context Indices with tf.name_scope('target_word'): self.p_idx = tf.gather(self.words, self.p_mask) self.p_rho = tf.squeeze(tf.gather(self.rho, self.p_idx)) # Negative samples with tf.name_scope('negative_samples'): unigram_logits = tf.tile( tf.expand_dims(tf.log(tf.constant(d.unigram)), [0]), [d.n_minibatch, 1]) self.n_idx = tf.multinomial(unigram_logits, d.ns) self.n_rho = tf.gather(self.rho, self.n_idx) with tf.name_scope('context'): self.ctx_idx = tf.squeeze( tf.gather(self.words, self.ctx_mask)) self.ctx_alphas = tf.gather(self.alpha, self.ctx_idx) # Natural parameter ctx_sum = tf.reduce_sum(self.ctx_alphas, [1]) self.p_eta = tf.expand_dims( tf.reduce_sum(tf.multiply(self.p_rho, ctx_sum), -1), 1) self.n_eta = tf.reduce_sum( tf.multiply( self.n_rho, tf.tile(tf.expand_dims(ctx_sum, 1), [1, d.ns, 1])), -1) # Conditional likelihood self.y_pos = Bernoulli(logits=self.p_eta) self.y_neg = Bernoulli(logits=self.n_eta) self.ll_pos = tf.reduce_sum(self.y_pos.log_prob(1.0)) self.ll_neg = tf.reduce_sum(self.y_neg.log_prob(0.0)) self.log_likelihood = self.ll_pos + self.ll_neg scale = 1.0 * d.N / d.n_minibatch self.loss = -(scale * self.log_likelihood + self.log_prior) # Training optimizer = tf.train.AdamOptimizer() self.train = optimizer.minimize(self.loss) with self.sess.as_default(): tf.global_variables_initializer().run() variable_summaries('rho', self.rho) variable_summaries('alpha', self.alpha) with tf.name_scope('objective'): tf.summary.scalar('loss', self.loss) tf.summary.scalar('priors', self.log_prior) tf.summary.scalar('ll_pos', self.ll_pos) tf.summary.scalar('ll_neg', self.ll_neg) self.summaries = tf.summary.merge_all() self.train_writer = tf.summary.FileWriter(self.logdir, self.sess.graph) self.saver = tf.train.Saver() config = projector.ProjectorConfig() alpha = config.embeddings.add() alpha.tensor_name = 'model/embeddings/alpha' alpha.metadata_path = '../vocab.tsv' rho = config.embeddings.add() rho.tensor_name = 'model/embeddings/rho' rho.metadata_path = '../vocab.tsv' projector.visualize_embeddings(self.train_writer, config)
def start_C(iteration,start = True): run_config = tf.ConfigProto() # run_config.gpu_options.allow_growth=True run_config.gpu_options.per_process_gpu_memory_fraction = 0.8 tf.logging.set_verbosity(tf.logging.DEBUG) tfrecords_path = './data_tf/' with tf.Graph().as_default(): mnist = input_data.read_data_sets('MNIST_data', one_hot=True) ckpt = tf.train.get_checkpoint_state(os.path.dirname('./checkpoint_pretrain/checkpoint')) sess = tf.InteractiveSession(config = run_config) global_step = slim.create_global_step() summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES)) mnist_net = Classification_Model() # mnist_net.change_dataset("5") # x, y_ = mnist_net.get_batch() x,y_ = mnist_net.get_batch_tf(tfrecords_path) # arg_scope = mnist_net.model_arg_scope() end_points = {} # with slim.arg_scope(arg_scope): logits, end_points = mnist_net.net(x) extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(extra_update_ops): losses = mnist_net.losses(logits, y_) train_step = mnist_net.optimizer(0.001).minimize(losses, global_step=global_step) embedding, config = mnist_net.get_embedding('./checkpoint_pretrain/') #total_loss = tf.losses.get_total_loss() summaries.add(tf.summary.image("img", tf.cast(x, tf.float32))) summaries.add(tf.summary.scalar('loss', losses)) for variable in tf.trainable_variables(): summaries.add(tf.summary.histogram(variable.op.name, variable)) train_writer = tf.summary.FileWriter('./checkpoint_pretrain/train',sess.graph) projector.visualize_embeddings(train_writer, config) correct_prediction = tf.equal(tf.argmax(end_points['Predictions'], 1), tf.argmax(y_, 1)) accuracy = tf.reduce_sum(tf.cast(correct_prediction, tf.float32)) / mnist_net.batch_size summaries.add(tf.summary.scalar('accuracy', accuracy)) summary_op = tf.summary.merge(list(summaries), name='summary_op') sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) saver = tf.train.Saver(max_to_keep=5, keep_checkpoint_every_n_hours=1.0, write_version=2, pad_step_number=False) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) save_test = [] x_test = tf.placeholder(tf.float32, shape=[None, 28, 28, 1]) y_test = tf.placeholder(tf.float32, shape=[None, 10]) logits_test, end_points_test = mnist_net.net(x_test,is_training=False, reuse = True) correct_prediction_test = tf.equal(tf.argmax(end_points_test['Predictions'], 1), tf.argmax(y_test, 1)) num_correct = tf.reduce_sum(tf.cast(correct_prediction_test,tf.float32)) assignment = embedding.assign(end_points_test['Features']) for i in range(iteration): if i %100 == 0 : # sum_accuracy_test = 0.0 # batch_size = 100 # for count in range(100): # test_image = np.reshape(mnist.test.images[count*batch_size:(count+1)*batch_size],(batch_size,28,28,1)) *2.0 -1 # test_label = np.reshape(mnist.test.labels[count*batch_size:(count+1)*batch_size],(batch_size,10)) # num_c = sess.run(num_correct, # feed_dict = {x_test:test_image, y_test:test_label}) # sum_accuracy_test += num_c test_batch_x = mnist.test.images[:10000] * 2.0 - 1 test_batch_y = mnist.test.labels[:10000] sum_accuracy_test, _ = sess.run([num_correct, assignment], feed_dict={x_test: np.reshape(test_batch_x, (-1, 28, 28, 1)), y_test: test_batch_y}) print ("test accuracy is: %f" % (sum_accuracy_test /10000.0 )) # saver.save(sess, "./checkpoint_pretrain/",global_step=global_step_str) print('****************************') print ("test accuracy is: %f" % (sum_accuracy_test /10000.0 )) print('****************************') if start: if not save_test: save_test.append(sum_accuracy_test) else : save_test.append(sum_accuracy_test) if sum_accuracy_test > save_test[0] : print ('u are getting better!!!!') print ('saving model') saver.save(sess, "./checkpoint_pretrain/",global_step= global_step.eval()) break else: print('ops, not this time ~!') else: if sum_accuracy_test/10000.0 >= 0.995: print ('saving model') saver.save(sess, "./checkpoint_pretrain/",global_step= global_step.eval()) break _,summary_str,current_accuracy = sess.run([train_step,summary_op,accuracy]) if i %10 == 0: train_writer.add_summary(summary_str,i) print('%diteration'%i,current_accuracy) coord.request_stop() coord.join(threads) # print ('saving model') # saver.save(sess, "./checkpoint_pretrain/",global_step= global_step.eval()) time.sleep(3)
def set_model(self, model): self.model = model self.sess = K.get_session() if self.histogram_freq and self.merged is None: for layer in self.model.layers: for weight in layer.weights: tf.summary.histogram(weight.name, weight) if self.write_grads: grads = model.optimizer.get_gradients(model.total_loss, weight) tf.summary.histogram('{}_grad'.format(weight.name), grads) if self.write_images: w_img = tf.squeeze(weight) shape = K.int_shape(w_img) if len(shape) == 2: # dense layer kernel case if shape[0] > shape[1]: w_img = tf.transpose(w_img) shape = K.int_shape(w_img) w_img = tf.reshape(w_img, [1, shape[0], shape[1], 1]) elif len(shape) == 3: # convnet case if K.image_data_format() == 'channels_last': # switch to channels_first to display # every kernel as a separate image w_img = tf.transpose(w_img, perm=[2, 0, 1]) shape = K.int_shape(w_img) w_img = tf.reshape(w_img, [shape[0], shape[1], shape[2], 1]) elif len(shape) == 1: # bias case w_img = tf.reshape(w_img, [1, shape[0], 1, 1]) else: # not possible to handle 3D convnets etc. continue shape = K.int_shape(w_img) assert len(shape) == 4 and shape[-1] in [1, 3, 4] tf.summary.image(weight.name, w_img) if hasattr(layer, 'output'): tf.summary.histogram('{}_out'.format(layer.name), layer.output) self.merged = tf.summary.merge_all() if self.write_graph: self.writer = tf.summary.FileWriter(self.log_dir, self.sess.graph) else: self.writer = tf.summary.FileWriter(self.log_dir) if self.embeddings_freq: self.saver = tf.train.Saver() embeddings_layer_names = self.embeddings_layer_names if not embeddings_layer_names: embeddings_layer_names = [layer.name for layer in self.model.layers if type(layer).__name__ == 'Embedding'] embeddings = {layer.name: layer.weights[0] for layer in self.model.layers if layer.name in embeddings_layer_names} embeddings_metadata = {} if not isinstance(self.embeddings_metadata, str): embeddings_metadata = self.embeddings_metadata else: embeddings_metadata = {layer_name: self.embeddings_metadata for layer_name in embeddings.keys()} config = projector.ProjectorConfig() self.embeddings_logs = [] for layer_name, tensor in embeddings.items(): embedding = config.embeddings.add() embedding.tensor_name = tensor.name self.embeddings_logs.append(os.path.join(self.log_dir, layer_name + '.ckpt')) if layer_name in embeddings_metadata: embedding.metadata_path = embeddings_metadata[layer_name] projector.visualize_embeddings(self.writer, config)
# train_step = tf.train.GradientDescentOptimizer(0.01).minimize(loss) train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss) correct_prediction = tf.equal(tf.argmax(y2,1), tf.argmax(y_,1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) variables_names = [v.name for v in tf.trainable_variables()] init = tf.global_variables_initializer() sess = tf.Session() sess.run(init) summary_writer = tf.summary.FileWriter(result_dir) projector.visualize_embeddings(summary_writer, config) saver = tf.train.Saver() print("Training...") #values = sess.run(variables_names) # for k, v in zip(variables_names, values): # print "Variable: ", k # print "Shape: ", v.shape # # print v
def main(args): # pass the args as params so the model_fn can use # the TPU specific args params = vars(args) if args.use_tpu: # additional configs required for using TPUs tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(args.tpu) tpu_config = tf.contrib.tpu.TPUConfig( num_shards=8, # using Cloud TPU v2-8 iterations_per_loop=args.save_checkpoints_steps) # use the TPU version of RunConfig config = tf.contrib.tpu.RunConfig( cluster=tpu_cluster_resolver, model_dir=args.model_dir, tpu_config=tpu_config, save_checkpoints_steps=args.save_checkpoints_steps, save_summary_steps=100) # TPUEstimator estimator = tf.contrib.tpu.TPUEstimator( model_fn=model_fn, config=config, params=params, train_batch_size=args.train_batch_size, # Calling TPUEstimator.predict requires setting predict_bath_size. predict_batch_size=PREDICT_BATCH_SIZE, eval_batch_size=32, export_to_tpu=False) else: config = tf.estimator.RunConfig(model_dir=args.model_dir) estimator = tf.estimator.Estimator( model_fn, config=config, params=params) estimator.train(train_input_fn, max_steps=args.max_steps) # After training, apply the learned embedding to the test data and visualize with tensorboard Projector. embeddings = next(estimator.predict(predict_input_fn, yield_single_examples=False))['embeddings'] # Put the embeddings into a variable to be visualized. embedding_var = tf.Variable(embeddings, name='test_embeddings') # Labels do not pass through the estimator.predict call, so we get it separately. _, (_, labels) = tf.keras.datasets.mnist.load_data() labels = labels[:PREDICT_BATCH_SIZE] # Write the metadata file for the projector. metadata_path = os.path.join(estimator.model_dir, 'metadata.tsv') with tf.gfile.GFile(metadata_path, 'w') as f: f.write('index\tlabel\n') for i, label in enumerate(labels): f.write('{}\t{}\n'.format(i, label)) # Configure the projector. projector_config = projector.ProjectorConfig() embedding_config = projector_config.embeddings.add() embedding_config.tensor_name = embedding_var.name # The metadata_path is relative to the summary_writer's log_dir. embedding_config.metadata_path = 'metadata.tsv' summary_writer = tf.summary.FileWriter(estimator.model_dir) projector.visualize_embeddings(summary_writer, projector_config) # Start a session to actually write the embeddings into a new checkpoint. sess = tf.Session() sess.run(tf.global_variables_initializer()) saver = tf.train.Saver() saver.save(sess, os.path.join(estimator.model_dir, 'model.ckpt'), args.max_steps+1)
def fit(self, trainFile=None, with_validation=general_config.with_validation, log_dir=general_config.log_dir + "/HAN", save_dir=general_config.save_dir + "/HAN", load_path=general_config.load_path_train, num_epochs=general_config.num_epochs, steps_every_epoch=general_config.steps_every_epoch, batch_size=general_config.batch_size, learning_rate=general_config.learning_rate, lr_changing=general_config.lr_changing, min_learning_rate=general_config.min_learning_rate, learning_rate_decay=general_config.learning_rate_decay, save_epochs=general_config.save_epochs, early_stopping=han_config.early_stopping, num_visual=general_config.num_visualize): self.learning_rate_value = learning_rate self.trainFile = trainFile self.validFile = None self.with_validation = with_validation if self.trainFile is None: if self.with_validation: self.trainFile = general_config.data_dir + "/train.txt" else: self.trainFile = general_config.data_dir + "/training_label_new.txt" if self.with_validation: self.validFile = self.trainFile.replace("train", "valid") tmp = os.path.join( os.path.dirname(self.trainFile), os.path.basename(self.trainFile).replace(".txt", "").split("_")[0]) self.int2vocabPath = tmp + "_i2v.json" self.vocab2intPath = tmp + "_v2i.json" metadataPath = {} metadataPath["nonstatic"] = "/home/leechen/code/python/TextSentimentClassification/" \ + self.int2vocabPath.replace("i2v.json", "metadata.tsv") train_loss = [] train_accuracy = [] valid_loss = [] valid_accuracy = [] # 训练过程中的日志保存文件以及模型保存路径 if self.with_validation: log_dir = ensure_dir_exist(log_dir + "/train_valid") train_dir = os.path.join(log_dir, "train") val_dir = os.path.join(log_dir, "valid") save_dir = ensure_dir_exist(save_dir + "/train_valid") else: log_dir = ensure_dir_exist(log_dir + "/train") train_dir = os.path.join(log_dir, "train") val_dir = None save_dir = ensure_dir_exist(save_dir + "/train") # 生成日志 logger = my_logger(log_dir + "/log_fit.txt") msg = "\n--state_size_word: %s\n"%self.state_size_word\ +"--attention_dim_word: %s\n"%self.attention_dim_word\ + "--state_size_sentence: %s\n" % self.state_size_sentence \ + "--attention_dim_sentence: %s\n" % self.attention_dim_sentence \ + "--fc_layer_size: %s\n" % self.fc_layer_size_list \ + "--embedding_size: %s\n" % self.embedding_size \ + "--dropout: %s\n" % self.dropout_value \ + "--max_l2_norm: %s\n" % self.max_l2_norm \ + "--grads_clip: %s\n" % self.grads_clip \ + "--learning_rate: %s\n" % self.learning_rate_value \ + "--lr_changing: %s\n" % lr_changing \ + "--min_learning_rate: %s\n" % min_learning_rate \ + "--learning_rate_decay: %s\n" % learning_rate_decay \ + "--load_path: %s\n" % load_path \ + "--num_epochs: %s\n" % num_epochs \ + "--steps_every_epoch: %s\n" % steps_every_epoch \ + "--batch_size: %s\n" % batch_size \ + "--save_epochs: %s\n" % save_epochs \ + "--early_stopping: %s\n" % early_stopping \ + "--num_visual: %s\n" % num_visual logger.info(msg) # 定义数据生成器 train_generator = BucketedDataIteratorForDoc( loadPath=self.trainFile, vocab2intPath=self.vocab2intPath) val_generator = None if self.validFile is None else BucketedDataIteratorForDoc( loadPath=self.validFile, vocab2intPath=self.vocab2intPath) os.environ["CUDA_VISIBLE_DEVICES"] = str(0) config = tf.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 0.8 with tf.Session(config=config, graph=self.graph) as sess: train_writer = tf.summary.FileWriter(train_dir, sess.graph) val_writer = None if val_dir is None else tf.summary.FileWriter( val_dir) saver = tf.train.Saver(max_to_keep=5) sess.run(tf.global_variables_initializer()) start = 0 if isinstance(load_path, str): if os.path.isdir(load_path): ckpt = tf.train.get_checkpoint_state(load_path) saver.restore(sess, ckpt.model_checkpoint_path) start = ckpt.model_checkpoint_path.split("-")[-1] else: saver.restore(sess, load_path) start = load_path.split("-")[-1] logger.info("Loading successfully, loading epoch is %s" % start) logger.info("The total number of trainable variables: %s" % get_num_params()) cur_early_stopping = 0 cur_max_acc = 0. logger.info('******* start training with %d *******' % start) epoch = 0 for epoch in range(start, num_epochs): if lr_changing: try: if (train_loss[-1] > train_loss[-2]): tmp = self.learning_rate_value * learning_rate_decay if (tmp >= min_learning_rate): self.learning_rate_value = tmp logger.info( "Learning rate multiplied by %s at epoch %s." % (learning_rate_decay, epoch + 1)) else: if (train_loss[-1] < train_loss[-2] - 0.015): self.learning_rate_value *= 1.05 logger.info( "Learning rate multiplied by 1.05 at epoch %s." % (epoch + 1)) except: pass avg_loss_t, avg_accuracy_t = 0, 0 avg_loss_v, avg_accuracy_v = 0, 0 for step in range(steps_every_epoch): _, batch_docs, batch_labels, batch_doc_lens, batch_sent_lens = train_generator.next( batch_size) sess.run(self.train_op, feed_dict=self._feed_dict_train( batch_x=batch_docs, batch_y=batch_labels, batch_doc_len=batch_doc_lens, batch_sent_len=batch_sent_lens)) loss_t, acc_t = sess.run( [self.loss_op, self.acc_op], feed_dict=self._feed_dict_valid( batch_x=batch_docs, batch_y=batch_labels, batch_doc_len=batch_doc_lens, batch_sent_len=batch_sent_lens)) avg_loss_t += loss_t avg_accuracy_t += acc_t avg_loss_t /= steps_every_epoch avg_accuracy_t /= steps_every_epoch train_loss.append(avg_loss_t) train_accuracy.append(avg_accuracy_t) self.loss_accuracy_summary.value[0].simple_value = avg_loss_t self.loss_accuracy_summary.value[ 1].simple_value = avg_accuracy_t train_writer.add_summary(summary=self.loss_accuracy_summary, global_step=epoch + 1) if self.with_validation: # 计算验证集上的表现 cur_loop = val_generator.loop _, batch_docs, batch_labels, batch_doc_lens, batch_sent_lens = val_generator.next( 1024, need_all=True) cur_count = 0 while (val_generator.loop == cur_loop): loss_v, acc_v = sess.run( [self.loss_op, self.acc_op], feed_dict=self._feed_dict_valid( batch_docs, batch_labels, batch_doc_len=batch_doc_lens, batch_sent_len=batch_sent_lens)) avg_loss_v += loss_v avg_accuracy_v += acc_v cur_count += 1 _, batch_docs, batch_labels, batch_doc_lens, batch_sent_lens = val_generator.next( 1024, need_all=True) avg_loss_v /= cur_count avg_accuracy_v /= cur_count valid_loss.append(avg_loss_v) valid_accuracy.append(avg_accuracy_v) self.loss_accuracy_summary.value[ 0].simple_value = avg_loss_v self.loss_accuracy_summary.value[ 1].simple_value = avg_accuracy_v val_writer.add_summary(summary=self.loss_accuracy_summary, global_step=epoch + 1) logger.info("Epoch: [%04d/%04d], " "Training Loss: %.4f, Training Accuracy: %.4f, " "Validation Loss: %.4f, Validation Accuracy: %.4f" \ % (epoch + 1, num_epochs, avg_loss_t, avg_accuracy_t, avg_loss_v, avg_accuracy_v)) # 如果验证集上的准确率连续低于历史最高准确率的次数超过early_stopping次,则提前停止迭代。 if (avg_accuracy_v > cur_max_acc): cur_max_acc = avg_accuracy_v cur_early_stopping = 0 logger.info("Saving model-%s" % (epoch + 1)) saver.save(sess, os.path.join(save_dir, 'model.ckpt'), global_step=epoch + 1) else: cur_early_stopping += 1 if cur_early_stopping > early_stopping: logger.info("Early stopping after epoch %s !" % (epoch + 1)) break else: logger.info("Epoch: [%04d/%04d], " "Training Loss: %.4f, Training Accuracy: %.4f " \ % (epoch + 1, num_epochs, avg_loss_t, avg_accuracy_t)) # 保存一次模型 if (epoch - start + 1) % save_epochs == 0: logger.info("Saving model-%s" % (epoch + 1)) saver.save(sess, os.path.join(save_dir, 'model.ckpt'), global_step=epoch + 1) if num_visual > 0: # 可视化最终词向量 config = projector.ProjectorConfig() final_embeddings = {} final_embeddings["nonstatic"] = self.embedding_matrix_ns.eval( )[:num_visual] for (name, final_embedding) in final_embeddings.items(): embedding_var = tf.Variable(final_embedding, name="word_embeddings_" + name) sess.run(embedding_var.initializer) saver = tf.train.Saver([embedding_var]) saver.save( sess, log_dir + "/embeddings_" + name + ".ckpt" + str(epoch + 1)) embedding = config.embeddings.add() embedding.tensor_name = embedding_var.name embedding.metadata_path = metadataPath[name] projector.visualize_embeddings(train_writer, config) return train_loss, train_accuracy, valid_loss, valid_accuracy
def set_model(self, model): self.model = model self.sess = K.get_session() if self.histogram_freq and self.merged is None: for layer in self.model.layers: for weight in layer.weights: mapped_weight_name = weight.name.replace(':', '_') tf.summary.histogram(mapped_weight_name, weight) if self.write_grads: grads = model.optimizer.get_gradients( model.total_loss, weight) def is_indexed_slices(grad): return type(grad).__name__ == 'IndexedSlices' grads = [ grad.values if is_indexed_slices(grad) else grad for grad in grads ] tf.summary.histogram( '{}_grad'.format(mapped_weight_name), grads) if self.write_images: w_img = tf.squeeze(weight) shape = K.int_shape(w_img) if len(shape) == 2: # dense layer kernel case if shape[0] > shape[1]: w_img = tf.transpose(w_img) shape = K.int_shape(w_img) w_img = tf.reshape(w_img, [1, shape[0], shape[1], 1]) elif len(shape) == 3: # convnet case if K.image_data_format() == 'channels_last': # switch to channels_first to display # every kernel as a separate image w_img = tf.transpose(w_img, perm=[2, 0, 1]) shape = K.int_shape(w_img) w_img = tf.reshape( w_img, [shape[0], shape[1], shape[2], 1]) elif len(shape) == 1: # bias case w_img = tf.reshape(w_img, [1, shape[0], 1, 1]) else: # not possible to handle 3D convnets etc. continue shape = K.int_shape(w_img) assert len(shape) == 4 and shape[-1] in [1, 3, 4] tf.summary.image(mapped_weight_name, w_img) if hasattr(layer, 'output'): tf.summary.histogram('{}_out'.format(layer.name), layer.output) self.merged = tf.summary.merge_all() if self.write_graph: self.writer = tf.summary.FileWriter(self.log_dir, self.sess.graph) else: self.writer = tf.summary.FileWriter(self.log_dir) if self.embeddings_freq: embeddings_layer_names = self.embeddings_layer_names if not embeddings_layer_names: embeddings_layer_names = [ layer.name for layer in self.model.layers if type(layer).__name__ == 'Embedding' ] embeddings = { layer.name: layer.weights[0] for layer in self.model.layers if layer.name in embeddings_layer_names } self.saver = tf.train.Saver(list(embeddings.values())) embeddings_metadata = {} if not isinstance(self.embeddings_metadata, str): embeddings_metadata = self.embeddings_metadata else: embeddings_metadata = { layer_name: self.embeddings_metadata for layer_name in embeddings.keys() } config = projector.ProjectorConfig() self.embeddings_ckpt_path = os.path.join(self.log_dir, 'keras_embedding.ckpt') for layer_name, tensor in embeddings.items(): embedding = config.embeddings.add() embedding.tensor_name = tensor.name if layer_name in embeddings_metadata: embedding.metadata_path = embeddings_metadata[layer_name] projector.visualize_embeddings(self.writer, config)
def train_model(model, batch_gen, num_train_steps, weights_fld): saver = tf.train.Saver( ) # defaults to saving all variables - in this case embed_matrix, nce_weight, nce_bias initial_step = 0 with tf.Session() as sess: sess.run(tf.global_variables_initializer()) ckpt = tf.train.get_checkpoint_state( os.path.dirname(logDir + '/checkpoints/checkpoint')) # if that checkpoint exists, restore from checkpoint if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) total_loss = 0.0 # we use this to calculate late average loss in the last SKIP_STEP steps writer = tf.summary.FileWriter(logDir + '/lr' + str(LEARNING_RATE), sess.graph) initial_step = model.global_step.eval() for index in range(initial_step, initial_step + num_train_steps): try: centers, targets = next(batch_gen) feed_dict = { model.center_words: centers, model.target_words: targets } loss_batch, _, summary = sess.run( [model.loss, model.optimizer, model.summary_op], feed_dict=feed_dict) writer.add_summary(summary, global_step=index) total_loss += loss_batch if (index + 1) % SKIP_STEP == 0: print('Average loss at step {}: {:5.1f}'.format( index, total_loss / SKIP_STEP)) total_loss = 0.0 saver.save(sess, logDir + '/checkpoints/skip-gram', index) except StopIteration: break writer.close() #################### # code to visualize the embeddings. uncomment the below to visualize embeddings final_embed_matrix = sess.run(model.embed_matrix) # it has to variable. constants don't work here. you can't reuse model.embed_matrix embedding_var = tf.Variable(final_embed_matrix[:1000], name='embedding') sess.run(embedding_var.initializer) config = projector.ProjectorConfig() summary_writer = tf.summary.FileWriter(logDir + '/processed') # add embedding to the config file embedding = config.embeddings.add() embedding.tensor_name = embedding_var.name # link this tensor to its metadata file, in this case the first 1000 words of vocab embedding.metadata_path = weights_fld + '/vocab_1000.tsv' # saves a configuration file that TensorBoard will read during startup. projector.visualize_embeddings(summary_writer, config) saver_embed = tf.train.Saver([embedding_var]) saver_embed.save(sess, logDir + '/processed/model3.ckpt', 1) summary_writer.close()