def visualize_char(model, path="/home/aegis/igor/LM4paper/tests/textchar.txt", ):
chars = open(path, 'r').read().splitlines()
embedding = np.empty(shape=(len(chars), model.hps.emb_char_size), dtype=np.float32)
for i, char in enumerate(chars):
embedding[i] = model.get_char_embedding(char)
print(embedding)
print(embedding.shape)
logdir = "/data/visualog/char/"
metadata = os.path.join(logdir, "metadata.tsv")
with open(metadata, "w") as metadata_file:
for c in chars:
metadata_file.write("%s\n" % c)
tf.reset_default_graph()
with tf.Session() as sess:
X = tf.Variable([0.0], name='embedding')
place = tf.placeholder(tf.float32, shape=embedding.shape)
set_x = tf.assign(X, place, validate_shape=False)
sess.run(tf.global_variables_initializer())
sess.run(set_x, feed_dict={place: embedding})
saver = tf.train.Saver([X])
saver.save(sess, os.path.join(logdir, 'char.ckpt'))
config = projector.ProjectorConfig()
# One can add multiple embeddings.
embedding = config.embeddings.add()
embedding.tensor_name = X.name
# Link this tensor to its metadata file (e.g. labels).
embedding.metadata_path = metadata
# Saves a config file that TensorBoard will read during startup.
projector.visualize_embeddings(tf.summary.FileWriter(logdir), config)
def generate_embeddings():
# Import data
mnist = input_data.read_data_sets(FLAGS.data_dir,
one_hot=True,
fake_data=FLAGS.fake_data)
sess = tf.InteractiveSession()
# Input set for Embedded TensorBoard visualization
# Performed with cpu to conserve memory and processing power
with tf.device("/cpu:0"):
embedding = tf.Variable(tf.stack(mnist.test.images[:FLAGS.max_steps], axis=0), trainable=False, name='embedding')
tf.global_variables_initializer().run()
saver = tf.train.Saver()
writer = tf.summary.FileWriter(FLAGS.log_dir + '/projector', sess.graph)
# Add embedding tensorboard visualization. Need tensorflow version
# >= 0.12.0RC0
config = projector.ProjectorConfig()
embed= config.embeddings.add()
embed.tensor_name = 'embedding:0'
embed.metadata_path = os.path.join(FLAGS.log_dir + '/projector/metadata.tsv')
embed.sprite.image_path = os.path.join(FLAGS.data_dir + '/mnist_10k_sprite.png')
# Specify the width and height of a single thumbnail.
embed.sprite.single_image_dim.extend([28, 28])
projector.visualize_embeddings(writer, config)
saver.save(sess, os.path.join(
FLAGS.log_dir, 'projector/a_model.ckpt'), global_step=FLAGS.max_steps)
def set_model(self, model):
if self.embeddings_freq:
self.saver = tf.train.Saver()
embeddings_layer_names = self.embeddings_layer_names
elayers = find_embedding_layers(model.layers)
if not embeddings_layer_names:
embeddings_layer_names = [layer.name for layer in elayers]
embeddings = {layer.name: layer.weights[0] for layer in elayers
if layer.name in embeddings_layer_names}
embeddings_metadata = {}
if not isinstance(self.embeddings_metadata, str):
embeddings_metadata = self.embeddings_metadata
else:
embeddings_metadata = {layer_name: self.embeddings_metadata
for layer_name in embeddings.keys()}
config = projector.ProjectorConfig()
self.embeddings_logs = []
for layer_name, tensor in embeddings.items():
embedding = config.embeddings.add()
embedding.tensor_name = tensor.name
self.embeddings_logs.append(os.path.join(self.log_dir,
layer_name + '.ckpt'))
if layer_name in embeddings_metadata:
embedding.metadata_path = embeddings_metadata[layer_name]
projector.visualize_embeddings(self.writer, config)
def visualize(self, visual_fld, num_visualize):
""" run "'tensorboard --logdir='visualization'" to see the embeddings """
# create the list of num_variable most common words to visualize
word2vec_utils.most_common_words(visual_fld, num_visualize)
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
ckpt = tf.train.get_checkpoint_state(os.path.dirname('checkpoints/checkpoint'))
# if that checkpoint exists, restore from checkpoint
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
final_embed_matrix = sess.run(self.embed_matrix)
# you have to store embeddings in a new variable
embedding_var = tf.Variable(final_embed_matrix[:num_visualize], name='embedding')
sess.run(embedding_var.initializer)
config = projector.ProjectorConfig()
summary_writer = tf.summary.FileWriter(visual_fld)
# add embedding to the config file
embedding = config.embeddings.add()
embedding.tensor_name = embedding_var.name
# link this tensor to its metadata file, in this case the first NUM_VISUALIZE words of vocab
embedding.metadata_path = 'vocab_' + str(num_visualize) + '.tsv'
# saves a configuration file that TensorBoard will read during startup.
projector.visualize_embeddings(summary_writer, config)
saver_embed = tf.train.Saver([embedding_var])
saver_embed.save(sess, os.path.join(visual_fld, 'model.ckpt'), 1)
def test(self, step, number=400): # 256 self.batch_size
session = sess = self.session
config = projector.ProjectorConfig()
if visualize_cluster:
embedding = config.embeddings.add() # You can add multiple embeddings. Here just one.
embedding.tensor_name = self.last_layer.name # last_dense
# embedding.tensor_path
# embedding.tensor_shape
embedding.sprite.image_path = PATH_TO_SPRITE_IMAGE
# help(embedding.sprite)
embedding.sprite.single_image_dim.extend([width, hight]) # if mnist thumbnail
# embedding.single_image_dim.extend([28, 28]) # if mnist thumbnail
# Link this tensor to its metadata file (e.g. labels).
embedding.metadata_path = os.path.join(LOG_DIR, 'metadata.tsv')
# Saves a configuration file that TensorBoard will read during startup.
projector.visualize_embeddings(self.summary_writer, config)
run_metadata = tf.RunMetadata()
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
# Calculate accuracy for 256 mnist test images
test_images, test_labels = self.next_batch(number, session, test=True)
feed_dict = {self.x: test_images, self.y: test_labels, self.keep_prob: 1., self.train_phase: False}
# accuracy,summary= self.session.run([self.accuracy, self.summaries], feed_dict=feed_dict)
accuracy, summary = session.run([self.accuracy, self.summaries], feed_dict, run_options, run_metadata)
print('\t' * 3 + "Test Accuracy: ", accuracy)
self.summary_writer.add_run_metadata(run_metadata, 'step #%03d' % step)
self.summary_writer.add_summary(summary, global_step=step)
def write_embeddings(self, Wv, name="WordVectors"):
"""Write embedding matrix to the right place.
Args:
Wv: (numpy.ndarray) |V| x d matrix of word embeddings
"""
with tf.Graph().as_default(), tf.Session() as session:
##
# Feed embeddings to tf, and save.
embedding_var = tf.Variable(Wv, name=name, dtype=tf.float32)
session.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.save(session, self.CHECKPOINT_FILE, 0)
##
# Save metadata
summary_writer = tf.summary.FileWriter(self.LOGDIR)
config = projector.ProjectorConfig()
embedding = config.embeddings.add()
embedding.tensor_name = embedding_var.name
embedding.metadata_path = self.VOCAB_FILE_BASE
projector.visualize_embeddings(summary_writer, config)
msg = "Saved {s0:d} x {s1:d} embedding matrix '{name}'"
msg += " to LOGDIR='{logdir}'"
print(msg.format(s0=Wv.shape[0], s1=Wv.shape[1], name=name,
logdir=self.LOGDIR))
print("To view, run:")
print("\n tensorboard --logdir=\"{logdir}\"\n".format(logdir=self.LOGDIR))
print("and navigate to the \"Embeddings\" tab in the web interface.")
def save_embeddings(model: adagram.VectorModel, output: Path, words: List[str]):
labels = []
senses = []
for word in words:
for sense, _ in model.word_sense_probs(word):
labels.append('{} #{}'.format(word, sense))
v = model.sense_vector(word, sense)
senses.append(v / np.linalg.norm(v))
output.mkdir(exist_ok=True)
labels_path = output.joinpath('labels.tsv')
labels_path.write_text('\n'.join(labels))
senses = np.array(senses)
with tf.Session() as session:
embedding_var = tf.Variable(senses, trainable=False, name='senses')
session.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.save(session, str(output.joinpath('model.ckpt')))
summary_writer = tf.train.SummaryWriter(str(output))
config = projector.ProjectorConfig()
embedding = config.embeddings.add()
embedding.tensor_name = embedding_var.name
embedding.metadata_path = str(labels_path)
projector.visualize_embeddings(summary_writer, config)
def save_tb_embeddings(embeddings_filename):
f = open(embeddings_filename, 'rb')
embeddings = pickle.load(f)
images = embeddings['images']
zs = embeddings['zs']
# overwrite Tensorboard log dir if necessary
if os.path.exists(TB_DIR):
shutil.rmtree(TB_DIR)
os.makedirs(TB_DIR)
# create grid image
img_width, img_height = save_sprite_image(images)
with tf.device('cpu:0'):
# create embedding var
embedding_var = tf.Variable(initial_value=zs)
# save projector config
summary_writer = tf.summary.FileWriter(TB_DIR)
config = projector.ProjectorConfig()
embedding = config.embeddings.add()
embedding.tensor_name = embedding_var.name
embedding.sprite.image_path = SPRITE_IMAGE_FILENAME
embedding.sprite.single_image_dim.extend([img_width, img_height])
projector.visualize_embeddings(summary_writer, config)
# save embeddings
sess = tf.Session()
sess.run(embedding_var.initializer)
saver = tf.train.Saver([embedding_var])
saver.save(sess, os.path.join(TB_DIR, 'model.ckpt'))
def main(vectors_loc, out_loc, name="spaCy_vectors"):
meta_file = "{}.tsv".format(name)
out_meta_file = path.join(out_loc, meta_file)
print('Loading spaCy vectors model: {}'.format(vectors_loc))
model = spacy.load(vectors_loc)
print('Finding lexemes with vectors attached: {}'.format(vectors_loc))
strings_stream = tqdm.tqdm(model.vocab.strings, total=len(model.vocab.strings), leave=False)
queries = [w for w in strings_stream if model.vocab.has_vector(w)]
vector_count = len(queries)
print('Building Tensorboard Projector metadata for ({}) vectors: {}'.format(vector_count, out_meta_file))
# Store vector data in a tensorflow variable
tf_vectors_variable = numpy.zeros((vector_count, model.vocab.vectors.shape[1]))
# Write a tab-separated file that contains information about the vectors for visualization
#
# Reference: https://www.tensorflow.org/programmers_guide/embedding#metadata
with open(out_meta_file, 'wb') as file_metadata:
# Define columns in the first row
file_metadata.write("Text\tFrequency\n".encode('utf-8'))
# Write out a row for each vector that we add to the tensorflow variable we created
vec_index = 0
for text in tqdm.tqdm(queries, total=len(queries), leave=False):
# https://github.com/tensorflow/tensorflow/issues/9094
text = '<Space>' if text.lstrip() == '' else text
lex = model.vocab[text]
# Store vector data and metadata
tf_vectors_variable[vec_index] = model.vocab.get_vector(text)
file_metadata.write("{}\t{}\n".format(text, math.exp(lex.prob) * vector_count).encode('utf-8'))
vec_index += 1
print('Running Tensorflow Session...')
sess = tf.InteractiveSession()
tf.Variable(tf_vectors_variable, trainable=False, name=name)
tf.global_variables_initializer().run()
saver = tf.train.Saver()
writer = tf.summary.FileWriter(out_loc, sess.graph)
# Link the embeddings into the config
config = ProjectorConfig()
embed = config.embeddings.add()
embed.tensor_name = name
embed.metadata_path = meta_file
# Tell the projector about the configured embeddings and metadata file
visualize_embeddings(writer, config)
# Save session and print run command to the output
print('Saving Tensorboard Session...')
saver.save(sess, path.join(out_loc, '{}.ckpt'.format(name)))
print('Done. Run `tensorboard --logdir={0}` to view in Tensorboard'.format(out_loc))
def _add_emb_vis(self, embedding_var): """Do setup so that we can view word embedding visualization in Tensorboard, as described here: https://www.tensorflow.org/get_started/embedding_viz Make the vocab metadata file, then make the projector config file pointing to it.""" train_dir = os.path.join(FLAGS.log_root, "train") vocab_metadata_path = os.path.join(train_dir, "vocab_metadata.tsv") self._vocab.write_metadata(vocab_metadata_path) # write metadata file summary_writer = tf.summary.FileWriter(train_dir) config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = embedding_var.name embedding.metadata_path = vocab_metadata_path projector.visualize_embeddings(summary_writer, config)
def testVisualizeEmbeddings(self):
# Create a dummy configuration.
config = projector_config_pb2.ProjectorConfig()
config.model_checkpoint_path = 'test'
emb1 = config.embeddings.add()
emb1.tensor_name = 'tensor1'
emb1.metadata_path = 'metadata1'
# Call the API method to save the configuration to a temporary dir.
temp_dir = self.get_temp_dir()
self.addCleanup(shutil.rmtree, temp_dir)
writer = writer_lib.FileWriter(temp_dir)
projector.visualize_embeddings(writer, config)
# Read the configuratin from disk and make sure it matches the original.
with gfile.GFile(os.path.join(temp_dir, 'projector_config.pbtxt')) as f:
config2 = projector_config_pb2.ProjectorConfig()
text_format.Parse(f.read(), config2)
self.assertEqual(config, config2)
def __get_tensorboard_writer(self, path):
tensorboard_writer = tf.summary.FileWriter(path, graph=self.graph, filename_suffix=".bot")
# set the projector's configuration to add the embedding summary also:
conf = projector.ProjectorConfig()
embedding_field = conf.embeddings.add()
embedding_content_label = conf.embeddings.add()
# set the tensors to these embedding matrices
embedding_field.tensor_name = self.field_embedding_matrix.name
embedding_content_label.tensor_name = self.content_label_embedding_matrix.name
# add the metadata paths to these embedding_summaries:
embedding_field.metadata_path = os.path.join("..", "Metadata/fields.vocab")
embedding_content_label.metadata_path = os.path.join("..", "Metadata/content_labels.vocab")
# save the configuration file for this
projector.visualize_embeddings(tensorboard_writer, conf)
# return the so created tensorboard_writer
return tensorboard_writer
def run_latent(model, n_images, data_out_path, sprite=True):
tensorboard_path = os.path.join(data_out_path, 'tensorboard')
saver = tf.train.Saver()
with tf.Session() as session:
# Initializer and restoring model.
session.run(tf.global_variables_initializer())
check = get_checkpoint(data_out_path)
saver.restore(session, check)
# Inputs for tensorboard.
tf_data = tf.Variable(tf.zeros((n_images, model.z_dim)), name='tf_data')
input_sample = tf.placeholder(tf.float32, shape=(n_images, model.z_dim))
set_tf_data = tf.assign(tf_data, input_sample, validate_shape=False)
if sprite:
# Sample images.
gen_samples, sample_z = show_generated(session=session, z_input=model.z_input, z_dim=model.z_dim, output_fake=model.output_gen, n_images=n_images, show=False)
# Generate sprite of images.
write_sprite_image(filename=os.path.join(data_out_path, 'gen_sprite.png'), data=gen_samples)
else:
sample_z = np.random.uniform(low=-1., high=1., size=(n_images, model.z_dim))
# Variable for embedding.
saver_latent = tf.train.Saver([tf_data])
session.run(set_tf_data, feed_dict={input_sample: sample_z})
saver_latent.save(sess=session, save_path=os.path.join(tensorboard_path, 'tf_data.ckpt'))
# Tensorflow embedding.
config = projector.ProjectorConfig()
embedding = config.embeddings.add()
embedding.tensor_name = tf_data.name
if sprite:
embedding.metadata_path = os.path.join(data_out_path, 'metadata.tsv')
embedding.sprite.image_path = os.path.join(data_out_path, 'gen_sprite.png')
embedding.sprite.single_image_dim.extend([model.image_height, model.image_width])
projector.visualize_embeddings(tf.summary.FileWriter(tensorboard_path), config)
def visualize_embeddings(self, sess, tensor, name):
"""
Visualises an embedding vector into Tensorboard
:param sess: Tensorflow session object
:param tensor: The embedding tensor to be visualizd
:param name: Name of the tensor
"""
# make directory if not exist
if not tf.os.path.exists(self.save_dir):
tf.os.makedirs(self.save_dir)
# summary writer
summary_writer = tf.summary.FileWriter(self.save_dir, graph=tf.get_default_graph())
# embedding visualizer
config = projector.ProjectorConfig()
emb = config.embeddings.add()
emb.tensor_name = name # tensor
emb.metadata_path = tf.os.path.join(self.DEFAULT_META_DATA_DIR, self.meta_file) # metadata file
print(tf.os.path.abspath(emb.metadata_path))
projector.visualize_embeddings(summary_writer, config)
def embedding_view(EMB, y, EMB_C, sess, opt):
EMB = [(x / np.linalg.norm(x)).tolist() for x in EMB]
EMB_C = [(x / np.linalg.norm(x) ).tolist() for x in EMB_C]
embedding_var = tf.Variable(EMB + EMB_C, name='Embedding_of_sentence')
sess.run(embedding_var.initializer)
EB_summary_writer = tf.summary.FileWriter(opt.log_path)
config = projector.ProjectorConfig()
embedding = config.embeddings.add()
embedding.metadata_path = os.path.join(opt.log_path, 'metadata.tsv')
projector.visualize_embeddings(EB_summary_writer, config)
saver = tf.train.Saver([embedding_var])
saver.save(sess, os.path.join(opt.log_path, 'model2.ckpt'), 1)
metadata_file = open(os.path.join(opt.log_path, 'metadata.tsv'), 'w')
metadata_file.write('ClassID\tClass\n')
for i in range(len(y)):
metadata_file.write('%06d\t%s\n' % (y[i], opt.class_name[y[i]]))
for i in range(opt.num_class):
metadata_file.write('%06d\t%s\n' % (i, "class_"+opt.class_name[i]))
metadata_file.close()
print("embedding created")
def visualize_embeddings(self) -> None:
"""Insert visualization of embeddings in TensorBoard.
Visualize the embeddings of `EmbeddedFactorSequence` objects specified
in the `main.visualize_embeddings` config attribute.
"""
tb_projector = projector.ProjectorConfig()
for sequence in self.model.visualize_embeddings:
for i, (vocabulary, emb_matrix) in enumerate(
zip(sequence.vocabularies, sequence.embedding_matrices)):
# TODO when vocabularies will have name parameter, change it
path = self.get_path("seq.{}-{}.tsv".format(sequence.name, i))
vocabulary.save_wordlist(path)
embedding = tb_projector.embeddings.add()
# pylint: disable=unsubscriptable-object
embedding.tensor_name = emb_matrix.name
embedding.metadata_path = path
# pylint: enable=unsubscriptable-object
summary_writer = tf.summary.FileWriter(self.model.output)
projector.visualize_embeddings(summary_writer, tb_projector)
def _model_fn(inputs, context_indices, mode):
if mode == ModeKeys.INFER:
sparse_index_tensor = tf.string_split(
[tf.read_file(vocab_file)],
delimiter='\n'
)
index_tensor = tf.squeeze(tf.sparse_to_dense(
sparse_index_tensor.indices,
[1, vocab_size],
sparse_index_tensor.values,
default_value='UNK'
))
reverse_index = tf.contrib.lookup.HashTable(
tf.contrib.lookup.KeyValueTensorInitializer(
index_tensor,
tf.constant(range(vocab_size), dtype=tf.int64)
),
0
)
target_indices = reverse_index.lookup(inputs)
else:
target_indices = inputs
with tf.device(tf.train.replica_device_setter()):
with tf.variable_scope('nce',
partitioner=tf.fixed_size_partitioner(
num_partitions)):
embeddings = tf.get_variable(
'embeddings',
shape=[vocab_size, embedding_size],
dtype=tf.float32,
initializer=tf.random_uniform_initializer(-1.0, 1.0)
)
if mode in [ModeKeys.TRAIN, ModeKeys.EVAL]:
nce_weights = tf.get_variable(
'nce_weights',
shape=[vocab_size, embedding_size],
dtype=tf.float32,
initializer=tf.truncated_normal_initializer(
stddev=1.0 / math.sqrt(embedding_size)
)
)
nce_biases = tf.get_variable(
'nce_biases',
initializer=tf.zeros_initializer([vocab_size]),
dtype=tf.float32
)
tensors, loss, train_op = ({}, None, None)
if mode in [ModeKeys.TRAIN, ModeKeys.EVAL]:
embedded = tf.nn.embedding_lookup(embeddings, target_indices)
loss = tf.reduce_mean(tf.nn.nce_loss(
nce_weights,
nce_biases,
embedded,
context_indices,
num_sampled,
vocab_size
))
tf.summary.scalar('loss', loss)
tf.summary.scalar('training/hptuning/metric', loss)
# Embedding Visualizer
embedding_writer = tf.summary.FileWriter(output_path)
config = projector.ProjectorConfig()
embedding = config.embeddings.add()
embedding.tensor_name = embeddings.name
embedding.metadata_path = vocab_file
projector.visualize_embeddings(embedding_writer, config)
if mode == ModeKeys.TRAIN:
train_op = tf.train.GradientDescentOptimizer(
learning_rate
).minimize(
loss,
global_step=tf.contrib.framework.get_or_create_global_step()
)
if mode == ModeKeys.INFER:
# Compute the cosine similarity between examples and embeddings.
norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
normalized_embeddings = embeddings / norm
valid_embeddings = tf.nn.embedding_lookup(
normalized_embeddings, tf.squeeze(target_indices))
similarity = tf.matmul(
valid_embeddings, normalized_embeddings, transpose_b=True)
tensors['values'], predictions = tf.nn.top_k(
similarity, sorted=True, k=num_sim)
index_tensor = tf.concat(0, [tf.constant(['UNK']), index_tensor])
tensors['predictions'] = tf.gather(index_tensor, predictions)
return tensors, loss, train_op
def word2vec_basic(log_dir):
"""Example of building, training and visualizing a word2vec model."""
# Create the directory for TensorBoard variables if there is not.
if not os.path.exists(log_dir):
os.makedirs(log_dir)
# Step 1: Download the data.
url = 'http://mattmahoney.net/dc/'
# pylint: disable=redefined-outer-name
def maybe_download(filename, expected_bytes):
"""Download a file if not present, and make sure it's the right size."""
local_filename = os.path.join(gettempdir(), filename)
if not os.path.exists(local_filename):
local_filename, _ = urllib.request.urlretrieve(url + filename,
local_filename)
statinfo = os.stat(local_filename)
if statinfo.st_size == expected_bytes:
print('Found and verified', filename)
else:
print(statinfo.st_size)
raise Exception('Failed to verify ' + local_filename +
'. Can you get to it with a browser?')
return local_filename
filename = maybe_download('text8.zip', 31344016)
# Read the data into a list of strings.
def read_data(filename):
"""Extract the first file enclosed in a zip file as a list of words."""
with zipfile.ZipFile(filename) as f:
data = tf.compat.as_str(f.read(f.namelist()[0])).split()
return data
vocabulary = read_data(filename)
print('Data size', len(vocabulary))
# Step 2: Build the dictionary and replace rare words with UNK token.
vocabulary_size = 50000
def build_dataset(words, n_words):
"""Process raw inputs into a dataset."""
count = [['UNK', -1]]
count.extend(collections.Counter(words).most_common(n_words - 1))
dictionary = {}
for word, _ in count:
dictionary[word] = len(dictionary)
data = []
unk_count = 0
for word in words:
index = dictionary.get(word, 0)
if index == 0: # dictionary['UNK']
unk_count += 1
data.append(index)
count[0][1] = unk_count
reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))
return data, count, dictionary, reversed_dictionary
# Filling 4 global variables:
# data - list of codes (integers from 0 to vocabulary_size-1).
# This is the original text but words are replaced by their codes
# count - map of words(strings) to count of occurrences
# dictionary - map of words(strings) to their codes(integers)
# reverse_dictionary - maps codes(integers) to words(strings)
data, count, unused_dictionary, reverse_dictionary = build_dataset(
vocabulary, vocabulary_size)
del vocabulary # Hint to reduce memory.
print('Most common words (+UNK)', count[:5])
print('Sample data', data[:10], [reverse_dictionary[i] for i in data[:10]])
# Step 3: Function to generate a training batch for the skip-gram model.
def generate_batch(batch_size, num_skips, skip_window):
global data_index
assert batch_size % num_skips == 0
assert num_skips <= 2 * skip_window
batch = np.ndarray(shape=(batch_size), dtype=np.int32)
labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
span = 2 * skip_window + 1 # [ skip_window target skip_window ]
buffer = collections.deque(maxlen=span) # pylint: disable=redefined-builtin
if data_index + span > len(data):
data_index = 0
buffer.extend(data[data_index:data_index + span])
data_index += span
for i in range(batch_size // num_skips):
context_words = [w for w in range(span) if w != skip_window]
words_to_use = random.sample(context_words, num_skips)
for j, context_word in enumerate(words_to_use):
batch[i * num_skips + j] = buffer[skip_window]
labels[i * num_skips + j, 0] = buffer[context_word]
if data_index == len(data):
buffer.extend(data[0:span])
data_index = span
else:
buffer.append(data[data_index])
data_index += 1
# Backtrack a little bit to avoid skipping words in the end of a batch
data_index = (data_index + len(data) - span) % len(data)
return batch, labels
batch, labels = generate_batch(batch_size=8, num_skips=2, skip_window=1)
for i in range(8):
print(batch[i], reverse_dictionary[batch[i]], '->', labels[i, 0],
reverse_dictionary[labels[i, 0]])
# Step 4: Build and train a skip-gram model.
batch_size = 128
embedding_size = 128 # Dimension of the embedding vector.
skip_window = 1 # How many words to consider left and right.
num_skips = 2 # How many times to reuse an input to generate a label.
num_sampled = 64 # Number of negative examples to sample.
# We pick a random validation set to sample nearest neighbors. Here we limit
# the validation samples to the words that have a low numeric ID, which by
# construction are also the most frequent. These 3 variables are used only for
# displaying model accuracy, they don't affect calculation.
valid_size = 16 # Random set of words to evaluate similarity on.
valid_window = 100 # Only pick dev samples in the head of the distribution.
valid_examples = np.random.choice(valid_window, valid_size, replace=False)
graph = tf.Graph()
with graph.as_default():
# Input data.
with tf.name_scope('inputs'):
train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
valid_dataset = tf.constant(valid_examples, dtype=tf.int32)
# Ops and variables pinned to the CPU because of missing GPU implementation
with tf.device('/cpu:0'):
# Look up embeddings for inputs.
with tf.name_scope('embeddings'):
embeddings = tf.Variable(
tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
embed = tf.nn.embedding_lookup(embeddings, train_inputs)
# Construct the variables for the NCE loss
with tf.name_scope('weights'):
nce_weights = tf.Variable(
tf.truncated_normal([vocabulary_size, embedding_size],
stddev=1.0 / math.sqrt(embedding_size)))
with tf.name_scope('biases'):
nce_biases = tf.Variable(tf.zeros([vocabulary_size]))
# Compute the average NCE loss for the batch.
# tf.nce_loss automatically draws a new sample of the negative labels each
# time we evaluate the loss.
# Explanation of the meaning of NCE loss:
# http://mccormickml.com/2016/04/19/word2vec-tutorial-the-skip-gram-model/
with tf.name_scope('loss'):
loss = tf.reduce_mean(
tf.nn.nce_loss(
weights=nce_weights,
biases=nce_biases,
labels=train_labels,
inputs=embed,
num_sampled=num_sampled,
num_classes=vocabulary_size))
# Add the loss value as a scalar to summary.
tf.summary.scalar('loss', loss)
# Construct the SGD optimizer using a learning rate of 1.0.
with tf.name_scope('optimizer'):
optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)
# Compute the cosine similarity between minibatch examples and all
# embeddings.
norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keepdims=True))
normalized_embeddings = embeddings / norm
valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings,
valid_dataset)
similarity = tf.matmul(
valid_embeddings, normalized_embeddings, transpose_b=True)
# Merge all summaries.
merged = tf.summary.merge_all()
# Add variable initializer.
init = tf.global_variables_initializer()
# Create a saver.
saver = tf.train.Saver()
# Step 5: Begin training.
num_steps = 100001
with tf.Session(graph=graph) as session:
# Open a writer to write summaries.
writer = tf.summary.FileWriter(log_dir, session.graph)
# We must initialize all variables before we use them.
init.run()
print('Initialized')
average_loss = 0
for step in xrange(num_steps):
batch_inputs, batch_labels = generate_batch(batch_size, num_skips,
skip_window)
feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels}
# Define metadata variable.
run_metadata = tf.RunMetadata()
# We perform one update step by evaluating the optimizer op (including it
# in the list of returned values for session.run()
# Also, evaluate the merged op to get all summaries from the returned
# "summary" variable. Feed metadata variable to session for visualizing
# the graph in TensorBoard.
_, summary, loss_val = session.run([optimizer, merged, loss],
feed_dict=feed_dict,
run_metadata=run_metadata)
average_loss += loss_val
# Add returned summaries to writer in each step.
writer.add_summary(summary, step)
# Add metadata to visualize the graph for the last run.
if step == (num_steps - 1):
writer.add_run_metadata(run_metadata, 'step%d' % step)
if step % 2000 == 0:
if step > 0:
average_loss /= 2000
# The average loss is an estimate of the loss over the last 2000
# batches.
print('Average loss at step ', step, ': ', average_loss)
average_loss = 0
# Note that this is expensive (~20% slowdown if computed every 500 steps)
if step % 10000 == 0:
sim = similarity.eval()
for i in xrange(valid_size):
valid_word = reverse_dictionary[valid_examples[i]]
top_k = 8 # number of nearest neighbors
nearest = (-sim[i, :]).argsort()[1:top_k + 1]
log_str = 'Nearest to %s:' % valid_word
for k in xrange(top_k):
close_word = reverse_dictionary[nearest[k]]
log_str = '%s %s,' % (log_str, close_word)
print(log_str)
final_embeddings = normalized_embeddings.eval()
# Write corresponding labels for the embeddings.
with open(log_dir + '/metadata.tsv', 'w') as f:
for i in xrange(vocabulary_size):
f.write(reverse_dictionary[i] + '\n')
# Save the model for checkpoints.
saver.save(session, os.path.join(log_dir, 'model.ckpt'))
# Create a configuration for visualizing embeddings with the labels in
# TensorBoard.
config = projector.ProjectorConfig()
embedding_conf = config.embeddings.add()
embedding_conf.tensor_name = embeddings.name
embedding_conf.metadata_path = os.path.join(log_dir, 'metadata.tsv')
projector.visualize_embeddings(writer, config)
writer.close()
# Step 6: Visualize the embeddings.
# pylint: disable=missing-docstring
# Function to draw visualization of distance between embeddings.
def plot_with_labels(low_dim_embs, labels, filename):
assert low_dim_embs.shape[0] >= len(labels), 'More labels than embeddings'
plt.figure(figsize=(18, 18)) # in inches
for i, label in enumerate(labels):
x, y = low_dim_embs[i, :]
plt.scatter(x, y)
plt.annotate(
label,
xy=(x, y),
xytext=(5, 2),
textcoords='offset points',
ha='right',
va='bottom')
plt.savefig(filename)
try:
# pylint: disable=g-import-not-at-top
from sklearn.manifold import TSNE
import matplotlib.pyplot as plt
tsne = TSNE(
perplexity=30, n_components=2, init='pca', n_iter=5000, method='exact')
plot_only = 500
low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only, :])
labels = [reverse_dictionary[i] for i in xrange(plot_only)]
plot_with_labels(low_dim_embs, labels, os.path.join(gettempdir(),
'tsne.png'))
except ImportError as ex:
print('Please install sklearn, matplotlib, and scipy to show embeddings.')
print(ex)
train_summary_writer = tf.summary.FileWriter('/tmp/train', session.graph)
test_images, test_labels = mnist.test.images, mnist.test.labels
for batch_no in range(total_batches):
image_batch = mnist.train.next_batch(100)
_, merged_summary = session.run([optimiser, merged_summary_operation], feed_dict={
x_input: image_batch[0],
y_input: image_batch[1],
dropout_bool: True
})
train_summary_writer.add_summary(merged_summary, batch_no)
work_dir = '' # change path
metadata_path = '/tmp/train/metadata.tsv'
with open(metadata_path, 'w') as metadata_file:
for i in range(no_embedding_data):
metadata_file.write('{}\n'.format(
np.nonzero(mnist.test.labels[::1])[1:][0][i]))
from tensorflow.contrib.tensorboard.plugins import projector
projector_config = projector.ProjectorConfig()
embedding_projection = projector_config.embeddings.add()
embedding_projection.tensor_name = embedding_variable.name
embedding_projection.metadata_path = metadata_path
embedding_projection.sprite.image_path = os.path.join(work_dir + '/mnist_10k_sprite.png')
embedding_projection.sprite.single_image_dim.extend([28, 28])
projector.visualize_embeddings(train_summary_writer, projector_config)
tf.train.Saver().save(session, '/tmp/train/model.ckpt', global_step=1)
log_dir = args.log_dir
if args.log_dir is None:
log_dir = os.path.join(os.getcwd(), "log_dir")
# Load data
data_frame = pd.read_csv(args.vector, index_col=False, header=None, sep=' ')
metadata = args.labels
# Generating PCA and
pca = PCA(n_components=50, random_state=123, svd_solver='auto')
df_pca = pd.DataFrame(pca.fit_transform(data_frame))
df_pca = df_pca.values
# Start tensorflow variable setup
tf_data = tf.Variable(df_pca)
# Start TF session
with tf.Session() as sess:
saver = tf.train.Saver([tf_data])
sess.run(tf_data.initializer)
saver.save(sess, os.path.join(log_dir, 'tf_data.ckpt'))
config = projector.ProjectorConfig()
embedding = config.embeddings.add()
embedding.tensor_name = tf_data.name
# Link this tensor to its metadata(Labels) file
embedding.metadata_path = metadata
# Saves a config file that TensorBoard will read during startup.
projector.visualize_embeddings(tf.summary.FileWriter(log_dir), config)
def create_embedding(self,
batch,
img_shape,
labels=None,
invert_colors=True):
""" This will eventually be called inside some convenient training
routine that will be exposed to the user. This creates the logs
and variables necessary to visualize the latency space with the
embedding visualizer in tensorboard.
batch - (array) Input to the network. Typically of shape
(batch_size, [input_dimensions])
img_shape - (array like) Can be an array, tuple or list containing
the dimensions of one image in (height, width) format.
For example, if using MNIST img_shape might be equal
to (28,28) or [28,28]
labels - (array) One dimensional array containing labels.
The element in the ith index of 'labels' is the
associated label for ith training element in batch
"""
EMBED_VAR_NAME = "Latent_Space"
EMBED_LOG_DIR = self.LOG_DIR
SPRITES_PATH = os.path.join(self.LOG_DIR, 'sprites.png')
METADATA_PATH = os.path.join(self.LOG_DIR, 'metadata.tsv')
img_h, img_w = img_shape
with tf.name_scope('Create_Embedding'):
# ----- Create the embedding variable to be visualized -----
# Paths to sprites and metadata must be relative to the location of
# the projector config file which is determined by the line below
embedding_writer = tf.summary.FileWriter(EMBED_LOG_DIR)
latent_var = self.sess.run(self.z,
feed_dict={self.network_input: batch})
embedding_var = tf.Variable(latent_var,
trainable=False,
name=EMBED_VAR_NAME)
# Initialize the newly created embedding variable
init_embedding_var_op = tf.variables_initializer([embedding_var])
self.sess.run(init_embedding_var_op)
# Create a projector configuartion object
config = projector.ProjectorConfig()
# Create an embedding
embedding = config.embeddings.add()
# This is where the name is important again. Specify
# which variable to embed
embedding.tensor_name = embedding_var.name
# Specify where you find the metadata
embedding.metadata_path = METADATA_PATH #'metadata.tsv'
# Specify where you find the sprite (we will create this later)
embedding.sprite.image_path = SPRITES_PATH #'mnistdigits.png'
embedding.sprite.single_image_dim.extend([img_h, img_w])
# Say that you want to visualise the embeddings
projector.visualize_embeddings(embedding_writer, config)
# ----- Construct the Sprites for visualization -----
images = np.reshape(batch, (-1, img_h, img_w))
# Maybe invert greyscale... MNIST looks prettier in tensorboard
if invert_colors:
images = 1 - images
batch_size = images.shape[0]
n_plots = int(np.ceil(np.sqrt(batch_size)))
spriteimage = np.ones((img_h * n_plots, img_w * n_plots))
for i in range(n_plots):
for j in range(n_plots):
this_filter = i * n_plots + j
if this_filter < images.shape[0]:
this_img = images[this_filter]
spriteimage[i * img_h:(i + 1) * img_h,
j * img_w:(j + 1) * img_w] = this_img
plt.imsave(SPRITES_PATH, spriteimage, cmap='gray')
# ----- Create the metadata file for visualization -----
with open(METADATA_PATH, 'w') as f:
if labels is None:
for index in range(batch_size):
f.write("%d\n" % (index))
else:
f.write("Index\tLabel\n")
for index, label in enumerate(labels):
f.write("%d\t%d\n" % (index, label))
embed_saver = tf.train.Saver()
embed_saver.save(self.sess, os.path.join(EMBED_LOG_DIR, "embed.ckpt"),
self.CHECKPOINT_COUNTER)
self.CHECKPOINT_COUNTER += 1
import pickle
from tensorflow.contrib.tensorboard.plugins import projector
LOG_DIR = 'logs'
metadata ='./data/metadata.tsv'
embed = pickle.load(open('./data/embedding.p','rb'))
vocabs = pickle.load(open('./data/vocab.p','rb'))
embeds = tf.Variable(embed, name='embeds')
with open(metadata, 'w', encoding='utf-8') as metadata_file:
metadata_file.write('Name\tPOS\n')
for row in vocabs:
metadata_file.write('%s\t%s\n' % (row[0],row[1]))
with tf.Session() as sess:
saver = tf.train.Saver([embeds])
sess.run(embeds.initializer)
saver.save(sess, os.path.join(LOG_DIR, 'embeds.ckpt'))
config = projector.ProjectorConfig()
# One can add multiple embeddings.
embedding = config.embeddings.add()
embedding.tensor_name = embeds.name
# Link this tensor to its metadata file (e.g. labels).
embedding.metadata_path = metadata
# Saves a config file that TensorBoard will read during startup.
projector.visualize_embeddings(tf.summary.FileWriter(LOG_DIR), config)
def main(args):
# pass the args as params so the model_fn can use
# the TPU specific args
params = vars(args)
if args.use_tpu:
# additional configs required for using TPUs
tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
args.tpu)
tpu_config = tf.contrib.tpu.TPUConfig(
num_shards=8, # using Cloud TPU v2-8
iterations_per_loop=args.save_checkpoints_steps)
# use the TPU version of RunConfig
config = tf.contrib.tpu.RunConfig(
cluster=tpu_cluster_resolver,
model_dir=args.model_dir,
tpu_config=tpu_config,
save_checkpoints_steps=args.save_checkpoints_steps,
save_summary_steps=100)
# TPUEstimator
estimator = tf.contrib.tpu.TPUEstimator(
model_fn=model_fn,
config=config,
params=params,
train_batch_size=args.train_batch_size,
# Calling TPUEstimator.predict requires setting predict_bath_size.
predict_batch_size=PREDICT_BATCH_SIZE,
eval_batch_size=32,
export_to_tpu=False)
else:
config = tf.estimator.RunConfig(model_dir=args.model_dir)
estimator = tf.estimator.Estimator(model_fn,
config=config,
params=params)
estimator.train(train_input_fn, max_steps=args.max_steps)
# After training, apply the learned embedding to the test data and visualize with tensorboard Projector.
embeddings = next(
estimator.predict(predict_input_fn,
yield_single_examples=False))['embeddings']
# Put the embeddings into a variable to be visualized.
embedding_var = tf.Variable(embeddings, name='test_embeddings')
# Labels do not pass through the estimator.predict call, so we get it separately.
_, (_, labels) = tf.keras.datasets.mnist.load_data()
labels = labels[:PREDICT_BATCH_SIZE]
# Write the metadata file for the projector.
metadata_path = os.path.join(estimator.model_dir, 'metadata.tsv')
with tf.gfile.GFile(metadata_path, 'w') as f:
f.write('index\tlabel\n')
for i, label in enumerate(labels):
f.write('{}\t{}\n'.format(i, label))
# Configure the projector.
projector_config = projector.ProjectorConfig()
embedding_config = projector_config.embeddings.add()
embedding_config.tensor_name = embedding_var.name
# The metadata_path is relative to the summary_writer's log_dir.
embedding_config.metadata_path = 'metadata.tsv'
summary_writer = tf.summary.FileWriter(estimator.model_dir)
projector.visualize_embeddings(summary_writer, projector_config)
# Start a session to actually write the embeddings into a new checkpoint.
sess = tf.Session()
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.save(sess, os.path.join(estimator.model_dir, 'model.ckpt'),
args.max_steps + 1)
def _model_fn(features, labels, mode, params):
"""A model function for item recommendation.
Two Tower Architecture:
Builds neural nets for users and items that learn n-dimensional
representations of each. The distance between these representations is used
to make a prediction for a binary classification.
Args:
features: a batch of features.
labels: a batch of labels or None if predicting.
mode: an instance of tf.estimator.ModeKeys.
params: a dict of additional params.
Returns:
A tf.estimator.EstimatorSpec that fully defines the model that will be run
by an Estimator.
"""
tft_output = tft.TFTransformOutput(params["tft_dir"])
tft_features = tft_output.transform_raw_features(features)
hparams = params["hparams"]
# Prediction op: Return the top k closest items for a given user or item.
if mode == tf.estimator.ModeKeys.PREDICT:
table = tf.contrib.lookup.index_to_string_table_from_file(
tft_output.vocabulary_file_by_name(constants.ITEM_VOCAB_NAME))
item_embedding = _get_embedding_matrix(hparams.embedding_size, tft_output,
constants.ITEM_VOCAB_NAME)
item_sims, item_top_k = _get_top_k(tft_features,
item_embedding,
constants.TFT_ITEM_KEY,
item_embedding)
user_embedding = _get_embedding_matrix(hparams.embedding_size, tft_output,
constants.USER_VOCAB_NAME)
user_sims, user_top_k = _get_top_k(tft_features,
user_embedding,
constants.TFT_USER_KEY,
item_embedding)
predictions = {
constants.USER_KEY: tf.identity(features[constants.USER_KEY]),
"user_top_k": table.lookup(tf.cast(user_top_k, tf.int64)),
"user_sims": user_sims,
constants.ITEM_KEY: tf.identity(features[constants.ITEM_KEY]),
"item_top_k": table.lookup(tf.cast(item_top_k, tf.int64)),
"item_sims": item_sims,
}
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
# Build user and item networks.
user_net, user_size = _make_input_layer(tft_features,
tft_output,
constants.TFT_USER_KEY,
constants.USER_VOCAB_NAME,
constants.USER_NUMERICAL_FEATURES,
constants.USER_CATEGORICAL_FEATURES,
constants.USER_CATEGORICAL_VOCABS,
hparams.user_embed_mult)
item_net, item_size = _make_input_layer(tft_features,
tft_output,
constants.TFT_ITEM_KEY,
constants.ITEM_VOCAB_NAME,
constants.ITEM_NUMERICAL_FEATURES,
constants.ITEM_CATEGORICAL_FEATURES,
constants.ITEM_CATEGORICAL_VOCABS,
hparams.item_embed_mult)
user_net, item_net = _resize_networks(user_net,
user_size,
item_net,
item_size,
hparams.num_layers,
hparams.embedding_size)
user_norm = tf.nn.l2_normalize(user_net, 1)
item_norm = tf.nn.l2_normalize(item_net, 1)
preds = tf.abs(tf.reduce_sum(tf.multiply(user_norm, item_norm), axis=1))
loss = tf.losses.mean_squared_error(labels, preds, weights=labels**.5)
user_embedding = _update_embedding_matrix(
tft_features[constants.TFT_USER_KEY],
user_norm,
hparams.embedding_size,
tft_output,
constants.USER_VOCAB_NAME)
item_embedding = _update_embedding_matrix(
tft_features[constants.TFT_ITEM_KEY],
item_norm,
hparams.embedding_size,
tft_output,
constants.ITEM_VOCAB_NAME)
# Eval op: Log the recall of the batch and save a sample of user+item shared
# embedding space for tensorboard projector.
item_sims = tf.matmul(user_norm, item_embedding, transpose_b=True)
metrics = {}
with tf.name_scope("recall"):
for k in constants.EVAL_RECALLS:
key = "recall_{0}".format(k)
recall = tf.metrics.recall_at_k(
tft_features[constants.TFT_TOP_10_KEY], item_sims, k)
metrics["recall/{0}".format(key)] = recall
tf.summary.scalar(key, recall[1])
tf.summary.merge_all()
if mode == tf.estimator.ModeKeys.EVAL:
samples, config = _get_projector_data(user_embedding,
params["projector_users"],
item_embedding,
params["projector_items"])
writer = tf.summary.FileWriter(params["model_dir"])
projector.visualize_embeddings(writer, config)
with tf.control_dependencies([samples]):
loss = tf.identity(loss)
return tf.estimator.EstimatorSpec(mode, loss=loss, eval_metric_ops=metrics)
# Training op: Update the weights via backpropagation.
num_samples = constants.NUM_PROJECTOR_USERS + constants.NUM_PROJECTOR_ITEMS
sample = tf.get_variable(constants.PROJECTOR_NAME,
[num_samples, hparams.embedding_size])
optimizer = tf.train.AdagradOptimizer(learning_rate=hparams.learning_rate)
with tf.control_dependencies([user_embedding, item_embedding, sample]):
train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)
def fit(self):
"""
Fit the model.
"""
parameters = self.parameters
conf_parameters = self.conf_parameters
dataset_filepaths = self.dataset_filepaths
modeldata = self.modeldata
dataset_brat_folders = self.dataset_brat_folders
sess = self.sess
model = self.model
transition_params_trained = self.transition_params_trained
stats_graph_folder, experiment_timestamp = self._create_stats_graph_folder(
parameters)
# Initialize and save execution details
start_time = time.time()
results = {}
results['epoch'] = {}
results['execution_details'] = {}
results['execution_details']['train_start'] = start_time
results['execution_details']['time_stamp'] = experiment_timestamp
results['execution_details']['early_stop'] = False
results['execution_details']['keyboard_interrupt'] = False
results['execution_details']['num_epochs'] = 0
results['model_options'] = copy.copy(parameters)
model_folder = os.path.join(stats_graph_folder, 'model')
utils.create_folder_if_not_exists(model_folder)
with open(os.path.join(model_folder, 'parameters.ini'),
'w') as parameters_file:
conf_parameters.write(parameters_file)
pickle.dump(modeldata,
open(os.path.join(model_folder, 'dataset.pickle'), 'wb'))
tensorboard_log_folder = os.path.join(stats_graph_folder,
'tensorboard_logs')
utils.create_folder_if_not_exists(tensorboard_log_folder)
tensorboard_log_folders = {}
for dataset_type in dataset_filepaths.keys():
tensorboard_log_folders[dataset_type] = os.path.join(
stats_graph_folder, 'tensorboard_logs', dataset_type)
utils.create_folder_if_not_exists(
tensorboard_log_folders[dataset_type])
# Instantiate the writers for TensorBoard
writers = {}
for dataset_type in dataset_filepaths.keys():
writers[dataset_type] = tf.summary.FileWriter(
tensorboard_log_folders[dataset_type], graph=sess.graph)
# embedding_writer has to write in model_folder, otherwise TensorBoard won't be able to view embeddings
embedding_writer = tf.summary.FileWriter(model_folder)
embeddings_projector_config = projector.ProjectorConfig()
tensorboard_token_embeddings = embeddings_projector_config.embeddings.add(
)
tensorboard_token_embeddings.tensor_name = model.token_embedding_weights.name
token_list_file_path = os.path.join(model_folder,
'tensorboard_metadata_tokens.tsv')
tensorboard_token_embeddings.metadata_path = os.path.relpath(
token_list_file_path, '.')
tensorboard_character_embeddings = embeddings_projector_config.embeddings.add(
)
tensorboard_character_embeddings.tensor_name = model.character_embedding_weights.name
character_list_file_path = os.path.join(
model_folder, 'tensorboard_metadata_characters.tsv')
tensorboard_character_embeddings.metadata_path = os.path.relpath(
character_list_file_path, '.')
projector.visualize_embeddings(embedding_writer,
embeddings_projector_config)
# Write metadata for TensorBoard embeddings
token_list_file = codecs.open(token_list_file_path, 'w', 'UTF-8')
for token_index in range(modeldata.vocabulary_size):
token_list_file.write('{0}\n'.format(
modeldata.index_to_token[token_index]))
token_list_file.close()
character_list_file = codecs.open(character_list_file_path, 'w',
'UTF-8')
for character_index in range(modeldata.alphabet_size):
if character_index == modeldata.PADDING_CHARACTER_INDEX:
character_list_file.write('PADDING\n')
else:
character_list_file.write('{0}\n'.format(
modeldata.index_to_character[character_index]))
character_list_file.close()
# Start training + evaluation loop. Each iteration corresponds to 1 epoch.
# number of epochs with no improvement on the validation test in terms of F1-score
bad_counter = 0
previous_best_valid_f1_score = 0
epoch_number = -1
try:
while True:
step = 0
epoch_number += 1
print('\nStarting epoch {0}'.format(epoch_number))
epoch_start_time = time.time()
if epoch_number != 0:
# Train model: loop over all sequences of training set with shuffling
sequence_numbers = list(
range(len(modeldata.token_indices['train'])))
random.shuffle(sequence_numbers)
for sequence_number in sequence_numbers:
transition_params_trained = train.train_step(
sess, modeldata, sequence_number, model,
parameters)
step += 1
if step % 10 == 0:
print('Training {0:.2f}% done'.format(
step / len(sequence_numbers) * 100),
end='\r',
flush=True)
epoch_elapsed_training_time = time.time() - epoch_start_time
print('Training completed in {0:.2f} seconds'.format(
epoch_elapsed_training_time),
flush=True)
y_pred, y_true, output_filepaths = train.predict_labels(
sess, model, transition_params_trained, parameters,
modeldata, epoch_number, stats_graph_folder,
dataset_filepaths)
# Evaluate model: save and plot results
evaluate.evaluate_model(results, modeldata, y_pred, y_true,
stats_graph_folder, epoch_number,
epoch_start_time, output_filepaths,
parameters)
if parameters['use_pretrained_model'] and not parameters[
'train_model']:
conll_to_brat.output_brat(output_filepaths,
dataset_brat_folders,
stats_graph_folder)
break
# Save model
model.saver.save(
sess,
os.path.join(model_folder,
'model_{0:05d}.ckpt'.format(epoch_number)))
# Save TensorBoard logs
summary = sess.run(model.summary_op, feed_dict=None)
writers['train'].add_summary(summary, epoch_number)
writers['train'].flush()
utils.copytree(writers['train'].get_logdir(), model_folder)
# Early stop
valid_f1_score = results['epoch'][epoch_number][0]['valid'][
'f1_score']['micro']
if valid_f1_score > previous_best_valid_f1_score:
bad_counter = 0
previous_best_valid_f1_score = valid_f1_score
conll_to_brat.output_brat(output_filepaths,
dataset_brat_folders,
stats_graph_folder,
overwrite=True)
self.transition_params_trained = transition_params_trained
else:
bad_counter += 1
print(
"The last {0} epochs have not shown improvements on the validation set."
.format(bad_counter))
if bad_counter >= parameters['patience']:
print('Early Stop!')
results['execution_details']['early_stop'] = True
break
if epoch_number >= parameters['maximum_number_of_epochs']:
break
except KeyboardInterrupt:
results['execution_details']['keyboard_interrupt'] = True
print('Training interrupted')
print('Finishing the experiment')
end_time = time.time()
results['execution_details']['train_duration'] = end_time - start_time
results['execution_details']['train_end'] = end_time
evaluate.save_results(results, stats_graph_folder)
for dataset_type in dataset_filepaths.keys():
writers[dataset_type].close()
def run_training():
"""Train MNIST for a number of steps."""
# Get the sets of images and labels for training, validation, and
# test on MNIST.
data_sets = input_data.read_data_sets(FLAGS.input_data_dir, FLAGS.fake_data)
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default():
# Generate placeholders for the images and labels.
images_placeholder, labels_placeholder = placeholder_inputs(
FLAGS.batch_size)
# Build a Graph that computes predictions from the inference model.
logits = mnist.inference(images_placeholder,
FLAGS.hidden1,
FLAGS.hidden2)
# Add to the Graph the Ops for loss calculation.
loss = mnist.loss(logits, labels_placeholder)
# Add to the Graph the Ops that calculate and apply gradients.
train_op = mnist.training(loss, FLAGS.learning_rate)
# Add the Op to compare the logits to the labels during evaluation.
eval_correct = mnist.evaluation(logits, labels_placeholder)
# Build the summary Tensor based on the TF collection of Summaries.
summary = tf.summary.merge_all()
# Add the variable initializer Op.
init = tf.global_variables_initializer()
# Create a saver for writing training checkpoints.
saver = tf.train.Saver()
# Create a session for running Ops on the Graph.
sess = tf.Session()
# Instantiate a SummaryWriter to output summaries and the Graph.
summary_writer = tf.summary.FileWriter(FLAGS.log_dir, sess.graph)
# And then after everything is built:
# Run the Op to initialize the variables.
sess.run(init)
# Start the training loop.
for step in xrange(FLAGS.max_steps):
start_time = time.time()
# Fill a feed dictionary with the actual set of images and labels
# for this particular training step.
feed_dict = fill_feed_dict(data_sets.train,
images_placeholder,
labels_placeholder)
# Run one step of the model. The return values are the activations
# from the `train_op` (which is discarded) and the `loss` Op. To
# inspect the values of your Ops or variables, you may include them
# in the list passed to sess.run() and the value tensors will be
# returned in the tuple from the call.
_, loss_value = sess.run([train_op, loss],
feed_dict=feed_dict)
duration = time.time() - start_time
# Write the summaries and print an overview fairly often.
if step % 100 == 0:
# Print status to stdout.
print('Step %d: loss = %.2f (%.3f sec)' % (step, loss_value, duration))
# Update the events file.
summary_str = sess.run(summary, feed_dict=feed_dict)
summary_writer.add_summary(summary_str, step)
summary_writer.flush()
# Save a checkpoint and evaluate the model periodically.
if (step + 1) % 1000 == 0 or (step + 1) == FLAGS.max_steps:
checkpoint_file = os.path.join(FLAGS.log_dir, 'model.ckpt')
saver.save(sess, checkpoint_file, global_step=step)
# Evaluate against the training set.
print('Training Data Eval:')
do_eval(sess,
eval_correct,
images_placeholder,
labels_placeholder,
data_sets.train)
# Evaluate against the validation set.
print('Validation Data Eval:')
do_eval(sess,
eval_correct,
images_placeholder,
labels_placeholder,
data_sets.validation)
# Evaluate against the test set.
print('Test Data Eval:')
do_eval(sess,
eval_correct,
images_placeholder,
labels_placeholder,
data_sets.test)
# Compute embeddings and save them.
thumbnail_size = int(np.sqrt(mnist.IMAGE_PIXELS))
for data_set, name in [
(data_sets.train, 'train'),
(data_sets.validation, 'validation'),
(data_sets.test, 'test')]:
output_path = os.path.join(FLAGS.log_dir, 'embed', name)
print('Computing %s Embedding' % name)
(all_images, all_labels, hidden1_vectors, hidden2_vectors) = do_eval(
sess,
eval_correct,
images_placeholder,
labels_placeholder,
data_set,
True)
embed_tensors = []
summary_writer = tf.summary.FileWriter(output_path, sess.graph)
config = projector.ProjectorConfig()
for layer, embed_vectors in enumerate([hidden1_vectors, hidden2_vectors]):
embed_tensor = tf.Variable(
np.array(embed_vectors).reshape(
len(embed_vectors) * embed_vectors[0].shape[0], -1),
name=('%s_layer_%s' % (name, layer)))
embed_tensors.append(embed_tensor)
sess.run(embed_tensor.initializer)
embedding = config.embeddings.add()
embedding.tensor_name = embed_tensor.name
embedding.metadata_path = os.path.join(output_path, 'labels.tsv')
embedding.sprite.image_path = os.path.join(output_path, 'sprite.png')
embedding.sprite.single_image_dim.extend(
[thumbnail_size, thumbnail_size])
projector.visualize_embeddings(summary_writer, config)
result = sess.run(embed_tensors)
saver = tf.train.Saver(embed_tensors)
saver.save(sess, os.path.join(output_path, 'model.ckpt'), layer)
# Make sprite and labels.
images = np.array(all_images).reshape(
-1, thumbnail_size, thumbnail_size).astype(np.float32)
sprite = images_to_sprite(images)
scipy.misc.imsave(os.path.join(output_path, 'sprite.png'), sprite)
all_labels = np.array(all_labels).flatten()
metadata_file = open(os.path.join(output_path, 'labels.tsv'), 'w')
metadata_file.write('Name\tClass\n')
for ll in xrange(len(all_labels)):
metadata_file.write('%06d\t%d\n' % (ll, all_labels[ll]))
metadata_file.close()
def train(self, dataset_list, config):
"""
Args:
dataset_list (<StockDataSet>)
config (tf.app.flags.FLAGS)
"""
assert len(dataset_list) > 0
self.merged_sum = tf.summary.merge_all()
# Set up the logs folder
self.writer = tf.summary.FileWriter(
os.path.join("./logs", self.model_name))
self.writer.add_graph(self.sess.graph)
if self.use_embed:
# Set up embedding visualization
# Format: tensorflow/tensorboard/plugins/projector/projector_config.proto
projector_config = projector.ProjectorConfig()
# You can add multiple embeddings. Here we add only one.
added_embed = projector_config.embeddings.add()
added_embed.tensor_name = self.embed_matrix.name
# Link this tensor to its metadata file (e.g. labels).
shutil.copyfile(os.path.join(self.logs_dir, "metadata.tsv"),
os.path.join(self.model_logs_dir, "metadata.tsv"))
added_embed.metadata_path = "metadata.tsv"
# The next line writes a projector_config.pbtxt in the LOG_DIR. TensorBoard will
# read this file during startup.
projector.visualize_embeddings(self.writer, projector_config)
tf.global_variables_initializer().run()
# Merged test data of different stocks.
merged_test_X = []
merged_test_y = []
merged_test_labels = []
for label_, d_ in enumerate(dataset_list):
merged_test_X += list(d_.test_X)
merged_test_y += list(d_.test_y)
merged_test_labels += [[label_]] * len(d_.test_X)
merged_test_X = np.array(merged_test_X)
merged_test_y = np.array(merged_test_y)
merged_test_labels = np.array(merged_test_labels)
print "len(merged_test_X) =", len(merged_test_X)
print "len(merged_test_y) =", len(merged_test_y)
print "len(merged_test_labels) =", len(merged_test_labels)
test_data_feed = {
self.learning_rate: 0.0,
self.keep_prob: 1.0,
self.inputs: merged_test_X,
self.targets: merged_test_y,
self.symbols: merged_test_labels,
}
global_step = 0
num_batches = sum(len(d_.train_X)
for d_ in dataset_list) // config.batch_size
random.seed(time.time())
# Select samples for plotting.
sample_labels = range(min(config.sample_size, len(dataset_list)))
sample_indices = {}
for l in sample_labels:
sym = dataset_list[l].stock_sym
target_indices = np.array([
i for i, sym_label in enumerate(merged_test_labels)
if sym_label[0] == l
])
sample_indices[sym] = target_indices
print sample_indices
print "Start training for stocks:", [d.stock_sym for d in dataset_list]
for epoch in xrange(config.max_epoch):
epoch_step = 0
learning_rate = config.init_learning_rate * (
config.learning_rate_decay**max(
float(epoch + 1 - config.init_epoch), 0.0))
for label_, d_ in enumerate(dataset_list):
for batch_X, batch_y in d_.generate_one_epoch(
config.batch_size):
global_step += 1
epoch_step += 1
batch_labels = np.array([[label_]] * len(batch_X))
train_data_feed = {
self.learning_rate: learning_rate,
self.keep_prob: config.keep_prob,
self.inputs: batch_X,
self.targets: batch_y,
self.symbols: batch_labels,
}
train_loss, _, train_merged_sum = self.sess.run(
[self.loss, self.optim, self.merged_sum],
train_data_feed)
self.writer.add_summary(train_merged_sum,
global_step=global_step)
if np.mod(global_step,
len(dataset_list) * 200 /
config.input_size) == 1:
test_loss, test_pred = self.sess.run(
[self.loss_test, self.pred], test_data_feed)
print "Step:%d [Epoch:%d] [Learning rate: %.6f] train_loss:%.6f test_loss:%.6f" % (
global_step, epoch, learning_rate, train_loss,
test_loss)
# Plot samples
for sample_sym, indices in sample_indices.iteritems():
image_path = os.path.join(
self.model_plots_dir,
"{}_epoch{:02d}_step{:04d}.png".format(
sample_sym, epoch, epoch_step))
sample_preds = test_pred[indices]
sample_truth = merged_test_y[indices]
self.plot_samples(sample_preds,
sample_truth,
image_path,
stock_sym=sample_sym)
self.save(global_step)
final_pred, final_loss = self.sess.run([self.pred, self.loss],
test_data_feed)
# Save the final model
self.save(global_step)
return final_pred
def set_model(self, model):
self.model = model
self.sess = K.get_session()
if self.histogram_freq and self.merged is None:
for layer in self.model.layers:
for weight in layer.weights:
tf.summary.histogram(weight.name, weight)
if self.write_images:
w_img = tf.squeeze(weight)
shape = w_img.get_shape()
if len(shape) > 1 and shape[0] > shape[1]:
w_img = tf.transpose(w_img)
if len(shape) == 1:
w_img = tf.expand_dims(w_img, 0)
w_img = tf.expand_dims(tf.expand_dims(w_img, 0), -1)
tf.summary.image(weight.name, w_img)
if hasattr(layer, 'output'):
tf.summary.histogram('{}_out'.format(layer.name),
layer.output)
self.merged = tf.summary.merge_all()
if self.write_graph:
self.writer = tf.summary.FileWriter(self.log_dir,
self.sess.graph)
else:
self.writer = tf.summary.FileWriter(self.log_dir)
if self.embeddings_freq:
self.saver = tf.train.Saver()
embeddings_layer_names = self.embeddings_layer_names
if not embeddings_layer_names:
embeddings_layer_names = [layer.name for layer in self.model.layers
if type(layer).__name__ == 'Embedding']
embeddings = {layer.name: layer.weights[0]
for layer in self.model.layers
if layer.name in embeddings_layer_names}
embeddings_metadata = {}
if not isinstance(self.embeddings_metadata, str):
embeddings_metadata = self.embeddings_metadata
else:
embeddings_metadata = {layer_name: self.embeddings_metadata
for layer_name in embeddings.keys()}
config = projector.ProjectorConfig()
self.embeddings_logs = []
for layer_name, tensor in embeddings.items():
embedding = config.embeddings.add()
embedding.tensor_name = tensor.name
self.embeddings_logs.append(os.path.join(self.log_dir,
layer_name + '.ckpt'))
if layer_name in embeddings_metadata:
embedding.metadata_path = embeddings_metadata[layer_name]
projector.visualize_embeddings(self.writer, config)
def run(self):
'''
Runs the model according to the specified settings
- If mode = Train: Train a GRU model using the training data
- If mode = Val: Load the saved GRU model and evaluate it on the validation fold
- If mode = Test: Load the saved GRU model and evaluate it on the blind test set
'''
self.is_train = (self.mode == 'Train')
if not os.path.exists(self.path):
os.mkdir(self.path)
# Load the training data
with open('train_data.pkl', 'rb') as f:
data_sequences = pkl.load(f)
with open('train_labels.pkl', 'rb') as f:
data_labels = pkl.load(f)
dictionary, reverse_dictionary, data_lengths, self.max_seq_len, enc_sequences = build_dictionary(
data_sequences)
self.dictionary = sorted(dictionary.items(),
key=operator.itemgetter(1))
print(self.dictionary)
self.vocabulary_size = len(dictionary)
self.val_size = len(data_sequences) // self.folds
fold = self.mask
print('Training fold number %d. Each fold of size %d' %
(fold, len(data_sequences) // self.folds))
# Truncates sequences at length 2000 and returns descriptive statistics.
# This is done by concatenating the first 1900 and the last 100 amino acids.
if self.is_train:
self.max_seq_len = 2000
original_lengths = copy(data_lengths)
data_sequences = enc_sequences[:, :self.max_seq_len]
for i in range(len(data_lengths)):
if data_lengths[i] > self.max_seq_len:
data_sequences[i] = np.concatenate(
(enc_sequences[i, :self.max_seq_len - 100],
enc_sequences[i, -100:]),
axis=0)
data_lengths[i] = self.max_seq_len
if self.folds == 1:
val_mask = np.array([False])
else:
val_mask = np.arange(self.val_size * (fold - 1),
self.val_size * (fold))
# Use seed to ensure same randomisation is applied for each fold
np.random.seed(4)
perm = np.random.permutation(len(data_sequences))
data_labels = np.array(data_labels)
data_sequences = data_sequences[perm]
data_labels = data_labels[perm]
data_lenghts = data_lengths[perm]
original_lengths = original_lengths[perm]
self.val_data = data_sequences[val_mask]
self.val_labels = data_labels[val_mask]
self.val_lengths = data_lengths[val_mask]
self.val_original_lengths = original_lengths[val_mask]
self.train_data = np.delete(data_sequences, val_mask, axis=0)
self.train_labels = np.delete(data_labels, val_mask, axis=0)
self.train_lengths = np.delete(data_lengths, val_mask, axis=0)
self.train_original_lengths = np.delete(original_lengths,
val_mask,
axis=0)
self.train_statistics, self.train_frame = self.summary_stats(
self.train_lengths, self.train_labels, 'train')
if self.folds == 1:
self.val_statistics = np.array([])
self.val_frame = np.array([])
self.val_original_lengths = np.array([])
else:
self.val_statistics, self.val_frame = self.summary_stats(
self.val_lengths, self.val_labels, 'validation')
this_data = [
self.train_data, self.train_labels, self.train_lengths,
self.val_data, self.val_labels, self.val_lengths,
self.train_statistics, self.train_frame, self.val_statistics,
self.val_frame, self.train_original_lengths,
self.val_original_lengths
]
with open(self.path + 'this_data.pkl', 'wb') as f:
pkl.dump(this_data, f)
else:
with open(self.path + 'this_data.pkl', 'rb') as f:
self.train_data, self.train_labels, self.train_lengths, self.val_data, self.val_labels, self.val_lengths, self.train_statistics, self.train_frame, self.val_statistics, self.val_frame, self.train_original_lengths, self.val_original_lengths = pkl.load(
f)
# Now construct the Tensorflow graph
print('\r~~~~~~~ Building model ~~~~~~~\r')
# Define placeholders and variables
initializer = tf.random_normal_initializer()
self.word_embeddings = tf.get_variable(
'embeddings', [self.vocabulary_size, self.embedding_size],
tf.float32,
initializer=initializer)
sequences = tf.placeholder(tf.int32, [None, None], "sequences")
sequences_lengths = tf.placeholder(tf.int32, [None],
"sequences_lengths")
labels = tf.placeholder(tf.int64, [None], "labels")
keep_prob_dropout = tf.placeholder(tf.float32, name='dropout')
global_step = tf.Variable(0, name='global_step', trainable=False)
# Embed and encode sequences
sequences_embedded = self.embed_data(sequences)
encoded_sequences = self.encoder(sequences_embedded,
sequences_lengths,
keep_prob_dropout,
bidirectional=self.bidirectional)
# Take last hidden state of GRU and put them through a nonlinear and a linear FC layer
with tf.name_scope('non_linear_layer'):
encoded_sentences_BN = self.batch_norm_wrapper(
encoded_sequences, self.is_train)
non_linear = tf.nn.dropout(tf.nn.relu(
tf.contrib.layers.linear(encoded_sentences_BN, 64)),
keep_prob=keep_prob_dropout)
with tf.name_scope('final_layer'):
non_linear_BN = self.batch_norm_wrapper(non_linear, self.is_train)
logits = tf.contrib.layers.linear(non_linear_BN, 4)
# Compute mean loss on this batch, consisting of cross entropy loss and L2 loss
CE_loss = self.get_CE_loss(labels, logits)
L2_loss = self.get_L2_loss()
loss = CE_loss + L2_loss
# Perform training operation
learning_rate = tf.train.exponential_decay(self.learning_rate,
global_step,
100,
0.96,
staircase=True)
opt_op = tf.contrib.layers.optimize_loss(loss=loss,
global_step=global_step,
learning_rate=learning_rate,
optimizer='Adam',
clip_gradients=2.0,
learning_rate_decay_fn=None,
summaries=None)
# Define scalars for Tensorboard
tf.summary.scalar('CE_loss', CE_loss)
tf.summary.scalar('L2_loss', L2_loss)
tf.summary.scalar('loss', loss)
tf.summary.scalar('learning_rate', learning_rate)
# Compute accuracy of prediction
probs = tf.nn.softmax(logits)
with tf.name_scope('accuracy'):
pred = tf.argmax(logits, 1)
correct_prediction = tf.equal(labels, pred)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('accuracy', accuracy)
# If in training mode:
# - shuffle data set before each epoch
# - train model using mini batches
# - track performance on train and validation set throughout training
if self.is_train == True:
with tf.Session() as session:
train_loss_writer = tf.summary.FileWriter(
str(self.path + 'tensorboard/train_loss'), session.graph)
train_summary_writer = tf.summary.FileWriter(
str(self.path + 'tensorboard/train_summary'),
session.graph)
val_summary_writer = tf.summary.FileWriter(
str(self.path + 'tensorboard/val_summary'), session.graph)
# Use the same LOG_DIR where you stored your checkpoint.
embedding_writer = tf.summary.FileWriter(
str(self.path + 'tensorboard/'), session.graph)
config = projector.ProjectorConfig()
embedding = config.embeddings.add()
embedding.tensor_name = self.word_embeddings.name
# Link this tensor to its metadata file (e.g. labels).
embedding.metadata_path = os.path.join('./metadata.tsv')
# Saves a configuration file that TensorBoard will read during startup.
projector.visualize_embeddings(embedding_writer, config)
merged = tf.summary.merge_all()
print('\r~~~~~~~ Initializing variables ~~~~~~~\r')
tf.global_variables_initializer().run()
start_time = time.time()
min_train_loss = np.inf
batch_times = []
n = self.train_data.shape[0]
print('\r~~~~~~~ Starting training ~~~~~~~\r')
try:
train_summaryIndex = -1
for epoch in range(self.num_epochs):
self.is_train = True
epoch_time = time.time()
print('----- Epoch', epoch, '-----')
print('Shuffling dataset')
perm = np.random.permutation(len(self.train_data))
self.train_data_perm = self.train_data[perm]
self.train_labels_perm = self.train_labels[perm]
self.train_lengths_perm = self.train_lengths[perm]
total_loss = 0
for i in range(n // self.batch_size):
batch_start = time.time()
batch_data = self.train_data_perm[i *
self.batch_size:
(i + 1) *
self.batch_size]
batch_lengths = self.train_lengths_perm[
i * self.batch_size:(i + 1) * self.batch_size]
batch_labels = self.train_labels_perm[
i * self.batch_size:(i + 1) * self.batch_size]
train_dict = {
sequences: batch_data,
sequences_lengths: batch_lengths,
labels: batch_labels,
keep_prob_dropout: self.keep_prob_dropout
}
_, batch_loss, batch_accuracy, batch_summary = session.run(
[opt_op, loss, accuracy, merged],
feed_dict=train_dict)
total_loss += batch_loss
batch_times.append(time.time() - batch_start)
train_loss_writer.add_summary(
batch_summary,
i + (n // self.batch_size) * epoch)
if i % 10 == 0 and i > 0:
# Print loss every 10 batches
time_per_epoch = np.mean(batch_times) * (
n // self.batch_size)
remaining_time = int(time_per_epoch -
time.time() + epoch_time)
string_out = '\rEnd of batch ' + str(
i) + ' Train loss: ' + str(
total_loss / (i * self.batch_size)
) + ' Accuracy: ' + str(
batch_accuracy)
string_out += ' Elapsed training time : ' + str(
int(time.time() - start_time)) + "s, "
string_out += str(
remaining_time
) + "s remaining for this epoch"
string_out += ' (' + str(
time_per_epoch * 100 / 60 // 1 /
100) + ' min/epoch)'
stdout.write(string_out)
# Train accuracy
train_dict = {
sequences: self.train_data_perm[:1000],
sequences_lengths: self.train_lengths_perm[:1000],
labels: self.train_labels_perm[:1000],
keep_prob_dropout: 1.0
}
train_summary, train_loss, train_accuracy = session.run(
[merged, loss, accuracy], feed_dict=train_dict)
train_summary_writer.add_summary(train_summary, epoch)
print('\nEpoch train loss: ', train_loss,
'Epoch train accuracy: ', train_accuracy)
# Val accuracy
val_dict = {
sequences: self.val_data,
sequences_lengths: self.val_lengths,
labels: self.val_labels,
keep_prob_dropout: 1.0
}
val_summary, val_loss, val_accuracy = session.run(
[merged, loss, accuracy], feed_dict=val_dict)
val_summary_writer.add_summary(val_summary, epoch)
print('\nEpoch val loss: ', val_loss,
'Epoch val accuracy: ', val_accuracy)
self.save_model(session, epoch)
saver = tf.train.Saver(
write_version=tf.train.SaverDef.V2)
saver.save(
session,
os.path.join(self.path + '/tensorboard/',
'model.ckpt'))
except KeyboardInterrupt:
save = input('save?')
if 'y' in save:
self.save_model(session, epoch)
# If in validation mode:
# - Load saved model and evaluate on validation fold
# - Return list containing confusion matrices, and accuracy measures such as FPR and TPR
elif self.mode == 'Val':
with tf.Session() as session:
print('Restoring model...')
saver = tf.train.Saver(write_version=tf.train.SaverDef.V2)
saver.restore(session, self.path + 'tensorboard/model.ckpt')
print('Model restored!')
val_dict = {
sequences: self.val_data,
sequences_lengths: self.val_lengths,
labels: self.val_labels,
keep_prob_dropout: 1.0
}
self.val_pred, self.val_accuracy, self.val_probs = session.run(
[pred, accuracy, probs], feed_dict=val_dict)
_ = self.summary_stats(self.val_lengths, self.val_labels,
'val')
print('\nConfusion matrix (all sequence lengths):')
val_confusion_1 = self.confusion(
gold=self.val_labels,
prediction=self.val_pred,
lengths=self.val_original_lengths,
min_length=0,
max_length=np.inf)
print(val_confusion_1)
print('\nConfusion matrix (sequence length < 2000):')
val_confusion_2 = self.confusion(
gold=self.val_labels,
prediction=self.val_pred,
lengths=self.val_original_lengths,
min_length=0,
max_length=2000)
print(val_confusion_2)
print('\nConfusion matrix (sequence length > 2000):')
val_confusion_3 = self.confusion(
gold=self.val_labels,
prediction=self.val_pred,
lengths=self.val_original_lengths,
min_length=2000,
max_length=np.inf)
print(val_confusion_3)
print('\n Val accuracy:', self.val_accuracy)
print(
'\n Val accuracy when length <2000:',
np.sum((self.val_pred == self.val_labels) *
(self.val_original_lengths <= 2000)) /
np.sum(self.val_original_lengths <= 2000))
print(
'\n Val accuracy when length >2000:',
np.sum((self.val_pred == self.val_labels) *
(self.val_original_lengths > 2000)) /
np.sum(self.val_original_lengths > 2000))
this_sum = np.zeros([3, 5])
this_auc = np.zeros([1, 5])
this_TPR = []
this_FPR = []
total_tp = 0
total_fp = 0
total_fn = 0
total_tn = 0
for i in range(4):
tp = np.sum((self.val_labels == i) * (self.val_pred == i))
fp = np.sum((self.val_labels != i) * (self.val_pred == i))
fn = np.sum((self.val_labels == i) * (self.val_pred != i))
tn = np.sum((self.val_labels != i) * (self.val_pred != i))
total_tp += tp
total_fp += fp
total_fn += fn
total_tn += tn
prec = tp / (tp + fp) if (tp + fp) > 0 else 0.0
recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0
f1 = 2 * prec * recall / (
prec + recall) if prec * recall > 0 else 0.0
this_sum[:, i] = np.array([prec, recall, f1])
this_auc[:, i] = roc_auc_score(self.val_labels == i,
self.val_pred == i)
if i < 4:
this_FPR.append(
roc_curve(self.val_labels == i,
self.val_probs[:, i])[0])
this_TPR.append(
roc_curve(self.val_labels == i,
self.val_probs[:, i])[1])
prec = total_tp / (total_tp + total_fp) if (
total_tp + total_fp) > 0 else 0.0
recall = total_tp / (total_tp + total_fn) if (
total_tp + total_fn) > 0 else 0.0
f1 = 2 * prec * recall / (prec +
recall) if prec * recall > 0 else 0.0
this_sum[:, 4] = np.array([prec, recall, f1])
this_sum = np.concatenate((this_sum, this_auc), 0)
self.this_sum = pd.DataFrame(this_sum)
self.this_sum.index = pd.Index(
['Precision', 'Recall', 'F1', 'AUC'])
self.this_sum.columns = pd.Index(
['cyto', 'secreted', 'mito', 'nucleus', 'Total'])
print(self.this_sum)
if self.is_train == False:
return [
val_confusion_1, val_confusion_2, val_confusion_3,
self.this_sum, this_FPR, this_TPR
]
# If in test model:
# - Load saved model and evaluate on test set
# - Print predicted probabilities for each protein in the test set
elif self.mode == 'Test':
with tf.Session() as session:
print('Restoring model...')
saver = tf.train.Saver(write_version=tf.train.SaverDef.V2)
saver.restore(session, self.path + 'model.checkpoint')
print('Model restored!')
with open('test_data.pkl', 'rb') as f:
test_sequences = pkl.load(f)
with open('test_labels.pkl', 'rb') as f:
test_labels = pkl.load(f)
_, _, data_lengths, _, enc_sequences = build_dictionary(
test_sequences, vocab=dictionary)
test_dict = {
sequences: enc_sequences,
sequences_lengths: data_lengths,
keep_prob_dropout: 1.0
}
self.probs, self.pred = session.run([probs, pred],
feed_dict=test_dict)
result = pd.DataFrame(
np.concatenate((self.probs, np.expand_dims(self.pred, 1)),
1))
result.columns = pd.Index(
['cyto', 'secreted', 'mito', 'nucleus', 'prediction'])
print(result)
def train(tfrecord_dir_pre,
model_pre,
logs_folder,
pic_size,
num,
forward,
end_epoch,
display=100,
base_lr=0.001):
"""placeholder的声明"""
input_image = tf.placeholder(
tf.float32,
shape=[config.BATCH_SIZE, pic_size, pic_size, 3],
name='input_image')
label = tf.placeholder(tf.float32, shape=[config.BATCH_SIZE], name='label')
box_target = tf.placeholder(tf.float32,
shape=[config.BATCH_SIZE, 4],
name='box_target')
mark_target = tf.placeholder(tf.float32,
shape=[config.BATCH_SIZE, 10],
name='mark_target')
"""forward op，train_op,lr_op结果是损失"""
input_image = image_color_distort(input_image)
cls_loss_op, box_loss_op,mark_loss_op, L2_loss_op,accuracy_op =\
forward(pic_size,input_image, label, box_target, mark_target, training = True)#前向传播
"""误差权重确认,以及整体误差"""
if pic_size == 12:
radio_cls_loss = 1.0
radio_box_loss = 0.5
radio_mark_loss = 0.5
elif pic_size == 24:
radio_cls_loss = 1.0
radio_box_loss = 0.5
radio_mark_loss = 0.5
elif pic_size == 48:
radio_cls_loss = 1.0
radio_box_loss = 0.5
radio_mark_loss = 1.0
total_loss_op = (radio_cls_loss * cls_loss_op +
radio_box_loss * box_loss_op +
radio_mark_loss * mark_loss_op + L2_loss_op)
"""对网络训练的操作，以及对学习率改变的操作"""
train_op, lr_op = train_lr_op(base_lr, total_loss_op, num)
"""tensorboard中记录的标量"""
tf.summary.scalar('cls_loss', cls_loss_op)
tf.summary.scalar('box_loss', box_loss_op)
tf.summary.scalar('landmark_loss', mark_loss_op)
tf.summary.scalar('cls_accuracy', accuracy_op)
tf.summary.scalar('total_loss', total_loss_op)
summary_op = tf.summary.merge_all()
"""tfrecord文件路径指定， 训练数据集从tfrecord读出"""
image_batch, label_batch, box_batch, mark_batch = read_tfrecord(
tfrecord_dir_pre, config.BATCH_SIZE, pic_size)
print('############################################################')
"""session 初始化"""
init = tf.global_variables_initializer()
sess = tf.Session()
saver = tf.train.Saver(max_to_keep=0)
sess.run(init)
writer = tf.summary.FileWriter(logs_folder, sess.graph)
projector_config = projector.ProjectorConfig()
projector.visualize_embeddings(writer, projector_config)
"""协同管理"""
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
batch_done_count = 0 #网络计算了批量数,每个epoch都重新计数
MAX_STEP = int(num / config.BATCH_SIZE + 1) * end_epoch
epoch_count = 0
try:
for step in range(1):
batch_done_count += 1
if coord.should_stop():
break
image_batch_array, label_batch_array,box_batch_array,mark_batch_array = \
sess.run([image_batch, label_batch,box_batch,mark_batch])
#随机旋转
image_batch_array,mark_batch_array = \
random_flip_images(image_batch_array, label_batch_array, mark_batch_array)
print(label_batch_array)
print(mark_batch_array[:100])
_, _, summary = sess.run(
[train_op, lr_op, summary_op],
feed_dict={
input_image: image_batch_array,
label: label_batch_array,
box_target: box_batch_array,
mark_target: mark_batch_array
})
writer.add_summary(summary, global_step=step)
"""满100就展示"""
if (step + 1) % (display) == 0:
cls_loss, box_loss, mark_loss, L2_loss, lr, acc = sess.run(
[
cls_loss_op, box_loss_op, mark_loss_op, L2_loss_op,
lr_op, accuracy_op
],
feed_dict={
input_image: image_batch_array,
label: label_batch_array,
box_target: box_batch_array,
mark_target: mark_batch_array
})
total_loss = radio_cls_loss * cls_loss + radio_box_loss * box_loss +\
radio_mark_loss * mark_loss + L2_loss
print(
"Step:%d/%d, acc:%3f, cls loss:%4f, box_loss:%4f, mark loss:%4f, L2 loss: %4f, Total Loss:%4f, lr:%f"
% (step + 1, MAX_STEP, acc, cls_loss, box_loss, mark_loss,
L2_loss, total_loss, lr))
"""下一轮迭代"""
if batch_done_count * config.BATCH_SIZE > num * 2:
epoch_count = epoch_count + 1
batch_done_count = 0
path_prefix = saver.save(sess,
model_pre,
global_step=epoch_count * 2)
print('path prefix is :', path_prefix)
except tf.errors.OutOfRangeError:
print("训练完成！！！")
finally:
coord.request_stop()
writer.close()
coord.join(threads)
sess.close()
def main(argv=None):
#maybe_download()
graph = load_graph()
#basedir = os.path.dirname(__file__)
# ensure log directory exists
logs_path = LOG_DIR
if not os.path.exists(logs_path):
os.makedirs(logs_path)
#generate metadata
with tf.Session(graph=graph) as sess:
pool3 = sess.graph.get_tensor_by_name('pool_3:0')
jpeg_data = tf.placeholder(tf.string)
outputs = []
# Create metadata
metadata_path = os.path.join(LOG_DIR, 'metadata.tsv')
metadata = open(metadata_path, 'w')
metadata.write("Name\tLabels\n")
for folder_name in os.listdir(IMAGE_DIR):
for file_name in os.listdir(IMAGE_DIR + '/' + folder_name):
if not file_name.endswith('.jpg'):
continue
print('process %s...' % file_name)
with open(
os.path.join(IMAGE_DIR + '/' + folder_name, file_name),
'rb') as f:
data = f.read()
results = sess.run(pool3, {
'DecodeJpeg/contents:0': data,
jpeg_data: data
})
#print("results", results)
outputs.append(results[0])
metadata.write('{}\t{}\n'.format(file_name, folder_name))
metadata.close()
embedding_var = tf.Variable(tf.stack([tf.squeeze(x) for x in outputs],
axis=0),
trainable=False,
name='embed')
# prepare projector config
config = projector.ProjectorConfig()
embedding = config.embeddings.add()
embedding.tensor_name = embedding_var.name
summary_writer = tf.summary.FileWriter(LOG_DIR)
# link metadata
embedding.metadata_path = metadata_path
# write to sprite image file
img = images_to_sprite(img_data)
cv2.imwrite(os.path.join(LOG_DIR, 'sprite_classes.png'), img)
#print("img", img)
embedding.sprite.image_path = os.path.join(LOG_DIR,
'sprite_classes.png')
embedding.sprite.single_image_dim.extend([100, 100])
# save embedding_var
projector.visualize_embeddings(summary_writer, config)
sess.run(tf.variables_initializer([embedding_var]))
saver = tf.train.Saver()
saver.save(sess, os.path.join(LOG_DIR, 'model.ckpt'))
def train(data):
data = shuffle(data)
numSamples = data.shape[0]
split = round(0.1 * numSamples)
ValidData = data[0:split, :]
TestData = data[split:2 * split, :]
TrainData = data[2 * split:numSamples, :]
dataset = tf.data.Dataset.from_tensor_slices(TrainData)
dataset = dataset.batch(BatchSize)
iterator = dataset.make_initializable_iterator()
next_element = iterator.get_next()
## Sparse autoencoder
ae = autoencoder()
rho_hat = tf.reduce_mean(ae['latent'])
kl = kl_divergence(rho, rho_hat)
sparse = beta * tf.reduce_sum(kl)
with tf.name_scope("Train"):
optimizer = tf.train.AdamOptimizer(LearningRate).minimize(ae['cost'] +
sparse)
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
merged_summary = tf.summary.merge_all()
writer = tf.summary.FileWriter(LogDir)
writer.add_graph(sess.graph)
TestWriter = tf.summary.FileWriter(LogDir + "_Test") # +"_Test")
TestWriter.add_graph(sess.graph)
# Training
prev_loss = 500000
epoch_idx = 0
trainingLoss = list()
validationLoss = list()
# testingLoss = list()
for i in range(1, NumEpochs):
sess.run(iterator.initializer)
loss = 0
num_batch = 0
saver = tf.train.Saver(max_to_keep=5)
while True:
try:
batch_x = sess.run(next_element)
_, l = sess.run([optimizer, ae['cost']],
feed_dict={ae['x']: batch_x})
loss += l
num_batch += 1
except tf.errors.OutOfRangeError:
s = sess.run(merged_summary, feed_dict={ae['x']: TrainData})
writer.add_summary(s, i)
trainLoss = loss / num_batch
print('Epoch = %i\tTraining Loss = %f' % (i, trainLoss))
trainingLoss.append(trainLoss)
break
if i % 10 == 0:
with tf.name_scope("TestingMeasurements"):
s, validation_cost = sess.run([merged_summary, ae['cost']],
feed_dict={ae['x']: ValidData})
print('Epoch = %i\t*** Validation Loss = %f' %
(i, validation_cost))
validationLoss.append(validation_cost)
if (validation_cost <
prev_loss): # Save best model based on validation loss
prev_loss = validation_cost
saver.save(sess,
os.path.join(LogDir, 'model.ckpt'),
global_step=i)
epoch_idx = i
TestWriter.add_summary(s, i)
## Apply best model to the test set
bestmodel = os.path.join(LogDir, 'model.ckpt-' + str(epoch_idx))
print("Best model occured at epoch = ", epoch_idx)
print("Best Model path = ", bestmodel)
saver.restore(sess, bestmodel)
s, test_cost = sess.run([merged_summary, ae['cost']],
feed_dict={ae['x']: TestData})
embedded_data = sess.run(ae['latent'], feed_dict={ae['x']: data})
np.savetxt(LogDir + "_trainingLoss.csv", trainingLoss, delimiter=",")
np.savetxt(LogDir + "_validationLoss.csv", validationLoss, delimiter=",")
csv = open(LogDir + "_testingLoss.csv", "w")
csv.write(str(test_cost))
### Visualize on Tensorboard
with tf.device("/cpu:0"):
embedding = tf.Variable(tf.stack(embedded_data, axis=0),
trainable=False,
name='embedding')
sess.run(tf.global_variables_initializer())
config = projector.ProjectorConfig()
embed = config.embeddings.add()
embed.tensor_name = 'embedding:0'
embed.metadata_path = os.path.join(LogDir + '/metadata.tsv')
projector.visualize_embeddings(writer, config)
sess.close()
return embedded_data
def main():
# Load configurations and write to config.txt
cfg = TrainConfig().parse()
print(cfg.name)
result_dir = os.path.join(
cfg.result_root,
cfg.name + '_' + datetime.strftime(datetime.now(), '%Y%m%d-%H%M%S'))
if not os.path.isdir(result_dir):
os.makedirs(result_dir)
utils.write_configure_to_file(cfg, result_dir)
np.random.seed(seed=cfg.seed)
# prepare dataset
train_session = cfg.train_session
train_set = prepare_dataset(cfg.feature_root, train_session, cfg.feat,
cfg.label_root)
train_set = train_set[:cfg.label_num]
batch_per_epoch = len(train_set) // cfg.sess_per_batch
val_session = cfg.val_session
val_set = prepare_dataset(cfg.feature_root, val_session, cfg.feat,
cfg.label_root)
# construct the graph
with tf.Graph().as_default():
tf.set_random_seed(cfg.seed)
global_step = tf.Variable(0, trainable=False)
lr_ph = tf.placeholder(tf.float32, name='learning_rate')
####################### Define model here ########################
with tf.variable_scope("modality_core"):
# load backbone model
if cfg.network == "tsn":
model_emb = networks.TSN(n_seg=cfg.num_seg,
emb_dim=cfg.emb_dim)
elif cfg.network == "rtsn":
model_emb = networks.RTSN(n_seg=cfg.num_seg,
emb_dim=cfg.emb_dim)
elif cfg.network == "convtsn":
model_emb = networks.ConvTSN(n_seg=cfg.num_seg,
emb_dim=cfg.emb_dim)
elif cfg.network == "convrtsn":
model_emb = networks.ConvRTSN(n_seg=cfg.num_seg,
emb_dim=cfg.emb_dim,
n_h=cfg.n_h,
n_w=cfg.n_w,
n_C=cfg.n_C,
n_input=cfg.n_input)
elif cfg.network == "convbirtsn":
model_emb = networks.ConvBiRTSN(n_seg=cfg.num_seg,
emb_dim=cfg.emb_dim)
else:
raise NotImplementedError
# get the embedding
if cfg.feat == "sensors" or cfg.feat == "segment":
input_ph = tf.placeholder(tf.float32,
shape=[None, cfg.num_seg, None])
elif cfg.feat == "resnet" or cfg.feat == "segment_down":
input_ph = tf.placeholder(
tf.float32, shape=[None, cfg.num_seg, None, None, None])
dropout_ph = tf.placeholder(tf.float32, shape=[])
model_emb.forward(input_ph, dropout_ph)
if cfg.normalized:
embedding = tf.nn.l2_normalize(model_emb.hidden,
axis=-1,
epsilon=1e-10)
else:
embedding = model_emb.hidden
# split embedding into anchor, positive and negative and calculate triplet loss
anchor, positive, negative = tf.unstack(
tf.reshape(embedding, [-1, 3, cfg.emb_dim]), 3, 1)
metric_loss = networks.triplet_loss(anchor, positive, negative,
cfg.alpha)
regularization_loss = tf.reduce_sum(
tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
total_loss = metric_loss + regularization_loss * cfg.lambda_l2
tf.summary.scalar('learning_rate', lr_ph)
train_op = utils.optimize(total_loss, global_step, cfg.optimizer,
lr_ph, tf.global_variables())
####################### Define data loader ############################
# session iterator for session sampling
feat_paths_ph = tf.placeholder(tf.string,
shape=[None, cfg.sess_per_batch])
label_paths_ph = tf.placeholder(tf.string,
shape=[None, cfg.sess_per_batch])
train_data = session_generator(feat_paths_ph,
label_paths_ph,
sess_per_batch=cfg.sess_per_batch,
num_threads=2,
shuffled=False,
preprocess_func=model_emb.prepare_input)
train_sess_iterator = train_data.make_initializable_iterator()
next_train = train_sess_iterator.get_next()
# prepare validation data
val_sess = []
val_feats = []
val_labels = []
val_boundaries = []
for session in val_set:
session_id = os.path.basename(session[1]).split('_')[0]
eve_batch, lab_batch, boundary = load_data_and_label(
session[0], session[-1], model_emb.prepare_input_test
) # use prepare_input_test for testing time
val_feats.append(eve_batch)
val_labels.append(lab_batch)
val_sess.extend([session_id] * eve_batch.shape[0])
val_boundaries.extend(boundary)
val_feats = np.concatenate(val_feats, axis=0)
val_labels = np.concatenate(val_labels, axis=0)
print("Shape of val_feats: ", val_feats.shape)
# generate metadata.tsv for visualize embedding
with open(os.path.join(result_dir, 'metadata_val.tsv'), 'w') as fout:
fout.write('id\tlabel\tsession_id\tstart\tend\n')
for i in range(len(val_sess)):
fout.write('{0}\t{1}\t{2}\t{3}\t{4}\n'.format(
i, val_labels[i, 0], val_sess[i], val_boundaries[i][0],
val_boundaries[i][1]))
# Variable for visualizing the embeddings
emb_var = tf.Variable(tf.zeros([val_feats.shape[0], cfg.emb_dim]),
name='embeddings')
set_emb = tf.assign(emb_var, embedding, validate_shape=False)
# calculated for monitoring all-pair embedding distance
diffs = utils.all_diffs_tf(embedding, embedding)
all_dist = utils.cdist_tf(diffs)
tf.summary.histogram('embedding_dists', all_dist)
summary_op = tf.summary.merge_all()
saver = tf.train.Saver(max_to_keep=10)
#########################################################################
# Start running the graph
if cfg.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = cfg.gpu
gpu_options = tf.GPUOptions(allow_growth=True)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
summary_writer = tf.summary.FileWriter(result_dir, sess.graph)
with sess.as_default():
sess.run(tf.global_variables_initializer())
# load pretrain model, if needed
if cfg.model_path:
print("Restoring pretrained model: %s" % cfg.model_path)
saver.restore(sess, cfg.model_path)
################## Training loop ##################
epoch = -1
while epoch < cfg.max_epochs - 1:
step = sess.run(global_step, feed_dict=None)
epoch = step // batch_per_epoch
# learning rate schedule, reference: "In defense of Triplet Loss"
if epoch < cfg.static_epochs:
learning_rate = cfg.learning_rate
else:
learning_rate = cfg.learning_rate * \
0.01**((epoch-cfg.static_epochs)/(cfg.max_epochs-cfg.static_epochs))
# prepare data for this epoch
random.shuffle(train_set)
feat_paths = [path[0] for path in train_set]
label_paths = [path[1] for path in train_set]
# reshape a list to list of list
# interesting hacky code from: https://stackoverflow.com/questions/10124751/convert-a-flat-list-to-list-of-list-in-python
feat_paths = list(zip(*[iter(feat_paths)] *
cfg.sess_per_batch))
label_paths = list(
zip(*[iter(label_paths)] * cfg.sess_per_batch))
sess.run(train_sess_iterator.initializer,
feed_dict={
feat_paths_ph: feat_paths,
label_paths_ph: label_paths
})
# for each epoch
batch_count = 1
while True:
try:
# Hierarchical sampling (same as fast rcnn)
start_time_select = time.time()
# First, sample sessions for a batch
eve, se, lab = sess.run(next_train)
# for memory concern, cfg.event_per_batch events are used in maximum
if eve.shape[0] > cfg.event_per_batch:
idx = np.random.permutation(
eve.shape[0])[:cfg.event_per_batch]
eve = eve[idx]
se = se[idx]
lab = lab[idx]
select_time1 = time.time() - start_time_select
# Get the embeddings of all events
eve_embedding = np.zeros((eve.shape[0], cfg.emb_dim),
dtype='float32')
for start, end in zip(
range(0, eve.shape[0], cfg.batch_size),
range(cfg.batch_size,
eve.shape[0] + cfg.batch_size,
cfg.batch_size)):
end = min(end, eve.shape[0])
emb = sess.run(embedding,
feed_dict={
input_ph: eve[start:end],
dropout_ph: 1.0
})
eve_embedding[start:end] = emb
# Second, sample triplets within sampled sessions
# get distance for all pairs
all_diff = utils.all_diffs(eve_embedding,
eve_embedding)
triplet_selected, active_count = utils.select_triplets_facenet(
lab, utils.cdist(all_diff, metric=cfg.metric),
cfg.triplet_per_batch, cfg.alpha)
select_time2 = time.time(
) - start_time_select - select_time1
# perform training on the selected triplets
start_time_train = time.time()
triplet_input_idx = [
idx for triplet in triplet_selected
for idx in triplet
]
triplet_input = eve[triplet_input_idx]
err, _, step, summ = sess.run(
[total_loss, train_op, global_step, summary_op],
feed_dict={
input_ph: triplet_input,
dropout_ph: cfg.keep_prob,
lr_ph: learning_rate
})
train_time = time.time() - start_time_train
print ("%s\tEpoch: [%d][%d/%d]\tEvent num: %d\tTriplet num: %d\tSelect_time1: %.3f\tSelect_time2: %.3f\tTrain_time: %.3f\tLoss %.4f" % \
(cfg.name, epoch+1, batch_count, batch_per_epoch, eve.shape[0], triplet_input.shape[0]//3, select_time1, select_time2, train_time, err))
summary = tf.Summary(value=[
tf.Summary.Value(tag="train_loss",
simple_value=err),
tf.Summary.Value(tag="active_count",
simple_value=active_count),
tf.Summary.Value(
tag="triplet_num",
simple_value=triplet_input.shape[0] // 3)
])
summary_writer.add_summary(summary, step)
summary_writer.add_summary(summ, step)
batch_count += 1
except tf.errors.OutOfRangeError:
print("Epoch %d done!" % (epoch + 1))
break
# validation on val_set
print("Evaluating on validation set...")
val_embeddings, _ = sess.run([embedding, set_emb],
feed_dict={
input_ph: val_feats,
dropout_ph: 1.0
})
mAP, mPrec, recall = utils.evaluate_simple(
val_embeddings, val_labels)
summary = tf.Summary(value=[
tf.Summary.Value(tag="Valiation mAP", simple_value=mAP),
tf.Summary.Value(tag="Validation Recall@1",
simple_value=recall),
tf.Summary.Value(tag="Validation [email protected]",
simple_value=mPrec)
])
summary_writer.add_summary(summary, step)
print("Epoch: [%d]\tmAP: %.4f\tmPrec: %.4f" %
(epoch + 1, mAP, mPrec))
# config for embedding visualization
config = projector.ProjectorConfig()
visual_embedding = config.embeddings.add()
visual_embedding.tensor_name = emb_var.name
visual_embedding.metadata_path = os.path.join(
result_dir, 'metadata_val.tsv')
projector.visualize_embeddings(summary_writer, config)
# save model
saver.save(sess,
os.path.join(result_dir, cfg.name + '.ckpt'),
global_step=step)
def fit_emb(reviews, batch_feeder, config):
do_log_save = False
do_profiling = False
log_save_path = 'log'
# separate a validation set
use_valid_set = True
if use_valid_set:
reviews, valid_reviews = separate_valid(reviews, 0.1)
# build model graph
with tf.device('/gpu:0'):
graph = tf.Graph()
with graph.as_default():
tf.set_random_seed(27)
builder = GraphBuilder()
problem_size = {'num_reviews': reviews.shape[0], 'num_items': reviews.shape[1]}
inputs, outputs, model_param = builder.construct_model_graph(problem_size, config, init_model=None, training=True)
model_vars = [model_param['alpha'], model_param['rho'], model_param['intercept'], model_param['prior_logit']]
optimizer = tf.train.AdagradOptimizer(0.1).minimize(outputs['objective'], var_list=model_vars)
if config['model'] in ['context_select']:
net_vars = builder.infer_net.param_list()
net_optimizer = tf.train.AdagradOptimizer(0.1).minimize(outputs['objective'], var_list=net_vars)
init = tf.global_variables_initializer()
# for visualization
vis_conf = projector.ProjectorConfig()
embedding = vis_conf.embeddings.add()
embedding.tensor_name = model_param['alpha'].name
# optimize the model
with tf.Session(graph=graph) as session:
# initialize all variables
init.run()
# Merge all the summaries and write them out to /tmp/mnist_logs (by defaul)
if do_log_save:
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(log_save_path,
session.graph)
projector.visualize_embeddings(train_writer, vis_conf)
else:
merged = []
if do_profiling:
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
nprint = config['nprint']
val_accum = np.array([0.0, 0.0])
count_accum = 0
train_logg = np.zeros([int(config['max_iter'] / nprint) + 1, 3])
review_size = reviews.shape[0]
for step in xrange(1, config['max_iter'] + 1):
rind = np.random.choice(review_size)
indices, labels = batch_feeder(reviews[rind])
if indices.shape[0] <= 1: # neglect views with only one entry
raise Exception('Row %d of the data has only one non-zero entry.' % rind)
feed_dict = {inputs['input_ind']: indices, inputs['input_label']: labels}
if config['model'] in ['context_select']:
_, net_debugv, summary = session.run((net_optimizer, outputs['debugv'], merged), feed_dict=feed_dict)
else:
net_debugv = ''
_, llh_val, nums, obj_val, debug_val, summary = session.run((optimizer, outputs['llh_sum'], outputs['num_items'], \
outputs['objective'], outputs['debugv'], merged), feed_dict=feed_dict)
if do_log_save:
train_writer.add_summary(summary, step)
# record llh, and objective
val_accum = val_accum + np.array([llh_val, obj_val])
count_accum = count_accum + (nums[0] + nums[1]);
# print loss every nprint iterations
if step % nprint == 0 or np.isnan(llh_val) or np.isinf(llh_val):
# do validation
valid_llh = 0.0
if use_valid_set:
valid_llh = validate(valid_reviews, batch_feeder, session, inputs, outputs)
# record the three values
ibatch = int(step / nprint)
train_logg[ibatch, :] = np.array([val_accum[0] / count_accum, val_accum[1] / nprint, valid_llh])
val_accum[:] = 0.0 # reset the accumulater
count_accum = 0
print("iteration[", step, "]: average llh, obj, valid_llh, and debug_val are ", train_logg[ibatch, :], debug_val, net_debugv)
#check nan value
if np.isnan(llh_val) or np.isinf(llh_val):
print('Loss value is ', llh_val, ', and the debug value is ', debug_val)
raise Exception('Bad values')
model = get_model(model_param, session, config)
if do_log_save:
tf.train.Saver().save(session, log_save_path, step)
# Create the Timeline object, and write it to a json
if do_profiling:
tl = timeline.Timeline(run_metadata.step_stats)
ctf = tl.generate_chrome_trace_format()
with open(log_save_path + '/timeline_step%d.json' % (step / nprint), 'w') as f:
f.write(ctf)
model = get_model(model_param, session, config)
return model, train_logg
def main(current_path, args):
print(args)
# Give a folder path as an argument with '--log_dir' to save
# TensorBoard summaries. Default is a log folder in current directory.
# Create the directory for TensorBoard variables if there is not.
if not os.path.exists(args.log_dir):
os.makedirs(args.log_dir)
filename = args.file
# preprocessing raw data and building vocabulary
vocabulary = read_data(filename)
print('Data size', len(vocabulary))
# Step 2: Build the dictionary and replace rare words with UNK token
# Filling 4 global variables:
# data - list of codes (integers from 0 to vocabulary_size-1).
# This is the original text but words are replaced by their codes
# count - map of words(strings) to count of occurrences
# dictionary - map of words(strings) to their codes(integers)
# reverse_dictionary - maps codes(integers) to words(strings)
data, count, dictionary, reverse_dictionary = build_dataset(vocabulary, 0)
del vocabulary # Hint to reduce memory.
vocabulary_size = len(dictionary)
num_sampled = args.neg_samples # Number of negative examples to sample.
train_args = {
'batch_size': args.train_bs,
'embedding_size': args.emb_size, # Dimension of the embedding vector.
'skip_window':
args.skip_window, # How many words to consider left and right.
'num_skips': args.
num_skips, # How many times to reuse an input to generate a label.
'num_steps': args.epochs, # Number of epochs
'log_dir': args.log_dir
}
model = Word2Vec(args.train_bs, args.emb_size, vocabulary_size,
num_sampled)
graph = model._graph
embeddings = None
with tf.Session(graph=graph) as session:
# Open a writer to write summaries.
writer = tf.summary.FileWriter(args.log_dir, session.graph)
embeddings = train_model(session, model, reverse_dictionary, writer,
data, **train_args)
#similarity = model.similarity.eval()
np.savetxt(args.emb_file, embeddings)
# Save the model for checkpoints.
model.savemodel.save(session, os.path.join(args.log_dir, 'model.ckpt'))
# Create a configuration for visualizing embeddings with the labels in TensorBoard.
config = projector.ProjectorConfig()
embedding_conf = config.embeddings.add()
embedding_conf.tensor_name = model.embeddings.name
embedding_conf.metadata_path = os.path.join(args.log_dir,
'metadata.tsv')
projector.visualize_embeddings(writer, config)
# Closing writer
writer.close()
print("Plotting items on TSNE plot...")
runTSNE(embeddings, reverse_dictionary, args.tsne_img_file)
def main(args):
tf.logging.set_verbosity(tf.logging.INFO)
embedding_dir_path = args.MODEL_CHPT_DIR_PATH + "-embedding"
if not os.path.exists(embedding_dir_path):
os.makedirs(embedding_dir_path)
json_path = os.path.join(args.MODEL_DIR_PATH, "params.json")
if not json_path:
tf.logging.error(
"No json configuration file found at {}".format(json_path))
params = get_params(json_path)
train_config = tf.estimator.RunConfig(
tf_random_seed=230,
model_dir=args.MODEL_CHPT_DIR_PATH,
save_summary_steps=params["save_summary_steps"])
estimator = tf.estimator.Estimator(model_fn,
params=params,
config=train_config)
#### prepare sprite.png and metadata.tsv
sprite_image_path = os.path.join(embedding_dir_path, "sprite.png")
if not os.path.exists(sprite_image_path):
save_sprite_image(embedding_dir_path, args.DATA_DIR_PATH)
metadata_path = os.path.join(embedding_dir_path, "metadata.tsv")
if not os.path.exists(metadata_path):
write_matadata_tsv(embedding_dir_path, args.DATA_DIR_PATH)
# Compute embeddings on the test set
tf.logging.info("Predicting")
predictions = estimator.predict(
lambda: test_input_fn(args.DATA_DIR_PATH, params))
embeddings = np.zeros((params["test_size"], params["embedding_size"]))
for i, p in enumerate(predictions):
embeddings[i] = p["embeddings"]
# Visualize test embeddings
embedding_var = tf.Variable(embeddings, name="mnist_embedding")
summary_writer = tf.summary.FileWriter(embedding_dir_path)
embedding_config = projector.ProjectorConfig()
embedding = embedding_config.embeddings.add()
embedding.tensor_name = embedding_var.name
# Specify where you find the metadata
embedding.metadata_path = "metadata.tsv" #'metadata.tsv'
# Specify where you find the sprite (we will create this later)
embedding.sprite.image_path = "sprite.png"
embedding.sprite.single_image_dim.extend(
[params["image_size"], params["image_size"]])
# Say that you want to visualise the embeddings
projector.visualize_embeddings(summary_writer, embedding_config)
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(embedding_var.initializer)
saver.save(sess, os.path.join(embedding_dir_path, "embeddings.ckpt"))
log_path = "tf_writer"
model_path = './singleNerualNode'
removeFileInDir(log_path)
[inputData, labels] = generateDataAndLabels(batchSize, numberOfInputDims)
combinedDataAndLabel = np.column_stack((inputData, labels))
np.savetxt('trainDataAndLabel.csv', combinedDataAndLabel, delimiter=",")
np.savetxt('label.csv', labels, delimiter=",")
embedding_var = tf.Variable(inputData, 'data_embeding')
config = projector.ProjectorConfig()
embedding = config.embeddings.add()
embedding.tensor_name = embedding_var.name
embedding.metadata_path = 'label.csv'
projector.visualize_embeddings(tf.summary.FileWriter(log_path), config)
inputTensor = tf.placeholder(tf.float32, [None, numberOfInputDims],
name='inputTensor')
labelTensor = tf.placeholder(tf.float32, [None, 1], name='LabelTensor')
with tf.name_scope('Nerual_Node'):
W = tf.Variable(tf.random_normal([numberOfInputDims, 1]),
name='weights')
tf.summary.histogram('weights', W)
b = tf.Variable(tf.zeros([1]), name='biases')
tf.summary.histogram('biases', b)
a = tf.nn.sigmoid(tf.matmul(inputTensor, W) + b, name='activation')
with tf.name_scope('evaluation'):
loss = tf.nn.l2_loss(a - labels, name='L2Loss') / batchSize
threshold = 0.5
def main(argv=None):
classify_image.maybe_download_and_extract()
classify_image.create_graph()
basedir = os.path.dirname(__file__)
with tf.Session() as sess:
pool3 = sess.graph.get_tensor_by_name('pool_3:0')
jpeg_data = tf.placeholder(tf.string)
thumbnail = tf.cast(
tf.image.resize_images(tf.image.decode_jpeg(jpeg_data),
[100, 100]), tf.uint8)
outputs = []
files = []
images = []
for filename in os.listdir('images'):
if not filename.endswith('.JPG'):
continue
print('process %s...' % filename)
files.append(filename)
with open(os.path.join(basedir, 'images', filename), 'rb') as f:
data = f.read()
results = sess.run([pool3, thumbnail], {
'DecodeJpeg/contents:0': data,
jpeg_data: data
})
outputs.append(results[0])
images.append(results[1])
embedding_var = tf.Variable(tf.pack([tf.squeeze(x) for x in outputs],
axis=0),
trainable=False,
name='pool3')
# prepare projector config
config = projector.ProjectorConfig()
embedding = config.embeddings.add()
embedding.tensor_name = embedding_var.name
summary_writer = tf.train.SummaryWriter(os.path.join(
basedir, 'logdir'))
# link metadata
metadata_path = os.path.join(basedir, 'logdir', 'metadata.tsv')
with open(metadata_path, 'w') as f:
for name in files:
f.write('%s\n' % name)
embedding.metadata_path = metadata_path
# write to sprite image file
image_path = os.path.join(basedir, 'logdir', 'sprite.jpg')
size = int(math.sqrt(len(images))) + 1
while len(images) < size * size:
images.append(np.zeros((100, 100, 3), dtype=np.uint8))
rows = []
for i in range(size):
rows.append(tf.concat(1, images[i * size:(i + 1) * size]))
jpeg = tf.image.encode_jpeg(tf.concat(0, rows))
with open(image_path, 'wb') as f:
f.write(sess.run(jpeg))
embedding.sprite.image_path = image_path
embedding.sprite.single_image_dim.extend([100, 100])
# save embedding_var
projector.visualize_embeddings(summary_writer, config)
sess.run(tf.variables_initializer([embedding_var]))
saver = tf.train.Saver([embedding_var])
saver.save(sess, os.path.join(basedir, 'logdir', 'model.ckpt'))
def main(_):
# Create training directories
now = datetime.datetime.now()
train_dir_name = now.strftime('alexnet_%Y%m%d_%H%M%S')
train_dir = os.path.join(FLAGS.train_root_dir, train_dir_name)
checkpoint_dir = os.path.join(train_dir, 'checkpoint')
tensorboard_dir = os.path.join(train_dir, 'tensorboard')
tensorboard_train_dir = os.path.join(tensorboard_dir, 'train')
tensorboard_val_dir = os.path.join(tensorboard_dir, 'val')
if not os.path.isdir(FLAGS.train_root_dir): os.mkdir(FLAGS.train_root_dir)
if not os.path.isdir(train_dir): os.mkdir(train_dir)
if not os.path.isdir(checkpoint_dir): os.mkdir(checkpoint_dir)
if not os.path.isdir(tensorboard_dir): os.mkdir(tensorboard_dir)
if not os.path.isdir(tensorboard_train_dir):
os.mkdir(tensorboard_train_dir)
if not os.path.isdir(tensorboard_val_dir): os.mkdir(tensorboard_val_dir)
# Write flags to txt
flags_file_path = os.path.join(train_dir, 'flags.txt')
flags_file = open(flags_file_path, 'w')
flags_file.write('learning_rate={}\n'.format(FLAGS.learning_rate))
flags_file.write('dropout_keep_prob={}\n'.format(FLAGS.dropout_keep_prob))
flags_file.write('num_epochs={}\n'.format(FLAGS.num_epochs))
flags_file.write('batch_size={}\n'.format(FLAGS.batch_size))
flags_file.write('train_layers={}\n'.format(FLAGS.train_layers))
flags_file.write('multi_scale={}\n'.format(FLAGS.multi_scale))
flags_file.write('train_root_dir={}\n'.format(FLAGS.train_root_dir))
flags_file.write('log_step={}\n'.format(FLAGS.log_step))
flags_file.close()
adlamb = tf.placeholder(tf.float32, name='adlamb')
num_update = tf.placeholder(tf.float32, name='num_update')
decay_learning_rate = tf.placeholder(tf.float32)
dropout_keep_prob = tf.placeholder(tf.float32)
is_training = tf.placeholder(tf.bool)
time = tf.placeholder(tf.float32, [1])
# Model
train_layers = FLAGS.train_layers.split(',')
model = LeNetModel(num_classes=NUM_CLASSES,
image_size=28,
is_training=is_training,
dropout_keep_prob=dropout_keep_prob)
# Placeholders
x_s = tf.placeholder(tf.float32, [None, 32, 32, 3], name='x')
x_t = tf.placeholder(tf.float32, [None, 28, 28, 1], name='xt')
x = preprocessing(x_s, model)
xt = preprocessing(x_t, model)
tf.summary.image('Source Images', x)
tf.summary.image('Target Images', xt)
print 'x_s ', x_s.get_shape()
print 'x ', x.get_shape()
print 'x_t ', x_t.get_shape()
print 'xt ', xt.get_shape()
y = tf.placeholder(tf.float32, [None, NUM_CLASSES], name='y')
yt = tf.placeholder(tf.float32, [None, NUM_CLASSES], name='yt')
loss = model.loss(x, y)
# Training accuracy of the model
source_correct_pred = tf.equal(tf.argmax(model.score, 1), tf.argmax(y, 1))
source_correct = tf.reduce_sum(tf.cast(source_correct_pred, tf.float32))
source_accuracy = tf.reduce_mean(tf.cast(source_correct_pred, tf.float32))
G_loss, D_loss, sc, tc = model.adloss(x, xt, y, yt)
# Testing accuracy of the model
correct_pred = tf.equal(tf.argmax(model.score, 1), tf.argmax(yt, 1))
correct = tf.reduce_sum(tf.cast(correct_pred, tf.float32))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
update_op = model.optimize(decay_learning_rate, train_layers, adlamb, sc,
tc)
D_op = model.adoptimize(decay_learning_rate, train_layers)
optimizer = tf.group(update_op, D_op)
train_writer = tf.summary.FileWriter('./log/tensorboard')
train_writer.add_graph(tf.get_default_graph())
config = projector.ProjectorConfig()
embedding = config.embeddings.add()
embedding.tensor_name = model.feature.name
embedding.metadata_path = 'domain.csv'
projector.visualize_embeddings(train_writer, config)
tf.summary.scalar('G_loss', model.G_loss)
tf.summary.scalar('D_loss', model.D_loss)
tf.summary.scalar('C_loss', model.loss)
tf.summary.scalar('SA_loss', model.Semanticloss)
tf.summary.scalar('Training Accuracy', source_accuracy)
tf.summary.scalar('Testing Accuracy', accuracy)
merged = tf.summary.merge_all()
print '============================GLOBAL TRAINABLE VARIABLES ============================'
print tf.trainable_variables()
#print '============================GLOBAL VARIABLES ======================================'
#print tf.global_variables()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
#saver.restore(sess,'log/checkpoint')
# Load the pretrained weights
#model.load_original_weights(sess, skip_layers=train_layers)
train_writer.add_graph(sess.graph)
# Directly restore (your model should be exactly the same with checkpoint)
# saver.restore(sess, "/Users/dgurkaynak/Projects/marvel-training/alexnet64-fc6/model_epoch10.ckpt")
print("{} Start training...".format(datetime.datetime.now()))
#print("{} Open Tensorboard at --logdir {}".format(datetime.datetime.now(), tensorboard_dir))
gd = 0
step = 1
for epoch in range(300000):
# Start training
gd += 1
lamb = adaptation_factor(gd * 1.0 / MAX_STEP)
#rate=decay(FLAGS.learning_rate,gd,MAX_STEP)
power = gd / 10000
rate = FLAGS.learning_rate
tt = pow(0.1, power)
batch_xs, batch_ys = TRAIN.next_batch(FLAGS.batch_size)
Tbatch_xs, Tbatch_ys = VALID.next_batch(FLAGS.batch_size)
#print batch_xs.shape
#print Tbatch_xs.shape
summary, _, closs, gloss, dloss, smloss = sess.run(
[
merged, optimizer, model.loss, model.G_loss, model.D_loss,
model.Semanticloss
],
feed_dict={
x_s: batch_xs,
x_t: Tbatch_xs,
time: [1.0 * gd],
decay_learning_rate: rate,
adlamb: lamb,
is_training: True,
y: batch_ys,
dropout_keep_prob: 0.5,
yt: Tbatch_ys
})
train_writer.add_summary(summary, gd)
step += 1
if gd % 250 == 0:
epoch = gd / (72357 / 100)
print 'lambda: ', lamb
print 'rate: ', rate
print 'Epoch {5:<10} Step {3:<10} C_loss {0:<10} G_loss {1:<10} D_loss {2:<10} Sem_loss {4:<10}'.format(
closs, gloss, dloss, gd, smloss, epoch)
print("{} Start validation".format(datetime.datetime.now()))
test_acc = 0.
test_count = 0
print 'test_iter ', len(TEST.labels)
for _ in xrange((len(TEST.labels)) / 5000):
batch_tx, batch_ty = TEST.next_batch(5000)
#print TEST.pointer,' ',TEST.shuffle
acc = sess.run(correct,
feed_dict={
x_t: batch_tx,
yt: batch_ty,
is_training: True,
dropout_keep_prob: 1.
})
test_acc += acc
test_count += 5000
print test_acc, test_count
test_acc /= test_count
if epoch == 300:
return
#batch_tx, batch_ty = TEST.next_batch(len(TEST.labels))
#test_acc=sess.run(accuracy,feed_dict={x_t:batch_tx,y:batch_ty,is_training:False,dropout_keep_prob:1.0})
print len(batch_tx)
print("{} Validation Accuracy = {:.4f}".format(
datetime.datetime.now(), test_acc))
if gd % 10000 == 0 and gd > 0:
#saver.save(sess,'./log/mstn2model'+str(gd)+'.ckpt')
#print 'tensorboard --logdir ./log/tensorboard'
#return
pass
def tensorboard_op():
# 载入数据集
mnist = input_data.read_data_sets("./../../../datas/mnist/",one_hot=True)
# 运行次数
max_steps = 10001
# 图片数量
image_num = 3000
# 文件路径
DIR = "./../../../datas/logs/share/mnist/fc/"
# 定义会话
sess = tf.Session()
# 载入图片
embedding = tf.Variable(tf.stack(mnist.test.images[:image_num]), trainable=False, name='embedding')
# 参数概要
def variable_summaries(var):
with tf.name_scope('summaries'):
mean = tf.reduce_mean(var)
tf.summary.scalar('mean', mean) # 平均值
with tf.name_scope('stddev'):
stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
tf.summary.scalar('stddev', stddev) # 标准差
tf.summary.scalar('max', tf.reduce_max(var)) # 最大值
tf.summary.scalar('min', tf.reduce_min(var)) # 最小值
tf.summary.histogram('histogram', var) # 直方图
# 命名空间
with tf.name_scope('input'):
# 这里的none表示第一个维度可以是任意的长度
x = tf.placeholder(tf.float32, [None, 784], name='x-input')
# 正确的标签
y = tf.placeholder(tf.float32, [None, 10], name='y-input')
keep_prob = tf.placeholder(tf.float32)
lr = tf.Variable(0.001, dtype=tf.float32)
# 显示图片
with tf.name_scope('input_reshape'):
image_shaped_input = tf.reshape(x, [-1, 28, 28, 1])
tf.summary.image('input', image_shaped_input, 10)
with tf.name_scope('layer1'):
# 创建一个简单神经网络
with tf.name_scope('weights1'):
W1 = tf.Variable(tf.truncated_normal([784, 500], stddev=0.1), name='W1')
variable_summaries(W1)
with tf.name_scope('biases1'):
b1 = tf.Variable(tf.zeros([500]) + 0.1, name='b1')
variable_summaries(b1)
with tf.name_scope('wx_plus_b1'):
wx_plus_b1 = tf.matmul(x, W1) + b1
with tf.name_scope('tanh1'):
L1 = tf.nn.tanh(wx_plus_b1)
with tf.name_scope('dropout1'):
L1_drop = tf.nn.dropout(L1, keep_prob)
with tf.name_scope('layer2'):
# 创建一个简单神经网络
with tf.name_scope('weights2'):
W2 = tf.Variable(tf.truncated_normal([500, 300], stddev=0.1), name='W2')
variable_summaries(W2)
with tf.name_scope('biases2'):
b2 = tf.Variable(tf.zeros([300]) + 0.1, name='b2')
variable_summaries(b2)
with tf.name_scope('wx_plus_b2'):
wx_plus_b2 = tf.matmul(L1_drop, W2) + b2
with tf.name_scope('tanh2'):
L2 = tf.nn.tanh(wx_plus_b2)
with tf.name_scope('dropout2'):
L2_drop = tf.nn.dropout(L2, keep_prob)
with tf.name_scope('layer3'):
# 创建一个简单神经网络
with tf.name_scope('weights3'):
W3 = tf.Variable(tf.truncated_normal([300, 10], stddev=0.1), name='W3')
variable_summaries(W3)
with tf.name_scope('biases3'):
b3 = tf.Variable(tf.zeros([10]) + 0.1, name='b3')
variable_summaries(b3)
with tf.name_scope('wx_plus_b3'):
wx_plus_b3 = tf.matmul(L2_drop, W3) + b3
with tf.name_scope('softmax'):
prediction = tf.nn.softmax(wx_plus_b3)
with tf.name_scope('loss'):
# 交叉熵代价函数
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=y, logits=prediction))
tf.summary.scalar('loss', loss)
with tf.name_scope('train'):
# 使用梯度下降法
train_step = tf.train.GradientDescentOptimizer(0.2).minimize(loss)
#train_step = tf.train.AdamOptimizer(lr).minimize(loss)
# 初始化变量
sess.run(tf.global_variables_initializer())
with tf.name_scope('accuracy'):
with tf.name_scope('correct_prediction'):
# 结果存放在一个布尔型列表中
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(prediction, 1)) # argmax返回一维张量中最大的值所在的位置
with tf.name_scope('accuracy'):
# 求准确率
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) # 把correct_prediction变为float32类型
tf.summary.scalar('accuracy', accuracy)
# 产生metadata文件
if tf.gfile.Exists(DIR + 'projector/projector/metadata.tsv'):
tf.gfile.DeleteRecursively(DIR + 'projector/projector/metadata.tsv')
with open(DIR + 'projector/projector/metadata.tsv', 'w') as f:
labels = sess.run(tf.argmax(mnist.test.labels[:], 1))
for i in range(image_num):
f.write(str(labels[i]) + '\n')
# 合并所有的summary
merged = tf.summary.merge_all()
projector_writer = tf.summary.FileWriter(DIR + 'projector/projector', sess.graph)
saver = tf.train.Saver()
config = projector.ProjectorConfig()
embed = config.embeddings.add()
embed.tensor_name = embedding.name
embed.metadata_path = 'metadata.tsv'
embed.sprite.image_path = 'mnist_10k_sprite.png'
embed.sprite.single_image_dim.extend([28, 28])
projector.visualize_embeddings(projector_writer, config)
count = 0
for i in range(max_steps):
# 每个批次100个样本
# if i % 20 == 0 and count < 50:
# count += 1
# sess.run(tf.assign(lr, 0.001 * (0.95 ** i)))
batch_xs, batch_ys = mnist.train.next_batch(100)
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
summary, _ = sess.run([merged, train_step], feed_dict={x: batch_xs, y: batch_ys,keep_prob:1.0}, options=run_options,
run_metadata=run_metadata)
projector_writer.add_run_metadata(run_metadata, 'step%03d' % i)
projector_writer.add_summary(summary, i)
if i % 100 == 0:
acc = sess.run(accuracy, feed_dict={x: mnist.test.images, y: mnist.test.labels,keep_prob:1.0})
print("Iter " + str(i) + ", Testing Accuracy= " + str(acc))
saver.save(sess, DIR + 'projector/projector/a_model.ckpt', global_step=max_steps)
projector_writer.close()
sess.close()
def train_main(args):
"""
trains model specfied in args.
main method for train subcommand.
"""
# load text
with open(args.text_path) as f:
text = f.read()
logger.info("corpus length: %s.", len(text))
# restore or build model
if args.restore:
load_path = args.checkpoint_path if args.restore is True else args.restore
with open("{}.json".format(args.checkpoint_path)) as f:
model_args = json.load(f)
logger.info("model restored: %s.", load_path)
else:
load_path = None
model_args = {
"batch_size": args.batch_size,
"vocab_size": VOCAB_SIZE,
"embedding_size": args.embedding_size,
"rnn_size": args.rnn_size,
"num_layers": args.num_layers,
"p_keep": 1 - args.drop_rate,
"learning_rate": args.learning_rate,
"clip_norm": args.clip_norm
}
# build train model
train_graph = tf.Graph()
with train_graph.as_default():
train_model = build_model(**model_args)
with tf.Session(graph=train_graph) as train_sess:
# restore or initialise model weights
if load_path is not None:
train_model["saver"].restore(train_sess, load_path)
logger.info("model weights restored: %s.", load_path)
else:
train_sess.run(train_model["init_op"])
# clear checkpoint directory
log_dir = make_dirs(args.checkpoint_path, empty=True)
# save model
with open("{}.json".format(args.checkpoint_path), "w") as f:
json.dump(train_model["args"], f, indent=2)
checkpoint_path = train_model["saver"].save(train_sess,
args.checkpoint_path)
logger.info("model saved: %s.", checkpoint_path)
# tensorboard logger
summary_writer = tf.summary.FileWriter(log_dir, train_sess.graph)
# embeddings visualisation
config = projector.ProjectorConfig()
embedding = config.embeddings.add()
embedding.tensor_name = "EmbedSequence/embeddings"
embedding.metadata_path = os.path.abspath(
os.path.join("data", "id2char.tsv"))
projector.visualize_embeddings(summary_writer, config)
logger.info("tensorboard set up.")
# build infer model
inference_graph = tf.Graph()
with inference_graph.as_default():
inference_model = load_inference_model(args.checkpoint_path)
# training start
num_batches = (len(text) - 1) // (args.batch_size * args.seq_len)
data_iter = batch_generator(encode_text(text), args.batch_size,
args.seq_len)
fetches = [
train_model["train_op"], train_model["output_state"],
train_model["loss"], train_model["summary"]
]
state = train_sess.run(train_model["input_state"])
logger.info("start of training.")
time_train = time.time()
for i in range(args.num_epochs):
epoch_losses = np.empty(num_batches)
time_epoch = time.time()
# training epoch
for j in tqdm(range(num_batches),
desc="epoch {}/{}".format(i + 1, args.num_epochs)):
x, y = next(data_iter)
feed_dict = {
train_model["X"]: x,
train_model["Y"]: y,
train_model["input_state"]: state
}
_, state, loss, summary_log = train_sess.run(
fetches, feed_dict)
epoch_losses[j] = loss
# logs
duration_epoch = time.time() - time_epoch
logger.info("epoch: %s, duration: %ds, loss: %.6g.", i + 1,
duration_epoch, epoch_losses.mean())
# tensorboard logs
summary_writer.add_summary(summary_log, i + 1)
summary_writer.flush()
# checkpoint
checkpoint_path = train_model["saver"].save(
train_sess, args.checkpoint_path)
logger.info("model saved: %s.", checkpoint_path)
# generate text
seed = generate_seed(text)
with tf.Session(graph=inference_graph) as infer_sess:
# restore weights
inference_model["saver"].restore(infer_sess, checkpoint_path)
generate_text(inference_model, infer_sess, seed)
# training end
duration_train = time.time() - time_train
logger.info("end of training, duration: %ds.", duration_train)
# generate text
seed = generate_seed(text)
with tf.Session(graph=inference_graph) as infer_sess:
# restore weights
inference_model["saver"].restore(infer_sess, checkpoint_path)
generate_text(inference_model, infer_sess, seed, 1024, 3)
return train_model
def main(_):
# Create training directories
now = datetime.datetime.now()
train_dir_name = now.strftime('alexnet_%Y%m%d_%H%M%S')
train_dir = os.path.join(FLAGS.train_root_dir, train_dir_name)
checkpoint_dir = os.path.join(train_dir, 'checkpoint')
tensorboard_dir = os.path.join(train_dir, 'tensorboard')
tensorboard_train_dir = os.path.join(tensorboard_dir, 'train')
tensorboard_val_dir = os.path.join(tensorboard_dir, 'val')
if not os.path.isdir(FLAGS.train_root_dir): os.mkdir(FLAGS.train_root_dir)
if not os.path.isdir(train_dir): os.mkdir(train_dir)
if not os.path.isdir(checkpoint_dir): os.mkdir(checkpoint_dir)
if not os.path.isdir(tensorboard_dir): os.mkdir(tensorboard_dir)
if not os.path.isdir(tensorboard_train_dir): os.mkdir(tensorboard_train_dir)
if not os.path.isdir(tensorboard_val_dir): os.mkdir(tensorboard_val_dir)
# Write flags to txt
flags_file_path = os.path.join(train_dir, 'flags.txt')
flags_file = open(flags_file_path, 'w')
flags_file.write('learning_rate={}\n'.format(FLAGS.learning_rate))
flags_file.write('dropout_keep_prob={}\n'.format(FLAGS.dropout_keep_prob))
flags_file.write('num_epochs={}\n'.format(FLAGS.num_epochs))
flags_file.write('batch_size={}\n'.format(FLAGS.batch_size))
flags_file.write('train_layers={}\n'.format(FLAGS.train_layers))
flags_file.write('multi_scale={}\n'.format(FLAGS.multi_scale))
flags_file.write('train_root_dir={}\n'.format(FLAGS.train_root_dir))
flags_file.write('log_step={}\n'.format(FLAGS.log_step))
flags_file.close()
adlamb=tf.placeholder(tf.float32,name='adlamb')
num_update=tf.placeholder(tf.float32,name='num_update')
decay_learning_rate=tf.placeholder(tf.float32)
dropout_keep_prob = tf.placeholder(tf.float32)
is_training=tf.placeholder(tf.bool)
time=tf.placeholder(tf.float32,[1])
# Model
train_layers = FLAGS.train_layers.split(',')
model = LeNetModel(num_classes=NUM_CLASSES, image_size=28,is_training=is_training,dropout_keep_prob=dropout_keep_prob)
# Placeholders
x_s = tf.placeholder(tf.float32, [None, 32, 32, 3],name='x')
x_t = tf.placeholder(tf.float32, [None, 28, 28, 1],name='xt')
x=preprocessing(x_s,model)
xt=preprocessing(x_t,model)
tf.summary.image('Source Images',x)
tf.summary.image('Target Images',xt)
print 'x_s ',x_s.get_shape()
print 'x ',x.get_shape()
print 'x_t ',x_t.get_shape()
print 'xt ',xt.get_shape()
y = tf.placeholder(tf.float32, [None, NUM_CLASSES],name='y')
yt = tf.placeholder(tf.float32, [None, NUM_CLASSES],name='yt')
loss = model.loss(x, y)
# Training accuracy of the model
source_correct_pred = tf.equal(tf.argmax(model.score, 1), tf.argmax(y, 1))
source_correct=tf.reduce_sum(tf.cast(source_correct_pred,tf.float32))
source_accuracy = tf.reduce_mean(tf.cast(source_correct_pred, tf.float32))
G_loss,D_loss,sc,tc=model.adloss(x,xt,y,yt)
# Testing accuracy of the model
correct_pred = tf.equal(tf.argmax(model.score, 1), tf.argmax(yt, 1))
correct=tf.reduce_sum(tf.cast(correct_pred,tf.float32))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
update_op = model.optimize(decay_learning_rate,train_layers,adlamb,sc,tc)
D_op=model.adoptimize(decay_learning_rate,train_layers)
optimizer=tf.group(update_op,D_op)
train_writer=tf.summary.FileWriter('./log/tensorboard')
train_writer.add_graph(tf.get_default_graph())
config=projector.ProjectorConfig()
embedding=config.embeddings.add()
embedding.tensor_name=model.feature.name
embedding.metadata_path='domain.csv'
projector.visualize_embeddings(train_writer,config)
tf.summary.scalar('G_loss',model.G_loss)
tf.summary.scalar('D_loss',model.D_loss)
tf.summary.scalar('C_loss',model.loss)
tf.summary.scalar('SA_loss',model.Semanticloss)
tf.summary.scalar('Training Accuracy',source_accuracy)
tf.summary.scalar('Testing Accuracy',accuracy)
merged=tf.summary.merge_all()
print '============================GLOBAL TRAINABLE VARIABLES ============================'
print tf.trainable_variables()
#print '============================GLOBAL VARIABLES ======================================'
#print tf.global_variables()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
saver=tf.train.Saver()
#saver.restore(sess,'log/checkpoint')
# Load the pretrained weights
#model.load_original_weights(sess, skip_layers=train_layers)
train_writer.add_graph(sess.graph)
# Directly restore (your model should be exactly the same with checkpoint)
# saver.restore(sess, "/Users/dgurkaynak/Projects/marvel-training/alexnet64-fc6/model_epoch10.ckpt")
print("{} Start training...".format(datetime.datetime.now()))
#print("{} Open Tensorboard at --logdir {}".format(datetime.datetime.now(), tensorboard_dir))
gd=0
step = 1
for epoch in range(300000):
# Start training
gd+=1
lamb=adaptation_factor(gd*1.0/MAX_STEP)
#rate=decay(FLAGS.learning_rate,gd,MAX_STEP)
power=gd/10000
rate=FLAGS.learning_rate
tt=pow(0.1,power)
batch_xs, batch_ys = TRAIN.next_batch(FLAGS.batch_size)
Tbatch_xs, Tbatch_ys = VALID.next_batch(FLAGS.batch_size)
#print batch_xs.shape
#print Tbatch_xs.shape
summary,_,closs,gloss,dloss,smloss=sess.run([merged,optimizer,model.loss,model.G_loss,model.D_loss,model.Semanticloss], feed_dict={x_s: batch_xs,x_t: Tbatch_xs,time:[1.0*gd],decay_learning_rate:rate,adlamb:lamb,is_training:True,y: batch_ys,dropout_keep_prob:0.5,yt:Tbatch_ys})
train_writer.add_summary(summary,gd)
step += 1
if gd%250==0:
epoch=gd/(72357/100)
print 'lambda: ',lamb
print 'rate: ',rate
print 'Epoch {5:<10} Step {3:<10} C_loss {0:<10} G_loss {1:<10} D_loss {2:<10} Sem_loss {4:<10}'.format(closs,gloss,dloss,gd,smloss,epoch)
print("{} Start validation".format(datetime.datetime.now()))
test_acc = 0.
test_count = 0
print 'test_iter ',len(TEST.labels)
for _ in xrange((len(TEST.labels))/5000):
batch_tx, batch_ty = TEST.next_batch(5000)
#print TEST.pointer,' ',TEST.shuffle
acc = sess.run(correct, feed_dict={x_t: batch_tx, yt: batch_ty, is_training:True,dropout_keep_prob: 1.})
test_acc += acc
test_count += 5000
print test_acc,test_count
test_acc /= test_count
if epoch==300:
return
#batch_tx, batch_ty = TEST.next_batch(len(TEST.labels))
#test_acc=sess.run(accuracy,feed_dict={x_t:batch_tx,y:batch_ty,is_training:False,dropout_keep_prob:1.0})
print len(batch_tx)
print("{} Validation Accuracy = {:.4f}".format(datetime.datetime.now(), test_acc))
if gd%10000==0 and gd>0:
#saver.save(sess,'./log/mstn2model'+str(gd)+'.ckpt')
#print 'tensorboard --logdir ./log/tensorboard'
#return
pass
def word2vec(batch_gen):
# Phase1: assemble your graph
# Step 1: define the placeholders for input and output
with tf.name_scope("data"):
center_words = tf.placeholder(tf.int32,
shape=[BATCH_SIZE],
name='center_words')
target_words = tf.placeholder(tf.int32,
shape=[BATCH_SIZE, 1],
name='target_words')
# Step 2: define weights
with tf.name_scope("weight"):
embed_matrix = tf.Variable(tf.random_uniform([VOCAB_SIZE, EMBED_SIZE],
-1.0, 1.0),
name='embed_matrix')
# Step 3: define inference model
with tf.name_scope("loss"):
embed = tf.nn.embedding_lookup(embed_matrix,
center_words,
name='embed')
nce_weight = tf.Variable(tf.truncated_normal([VOCAB_SIZE, EMBED_SIZE],
stddev=1.0 /
(EMBED_SIZE**0.5)),
name='nce_weight')
nce_bias = tf.Variable(tf.zeros([VOCAB_SIZE]), name='nce_bias')
# Step 4: define the loss function
loss = tf.reduce_mean(tf.nn.nce_loss(weights=nce_weight,
biases=nce_bias,
labels=target_words,
inputs=embed,
num_sampled=NUM_SAMPLED,
num_classes=VOCAB_SIZE),
name='loss')
# Step 5: define the optimizer
optimizer = tf.train.GradientDescentOptimizer(LEARNING_RATE).minimize(loss)
# Phase2: ececute the computation
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
writer = tf.summary.FileWriter('./graphs/', sess.graph)
total_loss = 0
for index in range(TRAIN_STEPS):
centers, targets = next(batch_gen)
batch_loss, _ = sess.run([loss, optimizer],
feed_dict={
center_words: centers,
target_words: targets
})
total_loss += batch_loss
if (index + 1) % DISPLAY_STEP == 0:
print('Average loss at step {}: {:5.1f}'.format(
index, total_loss / DISPLAY_STEP))
total_loss = 0.0
writer.close()
# Phase3: visulize
# code to visualize the embeddings. uncomment the below to visualize embeddings
# run "'tensorboard --logdir='processed'" to see the embeddings
final_embed_matrix = sess.run(
embed_matrix) #result of sess.run is constant rather than variable
# # it has to variable. constants don't work here. you can't reuse model.embed_matrix
embedding_var = tf.Variable(final_embed_matrix[:1000],
name='embedding')
sess.run(embedding_var.initializer)
config = projector.ProjectorConfig()
summary_writer = tf.summary.FileWriter('processed')
# # add embedding to the config file
embedding = config.embeddings.add()
embedding.tensor_name = embedding_var.name
# # link this tensor to its metadata file, in this case the first 500 words of vocab
embedding.metadata_path = 'processed/vocab_1000.tsv'
# # saves a configuration file that TensorBoard will read during startup.
projector.visualize_embeddings(summary_writer, config)
saver_embed = tf.train.Saver([embedding_var])
saver_embed.save(sess, 'processed/model3.ckpt', 1)
w2v_samples = np.zeros((args.samples, model.vector_size))
with open('{}/{}_metadata.tsv'.format(args.projector, args.prefix), 'w+') as file_metadata:
for i, word in enumerate(model.wv.index2word[:args.samples]):
w2v_samples[i] = model[word]
file_metadata.write(word + '\n')
# define the model without training
sess = tf.InteractiveSession()
with tf.device("/cpu:0"):
embedding = tf.Variable(w2v_samples, trainable=False, name='{}_embedding'.format(args.prefix))
tf.global_variables_initializer().run()
saver = tf.train.Saver()
writer = tf.summary.FileWriter(args.projector, sess.graph)
# adding into projector
config = projector.ProjectorConfig()
embed = config.embeddings.add()
embed.tensor_name = '{}_embedding'.format(args.prefix)
embed.metadata_path = './{}_metadata.tsv'.format(args.prefix)
# Specify the width and height of a single thumbnail.
projector.visualize_embeddings(writer, config)
saver.save(sess, '{}/{}_model.ckpt'.format(args.projector, args.prefix), global_step=args.samples)
print('Start tensorboard with: \'tensorboard --logdir=\"projector\"\'\n'
'and check http://localhost:6006/#embeddings to view your embedding')
final_embeddings = normalized_embeddings.eval()
# Write corresponding labels for the embeddings.
with open(FLAGS.log_dir + '/metadata.tsv', 'w') as f:
for i in xrange(vocabulary_size):
f.write(reverse_dictionary[i] + '\n')
# Save the model for checkpoints.
saver.save(session, os.path.join(FLAGS.log_dir, 'model.ckpt'))
# Create a configuration for visualizing embeddings with the labels in TensorBoard.
config = projector.ProjectorConfig()
embedding_conf = config.embeddings.add()
embedding_conf.tensor_name = embeddings.name
embedding_conf.metadata_path = os.path.join(FLAGS.log_dir, 'metadata.tsv')
projector.visualize_embeddings(writer, config)
writer.close()
# Step 6: Visualize the embeddings.
# pylint: disable=missing-docstring
# Function to draw visualization of distance between embeddings.
def plot_with_labels(low_dim_embs, labels, filename):
assert low_dim_embs.shape[0] >= len(labels), 'More labels than embeddings'
plt.figure(figsize=(18, 18)) # in inches
for i, label in enumerate(labels):
x, y = low_dim_embs[i, :]
plt.scatter(x, y)
plt.annotate(
def set_model(self, model):
self.model = model
if K.backend() == 'tensorflow':
self.sess = K.get_session()
if self.histogram_freq and self.merged is None:
for layer in self.model.layers:
for weight in layer.weights:
mapped_weight_name = weight.name.replace(':', '_')
tf.summary.histogram(mapped_weight_name, weight)
if self.write_grads:
grads = model.optimizer.get_gradients(model.total_loss,
weight)
def is_indexed_slices(grad):
return type(grad).__name__ == 'IndexedSlices'
grads = [
grad.values if is_indexed_slices(grad) else grad
for grad in grads]
tf.summary.histogram('{}_grad'.format(mapped_weight_name), grads)
if self.write_images:
w_img = tf.squeeze(weight)
shape = K.int_shape(w_img)
if len(shape) == 2: # dense layer kernel case
if shape[0] > shape[1]:
w_img = tf.transpose(w_img)
shape = K.int_shape(w_img)
w_img = tf.reshape(w_img, [1,
shape[0],
shape[1],
1])
elif len(shape) == 3: # convnet case
if K.image_data_format() == 'channels_last':
# switch to channels_first to display
# every kernel as a separate image
w_img = tf.transpose(w_img, perm=[2, 0, 1])
shape = K.int_shape(w_img)
w_img = tf.reshape(w_img, [shape[0],
shape[1],
shape[2],
1])
elif len(shape) == 1: # bias case
w_img = tf.reshape(w_img, [1,
shape[0],
1,
1])
else:
# not possible to handle 3D convnets etc.
continue
shape = K.int_shape(w_img)
assert len(shape) == 4 and shape[-1] in [1, 3, 4]
tf.summary.image(mapped_weight_name, w_img)
if hasattr(layer, 'output'):
if isinstance(layer.output, list):
for i, output in enumerate(layer.output):
tf.summary.histogram('{}_out_{}'.format(layer.name, i), output)
else:
tf.summary.histogram('{}_out'.format(layer.name),
layer.output)
self.merged = tf.summary.merge_all()
if self.write_graph:
self.writer = tf.summary.FileWriter(self.log_dir,
self.sess.graph)
else:
self.writer = tf.summary.FileWriter(self.log_dir)
if self.embeddings_freq and self.embeddings_data is not None:
self.embeddings_data = standardize_input_data(self.embeddings_data, model.input_names)
embeddings_layer_names = self.embeddings_layer_names
if not embeddings_layer_names:
embeddings_layer_names = [layer.name for layer in self.model.layers
if type(layer).__name__ == 'Embedding']
self.assign_embeddings = []
embeddings_vars = {}
self.batch_id = batch_id = tf.placeholder(tf.int32)
self.step = step = tf.placeholder(tf.int32)
for layer in self.model.layers:
if layer.name in embeddings_layer_names:
embedding_input = self.model.get_layer(layer.name).output
embedding_size = np.prod(embedding_input.shape[1:])
embedding_input = tf.reshape(embedding_input,
(step, int(embedding_size)))
shape = (self.embeddings_data[0].shape[0], int(embedding_size))
embedding = tf.Variable(tf.zeros(shape),
name=layer.name + '_embedding')
embeddings_vars[layer.name] = embedding
batch = tf.assign(embedding[batch_id:batch_id + step],
embedding_input)
self.assign_embeddings.append(batch)
self.saver = tf.train.Saver(list(embeddings_vars.values()))
embeddings_metadata = {}
if not isinstance(self.embeddings_metadata, str):
embeddings_metadata = self.embeddings_metadata
else:
embeddings_metadata = {layer_name: self.embeddings_metadata
for layer_name in embeddings_vars.keys()}
config = projector.ProjectorConfig()
for layer_name, tensor in embeddings_vars.items():
embedding = config.embeddings.add()
embedding.tensor_name = tensor.name
if layer_name in embeddings_metadata:
embedding.metadata_path = embeddings_metadata[layer_name]
projector.visualize_embeddings(self.writer, config)
def word2vec_basic(log_dir):
"""Example of building, training and visualizing a word2vec model."""
# Create the directory for TensorBoard variables if there is not.
if not os.path.exists(log_dir):
os.makedirs(log_dir)
filename = 'dataset.txt'
# Read the data into a list of strings.
def read_data(filename):
dir_path = os.path.dirname(os.path.realpath(__file__))
with open(os.path.join(dir_path, filename)) as f:
data = tf.compat.as_str(f.read()).split()
return data
vocabulary = read_data(filename)
print(vocabulary)
print('Data size', len(vocabulary))
# sys.exit()
# Step 2: Build the dictionary and replace rare words with UNK token.
vocabulary_size = 100
def build_dataset(words, n_words):
"""Process raw inputs into a dataset."""
count = [['UNK', -1]]
count.extend(collections.Counter(words).most_common(n_words - 1))
dictionary = {}
for word, _ in count:
dictionary[word] = len(dictionary)
data = []
unk_count = 0
for word in words:
index = dictionary.get(word, 0)
if index == 0: # dictionary['UNK']
unk_count += 1
data.append(index)
count[0][1] = unk_count
reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))
return data, count, dictionary, reversed_dictionary
# Filling 4 global variables:
# data - list of codes (integers from 0 to vocabulary_size-1).
# This is the original text but words are replaced by their codes
# count - map of words(strings) to count of occurrences
# dictionary - map of words(strings) to their codes(integers)
# reverse_dictionary - maps codes(integers) to words(strings)
data, count, unused_dictionary, reverse_dictionary = build_dataset(
vocabulary, vocabulary_size)
del vocabulary # Hint to reduce memory.
print('Most common words (+UNK)', count[:5])
print('Sample data', data[:10], [reverse_dictionary[i] for i in data[:10]])
# Step 3: Function to generate a training batch for the skip-gram model.
def generate_batch(batch_size, num_skips, skip_window):
global data_index
assert batch_size % num_skips == 0
assert num_skips <= 2 * skip_window
batch = np.ndarray(shape=(batch_size), dtype=np.int32)
labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
span = 2 * skip_window + 1 # [ skip_window target skip_window ]
buffer = collections.deque(maxlen=span) # pylint: disable=redefined-builtin
if data_index + span > len(data):
data_index = 0
buffer.extend(data[data_index:data_index + span])
data_index += span
for i in range(batch_size // num_skips):
context_words = [w for w in range(span) if w != skip_window]
words_to_use = random.sample(context_words, num_skips)
for j, context_word in enumerate(words_to_use):
batch[i * num_skips + j] = buffer[skip_window]
labels[i * num_skips + j, 0] = buffer[context_word]
if data_index == len(data):
buffer.extend(data[0:span])
data_index = span
else:
buffer.append(data[data_index])
data_index += 1
# Backtrack a little bit to avoid skipping words in the end of a batch
data_index = (data_index + len(data) - span) % len(data)
return batch, labels
batch, labels = generate_batch(batch_size=8, num_skips=2, skip_window=1)
for i in range(8):
print(batch[i], reverse_dictionary[batch[i]], '->', labels[i, 0],
reverse_dictionary[labels[i, 0]])
# Step 4: Build and train a skip-gram model.
batch_size = 64 # 128
embedding_size = 64 # Dimension of the embedding vector. 128
skip_window = 1 # How many words to consider left and right.
num_skips = 2 # How many times to reuse an input to generate a label.
num_sampled = 64 # Number of negative examples to sample.
# We pick a random validation set to sample nearest neighbors. Here we limit
# the validation samples to the words that have a low numeric ID, which by
# construction are also the most frequent. These 3 variables are used only for
# displaying model accuracy, they don't affect calculation.
valid_size = 16 # Random set of words to evaluate similarity on.
valid_window = 20 # Only pick dev samples in the head of the distribution. 100
valid_examples = np.random.choice(valid_window, valid_size, replace=False)
graph = tf.Graph()
with graph.as_default():
# Input data.
with tf.name_scope('inputs'):
train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
valid_dataset = tf.constant(valid_examples, dtype=tf.int32)
# Ops and variables pinned to the CPU because of missing GPU implementation
with tf.device('/cpu:0'):
# Look up embeddings for inputs.
with tf.name_scope('embeddings'):
embeddings = tf.Variable(
tf.random_uniform([vocabulary_size, embedding_size], -1.0,
1.0))
embed = tf.nn.embedding_lookup(embeddings, train_inputs)
# Construct the variables for the NCE loss
with tf.name_scope('weights'):
nce_weights = tf.Variable(
tf.truncated_normal([vocabulary_size, embedding_size],
stddev=1.0 /
math.sqrt(embedding_size)))
with tf.name_scope('biases'):
nce_biases = tf.Variable(tf.zeros([vocabulary_size]))
# Compute the average NCE loss for the batch.
# tf.nce_loss automatically draws a new sample of the negative labels each
# time we evaluate the loss.
# Explanation of the meaning of NCE loss:
# http://mccormickml.com/2016/04/19/word2vec-tutorial-the-skip-gram-model/
with tf.name_scope('loss'):
loss = tf.reduce_mean(
tf.nn.nce_loss(weights=nce_weights,
biases=nce_biases,
labels=train_labels,
inputs=embed,
num_sampled=num_sampled,
num_classes=vocabulary_size))
# Add the loss value as a scalar to summary.
tf.summary.scalar('loss', loss)
# Construct the SGD optimizer using a learning rate of 1.0.
with tf.name_scope('optimizer'):
optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)
# Compute the cosine similarity between minibatch examples and all
# embeddings.
norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keepdims=True))
normalized_embeddings = embeddings / norm
valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings,
valid_dataset)
similarity = tf.matmul(valid_embeddings,
normalized_embeddings,
transpose_b=True)
# Merge all summaries.
merged = tf.summary.merge_all()
# Add variable initializer.
init = tf.global_variables_initializer()
# Create a saver.
saver = tf.train.Saver()
# Step 5: Begin training.
num_steps = 20001 # 100001
with tf.Session(graph=graph) as session:
# Open a writer to write summaries.
writer = tf.summary.FileWriter(log_dir, session.graph)
# We must initialize all variables before we use them.
init.run()
print('Initialized')
average_loss = 0
for step in xrange(num_steps):
batch_inputs, batch_labels = generate_batch(
batch_size, num_skips, skip_window)
feed_dict = {
train_inputs: batch_inputs,
train_labels: batch_labels
}
# Define metadata variable.
run_metadata = tf.RunMetadata()
# We perform one update step by evaluating the optimizer op (including it
# in the list of returned values for session.run()
# Also, evaluate the merged op to get all summaries from the returned
# "summary" variable. Feed metadata variable to session for visualizing
# the graph in TensorBoard.
_, summary, loss_val = session.run([optimizer, merged, loss],
feed_dict=feed_dict,
run_metadata=run_metadata)
average_loss += loss_val
# Add returned summaries to writer in each step.
writer.add_summary(summary, step)
# Add metadata to visualize the graph for the last run.
if step == (num_steps - 1):
writer.add_run_metadata(run_metadata, 'step%d' % step)
if step % 2000 == 0:
if step > 0:
average_loss /= 2000
# The average loss is an estimate of the loss over the last 2000
# batches.
print('Average loss at step ', step, ': ', average_loss)
average_loss = 0
# Note that this is expensive (~20% slowdown if computed every 500 steps)
if step % 10000 == 0:
sim = similarity.eval()
for i in xrange(valid_size):
valid_word = reverse_dictionary[valid_examples[i]]
top_k = 8 # number of nearest neighbors
nearest = (-sim[i, :]).argsort()[1:top_k + 1]
log_str = 'Nearest to %s:' % valid_word
for k in xrange(top_k):
close_word = reverse_dictionary[nearest[k]]
log_str = '%s %s,' % (log_str, close_word)
print(log_str)
final_embeddings = normalized_embeddings.eval()
# Write corresponding labels for the embeddings.
with open(log_dir + '/metadata.tsv', 'w') as f:
for i in xrange(vocabulary_size):
f.write(reverse_dictionary[i] + '\n')
# Save the model for checkpoints.
saver.save(session, os.path.join(log_dir, 'model.ckpt'))
# Create a configuration for visualizing embeddings with the labels in
# TensorBoard.
config = projector.ProjectorConfig()
embedding_conf = config.embeddings.add()
embedding_conf.tensor_name = embeddings.name
embedding_conf.metadata_path = os.path.join(log_dir, 'metadata.tsv')
projector.visualize_embeddings(writer, config)
writer.close()
def save_tsne(low_dim_embs, labels, filename):
import json
tsne = {}
xs = []
ys = []
for i, label in enumerate(labels):
x, y = low_dim_embs[i, :]
xs.append(x)
ys.append(y)
for i, label in enumerate(labels):
x, y = low_dim_embs[i, :]
j = np.interp(x, [min(xs), max(xs)], [0, 700])
k = np.interp(y, [min(ys), max(ys)], [0, 700])
tsne[label] = [float(j), float(k)]
with open(filename, 'w', encoding='utf-8') as f:
json.dump(tsne, f, ensure_ascii=False, indent=2)
# Step 6: Visualize the embeddings.
# pylint: disable=missing-docstring
# Function to draw visualization of distance between embeddings.
def plot_with_labels(low_dim_embs, labels, filename):
assert low_dim_embs.shape[0] >= len(
labels), 'More labels than embeddings'
plt.figure(figsize=(18, 18)) # in inches
for i, label in enumerate(labels):
x, y = low_dim_embs[i, :]
plt.scatter(x, y)
plt.annotate(label,
xy=(x, y),
xytext=(5, 2),
textcoords='offset points',
ha='right',
va='bottom')
plt.savefig(filename)
try:
# pylint: disable=g-import-not-at-top
from sklearn.manifold import TSNE
import matplotlib.pyplot as plt
tsne = TSNE(perplexity=30,
n_components=2,
init='pca',
n_iter=5000,
method='exact')
plot_only = 100
low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only, :])
labels = [reverse_dictionary[i] for i in xrange(plot_only)]
dir_path = os.path.dirname(os.path.realpath(__file__))
plot_with_labels(low_dim_embs, labels,
os.path.join(dir_path, 'tsne.png'))
save_tsne(low_dim_embs, labels, os.path.join(dir_path, 'tsne.json'))
except ImportError as ex:
print(
'Please install sklearn, matplotlib, and scipy to show embeddings.'
)
print(ex)
def main(_):
"""Runs main labeled eval loop."""
# Parse config dict from yaml config files / command line flags.
config = util.ParseConfigsToLuaTable(FLAGS.config_paths, FLAGS.model_params)
# Choose an estimator based on training strategy.
checkpointdir = FLAGS.checkpointdir
checkpoint_path = os.path.join(
'%s/model.ckpt-%s' % (checkpointdir, FLAGS.checkpoint_iter))
estimator = get_estimator(config, checkpointdir)
# Get records to embed.
validation_dir = FLAGS.embedding_records
validation_records = util.GetFilesRecursively(validation_dir)
sequences_to_data = {}
for (view_embeddings, view_raw_image_strings, seqname) in estimator.inference(
validation_records, checkpoint_path, config.data.embed_batch_size,
num_sequences=FLAGS.num_sequences):
sequences_to_data[seqname] = {
'embeddings': view_embeddings,
'images': view_raw_image_strings,
}
all_embeddings = np.zeros((0, config.embedding_size))
all_ims = []
all_seqnames = []
num_embeddings = FLAGS.num_embed
# Concatenate all views from all sequences into a big flat list.
for seqname, data in sequences_to_data.iteritems():
embs = data['embeddings']
ims = data['images']
for v in range(config.data.num_views):
for (emb, im) in zip(embs[v], ims[v]):
all_embeddings = np.append(all_embeddings, [emb], axis=0)
all_ims.append(im)
all_seqnames.append(seqname)
# Choose N indices uniformly from all images.
random_indices = range(all_embeddings.shape[0])
random.shuffle(random_indices)
viz_indices = random_indices[:num_embeddings]
# Extract embs.
viz_embs = np.array(all_embeddings[viz_indices])
# Extract and decode ims.
viz_ims = list(np.array(all_ims)[viz_indices])
decoded_ims = []
sprite_dim = FLAGS.sprite_dim
for i, im in enumerate(viz_ims):
if i % 100 == 0:
print('Decoding image %d/%d.' % (i, num_embeddings))
nparr_i = np.fromstring(str(im), np.uint8)
img_np = cv2.imdecode(nparr_i, 1)
img_np = img_np[..., [2, 1, 0]]
img_np = imresize(img_np, [sprite_dim, sprite_dim, 3])
decoded_ims.append(img_np)
decoded_ims = np.array(decoded_ims)
# Extract sequence names.
outdir = FLAGS.outdir
# The embedding variable, which needs to be stored
# Note this must a Variable not a Tensor!
embedding_var = tf.Variable(viz_embs, name='viz_embs')
with tf.Session() as sess:
sess.run(embedding_var.initializer)
summary_writer = tf.summary.FileWriter(outdir)
config = projector.ProjectorConfig()
embedding = config.embeddings.add()
embedding.tensor_name = embedding_var.name
# Comment out if you don't want sprites
embedding.sprite.image_path = os.path.join(outdir, 'sprite.png')
embedding.sprite.single_image_dim.extend(
[decoded_ims.shape[1], decoded_ims.shape[1]])
projector.visualize_embeddings(summary_writer, config)
saver = tf.train.Saver([embedding_var])
saver.save(sess, os.path.join(outdir, 'model2.ckpt'), 1)
sprite = images_to_sprite(decoded_ims)
imsave(os.path.join(outdir, 'sprite.png'), sprite)
def main(_):
if tf.gfile.Exists(FLAGS.log_dir):
tf.gfile.DeleteRecursively(FLAGS.log_dir)
tf.gfile.MakeDirs(FLAGS.log_dir)
# Hyper-parameters
width, height = 32, 32
size = (width, height)
classes = 1721
batch_size = 50
steps = 10000
save_location = "/tmp/tensorflow/kanji_simple/1"
# Import data
training, t_labels, validation, v_labels = prep.data_from_base(
'train_val_test_data_32')
t_labels = onehot_labels(t_labels, classes)
v_labels = onehot_labels(v_labels, classes)
print('data imported')
sess = tf.InteractiveSession()
# Create the model
with tf.name_scope('input'):
x = tf.placeholder(tf.float32, [None, width * height])
y_ = tf.placeholder(tf.float32, [None, classes])
with tf.name_scope('input_reshape'):
x_image = tf.reshape(x, [-1, width, height, 1])
tf.summary.image('input', x_image, classes)
# setting up the cnn
def weight_variable(shape, nme):
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(initial, name=nme)
def bias_variable(shape, nme):
initial = tf.constant(0.1, shape=shape)
return tf.Variable(initial, name=nme)
def conv2d(x, W):
return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')
def max_pool_2x2(x):
return tf.nn.max_pool(x,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
def variable_summaries(var):
with tf.name_scope('summaries'):
mean = tf.reduce_mean(var)
tf.summary.scalar('mean', mean)
tf.summary.scalar('max', tf.reduce_max(var))
tf.summary.scalar('min', tf.reduce_min(var))
tf.summary.histogram('histogram', var)
with tf.name_scope('stddev'):
stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
tf.summary.scalar('stddev', stddev)
# adding the first convolutional layer
with tf.name_scope('conv_layer1'):
with tf.name_scope('weights'):
W_conv1 = weight_variable([5, 5, 1, 32], "w1")
variable_summaries(W_conv1)
with tf.name_scope('biases'):
b_conv1 = bias_variable([32], "b1")
variable_summaries(b_conv1)
with tf.name_scope('activation'):
h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)
tf.summary.histogram('activations', h_conv1)
# adding the first pooling layer
with tf.name_scope('pooling1'):
h_pool1 = max_pool_2x2(h_conv1)
pool1_img = tf.reshape(h_pool1, [-1, width, height, 1])
tf.summary.image('pool1', pool1_img, classes)
# adding the third convolutional layer
with tf.name_scope('conv_layer2'):
with tf.name_scope('weights'):
W_conv3 = weight_variable([5, 5, 32, 64], "w3")
variable_summaries(W_conv3)
with tf.name_scope('biases'):
b_conv3 = bias_variable([64], "b3")
variable_summaries(b_conv3)
with tf.name_scope('activation'):
h_conv3 = tf.nn.relu(conv2d(h_pool1, W_conv3) + b_conv3)
tf.summary.histogram('activations', h_conv3)
# the second pooling layer
with tf.name_scope('pooling2'):
h_pool2 = max_pool_2x2(h_conv3)
pool2_img = tf.reshape(h_pool2, [-1, width, height, 1])
tf.summary.image('pool2', pool2_img, classes)
# adding the fifth convolutional layer
with tf.name_scope('conv_layer3'):
with tf.name_scope('weights'):
W_conv5 = weight_variable([5, 5, 64, 64], "w5")
variable_summaries(W_conv5)
with tf.name_scope('biases'):
b_conv5 = bias_variable([64], "b5")
variable_summaries(b_conv5)
with tf.name_scope('activation'):
h_conv5 = tf.nn.relu(conv2d(h_pool2, W_conv5) + b_conv5)
tf.summary.histogram('activations', h_conv5)
# the third pooling layer
h_pool3 = max_pool_2x2(h_conv5)
#adding the final layer
with tf.name_scope('final_layer'):
with tf.name_scope('weights'):
W_fc1 = weight_variable([4 * 4 * 64, 1024], "W_fc1")
variable_summaries(W_fc1)
with tf.name_scope('biases'):
b_fc1 = bias_variable([1024], "b_fc1")
variable_summaries(b_fc1)
h_pool3_flat = tf.reshape(h_pool3, [-1, 4 * 4 * 64])
with tf.name_scope('Wx_plus_b'):
preactivate = tf.matmul(h_pool3_flat, W_fc1) + b_fc1
tf.summary.histogram('pre_activations', preactivate)
h_fc1 = tf.nn.relu(preactivate)
tf.summary.histogram('activations', h_fc1)
# adding the dropout
with tf.name_scope('dropout'):
keep_prob = tf.placeholder(tf.float32)
tf.summary.scalar('dropout_keep_probability', keep_prob)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
# adding the readout layer
with tf.name_scope('readout_layer'):
with tf.name_scope('weights'):
W_fc3 = weight_variable([1024, classes], "w_read")
variable_summaries(W_fc3)
with tf.name_scope('biases'):
b_fc3 = bias_variable([classes], "b_read")
variable_summaries(b_fc3)
with tf.name_scope('Wx_plus_b'):
y_conv = tf.matmul(h_fc1_drop, W_fc3) + b_fc3
tf.summary.histogram('activations', y_conv)
with tf.name_scope('cross_entropy'):
# The raw formulation of cross-entropy,
#
# tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),
# reduction_indices=[1]))
#
# can be numerically unstable.
#
# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw
# outputs of 'y', and then average across the batch.
diff = tf.nn.softmax_cross_entropy_with_logits(labels=y_,
logits=y_conv)
with tf.name_scope('total'):
cross_entropy = tf.reduce_mean(diff)
tf.summary.scalar('cross_entropy', cross_entropy)
with tf.name_scope('train'):
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
# Test trained model
with tf.name_scope('accuracy'):
with tf.name_scope('correct_prediction'):
correct_prediction = tf.equal(tf.argmax(y_conv, 1),
tf.argmax(y_, 1))
with tf.name_scope('accuracy'):
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('accuracy', accuracy)
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(
'/tmp/tensorflow/kanji_simple/1/logs/kanji_with_summaries/train',
sess.graph)
test_writer = tf.summary.FileWriter(
'/tmp/tensorflow/kanji_simple/1/logs/kanji_with_summaries/test')
tf.global_variables_initializer().run()
saver = tf.train.Saver()
# if os.path.exists(os.path.join(save_location)):
# saver.restore(sess, save_location + "/model.ckpt")
epoch = -1
test_batch = 500
# Train
for i in range(steps):
a = i * batch_size % len(training)
batchx = training[a:a + batch_size]
batchy = t_labels[a:a + batch_size]
summary, _ = sess.run([merged, train_step],
feed_dict={
x: batchx,
y_: batchy,
keep_prob: 0.5
})
train_writer.add_summary(summary, i)
if i % 100 == 0:
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
summary, _ = sess.run([merged, train_step],
feed_dict={
x: batchx,
y_: batchy,
keep_prob: 0.5
},
options=run_options,
run_metadata=run_metadata)
train_writer.add_run_metadata(run_metadata, 'step%03d' % i)
train_writer.add_summary(summary, i)
print('Adding run metadata for', i)
summary, acc = sess.run([merged, accuracy],
feed_dict={
x: validation,
y_: v_labels,
keep_prob: 1.0
})
test_writer.add_summary(summary, i)
print('Accuracy at step %s: %s' % (i, acc))
save_path = saver.save(sess, save_location + "/model.ckpt", i)
save_path = saver.save(sess, save_location + "/model.ckpt", steps)
summary, acc = sess.run([merged, accuracy],
feed_dict={
x: validation,
y_: v_labels,
keep_prob: 1.0
})
test_writer.add_summary(summary, i)
print('Accuracy at step %s: %s' % (i, acc))
prediction = tf.argmax(y_conv, 1)
print(
"predictions",
prediction.eval(feed_dict={
x: validation,
y_: v_labels,
keep_prob: 1.0
},
session=sess))
print(
"correct predictions",
correct_prediction.eval(feed_dict={
x: validation,
y_: v_labels,
keep_prob: 1.0
},
session=sess))
# Create randomly initialized embedding weights which will be trained.
N = classes # Number of items (classes).
D = 200 # Dimensionality of the embedding.
embedding_var = tf.Variable(tf.random_normal([N, D]),
name='image_embedding')
# Format: tensorflow/tensorboard/plugins/projector/projector_config.proto
config = projector.ProjectorConfig()
# You can add multiple embeddings. Here we add only one.
embedding = config.embeddings.add()
embedding.sprite.image_path = 'home/workspace/kanji_tests/sprites20/master.jpg'
# Specify the width and height of a single thumbnail.
embedding.sprite.single_image_dim.extend([20, 20])
embedding.tensor_name = embedding_var.name
# Link this tensor to its metadata file (e.g. labels).
# embedding.metadata_path = os.path.join('sprites20/', 'labels.tsv')
# Use the same LOG_DIR where you stored your checkpoint.
summary_writer = tf.summary.FileWriter(save_location)
# The next line writes a projector_config.pbtxt in the LOG_DIR. TensorBoard will
# read this file during startup.
projector.visualize_embeddings(summary_writer, config)
train_writer.close()
test_writer.close()
np.savetxt('Xtest.tsv', X, fmt='%.6e', delimiter='\t')
np.savetxt('Ytest.tsv', Y_str, fmt='%s')
plt.imsave('zalando-mnist-sprite.png', get_sprite_image(X), cmap='gray')
exit()
embedding_var = tf.Variable(X, name='mnist_pixels')
# Format: tensorflow/tensorboard/plugins/projector/projector_config.proto
config = projector.ProjectorConfig()
# You can add multiple embeddings. Here we add only one.
embedding = config.embeddings.add()
embedding.tensor_name = embedding_var.name
# Link this tensor to its metadata file (e.g. labels).
embedding.metadata_path = VIS_DIR + 'Ytest.tsv'
embedding.sprite.image_path = VIS_DIR + 'zalando-mnist-sprite.png'
# Specify the width and height of a single thumbnail.
embedding.sprite.single_image_dim.extend([28, 28])
# Use the same LOG_DIR where you stored your checkpoint.
summary_writer = tf.summary.FileWriter(LOG_DIR + 'visualization')
# The next line writes a projector_config.pbtxt in the LOG_DIR. TensorBoard will
# read this file during startup.
projector.visualize_embeddings(summary_writer, config)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.save(sess, LOG_DIR + 'visualization/model.ckpt', 0)
data = np.pad(data, padding, mode='constant',
constant_values=0)
# Tile the individual thumbnails into an image.
data = data.reshape((n, n) + data.shape[1:]).transpose((0, 2, 1, 3)
+ tuple(range(4, data.ndim + 1)))
data = data.reshape((n * data.shape[1], n * data.shape[3]) + data.shape[4:])
data = (data * 225).astype(np.uint8)
return data
#%%
sprite = images_to_sprite(img_data)
cv2.imwrite(os.path.join(LOG_DIR, 'sprite_24_classes.png'), sprite)
#%% modified Prarthana
with tf.Session() as sess:
saver = tf.train.Saver([features])
sess.run(features.initializer)
saver.save(sess, os.path.join(LOG_DIR, 'images_24_classes.ckpt'))
config = projector.ProjectorConfig()
# One can add multiple embeddings.
embedding = config.embeddings.add()
embedding.tensor_name = features.name
# Link this tensor to its metadata file (e.g. labels).
embedding.metadata_path = os.path.join(LOG_DIR, 'metadata_24_classes.tsv')
# Comment out if you don't want sprites
embedding.sprite.image_path = os.path.join(LOG_DIR, 'sprite_24_classes.png')
embedding.sprite.single_image_dim.extend([img_data.shape[1], img_data.shape[1]])
# Saves a config file that TensorBoard will read during startup.
projector.visualize_embeddings(tf.summary.FileWriter(LOG_DIR), config)
def __init__(self, d, K, sig, sess, logdir):
self.K = K
self.sig = sig
self.sess = sess
self.logdir = logdir
with tf.name_scope('model'):
# Data Placeholder
with tf.name_scope('input'):
self.placeholders = tf.placeholder(tf.int32)
self.words = self.placeholders
# Index Masks
with tf.name_scope('context_mask'):
self.p_mask = tf.cast(
tf.range(d.cs / 2, d.n_minibatch + d.cs / 2), tf.int32)
rows = tf.cast(
tf.tile(tf.expand_dims(tf.range(0, d.cs / 2), [0]),
[d.n_minibatch, 1]), tf.int32)
columns = tf.cast(
tf.tile(tf.expand_dims(tf.range(0, d.n_minibatch), [1]),
[1, d.cs / 2]), tf.int32)
self.ctx_mask = tf.concat(
[rows + columns, rows + columns + d.cs / 2 + 1], 1)
with tf.name_scope('embeddings'):
# Embedding vectors
self.rho = tf.Variable(tf.random_normal([d.L, self.K]) /
self.K,
name='rho')
# Context vectors
self.alpha = tf.Variable(tf.random_normal([d.L, self.K]) /
self.K,
name='alpha')
with tf.name_scope('priors'):
prior = Normal(loc=0.0, scale=self.sig)
self.log_prior = tf.reduce_sum(
prior.log_prob(self.rho) + prior.log_prob(self.alpha))
with tf.name_scope('natural_param'):
# Taget and Context Indices
with tf.name_scope('target_word'):
self.p_idx = tf.gather(self.words, self.p_mask)
self.p_rho = tf.squeeze(tf.gather(self.rho, self.p_idx))
# Negative samples
with tf.name_scope('negative_samples'):
unigram_logits = tf.tile(
tf.expand_dims(tf.log(tf.constant(d.unigram)), [0]),
[d.n_minibatch, 1])
self.n_idx = tf.multinomial(unigram_logits, d.ns)
self.n_rho = tf.gather(self.rho, self.n_idx)
with tf.name_scope('context'):
self.ctx_idx = tf.squeeze(
tf.gather(self.words, self.ctx_mask))
self.ctx_alphas = tf.gather(self.alpha, self.ctx_idx)
# Natural parameter
ctx_sum = tf.reduce_sum(self.ctx_alphas, [1])
self.p_eta = tf.expand_dims(
tf.reduce_sum(tf.multiply(self.p_rho, ctx_sum), -1), 1)
self.n_eta = tf.reduce_sum(
tf.multiply(
self.n_rho,
tf.tile(tf.expand_dims(ctx_sum, 1), [1, d.ns, 1])), -1)
# Conditional likelihood
self.y_pos = Bernoulli(logits=self.p_eta)
self.y_neg = Bernoulli(logits=self.n_eta)
self.ll_pos = tf.reduce_sum(self.y_pos.log_prob(1.0))
self.ll_neg = tf.reduce_sum(self.y_neg.log_prob(0.0))
self.log_likelihood = self.ll_pos + self.ll_neg
scale = 1.0 * d.N / d.n_minibatch
self.loss = -(scale * self.log_likelihood + self.log_prior)
# Training
optimizer = tf.train.AdamOptimizer()
self.train = optimizer.minimize(self.loss)
with self.sess.as_default():
tf.global_variables_initializer().run()
variable_summaries('rho', self.rho)
variable_summaries('alpha', self.alpha)
with tf.name_scope('objective'):
tf.summary.scalar('loss', self.loss)
tf.summary.scalar('priors', self.log_prior)
tf.summary.scalar('ll_pos', self.ll_pos)
tf.summary.scalar('ll_neg', self.ll_neg)
self.summaries = tf.summary.merge_all()
self.train_writer = tf.summary.FileWriter(self.logdir,
self.sess.graph)
self.saver = tf.train.Saver()
config = projector.ProjectorConfig()
alpha = config.embeddings.add()
alpha.tensor_name = 'model/embeddings/alpha'
alpha.metadata_path = '../vocab.tsv'
rho = config.embeddings.add()
rho.tensor_name = 'model/embeddings/rho'
rho.metadata_path = '../vocab.tsv'
projector.visualize_embeddings(self.train_writer, config)
def start_C(iteration,start = True):
run_config = tf.ConfigProto()
# run_config.gpu_options.allow_growth=True
run_config.gpu_options.per_process_gpu_memory_fraction = 0.8
tf.logging.set_verbosity(tf.logging.DEBUG)
tfrecords_path = './data_tf/'
with tf.Graph().as_default():
mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
ckpt = tf.train.get_checkpoint_state(os.path.dirname('./checkpoint_pretrain/checkpoint'))
sess = tf.InteractiveSession(config = run_config)
global_step = slim.create_global_step()
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
mnist_net = Classification_Model()
# mnist_net.change_dataset("5")
# x, y_ = mnist_net.get_batch()
x,y_ = mnist_net.get_batch_tf(tfrecords_path)
# arg_scope = mnist_net.model_arg_scope()
end_points = {}
# with slim.arg_scope(arg_scope):
logits, end_points = mnist_net.net(x)
extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(extra_update_ops):
losses = mnist_net.losses(logits, y_)
train_step = mnist_net.optimizer(0.001).minimize(losses, global_step=global_step)
embedding, config = mnist_net.get_embedding('./checkpoint_pretrain/')
#total_loss = tf.losses.get_total_loss()
summaries.add(tf.summary.image("img", tf.cast(x, tf.float32)))
summaries.add(tf.summary.scalar('loss', losses))
for variable in tf.trainable_variables():
summaries.add(tf.summary.histogram(variable.op.name, variable))
train_writer = tf.summary.FileWriter('./checkpoint_pretrain/train',sess.graph)
projector.visualize_embeddings(train_writer, config)
correct_prediction = tf.equal(tf.argmax(end_points['Predictions'], 1), tf.argmax(y_, 1))
accuracy = tf.reduce_sum(tf.cast(correct_prediction, tf.float32)) / mnist_net.batch_size
summaries.add(tf.summary.scalar('accuracy', accuracy))
summary_op = tf.summary.merge(list(summaries), name='summary_op')
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
saver = tf.train.Saver(max_to_keep=5,
keep_checkpoint_every_n_hours=1.0,
write_version=2,
pad_step_number=False)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
save_test = []
x_test = tf.placeholder(tf.float32, shape=[None, 28, 28, 1])
y_test = tf.placeholder(tf.float32, shape=[None, 10])
logits_test, end_points_test = mnist_net.net(x_test,is_training=False, reuse = True)
correct_prediction_test = tf.equal(tf.argmax(end_points_test['Predictions'], 1), tf.argmax(y_test, 1))
num_correct = tf.reduce_sum(tf.cast(correct_prediction_test,tf.float32))
assignment = embedding.assign(end_points_test['Features'])
for i in range(iteration):
if i %100 == 0 :
# sum_accuracy_test = 0.0
# batch_size = 100
# for count in range(100):
# test_image = np.reshape(mnist.test.images[count*batch_size:(count+1)*batch_size],(batch_size,28,28,1)) *2.0 -1
# test_label = np.reshape(mnist.test.labels[count*batch_size:(count+1)*batch_size],(batch_size,10))
# num_c = sess.run(num_correct,
# feed_dict = {x_test:test_image, y_test:test_label})
# sum_accuracy_test += num_c
test_batch_x = mnist.test.images[:10000] * 2.0 - 1
test_batch_y = mnist.test.labels[:10000]
sum_accuracy_test, _ = sess.run([num_correct, assignment],
feed_dict={x_test: np.reshape(test_batch_x, (-1, 28, 28, 1)),
y_test: test_batch_y})
print ("test accuracy is: %f" % (sum_accuracy_test /10000.0 ))
# saver.save(sess, "./checkpoint_pretrain/",global_step=global_step_str)
print('****************************')
print ("test accuracy is: %f" % (sum_accuracy_test /10000.0 ))
print('****************************')
if start:
if not save_test:
save_test.append(sum_accuracy_test)
else :
save_test.append(sum_accuracy_test)
if sum_accuracy_test > save_test[0] :
print ('u are getting better!!!!')
print ('saving model')
saver.save(sess, "./checkpoint_pretrain/",global_step= global_step.eval())
break
else:
print('ops, not this time ~!')
else:
if sum_accuracy_test/10000.0 >= 0.995:
print ('saving model')
saver.save(sess, "./checkpoint_pretrain/",global_step= global_step.eval())
break
_,summary_str,current_accuracy = sess.run([train_step,summary_op,accuracy])
if i %10 == 0:
train_writer.add_summary(summary_str,i)
print('%diteration'%i,current_accuracy)
coord.request_stop()
coord.join(threads)
# print ('saving model')
# saver.save(sess, "./checkpoint_pretrain/",global_step= global_step.eval())
time.sleep(3)
def set_model(self, model):
self.model = model
self.sess = K.get_session()
if self.histogram_freq and self.merged is None:
for layer in self.model.layers:
for weight in layer.weights:
tf.summary.histogram(weight.name, weight)
if self.write_grads:
grads = model.optimizer.get_gradients(model.total_loss,
weight)
tf.summary.histogram('{}_grad'.format(weight.name), grads)
if self.write_images:
w_img = tf.squeeze(weight)
shape = K.int_shape(w_img)
if len(shape) == 2: # dense layer kernel case
if shape[0] > shape[1]:
w_img = tf.transpose(w_img)
shape = K.int_shape(w_img)
w_img = tf.reshape(w_img, [1,
shape[0],
shape[1],
1])
elif len(shape) == 3: # convnet case
if K.image_data_format() == 'channels_last':
# switch to channels_first to display
# every kernel as a separate image
w_img = tf.transpose(w_img, perm=[2, 0, 1])
shape = K.int_shape(w_img)
w_img = tf.reshape(w_img, [shape[0],
shape[1],
shape[2],
1])
elif len(shape) == 1: # bias case
w_img = tf.reshape(w_img, [1,
shape[0],
1,
1])
else:
# not possible to handle 3D convnets etc.
continue
shape = K.int_shape(w_img)
assert len(shape) == 4 and shape[-1] in [1, 3, 4]
tf.summary.image(weight.name, w_img)
if hasattr(layer, 'output'):
tf.summary.histogram('{}_out'.format(layer.name),
layer.output)
self.merged = tf.summary.merge_all()
if self.write_graph:
self.writer = tf.summary.FileWriter(self.log_dir,
self.sess.graph)
else:
self.writer = tf.summary.FileWriter(self.log_dir)
if self.embeddings_freq:
self.saver = tf.train.Saver()
embeddings_layer_names = self.embeddings_layer_names
if not embeddings_layer_names:
embeddings_layer_names = [layer.name for layer in self.model.layers
if type(layer).__name__ == 'Embedding']
embeddings = {layer.name: layer.weights[0]
for layer in self.model.layers
if layer.name in embeddings_layer_names}
embeddings_metadata = {}
if not isinstance(self.embeddings_metadata, str):
embeddings_metadata = self.embeddings_metadata
else:
embeddings_metadata = {layer_name: self.embeddings_metadata
for layer_name in embeddings.keys()}
config = projector.ProjectorConfig()
self.embeddings_logs = []
for layer_name, tensor in embeddings.items():
embedding = config.embeddings.add()
embedding.tensor_name = tensor.name
self.embeddings_logs.append(os.path.join(self.log_dir,
layer_name + '.ckpt'))
if layer_name in embeddings_metadata:
embedding.metadata_path = embeddings_metadata[layer_name]
projector.visualize_embeddings(self.writer, config)
# train_step = tf.train.GradientDescentOptimizer(0.01).minimize(loss)
train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss)
correct_prediction = tf.equal(tf.argmax(y2,1), tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
variables_names = [v.name for v in tf.trainable_variables()]
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
summary_writer = tf.summary.FileWriter(result_dir)
projector.visualize_embeddings(summary_writer, config)
saver = tf.train.Saver()
print("Training...")
#values = sess.run(variables_names)
# for k, v in zip(variables_names, values):
# print "Variable: ", k
# print "Shape: ", v.shape
# # print v
def main(args):
# pass the args as params so the model_fn can use
# the TPU specific args
params = vars(args)
if args.use_tpu:
# additional configs required for using TPUs
tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(args.tpu)
tpu_config = tf.contrib.tpu.TPUConfig(
num_shards=8, # using Cloud TPU v2-8
iterations_per_loop=args.save_checkpoints_steps)
# use the TPU version of RunConfig
config = tf.contrib.tpu.RunConfig(
cluster=tpu_cluster_resolver,
model_dir=args.model_dir,
tpu_config=tpu_config,
save_checkpoints_steps=args.save_checkpoints_steps,
save_summary_steps=100)
# TPUEstimator
estimator = tf.contrib.tpu.TPUEstimator(
model_fn=model_fn,
config=config,
params=params,
train_batch_size=args.train_batch_size,
# Calling TPUEstimator.predict requires setting predict_bath_size.
predict_batch_size=PREDICT_BATCH_SIZE,
eval_batch_size=32,
export_to_tpu=False)
else:
config = tf.estimator.RunConfig(model_dir=args.model_dir)
estimator = tf.estimator.Estimator(
model_fn,
config=config,
params=params)
estimator.train(train_input_fn, max_steps=args.max_steps)
# After training, apply the learned embedding to the test data and visualize with tensorboard Projector.
embeddings = next(estimator.predict(predict_input_fn, yield_single_examples=False))['embeddings']
# Put the embeddings into a variable to be visualized.
embedding_var = tf.Variable(embeddings, name='test_embeddings')
# Labels do not pass through the estimator.predict call, so we get it separately.
_, (_, labels) = tf.keras.datasets.mnist.load_data()
labels = labels[:PREDICT_BATCH_SIZE]
# Write the metadata file for the projector.
metadata_path = os.path.join(estimator.model_dir, 'metadata.tsv')
with tf.gfile.GFile(metadata_path, 'w') as f:
f.write('index\tlabel\n')
for i, label in enumerate(labels):
f.write('{}\t{}\n'.format(i, label))
# Configure the projector.
projector_config = projector.ProjectorConfig()
embedding_config = projector_config.embeddings.add()
embedding_config.tensor_name = embedding_var.name
# The metadata_path is relative to the summary_writer's log_dir.
embedding_config.metadata_path = 'metadata.tsv'
summary_writer = tf.summary.FileWriter(estimator.model_dir)
projector.visualize_embeddings(summary_writer, projector_config)
# Start a session to actually write the embeddings into a new checkpoint.
sess = tf.Session()
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.save(sess, os.path.join(estimator.model_dir, 'model.ckpt'), args.max_steps+1)
def fit(self,
trainFile=None,
with_validation=general_config.with_validation,
log_dir=general_config.log_dir + "/HAN",
save_dir=general_config.save_dir + "/HAN",
load_path=general_config.load_path_train,
num_epochs=general_config.num_epochs,
steps_every_epoch=general_config.steps_every_epoch,
batch_size=general_config.batch_size,
learning_rate=general_config.learning_rate,
lr_changing=general_config.lr_changing,
min_learning_rate=general_config.min_learning_rate,
learning_rate_decay=general_config.learning_rate_decay,
save_epochs=general_config.save_epochs,
early_stopping=han_config.early_stopping,
num_visual=general_config.num_visualize):
self.learning_rate_value = learning_rate
self.trainFile = trainFile
self.validFile = None
self.with_validation = with_validation
if self.trainFile is None:
if self.with_validation:
self.trainFile = general_config.data_dir + "/train.txt"
else:
self.trainFile = general_config.data_dir + "/training_label_new.txt"
if self.with_validation:
self.validFile = self.trainFile.replace("train", "valid")
tmp = os.path.join(
os.path.dirname(self.trainFile),
os.path.basename(self.trainFile).replace(".txt", "").split("_")[0])
self.int2vocabPath = tmp + "_i2v.json"
self.vocab2intPath = tmp + "_v2i.json"
metadataPath = {}
metadataPath["nonstatic"] = "/home/leechen/code/python/TextSentimentClassification/" \
+ self.int2vocabPath.replace("i2v.json", "metadata.tsv")
train_loss = []
train_accuracy = []
valid_loss = []
valid_accuracy = []
# 训练过程中的日志保存文件以及模型保存路径
if self.with_validation:
log_dir = ensure_dir_exist(log_dir + "/train_valid")
train_dir = os.path.join(log_dir, "train")
val_dir = os.path.join(log_dir, "valid")
save_dir = ensure_dir_exist(save_dir + "/train_valid")
else:
log_dir = ensure_dir_exist(log_dir + "/train")
train_dir = os.path.join(log_dir, "train")
val_dir = None
save_dir = ensure_dir_exist(save_dir + "/train")
# 生成日志
logger = my_logger(log_dir + "/log_fit.txt")
msg = "\n--state_size_word: %s\n"%self.state_size_word\
+"--attention_dim_word: %s\n"%self.attention_dim_word\
+ "--state_size_sentence: %s\n" % self.state_size_sentence \
+ "--attention_dim_sentence: %s\n" % self.attention_dim_sentence \
+ "--fc_layer_size: %s\n" % self.fc_layer_size_list \
+ "--embedding_size: %s\n" % self.embedding_size \
+ "--dropout: %s\n" % self.dropout_value \
+ "--max_l2_norm: %s\n" % self.max_l2_norm \
+ "--grads_clip: %s\n" % self.grads_clip \
+ "--learning_rate: %s\n" % self.learning_rate_value \
+ "--lr_changing: %s\n" % lr_changing \
+ "--min_learning_rate: %s\n" % min_learning_rate \
+ "--learning_rate_decay: %s\n" % learning_rate_decay \
+ "--load_path: %s\n" % load_path \
+ "--num_epochs: %s\n" % num_epochs \
+ "--steps_every_epoch: %s\n" % steps_every_epoch \
+ "--batch_size: %s\n" % batch_size \
+ "--save_epochs: %s\n" % save_epochs \
+ "--early_stopping: %s\n" % early_stopping \
+ "--num_visual: %s\n" % num_visual
logger.info(msg)
# 定义数据生成器
train_generator = BucketedDataIteratorForDoc(
loadPath=self.trainFile, vocab2intPath=self.vocab2intPath)
val_generator = None if self.validFile is None else BucketedDataIteratorForDoc(
loadPath=self.validFile, vocab2intPath=self.vocab2intPath)
os.environ["CUDA_VISIBLE_DEVICES"] = str(0)
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.8
with tf.Session(config=config, graph=self.graph) as sess:
train_writer = tf.summary.FileWriter(train_dir, sess.graph)
val_writer = None if val_dir is None else tf.summary.FileWriter(
val_dir)
saver = tf.train.Saver(max_to_keep=5)
sess.run(tf.global_variables_initializer())
start = 0
if isinstance(load_path, str):
if os.path.isdir(load_path):
ckpt = tf.train.get_checkpoint_state(load_path)
saver.restore(sess, ckpt.model_checkpoint_path)
start = ckpt.model_checkpoint_path.split("-")[-1]
else:
saver.restore(sess, load_path)
start = load_path.split("-")[-1]
logger.info("Loading successfully, loading epoch is %s" %
start)
logger.info("The total number of trainable variables: %s" %
get_num_params())
cur_early_stopping = 0
cur_max_acc = 0.
logger.info('******* start training with %d *******' % start)
epoch = 0
for epoch in range(start, num_epochs):
if lr_changing:
try:
if (train_loss[-1] > train_loss[-2]):
tmp = self.learning_rate_value * learning_rate_decay
if (tmp >= min_learning_rate):
self.learning_rate_value = tmp
logger.info(
"Learning rate multiplied by %s at epoch %s."
% (learning_rate_decay, epoch + 1))
else:
if (train_loss[-1] < train_loss[-2] - 0.015):
self.learning_rate_value *= 1.05
logger.info(
"Learning rate multiplied by 1.05 at epoch %s."
% (epoch + 1))
except:
pass
avg_loss_t, avg_accuracy_t = 0, 0
avg_loss_v, avg_accuracy_v = 0, 0
for step in range(steps_every_epoch):
_, batch_docs, batch_labels, batch_doc_lens, batch_sent_lens = train_generator.next(
batch_size)
sess.run(self.train_op,
feed_dict=self._feed_dict_train(
batch_x=batch_docs,
batch_y=batch_labels,
batch_doc_len=batch_doc_lens,
batch_sent_len=batch_sent_lens))
loss_t, acc_t = sess.run(
[self.loss_op, self.acc_op],
feed_dict=self._feed_dict_valid(
batch_x=batch_docs,
batch_y=batch_labels,
batch_doc_len=batch_doc_lens,
batch_sent_len=batch_sent_lens))
avg_loss_t += loss_t
avg_accuracy_t += acc_t
avg_loss_t /= steps_every_epoch
avg_accuracy_t /= steps_every_epoch
train_loss.append(avg_loss_t)
train_accuracy.append(avg_accuracy_t)
self.loss_accuracy_summary.value[0].simple_value = avg_loss_t
self.loss_accuracy_summary.value[
1].simple_value = avg_accuracy_t
train_writer.add_summary(summary=self.loss_accuracy_summary,
global_step=epoch + 1)
if self.with_validation:
# 计算验证集上的表现
cur_loop = val_generator.loop
_, batch_docs, batch_labels, batch_doc_lens, batch_sent_lens = val_generator.next(
1024, need_all=True)
cur_count = 0
while (val_generator.loop == cur_loop):
loss_v, acc_v = sess.run(
[self.loss_op, self.acc_op],
feed_dict=self._feed_dict_valid(
batch_docs,
batch_labels,
batch_doc_len=batch_doc_lens,
batch_sent_len=batch_sent_lens))
avg_loss_v += loss_v
avg_accuracy_v += acc_v
cur_count += 1
_, batch_docs, batch_labels, batch_doc_lens, batch_sent_lens = val_generator.next(
1024, need_all=True)
avg_loss_v /= cur_count
avg_accuracy_v /= cur_count
valid_loss.append(avg_loss_v)
valid_accuracy.append(avg_accuracy_v)
self.loss_accuracy_summary.value[
0].simple_value = avg_loss_v
self.loss_accuracy_summary.value[
1].simple_value = avg_accuracy_v
val_writer.add_summary(summary=self.loss_accuracy_summary,
global_step=epoch + 1)
logger.info("Epoch: [%04d/%04d], "
"Training Loss: %.4f, Training Accuracy: %.4f, "
"Validation Loss: %.4f, Validation Accuracy: %.4f" \
% (epoch + 1, num_epochs,
avg_loss_t, avg_accuracy_t, avg_loss_v, avg_accuracy_v))
# 如果验证集上的准确率连续低于历史最高准确率的次数超过early_stopping次，则提前停止迭代。
if (avg_accuracy_v > cur_max_acc):
cur_max_acc = avg_accuracy_v
cur_early_stopping = 0
logger.info("Saving model-%s" % (epoch + 1))
saver.save(sess,
os.path.join(save_dir, 'model.ckpt'),
global_step=epoch + 1)
else:
cur_early_stopping += 1
if cur_early_stopping > early_stopping:
logger.info("Early stopping after epoch %s !" %
(epoch + 1))
break
else:
logger.info("Epoch: [%04d/%04d], "
"Training Loss: %.4f, Training Accuracy: %.4f " \
% (epoch + 1, num_epochs, avg_loss_t, avg_accuracy_t))
# 保存一次模型
if (epoch - start + 1) % save_epochs == 0:
logger.info("Saving model-%s" % (epoch + 1))
saver.save(sess,
os.path.join(save_dir, 'model.ckpt'),
global_step=epoch + 1)
if num_visual > 0:
# 可视化最终词向量
config = projector.ProjectorConfig()
final_embeddings = {}
final_embeddings["nonstatic"] = self.embedding_matrix_ns.eval(
)[:num_visual]
for (name, final_embedding) in final_embeddings.items():
embedding_var = tf.Variable(final_embedding,
name="word_embeddings_" + name)
sess.run(embedding_var.initializer)
saver = tf.train.Saver([embedding_var])
saver.save(
sess, log_dir + "/embeddings_" + name + ".ckpt" +
str(epoch + 1))
embedding = config.embeddings.add()
embedding.tensor_name = embedding_var.name
embedding.metadata_path = metadataPath[name]
projector.visualize_embeddings(train_writer, config)
return train_loss, train_accuracy, valid_loss, valid_accuracy
def set_model(self, model):
self.model = model
self.sess = K.get_session()
if self.histogram_freq and self.merged is None:
for layer in self.model.layers:
for weight in layer.weights:
mapped_weight_name = weight.name.replace(':', '_')
tf.summary.histogram(mapped_weight_name, weight)
if self.write_grads:
grads = model.optimizer.get_gradients(
model.total_loss, weight)
def is_indexed_slices(grad):
return type(grad).__name__ == 'IndexedSlices'
grads = [
grad.values if is_indexed_slices(grad) else grad
for grad in grads
]
tf.summary.histogram(
'{}_grad'.format(mapped_weight_name), grads)
if self.write_images:
w_img = tf.squeeze(weight)
shape = K.int_shape(w_img)
if len(shape) == 2: # dense layer kernel case
if shape[0] > shape[1]:
w_img = tf.transpose(w_img)
shape = K.int_shape(w_img)
w_img = tf.reshape(w_img,
[1, shape[0], shape[1], 1])
elif len(shape) == 3: # convnet case
if K.image_data_format() == 'channels_last':
# switch to channels_first to display
# every kernel as a separate image
w_img = tf.transpose(w_img, perm=[2, 0, 1])
shape = K.int_shape(w_img)
w_img = tf.reshape(
w_img, [shape[0], shape[1], shape[2], 1])
elif len(shape) == 1: # bias case
w_img = tf.reshape(w_img, [1, shape[0], 1, 1])
else:
# not possible to handle 3D convnets etc.
continue
shape = K.int_shape(w_img)
assert len(shape) == 4 and shape[-1] in [1, 3, 4]
tf.summary.image(mapped_weight_name, w_img)
if hasattr(layer, 'output'):
tf.summary.histogram('{}_out'.format(layer.name),
layer.output)
self.merged = tf.summary.merge_all()
if self.write_graph:
self.writer = tf.summary.FileWriter(self.log_dir, self.sess.graph)
else:
self.writer = tf.summary.FileWriter(self.log_dir)
if self.embeddings_freq:
embeddings_layer_names = self.embeddings_layer_names
if not embeddings_layer_names:
embeddings_layer_names = [
layer.name for layer in self.model.layers
if type(layer).__name__ == 'Embedding'
]
embeddings = {
layer.name: layer.weights[0]
for layer in self.model.layers
if layer.name in embeddings_layer_names
}
self.saver = tf.train.Saver(list(embeddings.values()))
embeddings_metadata = {}
if not isinstance(self.embeddings_metadata, str):
embeddings_metadata = self.embeddings_metadata
else:
embeddings_metadata = {
layer_name: self.embeddings_metadata
for layer_name in embeddings.keys()
}
config = projector.ProjectorConfig()
self.embeddings_ckpt_path = os.path.join(self.log_dir,
'keras_embedding.ckpt')
for layer_name, tensor in embeddings.items():
embedding = config.embeddings.add()
embedding.tensor_name = tensor.name
if layer_name in embeddings_metadata:
embedding.metadata_path = embeddings_metadata[layer_name]
projector.visualize_embeddings(self.writer, config)
def train_model(model, batch_gen, num_train_steps, weights_fld):
saver = tf.train.Saver(
) # defaults to saving all variables - in this case embed_matrix, nce_weight, nce_bias
initial_step = 0
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
ckpt = tf.train.get_checkpoint_state(
os.path.dirname(logDir + '/checkpoints/checkpoint'))
# if that checkpoint exists, restore from checkpoint
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
total_loss = 0.0 # we use this to calculate late average loss in the last SKIP_STEP steps
writer = tf.summary.FileWriter(logDir + '/lr' + str(LEARNING_RATE),
sess.graph)
initial_step = model.global_step.eval()
for index in range(initial_step, initial_step + num_train_steps):
try:
centers, targets = next(batch_gen)
feed_dict = {
model.center_words: centers,
model.target_words: targets
}
loss_batch, _, summary = sess.run(
[model.loss, model.optimizer, model.summary_op],
feed_dict=feed_dict)
writer.add_summary(summary, global_step=index)
total_loss += loss_batch
if (index + 1) % SKIP_STEP == 0:
print('Average loss at step {}: {:5.1f}'.format(
index, total_loss / SKIP_STEP))
total_loss = 0.0
saver.save(sess, logDir + '/checkpoints/skip-gram', index)
except StopIteration:
break
writer.close()
####################
# code to visualize the embeddings. uncomment the below to visualize embeddings
final_embed_matrix = sess.run(model.embed_matrix)
# it has to variable. constants don't work here. you can't reuse model.embed_matrix
embedding_var = tf.Variable(final_embed_matrix[:1000],
name='embedding')
sess.run(embedding_var.initializer)
config = projector.ProjectorConfig()
summary_writer = tf.summary.FileWriter(logDir + '/processed')
# add embedding to the config file
embedding = config.embeddings.add()
embedding.tensor_name = embedding_var.name
# link this tensor to its metadata file, in this case the first 1000 words of vocab
embedding.metadata_path = weights_fld + '/vocab_1000.tsv'
# saves a configuration file that TensorBoard will read during startup.
projector.visualize_embeddings(summary_writer, config)
saver_embed = tf.train.Saver([embedding_var])
saver_embed.save(sess, logDir + '/processed/model3.ckpt', 1)
summary_writer.close()
