def visualize_embeddings(self, sess, tensor, name): """ Visualises an embedding vector into Tensorboard :param sess: Tensorflow session object :param tensor: The embedding tensor to be visualizd :param name: Name of the tensor """ # make directory if not exist if not tf.os.path.exists(self.save_dir): tf.os.makedirs(self.save_dir) # summary writer summary_writer = tf.summary.FileWriter(self.save_dir, graph=tf.get_default_graph()) # embedding visualizer config = projector.ProjectorConfig() emb = config.embeddings.add() emb.tensor_name = name # tensor emb.metadata_path = tf.os.path.join(self.DEFAULT_META_DATA_DIR, self.meta_file) # metadata file print(tf.os.path.abspath(emb.metadata_path)) projector.visualize_embeddings(summary_writer, config)
def visualize_embeddings(self, sess, tensor, name): """ Visualises an embedding vector into Tensorboard :param sess: Tensorflow session object :param tensor: The embedding tensor to be visualizd :param name: Name of the tensor """ # make directory if not exist if not tf.os.path.exists(self.save_dir): tf.os.makedirs(self.save_dir) # summary writer summary_writer = tf.summary.FileWriter(self.save_dir, graph=tf.get_default_graph()) # embedding visualizer config = projector.ProjectorConfig() emb = config.embeddings.add() emb.tensor_name = name # tensor emb.metadata_path = tf.os.path.join(self.DEFAULT_META_DATA_DIR, self.meta_file) # metadata file print(tf.os.path.abspath(emb.metadata_path)) projector.visualize_embeddings(summary_writer, config)
voca_size = data.voca_size x = tf.placeholder(dtype=tf.sg_floatx, shape=(batch_size, None, 20)) # sequence length except zero-padding seq_len = tf.not_equal(x.sg_sum(axis=2), 0.).sg_int().sg_sum(axis=1) # encode audio feature logit = get_logit(x, voca_size) # ctc decoding decoded, _ = tf.nn.ctc_beam_search_decoder(logit.sg_transpose(perm=[1, 0, 2]), seq_len, merge_repeated=False) # to dense tensor y = tf.add(tf.sparse_to_dense(decoded[0].indices, decoded[0].dense_shape, decoded[0].values), 1, name="output") with tf.Session() as sess: tf.sg_init(sess) saver = tf.train.Saver() saver.restore(sess, tf.train.latest_checkpoint('asset/train')) graph = tf.get_default_graph() input_graph_def = graph.as_graph_def() with tf.Session() as sess: tf.sg_init(sess) saver = tf.train.Saver() saver.restore(sess, tf.train.latest_checkpoint('asset/train')) # Output model's graph details for reference. tf.train.write_graph(sess.graph_def, '/root/speech-to-text-wavenet/asset/train', 'graph.txt', as_text=True) # Freeze the output graph. output_graph_def = graph_util.convert_variables_to_constants(sess,input_graph_def,"output".split(",")) # Write it into .pb file. with tfw.gfile.GFile("/root/speech-to-text-wavenet/asset/train/wavenet_model.pb", "wb") as f: f.write(output_graph_def.SerializeToString())
def wrapper(**kwargs): opt = tf.sg_opt(kwargs) # default training options opt += tf.sg_opt(lr=0.001, save_dir='asset/train', max_ep=1000, ep_size=100000, save_interval=600, log_interval=60, early_stop=True, lr_reset=False, eval_metric=[], max_keep=5, keep_interval=1, tqdm=True, console_log=False) # make directory if not exist if not os.path.exists(opt.save_dir + '/log'): os.makedirs(opt.save_dir + '/log') if not os.path.exists(opt.save_dir + '/ckpt'): os.makedirs(opt.save_dir + '/ckpt') # find last checkpoint last_file = tf.train.latest_checkpoint(opt.save_dir + '/ckpt') if last_file: ep = start_ep = int(last_file.split('-')[1]) + 1 start_step = int(last_file.split('-')[2]) else: ep = start_ep = 1 start_step = 0 # checkpoint saver saver = tf.train.Saver(max_to_keep=opt.max_keep, keep_checkpoint_every_n_hours=opt.keep_interval) # summary writer summary_writer = tf.train.SummaryWriter(opt.save_dir + '/log', graph=tf.get_default_graph()) # add learning rate summary with tf.name_scope('summary'): tf.scalar_summary('60. learning_rate/learning_rate', _learning_rate) # add evaluation metric summary for m in opt.eval_metric: tf.sg_summary_metric(m) # summary op summary_op = tf.merge_all_summaries() # create session if opt.sess: sess = opt.sess else: # session with multiple GPU support sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) # initialize variables sg_init(sess) # restore last checkpoint if last_file: saver.restore(sess, last_file) # set learning rate if start_ep == 1 or opt.lr_reset: sess.run(_learning_rate.assign(opt.lr)) # logging tf.sg_info('Training started from epoch[%03d]-step[%d].' % (start_ep, start_step)) try: # start data queue runner with tf.sg_queue_context(sess): # set session mode to train tf.sg_set_train(sess) # loss history for learning rate decay loss, loss_prev, early_stopped = None, None, False # time stamp for saving and logging last_saved = last_logged = time.time() # epoch loop for ep in range(start_ep, opt.max_ep + 1): # show progressbar if opt.tqdm: iterator = tqdm(range(opt.ep_size), desc='train', ncols=70, unit='b', leave=False) else: iterator = range(opt.ep_size) # batch loop for _ in iterator: # call train function batch_loss = func(sess, opt) # loss history update if batch_loss is not None: if loss is None: loss = np.mean(batch_loss) else: loss = loss * 0.9 + np.mean(batch_loss) * 0.1 # saving if time.time() - last_saved > opt.save_interval: last_saved = time.time() saver.save(sess, opt.save_dir + '/ckpt/model-%03d' % ep, write_meta_graph=False, global_step=sess.run( tf.sg_global_step())) # logging if time.time() - last_logged > opt.log_interval: last_logged = time.time() # set session mode to infer tf.sg_set_infer(sess) # run evaluation op if len(opt.eval_metric) > 0: sess.run(opt.eval_metric) if opt.console_log: # console logging # log epoch information tf.sg_info( '\tEpoch[%03d:lr=%7.5f:gs=%d] - loss = %s' % (ep, sess.run(_learning_rate), sess.run(tf.sg_global_step()), ('NA' if loss is None else '%8.6f' % loss))) else: # tensorboard logging # run logging op summary_writer.add_summary( sess.run(summary_op), global_step=sess.run(tf.sg_global_step())) # learning rate decay if opt.early_stop and loss_prev: # if loss stalling if loss >= 0.95 * loss_prev: # early stopping current_lr = sess.run(_learning_rate) if current_lr < 5e-6: early_stopped = True break else: # decrease learning rate by half sess.run( _learning_rate.assign(current_lr / 2.)) # update loss history loss_prev = loss # revert session mode to train tf.sg_set_train(sess) # log epoch information if not opt.console_log: tf.sg_info( '\tEpoch[%03d:lr=%7.5f:gs=%d] - loss = %s' % (ep, sess.run(_learning_rate), sess.run(tf.sg_global_step()), ('NA' if loss is None else '%8.6f' % loss))) if early_stopped: tf.sg_info('\tEarly stopped ( no loss progress ).') break finally: # save last epoch saver.save(sess, opt.save_dir + '/ckpt/model-%03d' % ep, write_meta_graph=False, global_step=sess.run(tf.sg_global_step())) # set session mode to infer tf.sg_set_infer(sess) # logging tf.sg_info('Training finished at epoch[%d]-step[%d].' % (ep, sess.run(tf.sg_global_step()))) # close session if opt.sess is None: sess.close()
def wrapper(**kwargs): r""" Manages arguments of `tf.sg_opt`. Args: **kwargs: lr: A Python Scalar (optional). Learning rate. Default is .001. eval_metric: A list of tensors containing the value to evaluate. Default is []. early_stop: Boolean. If True (default), the training should stop when the following two conditions are met. i. Current loss is less than .95 * previous loss. ii. Current learning rate is less than 5e-6. lr_reset: Boolean. If True, learning rate is set to opt.lr. when training restarts. Otherwise (Default), the value of the stored `_learning_rate` is taken. save_dir: A string. The root path to which checkpoint and log files are saved. Default is `asset/train`. max_ep: A positive integer. Maximum number of epochs. Default is 1000. ep_size: A positive integer. Number of Total batches in an epoch. For proper display of log. Default is 1e5. save_interval: A Python scalar. The interval of saving checkpoint files. By default, for every 600 seconds, a checkpoint file is written. log_interval: A Python scalar. The interval of recoding logs. By default, for every 60 seconds, logging is executed. max_keep: A positive integer. Maximum number of recent checkpoints to keep. Default is 5. keep_interval: A Python scalar. How often to keep checkpoints. Default is 1 hour. tqdm: Boolean. If True (Default), progress bars are shown. console_log: Boolean. If True, a series of loss will be shown on the console instead of tensorboard. Default is False. """ opt = tf.sg_opt(kwargs) # default training options opt += tf.sg_opt(lr=0.001, save_dir='asset/train', max_ep=1000, ep_size=100000, save_interval=600, log_interval=60, early_stop=True, lr_reset=False, eval_metric=[], max_keep=5, keep_interval=1, tqdm=True, console_log=False) # make directory if not exist if not os.path.exists(opt.save_dir): os.makedirs(opt.save_dir) # find last checkpoint last_file = tf.train.latest_checkpoint(opt.save_dir) if last_file: ep = start_ep = int(last_file.split('-')[1]) + 1 start_step = int(last_file.split('-')[2]) else: ep = start_ep = 1 start_step = 0 # checkpoint saver saver = tf.train.Saver(max_to_keep=opt.max_keep, keep_checkpoint_every_n_hours=opt.keep_interval) # summary writer summary_writer = tf.summary.FileWriter(opt.save_dir, graph=tf.get_default_graph()) # add learning rate summary tf.summary.scalar('learning_r', _learning_rate) # add evaluation metric summary for m in opt.eval_metric: tf.sg_summary_metric(m) # summary op summary_op = tf.summary.merge_all() # create session if opt.sess: sess = opt.sess else: # session with multiple GPU support sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) # initialize variables sg_init(sess) # restore last checkpoint if last_file: saver.restore(sess, last_file) # set learning rate if start_ep == 1 or opt.lr_reset: sess.run(_learning_rate.assign(opt.lr)) # logging tf.sg_info('Training started from epoch[%03d]-step[%d].' % (start_ep, start_step)) try: # start data queue runner with tf.sg_queue_context(sess): # set session mode to train tf.sg_set_train(sess) # loss history for learning rate decay loss, loss_prev, early_stopped = None, None, False # time stamp for saving and logging last_saved = last_logged = time.time() # epoch loop for ep in range(start_ep, opt.max_ep + 1): # show progressbar if opt.tqdm: iterator = tqdm(range(opt.ep_size), desc='train', ncols=70, unit='b', leave=False) else: iterator = range(opt.ep_size) # batch loop for _ in iterator: # call train function batch_loss = func(sess, opt) # loss history update if batch_loss is not None: if loss is None: loss = np.mean(batch_loss) else: loss = loss * 0.9 + np.mean(batch_loss) * 0.1 # saving if time.time() - last_saved > opt.save_interval: last_saved = time.time() saver.save(sess, opt.save_dir + '/model-%03d' % ep, write_meta_graph=False, global_step=sess.run(tf.sg_global_step())) # logging if time.time() - last_logged > opt.log_interval: last_logged = time.time() # set session mode to infer tf.sg_set_infer(sess) # run evaluation op if len(opt.eval_metric) > 0: sess.run(opt.eval_metric) if opt.console_log: # console logging # log epoch information tf.sg_info('\tEpoch[%03d:lr=%7.5f:gs=%d] - loss = %s' % (ep, sess.run(_learning_rate), sess.run(tf.sg_global_step()), ('NA' if loss is None else '%8.6f' % loss))) else: # tensorboard logging # run logging op summary_writer.add_summary(sess.run(summary_op), global_step=sess.run(tf.sg_global_step())) # learning rate decay if opt.early_stop and loss_prev: # if loss stalling if loss >= 0.95 * loss_prev: # early stopping current_lr = sess.run(_learning_rate) if current_lr < 5e-6: early_stopped = True break else: # decrease learning rate by half sess.run(_learning_rate.assign(current_lr / 2.)) # update loss history loss_prev = loss # revert session mode to train tf.sg_set_train(sess) # log epoch information if not opt.console_log: tf.sg_info('\tEpoch[%03d:lr=%7.5f:gs=%d] - loss = %s' % (ep, sess.run(_learning_rate), sess.run(tf.sg_global_step()), ('NA' if loss is None else '%8.6f' % loss))) if early_stopped: tf.sg_info('\tEarly stopped ( no loss progress ).') break finally: # save last epoch saver.save(sess, opt.save_dir + '/model-%03d' % ep, write_meta_graph=False, global_step=sess.run(tf.sg_global_step())) # set session mode to infer tf.sg_set_infer(sess) # logging tf.sg_info('Training finished at epoch[%d]-step[%d].' % (ep, sess.run(tf.sg_global_step()))) # close session if opt.sess is None: sess.close()
def sg_tsne(tensor, meta_file='metadata.tsv', save_dir='asset/tsne'): r""" Manages arguments of `tf.sg_opt`. Args: **kwargs: lr: A Python Scalar (optional). Learning rate. Default is .001. eval_metric: A list of tensors containing the value to evaluate. Default is []. early_stop: Boolean. If True (default), the training should stop when the following two conditions are met. i. Current loss is less than .95 * previous loss. ii. Current learning rate is less than 5e-6. lr_reset: Boolean. If True, learning rate is set to opt.lr. when training restarts. Otherwise (Default), the value of the stored `_learning_rate` is taken. save_dir: A string. The root path to which checkpoint and log files are saved. Default is `asset/train`. max_ep: A positive integer. Maximum number of epochs. Default is 1000. ep_size: A positive integer. Number of Total batches in an epoch. For proper display of log. Default is 1e5. save_interval: A Python scalar. The interval of saving checkpoint files. By default, for every 600 seconds, a checkpoint file is written. log_interval: A Python scalar. The interval of recoding logs. By default, for every 60 seconds, logging is executed. max_keep: A positive integer. Maximum number of recent checkpoints to keep. Default is 5. keep_interval: A Python scalar. How often to keep checkpoints. Default is 1 hour. tqdm: Boolean. If True (Default), progress bars are shown. console_log: Boolean. If True, a series of loss will be shown on the console instead of tensorboard. Default is False. embeds: List of Tuple. [(Tensor to add, metadata path)] """ # make directory if not exist if not os.path.exists(save_dir): os.makedirs(save_dir) # checkpoint saver saver = tf.train.Saver() # summary writer summary_writer = tf.summary.FileWriter(save_dir, graph=tf.get_default_graph()) # embedding visualizer config = projector.ProjectorConfig() emb = config.embeddings.add() emb.tensor_name = tensor.name # tensor # emb.metadata_path = os.path.join(save_dir, meta_file) # metadata file projector.visualize_embeddings(summary_writer, config) # create session sess = tf.Session() # initialize variables sg_init(sess) # save tsne saver.save(sess, save_dir + '/model-tsne') # logging tf.sg_info('Tsne saved at %s' % (save_dir + '/model-tsne')) # close session sess.close()
# ctc decoding decoded, _ = tf.nn.ctc_beam_search_decoder(logit.sg_transpose(perm=[1, 0, 2]), seq_len, merge_repeated=False) # to dense tensor y = tf.add(tf.sparse_to_dense(decoded[0].indices, decoded[0].dense_shape, decoded[0].values), 1, name="output") with tf.Session() as sess: tf.sg_init(sess) saver = tf.train.Saver() saver.restore(sess, tf.train.latest_checkpoint('asset/train')) graph = tf.get_default_graph() input_graph_def = graph.as_graph_def() with tf.Session() as sess: tf.sg_init(sess) saver = tf.train.Saver() saver.restore(sess, tf.train.latest_checkpoint('asset/train')) # Output model's graph details for reference. tf.train.write_graph(sess.graph_def, '/root/speech-to-text-wavenet/asset/train', 'graph.txt', as_text=True) # Freeze the output graph. output_graph_def = graph_util.convert_variables_to_constants( sess, input_graph_def, "output".split(",")) # Write it into .pb file.