def run_generator(fig_name, cval): with tf.Session() as sess: # init session tf.sg_init(sess) # restore parameters saver = tf.train.Saver() saver.restore(sess, tf.train.latest_checkpoint('asset/train')) feed_dic = {} for t, c in zip(target_cval, cval): feed_dic[t] = c # run generator imgs = sess.run(gen, feed_dic) # plot result _, ax = plt.subplots(10, 10, sharex=True, sharey=True) for i in range(10): for j in range(10): ax[i][j].plot(imgs[i * 10 + j]) ax[i][j].set_axis_off() plt.savefig('asset/train/' + fig_name, dpi=600) tf.sg_info('Sample image saved to "asset/train/%s"' % fig_name) plt.close()
def __init__(self, batch_size=16, data_path='/home/murugesan/VCTK-Corpus/', vocabulary_loading=False): if vocabulary_loading: vocabulary_file = 'muru_works' + self.__class__.__name__ + 'vocabulary.npy' if os.path.exists(vocabulary_file): self.index2byte = np.load(vocabulary_file) self.byte2index = {} for i, b in enumerate(self.index2byte): self.byte2index[b] = i self.voca_size = len(self.index2byte) tf.sg_info('Vocabulary loaded') return labels, wave_files = self.load_corpus(data_path) label = theano.tensor.as_tensor_variable(labels) wave_file = theano.tensor.as_tensor_variable(wave_file) label, wave_file = tf.train.slice_input_producer([labels, wave_file], shuffle=True) def load_mfcc(self, source): lable, wav = source lab = np.fromstring(lable, np.int) mfcc = mfcc.mfcc_generate(wav) return lab, mfcc
def __init__(self, batch_size=32, name='train'): # load train corpus sources, targets = self._load_corpus(mode='train') # to constant tensor source = tf.convert_to_tensor(sources) target = tf.convert_to_tensor(targets) # create queue from constant tensor source, target = tf.train.slice_input_producer([source, target]) # create batch queue batch_queue = tf.train.shuffle_batch([source, target], batch_size, num_threads=32, capacity=batch_size * 64, min_after_dequeue=batch_size * 32, name=name) # split data self.source, self.target = batch_queue # calc total batch count self.num_batch = len(sources) // batch_size # print info tf.sg_info('Train data loaded.(total data=%d, total batch=%d)' % (len(sources), self.num_batch))
def run_generator(num, x1, x2, fig_name='sample.png'): with tf.Session() as sess: tf.sg_init(sess) # restore parameters tf.sg_restore(sess, tf.train.latest_checkpoint('asset/train/infogan'), category='generator') # run generator imgs = sess.run(gen, { target_num: num, target_cval_1: x1, target_cval_2: x2 }) # plot result _, ax = plt.subplots(10, 10, sharex=True, sharey=True) for i in range(10): for j in range(10): ax[i][j].imshow(imgs[i * 10 + j], 'gray') ax[i][j].set_axis_off() plt.savefig('asset/train/infogan/' + fig_name, dpi=600) tf.sg_info('Sample image saved to "asset/train/infogan/%s"' % fig_name) plt.close()
def run_generator(num, x1, x2, fig_name='sample.png'): with tf.Session() as sess: tf.sg_init(sess) # restore parameters saver = tf.train.Saver() saver.restore(sess, tf.train.latest_checkpoint('asset/train/ckpt')) # run generator imgs = sess.run(gen, {target_num: num, target_cval_1: x1, target_cval_2: x2}) # plot result _, ax = plt.subplots(10, 10, sharex=True, sharey=True) for i in range(10): for j in range(10): ax[i][j].plot(imgs[i * 10 + j, :, 0], color='b', linewidth=0.25) # Turn off tick labels only # ax[i][j].set_axis_off() ax[i][j].set_xticks([]) ax[i][j].set_yticks([]) plt.savefig('asset/train/' + fig_name, dpi=600) tf.sg_info('Sample image saved to "asset/train/%s"' % fig_name) plt.close()
def build_input2(batch_size=16, set_name='train_sort', mode='train'): # load meta file label, mfcc_file, len1 = [], [], [] with open(_data_path + 'preprocess/meta/%s.csv' % set_name) as csv_file: reader = csv.reader(csv_file, delimiter=',') for i, row in enumerate(reader): # mfcc file mfcc_file.append(_data_path + 'preprocess/mfcc/' + row[0] + '.npy') # label info ( convert to string object for variable-length support ) #print 'shape is:', np.shape(label_temp), np.shape(indices) str1 = np.asarray(row[1:], dtype=np.int32) len_buf = np.asarray([len(str1)], dtype=np.int32) label.append(str1.tostring()) #print str1, len_buf len1.append(len_buf.tostring()) if i >= 1999: break print np.shape(len1) # to constant tensor label_t = tf.convert_to_tensor(label) mfcc_file_t = tf.convert_to_tensor(mfcc_file) len_t = tf.convert_to_tensor(len1) # create queue from constant tensor label_q, mfcc_file_q, len_q \ = tf.train.slice_input_producer([label_t, mfcc_file_t, len_t], shuffle=False) # create label, mfcc queue label_q, mfcc_q, len_q = _load_mfcc( source=[label_q, mfcc_file_q, len_q], dtypes=[tf.int32, tf.float32, tf.int32], capacity=batch_size * 16, num_threads=16) # create batch queue with dynamic pad label_, mfcc_, len_ = tf.train.batch([label_q, mfcc_q, len_q], batch_size, shapes=[(None, ), (20, None), (1, )], num_threads=16, capacity=batch_size * 16, dynamic_pad=True, allow_smaller_final_batch=False) #label_ = tf.deserialize_many_sparse(label_, tf.int32) # batch * time * dim mfcc_ = tf.transpose(mfcc_, perm=[0, 2, 1]) # calc total batch count num_batch = len(label) // batch_size len_ = tf.reshape(len_, [-1]) assert label_.get_shape()[0] == batch_size #convert to sparse tensor # print info stf.sg_info('%s set loaded.(total data=%d, total batch=%d)' % (set_name.upper(), len(label), num_batch)) return mfcc_, label_, len_
def __init__(self, batch_size=16, data_path='asset/data/', vocabulary_loading=False): @tf.sg_producer_func def _load_mfcc(src_list): lab, wav = src_list # label, wave_file # decode string to integer lab = np.fromstring(lab, np.int) # load wave file wav, sr = librosa.load(wav, mono=True) # mfcc mfcc = librosa.feature.mfcc(wav, sr) # return result return lab, mfcc # path for loading just vocabulary if vocabulary_loading: vocabulary_file = __vocabulary_save_dir__ + self.__class__.__name__ + '_vocabulary.npy' if os.path.exists(vocabulary_file): self.index2byte = np.load(vocabulary_file) self.byte2index = {} for i, b in enumerate(self.index2byte): self.byte2index[b] = i self.voca_size = len(self.index2byte) tf.sg_info('VCTK vocabulary loaded.') return # load corpus labels, wave_files = self._load_corpus(data_path) # to constant tensor label = tf.convert_to_tensor(labels) wave_file = tf.convert_to_tensor(wave_files) # create queue from constant tensor label, wave_file = tf.train.slice_input_producer([label, wave_file], shuffle=True) # decode wave file label, mfcc = _load_mfcc(source=[label, wave_file], dtypes=[tf.sg_intx, tf.sg_floatx], capacity=128, num_threads=32) # create batch queue with dynamic pad batch_queue = tf.train.batch([label, mfcc], batch_size, shapes=[(None,), (20, None)], num_threads=32, capacity=batch_size*48, dynamic_pad=True) # split data self.label, self.mfcc = batch_queue # batch * time * dim self.mfcc = self.mfcc.sg_transpose(perm=[0, 2, 1]) # calc total batch count self.num_batch = len(labels) // batch_size # print info tf.sg_info('VCTK corpus loaded.(total data=%d, total batch=%d)' % (len(labels), self.num_batch))
def plot_images(imgs): # plot result _, ax = plt.subplots(10, 10, sharex=True, sharey=True) for i in range(10): for j in range(10): ax[i][j].imshow(imgs[i * 10 + j], 'gray') ax[i][j].set_axis_off() plt.savefig(get_next_filename(), dpi=600) tf.sg_info('Sample image saved to "asset/train/sample.png"') plt.close()
def __init__(self, batch_size=16, data_path='asset/data/', mode='train'): @tf.sg_producer_func def _load_mfcc(src_list): lab, wav = src_list # label, wave_file # decode string to integer lab = np.fromstring(lab, np.int) # load wave file wav, sr = librosa.load(wav, mono=True) # mfcc hl = 512 mfcc = librosa.feature.mfcc(wav, sr, n_mfcc=40,hop_length=hl) mfcc = mfcc[:,:100] # return result return lab, mfcc print("Mode: %s" % mode) # load corpus labels, wave_files, accent_labels = self._load_corpus(data_path, mode=='train') labels = accent_labels labels = np.array(labels) self.labels = labels self.wave_files = wave_files # to constant tensor label = tf.convert_to_tensor(labels) #label = tf.convert_to_tensor(accent_labels) wave_file = tf.convert_to_tensor(wave_files) # create queue from constant tensor label, wave_file = tf.train.slice_input_producer([label, wave_file], shuffle=True) # decode wave file label, mfcc = _load_mfcc(source=[label, wave_file], dtypes=[tf.sg_intx, tf.sg_floatx], capacity=128, num_threads=32) # create batch queue with dynamic pad batch_queue = tf.train.batch([label, mfcc], batch_size, shapes=[(None,), (40, None)], num_threads=32, capacity=batch_size*48, dynamic_pad=True) # split data self.label, self.mfcc = batch_queue # batch * time * dim self.mfcc = self.mfcc.sg_transpose(perm=[0, 2, 1]) # calc total batch count self.num_batch = len(labels) // batch_size # print info tf.sg_info('VCTK corpus loaded.(total data=%d, total batch=%d)' % (len(labels), self.num_batch))
def __init__(self, batch_size=16, set_name='train'): # load meta file label, mfcc_file, filenames = [], [], [] with open(_data_path + 'preprocess/meta/%s.csv' % set_name) as csv_file: reader = csv.reader(csv_file, delimiter=',') for row in reader: # mfcc file filenames.append(row[0]) mfcc_file.append(_data_path + 'preprocess/mfcc/' + row[0] + '.npy') #mfcc_file.append(_data_path + 'preprocess/mfcc-one/' + row[0] + '.npy') # label info ( convert to string object for variable-length support ) label.append(np.asarray(row[1:], dtype=np.int).tostring()) # to constant tensor label_t = tf.convert_to_tensor(label) mfcc_file_t = tf.convert_to_tensor(mfcc_file) filenames_t = tf.convert_to_tensor(filenames) # create queue from constant tensor label_q, mfcc_file_q, filenames_q \ = tf.train.slice_input_producer([label_t, mfcc_file_t, filenames_t], shuffle=False) #= tf.train.slice_input_producer([label_t, mfcc_file_t, filenames_t], shuffle=True) # create label, mfcc queue label_q, mfcc_q, filenames_q = _load_mfcc( source=[label_q, mfcc_file_q, filenames_q], dtypes=[tf.sg_intx, tf.sg_floatx, tf.string], capacity=256, num_threads=64) # create batch queue with dynamic pad batch_queue = tf.train.batch([label_q, mfcc_q, filenames_q], batch_size, shapes=[(None, ), (20, None), ()], num_threads=64, capacity=batch_size * 32, dynamic_pad=True) # split data self.label, self.mfcc, self.filenames = batch_queue # batch * time * dim self.mfcc = self.mfcc.sg_transpose(perm=[0, 2, 1]) # calc total batch count self.num_batch = len(label) // batch_size # print info tf.sg_info('%s set loaded.(total data=%d, total batch=%d)' % (set_name.upper(), len(label), self.num_batch))
def __init__(self, batch_size=16, set_name='learning_batch.csv', data_path="MFCC/"): self._data_path = data_path # load meta file label, mfcc_file = [], [] csv_file = pd.read_csv("learning_batch.csv", header=None) for row in csv_file.iterrows(): file_name, label_temp = row[1] label_temp = map(int, label_temp.split()) mfcc_file.append(self._data_path + file_name) # label info ( convert to string object for variable-length support ) label.append(np.asarray(label_temp, dtype=np.int).tostring()) # to constant tensor label_t = tf.convert_to_tensor(label) mfcc_file_t = tf.convert_to_tensor(mfcc_file) # create queue from constant tensor label_q, mfcc_file_q = tf.train.slice_input_producer( [label_t, mfcc_file_t], shuffle=True) # create label, mfcc queue label_q, mfcc_q = _load_mfcc(source=[label_q, mfcc_file_q], dtypes=[tf.sg_intx, tf.sg_floatx], capacity=256, num_threads=64) # create batch queue with dynamic pad batch_queue = tf.train.batch([label_q, mfcc_q], batch_size, shapes=[(None, ), (20, None)], num_threads=64, capacity=batch_size * 32, dynamic_pad=True) # split data self.label, self.mfcc = batch_queue # batch * time * dim self.mfcc = self.mfcc.sg_transpose(perm=[0, 2, 1]) # calc total batch count self.num_batch = len(label) // batch_size # print info tf.sg_info('%s set loaded.(total data=%d, total batch=%d)' % (set_name.upper(), len(label), self.num_batch))
def __init__(self): data, labels = [], [] with open('data/Housing.csv') as csv_file: reader = csv.reader(csv_file) i = 0 for row in reader: if i != 0: labels.append(row[1]) cols = [] for col in row[2:]: if col == 'yes': col = '1' elif col == 'no': col = '0' cols.append(col) data.append(cols) i += 1 label_t = tf.convert_to_tensor(labels) data_t = tf.convert_to_tensor(data) label_q, data_q\ = tf.train.slice_input_producer([label_t, data_t], shuffle=True) label_q, data_q = _load_data(source=[label_q, data_q], dtypes=[sg.sg_intx, sg.sg_intx], capacity=256, num_threads=64) batch_queue = tf.train.batch([label_q, data_q], batch_size=32, shapes=[(), (11)], num_threads=64, capacity=32 * 32, dynamic_pad=False) self.label, self.data = batch_queue self.num_batch = len(labels) // 32 sg.sg_info('%s set loaded.(total data=%d, total batch=%d)' % ('train', len(labels), self.num_batch))
def __init__(self, batch_size=16, set_name='train'): # load meta file label, mfcc_file = [], [] with open(_data_path + 'preprocess/meta/%s.csv' % set_name) as csv_file: reader = csv.reader(csv_file, delimiter=',') for row in reader: # mfcc file mfcc_file.append(_data_path + 'preprocess/mfcc/' + row[0] + '.npy') # label info ( convert to string object for variable-length support ) label.append(np.asarray(row[1:], dtype=np.int).tostring()) # to constant tensor label_t = tf.convert_to_tensor(label) mfcc_file_t = tf.convert_to_tensor(mfcc_file) # create queue from constant tensor label_q, mfcc_file_q \ = tf.train.slice_input_producer([label_t, mfcc_file_t], shuffle=True) # create label, mfcc queue label_q, mfcc_q = _load_mfcc(source=[label_q, mfcc_file_q], dtypes=[tf.sg_intx, tf.sg_floatx], capacity=256, num_threads=64) # create batch queue with dynamic pad batch_queue = tf.train.batch([label_q, mfcc_q], batch_size, shapes=[(None,), (20, None)], num_threads=64, capacity=batch_size*32, dynamic_pad=True) # split data self.label, self.mfcc = batch_queue # batch * time * dim self.mfcc = self.mfcc.sg_transpose(perm=[0, 2, 1]) # calc total batch count self.num_batch = len(label) // batch_size # print info tf.sg_info('%s set loaded.(total data=%d, total batch=%d)' % (set_name.upper(), len(label), self.num_batch))
def run_generator(num, x1, x2, fig_name='sample.png'): with tf.Session() as sess: tf.sg_init(sess) # restore parameters saver = tf.train.Saver() saver.restore(sess, tf.train.latest_checkpoint('asset/train/ckpt')) # run generator imgs = sess.run(gen, {target_num: num, target_cval_1: x1, target_cval_2: x2}) # plot result _, ax = plt.subplots(10, 10, sharex=True, sharey=True) for i in range(10): for j in range(10): ax[i][j].plot(imgs[i * 10 + j, :, 0]) ax[i][j].plot(imgs[i * 10 + j, :, 1]) ax[i][j].set_axis_off() plt.savefig('asset/train/' + fig_name, dpi=600) tf.sg_info('Sample image saved to "asset/train/%s"' % fig_name) plt.close()
# # run network # with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess: # init variables tf.sg_init(sess) # restore parameters saver = tf.train.Saver() saver.restore(sess, tf.train.latest_checkpoint('asset/train')) # logging tf.sg_info('Testing started on %s set at global step[%08d].' % (tf.sg_arg().set.upper(), sess.run(tf.sg_global_step()))) with tf.sg_queue_context(): # create progress bar iterator = tqdm(range(0, int(data.num_batch * tf.sg_arg().frac)), total=int(data.num_batch * tf.sg_arg().frac), initial=0, desc='test', ncols=70, unit='b', leave=False) # batch loop loss_avg = 0. for _ in iterator: # run session batch_loss = sess.run(loss) # loss history update
def console_log(sess_): if epoch >= 0: tf.sg_info('\tEpoch[%03d:gs=%d] - loss = %s' % (epoch, sess_.run(tf.sg_global_step()), ('NA' if loss is None else '%8.6f' % loss)))
plt.figure(figsize=(4, 3)) #plt.set_axis_off() hr = plt.imshow(sr[0], 'gray') plt.axis('off') #ax.set_axis_off() #ax.thumbnail(size, Image.ANTIALIAS) #for i in range(1): #for j in range(1): #ax[i][j*4].imshow(low[i*3+j], 'gray') #ax[i][j*4].set_axis_off() #ax[i][j*4+1].imshow(bicubic[i*3+j], 'gray') #ax[i][j*4+1].set_axis_off() #ax[i][j*4+2].imshow(sr[i*3+j], 'gray') #ax.imshow(sr[0], 'gray') #ax.set_axis_off() #ax[i][j*4+2].set_axis_off() #ax[i][j*4+3].imshow(gt[i*3+j], 'gray') #ax[i][j*4+3].set_axis_off() #plt.savefig(fig_name,bbox_inches='tight',dpi=300) plt.savefig(fig_name, dpi=600) tf.sg_info('Sample image saved to "%s"' % fig_name) plt.close() plt.figure(figsize=(4, 3)) lr = plt.imshow(gt[0], 'gray') plt.axis('off') plt.savefig(fig_name2, dpi=600) tf.sg_info('Sample image saved to "%s"' % fig_name2) plt.close()
def wrapper(**kwargs): r""" Manages arguments of `tf.sg_opt`. Args: **kwargs: lr: A Python Scalar (optional). Learning rate. Default is .001. save_dir: A string. The root path to which checkpoint and log files are saved. Default is `asset/train`. max_ep: A positive integer. Maximum number of epochs. Default is 1000. ep_size: A positive integer. Number of Total batches in an epoch. For proper display of log. Default is 1e5. save_interval: A Python scalar. The interval of saving checkpoint files. By default, for every 600 seconds, a checkpoint file is written. log_interval: A Python scalar. The interval of recoding logs. By default, for every 60 seconds, logging is executed. max_keep: A positive integer. Maximum number of recent checkpoints to keep. Default is 5. keep_interval: A Python scalar. How often to keep checkpoints. Default is 1 hour. eval_metric: A list of tensors containing the value to evaluate. Default is []. tqdm: Boolean. If True (Default), progress bars are shown. If False, a series of loss will be shown on the console. """ opt = tf.sg_opt(kwargs) # default training options opt += tf.sg_opt(lr=0.001, save_dir='asset/train', max_ep=1000, ep_size=100000, save_interval=600, log_interval=60, eval_metric=[], max_keep=5, keep_interval=1, tqdm=True) # training epoch and loss epoch, loss = -1, None # checkpoint saver saver = tf.train.Saver(max_to_keep=opt.max_keep, keep_checkpoint_every_n_hours=opt.keep_interval) # add evaluation summary for m in opt.eval_metric: tf.sg_summary_metric(m) # summary writer log_dir = opt.save_dir + '/run-%02d%02d-%02d%02d' % tuple(tf.time.localtime(tf.time.time()))[1:5] summary_writer = tf.summary.FileWriter(log_dir) # console logging function def console_log(sess_): if epoch >= 0: tf.sg_info('\tEpoch[%03d:gs=%d] - loss = %s' % (epoch, sess_.run(tf.sg_global_step()), ('NA' if loss is None else '%8.6f' % loss))) local_init_op = tf.group(tf.sg_phase().assign(True), tf.tables_initializer(), tf.local_variables_initializer()) # create supervisor sv = tf.train.Supervisor(logdir=opt.save_dir, saver=saver, save_model_secs=opt.save_interval, summary_writer=summary_writer, save_summaries_secs=opt.log_interval, global_step=tf.sg_global_step(), local_init_op=local_init_op) # create session with sv.managed_session(config=tf.ConfigProto(allow_soft_placement=True)) as sess: # console logging loop if not opt.tqdm: sv.loop(opt.log_interval, console_log, args=(sess,)) # get start epoch _step = sess.run(tf.sg_global_step()) ep = _step // opt.ep_size best_f1 = 0 # check if already finished if ep <= opt.max_ep: # logging tf.sg_info('Training started from epoch[%03d]-step[%d].' % (ep, _step)) # epoch loop for ep in range(ep, opt.max_ep + 1): # update epoch info start_step = sess.run(tf.sg_global_step()) % opt.ep_size epoch = ep # create progressbar iterator if opt.tqdm: iterator = tf.tqdm(range(start_step, opt.ep_size), total=opt.ep_size, initial=start_step, desc='train', ncols=70, unit='b', leave=False) else: iterator = range(start_step, opt.ep_size) # batch loop for _ in iterator: # exit loop if sv.should_stop(): break # call train function batch_loss = func(sess, opt) # loss history update if batch_loss is not None and \ not np.isnan(batch_loss.all()) and not np.isinf(batch_loss.all()): if loss is None: loss = np.mean(batch_loss) else: loss = loss * 0.9 + np.mean(batch_loss) * 0.1 # log epoch information console_log(sess) f1_stat = show_metrics(sv, sess, opt.eval_metric[2], opt.eval_metric[3], ep, opt.val_ep_size, 'val', use_tqdm=True) if f1_stat > best_f1: best_f1 = f1_stat max_model_file = opt.save_dir + max_model_name # save last version saver.save(sess, max_model_file) print("Improved F1 score, max model saved in file: %s" % max_model_file) print('Test metrics:') show_metrics(sv, sess, opt.test_metric[0], opt.test_metric[1], ep, opt.test_ep_size, 'test', use_tqdm=True) # save last version saver.save(sess, opt.save_dir + '/model.ckpt', global_step=sess.run(tf.sg_global_step())) # logging tf.sg_info('Training finished at epoch[%d]-step[%d].' % (ep, sess.run(tf.sg_global_step()))) else: tf.sg_info('Training already finished at epoch[%d]-step[%d].' % (ep - 1, sess.run(tf.sg_global_step())))
z = tf.random_normal((batch_size, rand_dim)) # generator gen = generator(z).sg_squeeze() # # draw samples # with tf.Session() as sess: tf.sg_init(sess) # restore parameters tf.sg_restore(sess, tf.train.latest_checkpoint('asset/train/gan'), category='generator') # run generator imgs = sess.run(gen) # plot result _, ax = plt.subplots(10, 10, sharex=True, sharey=True) for i in range(10): for j in range(10): ax[i][j].imshow(imgs[i * 10 + j], 'gray') ax[i][j].set_axis_off() plt.savefig('asset/train/gan/sample.png', dpi=600) tf.sg_info('Sample image saved to "asset/train/gan/sample.png"') plt.close()
def sg_tsne(tensor, meta_file='metadata.tsv', save_dir='asset/tsne'): r""" Manages arguments of `tf.sg_opt`. Args: **kwargs: lr: A Python Scalar (optional). Learning rate. Default is .001. eval_metric: A list of tensors containing the value to evaluate. Default is []. early_stop: Boolean. If True (default), the training should stop when the following two conditions are met. i. Current loss is less than .95 * previous loss. ii. Current learning rate is less than 5e-6. lr_reset: Boolean. If True, learning rate is set to opt.lr. when training restarts. Otherwise (Default), the value of the stored `_learning_rate` is taken. save_dir: A string. The root path to which checkpoint and log files are saved. Default is `asset/train`. max_ep: A positive integer. Maximum number of epochs. Default is 1000. ep_size: A positive integer. Number of Total batches in an epoch. For proper display of log. Default is 1e5. save_interval: A Python scalar. The interval of saving checkpoint files. By default, for every 600 seconds, a checkpoint file is written. log_interval: A Python scalar. The interval of recoding logs. By default, for every 60 seconds, logging is executed. max_keep: A positive integer. Maximum number of recent checkpoints to keep. Default is 5. keep_interval: A Python scalar. How often to keep checkpoints. Default is 1 hour. tqdm: Boolean. If True (Default), progress bars are shown. console_log: Boolean. If True, a series of loss will be shown on the console instead of tensorboard. Default is False. embeds: List of Tuple. [(Tensor to add, metadata path)] """ # make directory if not exist if not os.path.exists(save_dir): os.makedirs(save_dir) # checkpoint saver saver = tf.train.Saver() # summary writer summary_writer = tf.summary.FileWriter(save_dir, graph=tf.get_default_graph()) # embedding visualizer config = projector.ProjectorConfig() emb = config.embeddings.add() emb.tensor_name = tensor.name # tensor # emb.metadata_path = os.path.join(save_dir, meta_file) # metadata file projector.visualize_embeddings(summary_writer, config) # create session sess = tf.Session() # initialize variables sg_init(sess) # save tsne saver.save(sess, save_dir + '/model-tsne') # logging tf.sg_info('Tsne saved at %s' % (save_dir + '/model-tsne')) # close session sess.close()
# # run network # with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess: # init variables tf.sg_init(sess) # restore parameters saver = tf.train.Saver() saver.restore(sess, tf.train.latest_checkpoint('asset/train')) # logging tf.sg_info('Testing started on %s set at global step[%08d].' % (tf.sg_arg().set.upper(), sess.run(tf.sg_global_step()))) with tf.sg_queue_context(): # create progress bar iterator = tqdm(range(0, int(data.num_batch * tf.sg_arg().frac)), total=int(data.num_batch * tf.sg_arg().frac), initial=0, desc='test', ncols=70, unit='b', leave=False) # batch loop loss_avg = 0. for _ in iterator: # run session batch_loss = sess.run(loss) # loss history update
def __init__(self, batch_size=16, set_name='train'): # load meta file label, mfcc_file = [], [] self.daniter_label = [] count = 0 with open(_data_path + 'preprocess/meta/%s.csv' % set_name) as csv_file: reader = csv.reader(csv_file, delimiter=',') for row in reader: # DANITER magic small dataset if count not in [0, 24, 25, 26, 27]: count += 1 if count == max_count: break continue # mfcc file mfcc_file.append(_data_path + 'preprocess/mfcc/' + row[0] + '.npy') # label info ( convert to string object for variable-length support ) label.append(np.asarray(row[1:], dtype=np.int).tostring()) self.daniter_label.append(np.asarray(row[1:], dtype=np.int)) # DANITER print "\n\n\n##### Label ######", count print "".join([ index2byte[letter] for letter in np.asarray(row[1:], dtype=np.int) ]) # DANITER count += 1 if count == max_count: break self.mfcc_file = mfcc_file # to constant tensor label_t = tf.convert_to_tensor(label) mfcc_file_t = tf.convert_to_tensor(mfcc_file) # create queue from constant tensor label_q, mfcc_file_q \ = tf.train.slice_input_producer([label_t, mfcc_file_t], shuffle=True) # create label, mfcc queue label_q, mfcc_q = _load_mfcc(source=[label_q, mfcc_file_q], dtypes=[tf.sg_intx, tf.sg_floatx], capacity=256, num_threads=64) # create batch queue with dynamic pad batch_queue = tf.train.batch([label_q, mfcc_q], batch_size, shapes=[(None, ), (20, None)], num_threads=64, capacity=batch_size * 32, dynamic_pad=True) # split data self.label, self.mfcc = batch_queue # batch * time * dim self.mfcc = self.mfcc.sg_transpose(perm=[0, 2, 1]) # calc total batch count self.num_batch = len(label) // batch_size # print info tf.sg_info('%s set loaded.(total data=%d, total batch=%d)' % (set_name.upper(), len(label), self.num_batch))
def wrapper(**kwargs): opt = tf.sg_opt(kwargs) # default training options opt += tf.sg_opt(lr=0.001, save_dir='asset/train', max_ep=1000, ep_size=100000, save_interval=600, log_interval=60, early_stop=True, lr_reset=False, eval_metric=[], max_keep=5, keep_interval=1, tqdm=True, console_log=False) # make directory if not exist if not os.path.exists(opt.save_dir + '/log'): os.makedirs(opt.save_dir + '/log') if not os.path.exists(opt.save_dir + '/ckpt'): os.makedirs(opt.save_dir + '/ckpt') # find last checkpoint last_file = tf.train.latest_checkpoint(opt.save_dir + '/ckpt') if last_file: ep = start_ep = int(last_file.split('-')[1]) + 1 start_step = int(last_file.split('-')[2]) else: ep = start_ep = 1 start_step = 0 # checkpoint saver saver = tf.train.Saver(max_to_keep=opt.max_keep, keep_checkpoint_every_n_hours=opt.keep_interval) # summary writer summary_writer = tf.train.SummaryWriter(opt.save_dir + '/log', graph=tf.get_default_graph()) # add learning rate summary with tf.name_scope('summary'): tf.scalar_summary('60. learning_rate/learning_rate', _learning_rate) # add evaluation metric summary for m in opt.eval_metric: tf.sg_summary_metric(m) # summary op summary_op = tf.merge_all_summaries() # create session if opt.sess: sess = opt.sess else: # session with multiple GPU support sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) # initialize variables sg_init(sess) # restore last checkpoint if last_file: saver.restore(sess, last_file) # set learning rate if start_ep == 1 or opt.lr_reset: sess.run(_learning_rate.assign(opt.lr)) # logging tf.sg_info('Training started from epoch[%03d]-step[%d].' % (start_ep, start_step)) try: # start data queue runner with tf.sg_queue_context(sess): # set session mode to train tf.sg_set_train(sess) # loss history for learning rate decay loss, loss_prev, early_stopped = None, None, False # time stamp for saving and logging last_saved = last_logged = time.time() # epoch loop for ep in range(start_ep, opt.max_ep + 1): # show progressbar if opt.tqdm: iterator = tqdm(range(opt.ep_size), desc='train', ncols=70, unit='b', leave=False) else: iterator = range(opt.ep_size) # batch loop for _ in iterator: # call train function batch_loss = func(sess, opt) # loss history update if batch_loss is not None: if loss is None: loss = np.mean(batch_loss) else: loss = loss * 0.9 + np.mean(batch_loss) * 0.1 # saving if time.time() - last_saved > opt.save_interval: last_saved = time.time() saver.save(sess, opt.save_dir + '/ckpt/model-%03d' % ep, write_meta_graph=False, global_step=sess.run( tf.sg_global_step())) # logging if time.time() - last_logged > opt.log_interval: last_logged = time.time() # set session mode to infer tf.sg_set_infer(sess) # run evaluation op if len(opt.eval_metric) > 0: sess.run(opt.eval_metric) if opt.console_log: # console logging # log epoch information tf.sg_info( '\tEpoch[%03d:lr=%7.5f:gs=%d] - loss = %s' % (ep, sess.run(_learning_rate), sess.run(tf.sg_global_step()), ('NA' if loss is None else '%8.6f' % loss))) else: # tensorboard logging # run logging op summary_writer.add_summary( sess.run(summary_op), global_step=sess.run(tf.sg_global_step())) # learning rate decay if opt.early_stop and loss_prev: # if loss stalling if loss >= 0.95 * loss_prev: # early stopping current_lr = sess.run(_learning_rate) if current_lr < 5e-6: early_stopped = True break else: # decrease learning rate by half sess.run( _learning_rate.assign(current_lr / 2.)) # update loss history loss_prev = loss # revert session mode to train tf.sg_set_train(sess) # log epoch information if not opt.console_log: tf.sg_info( '\tEpoch[%03d:lr=%7.5f:gs=%d] - loss = %s' % (ep, sess.run(_learning_rate), sess.run(tf.sg_global_step()), ('NA' if loss is None else '%8.6f' % loss))) if early_stopped: tf.sg_info('\tEarly stopped ( no loss progress ).') break finally: # save last epoch saver.save(sess, opt.save_dir + '/ckpt/model-%03d' % ep, write_meta_graph=False, global_step=sess.run(tf.sg_global_step())) # set session mode to infer tf.sg_set_infer(sess) # logging tf.sg_info('Training finished at epoch[%d]-step[%d].' % (ep, sess.run(tf.sg_global_step()))) # close session if opt.sess is None: sess.close()
def wrapper(**kwargs): r""" Manages arguments of `tf.sg_opt`. Args: **kwargs: lr: A Python Scalar (optional). Learning rate. Default is .001. eval_metric: A list of tensors containing the value to evaluate. Default is []. early_stop: Boolean. If True (default), the training should stop when the following two conditions are met. i. Current loss is less than .95 * previous loss. ii. Current learning rate is less than 5e-6. lr_reset: Boolean. If True, learning rate is set to opt.lr. when training restarts. Otherwise (Default), the value of the stored `_learning_rate` is taken. save_dir: A string. The root path to which checkpoint and log files are saved. Default is `asset/train`. max_ep: A positive integer. Maximum number of epochs. Default is 1000. ep_size: A positive integer. Number of Total batches in an epoch. For proper display of log. Default is 1e5. save_interval: A Python scalar. The interval of saving checkpoint files. By default, for every 600 seconds, a checkpoint file is written. log_interval: A Python scalar. The interval of recoding logs. By default, for every 60 seconds, logging is executed. max_keep: A positive integer. Maximum number of recent checkpoints to keep. Default is 5. keep_interval: A Python scalar. How often to keep checkpoints. Default is 1 hour. tqdm: Boolean. If True (Default), progress bars are shown. console_log: Boolean. If True, a series of loss will be shown on the console instead of tensorboard. Default is False. """ opt = tf.sg_opt(kwargs) # default training options opt += tf.sg_opt(lr=0.001, save_dir='asset/train', max_ep=1000, ep_size=100000, save_interval=600, log_interval=60, early_stop=True, lr_reset=False, eval_metric=[], max_keep=5, keep_interval=1, tqdm=True, console_log=False) # make directory if not exist if not os.path.exists(opt.save_dir): os.makedirs(opt.save_dir) # find last checkpoint last_file = tf.train.latest_checkpoint(opt.save_dir) if last_file: ep = start_ep = int(last_file.split('-')[1]) + 1 start_step = int(last_file.split('-')[2]) else: ep = start_ep = 1 start_step = 0 # checkpoint saver saver = tf.train.Saver(max_to_keep=opt.max_keep, keep_checkpoint_every_n_hours=opt.keep_interval) # summary writer summary_writer = tf.summary.FileWriter(opt.save_dir, graph=tf.get_default_graph()) # add learning rate summary tf.summary.scalar('learning_r', _learning_rate) # add evaluation metric summary for m in opt.eval_metric: tf.sg_summary_metric(m) # summary op summary_op = tf.summary.merge_all() # create session if opt.sess: sess = opt.sess else: # session with multiple GPU support sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) # initialize variables sg_init(sess) # restore last checkpoint if last_file: saver.restore(sess, last_file) # set learning rate if start_ep == 1 or opt.lr_reset: sess.run(_learning_rate.assign(opt.lr)) # logging tf.sg_info('Training started from epoch[%03d]-step[%d].' % (start_ep, start_step)) try: # start data queue runner with tf.sg_queue_context(sess): # set session mode to train tf.sg_set_train(sess) # loss history for learning rate decay loss, loss_prev, early_stopped = None, None, False # time stamp for saving and logging last_saved = last_logged = time.time() # epoch loop for ep in range(start_ep, opt.max_ep + 1): # show progressbar if opt.tqdm: iterator = tqdm(range(opt.ep_size), desc='train', ncols=70, unit='b', leave=False) else: iterator = range(opt.ep_size) # batch loop for _ in iterator: # call train function batch_loss = func(sess, opt) # loss history update if batch_loss is not None: if loss is None: loss = np.mean(batch_loss) else: loss = loss * 0.9 + np.mean(batch_loss) * 0.1 # saving if time.time() - last_saved > opt.save_interval: last_saved = time.time() saver.save(sess, opt.save_dir + '/model-%03d' % ep, write_meta_graph=False, global_step=sess.run(tf.sg_global_step())) # logging if time.time() - last_logged > opt.log_interval: last_logged = time.time() # set session mode to infer tf.sg_set_infer(sess) # run evaluation op if len(opt.eval_metric) > 0: sess.run(opt.eval_metric) if opt.console_log: # console logging # log epoch information tf.sg_info('\tEpoch[%03d:lr=%7.5f:gs=%d] - loss = %s' % (ep, sess.run(_learning_rate), sess.run(tf.sg_global_step()), ('NA' if loss is None else '%8.6f' % loss))) else: # tensorboard logging # run logging op summary_writer.add_summary(sess.run(summary_op), global_step=sess.run(tf.sg_global_step())) # learning rate decay if opt.early_stop and loss_prev: # if loss stalling if loss >= 0.95 * loss_prev: # early stopping current_lr = sess.run(_learning_rate) if current_lr < 5e-6: early_stopped = True break else: # decrease learning rate by half sess.run(_learning_rate.assign(current_lr / 2.)) # update loss history loss_prev = loss # revert session mode to train tf.sg_set_train(sess) # log epoch information if not opt.console_log: tf.sg_info('\tEpoch[%03d:lr=%7.5f:gs=%d] - loss = %s' % (ep, sess.run(_learning_rate), sess.run(tf.sg_global_step()), ('NA' if loss is None else '%8.6f' % loss))) if early_stopped: tf.sg_info('\tEarly stopped ( no loss progress ).') break finally: # save last epoch saver.save(sess, opt.save_dir + '/model-%03d' % ep, write_meta_graph=False, global_step=sess.run(tf.sg_global_step())) # set session mode to infer tf.sg_set_infer(sess) # logging tf.sg_info('Training finished at epoch[%d]-step[%d].' % (ep, sess.run(tf.sg_global_step()))) # close session if opt.sess is None: sess.close()
# create session print "Starting session..." with sv.managed_session(config=tf.ConfigProto(allow_soft_placement=True)) as sess: # console logging loop if not opt.tqdm: sv.loop(opt.log_interval, console_log, args=(sess, )) # get start epoch _step = sess.run(tf.sg_global_step()) ep = 1 # check if already finished if ep <= opt.max_ep: # logging tf.sg_info('Training started from epoch[%03d]-step[%d].' % (ep, _step)) # epoch loop for ep in range(ep, opt.max_ep + 1): # update epoch info start_step = 1 epoch = ep # create progressbar iterator if opt.tqdm: iterator = tqdm(range(start_step, opt.ep_size), total=opt.ep_size, initial=start_step, desc='train', ncols=70, unit='b', leave=False) else: iterator = range(start_step, opt.ep_size)
fig_name = 'asset/train/sample.png' with tf.Session() as sess: with tf.sg_queue_context(sess): tf.sg_init(sess) # restore parameters saver = tf.train.Saver() saver.restore(sess, tf.train.latest_checkpoint('asset/train/ckpt')) # run generator gt, low, bicubic, sr = sess.run([x.sg_squeeze(), x_nearest, x_bicubic, gen]) # plot result _, ax = plt.subplots(10, 12, sharex=True, sharey=True) for i in range(10): for j in range(3): ax[i][j*4].imshow(low[i*3+j], 'gray') ax[i][j*4].set_axis_off() ax[i][j*4+1].imshow(bicubic[i*3+j], 'gray') ax[i][j*4+1].set_axis_off() ax[i][j*4+2].imshow(sr[i*3+j], 'gray') ax[i][j*4+2].set_axis_off() ax[i][j*4+3].imshow(gt[i*3+j], 'gray') ax[i][j*4+3].set_axis_off() plt.savefig(fig_name, dpi=600) tf.sg_info('Sample image saved to "%s"' % fig_name) plt.close()
def __init__(self, batch_size=8, name='train', path=join(DATA_DIR, 'out_data.data'), b2i={}): if name == "train": print("Loading corpus...") # load train corpus if not b2i: sources, targets = self._load_corpus(mode='train', path=path) else: sources, targets = load_corpus_dict(path, b2i, 0, 150) print("Converting source to tensors...") # to constant tensor source = tf.convert_to_tensor(sources) print("Converting target to tensors...") target = tf.convert_to_tensor(targets) # create queue from constant tensor source, target = tf.train.slice_input_producer([source, target]) # create batch queue batch_queue = tf.train.shuffle_batch([source, target], batch_size, num_threads=4, capacity=batch_size * 8, min_after_dequeue=batch_size * 4, name=name) # split data self.source, self.target = batch_queue # calc total batch count self.num_batch = len(sources) // batch_size # print info tf.sg_info('Train data loaded.(total data=%d, total batch=%d)' % (len(sources), self.num_batch)) if name == "test": print("Loading test corpus...") sources = self._load_corpus(mode='test', path=path) targets = np.zeros_like(sources) print("Converting source to tensors...") # to constant tensor source = tf.convert_to_tensor(sources) print("Converting target to tensors...") target = tf.convert_to_tensor(targets) # create queue from constant tensor source, target = tf.train.slice_input_producer([source, target]) # create batch queue batch_queue = tf.train.shuffle_batch([source, target], batch_size, num_threads=4, capacity=batch_size * 8, min_after_dequeue=batch_size * 4, name=name) # split data self.source, self.target = batch_queue # calc total batch count self.num_batch = len(sources) // batch_size # print info tf.sg_info('Train data loaded.(total data=%d, total batch=%d)' % (len(sources), self.num_batch))