def train(self): # train baseline model input_ph = tf.placeholder(shape=[batch_size, 28, 28, 1], dtype=tf.float32) label_ph = tf.placeholder(shape=[ batch_size, ], dtype=tf.int32) predict = self.forward(input_ph) loss_tensor = tf.reduce_mean(predict.sg_ce(target=label_ph)) # use to update network parameters optim = tf.sg_optim(loss_tensor, optim='Adam', lr=1e-3) # use saver to save a new model saver = tf.train.Saver() sess = tf.Session() with tf.sg_queue_context(sess): # inital tf.sg_init(sess) # validation acc = (predict.sg_reuse( input=Mnist.valid.image).sg_softmax().sg_accuracy( target=Mnist.valid.label, name='validation')) tf.sg_train(loss=loss, eval_metric=[acc], max_ep=max_ep, save_dir=save_dir, ep_size=Mnist.train.num_batch, log_interval=10)
def sg_input(shape=None, dtype=sg_floatx, name=None): if shape is None: return tf.placeholder(dtype, shape=None, name=name) else: if not isinstance(shape, (list, tuple)): shape = [shape] return tf.placeholder(dtype, shape=[None] + list(shape), name=name)
def __init__(self, mode="train"): ''' Args: mode: A string. Either "train" , "val", or "test" ''' if mode == 'train': self.X_batch, self.Y_batch, self.num_batch = get_batch_data( 'train') else: self.X_batch = tf.placeholder( tf.int32, [Hyperparams.batch_size, Hyperparams.maxlen]) self.Y_batch = tf.placeholder( tf.int32, [Hyperparams.batch_size, Hyperparams.maxlen]) self.X_batch_rev = self.X_batch.sg_reverse_seq() # (8, 100) # make embedding matrix for input characters embed_mat = tf.convert_to_tensor(load_embed_lookup_table()) # embed table lookup X_batch_3d = self.X_batch.sg_lookup( emb=embed_mat).sg_float() # (8, 100, 200) X_batch_rev_3d = self.X_batch_rev.sg_lookup( emb=embed_mat).sg_float() # (8, 100, 200) # 1st biGRU layer gru_fw1 = X_batch_3d.sg_gru(dim=Hyperparams.hidden_dim, ln=True) # (8, 100, 200) gru_bw1 = X_batch_rev_3d.sg_gru(dim=Hyperparams.hidden_dim, ln=True) # (8, 100, 200) gru1 = gru_fw1.sg_concat(target=gru_bw1) # (8, 100, 400) # 2nd biGRU layer gru_fw2 = gru1.sg_gru(dim=Hyperparams.hidden_dim * 2, ln=True) # (8, 100, 400) gru_bw2 = gru1.sg_gru(dim=Hyperparams.hidden_dim * 2, ln=True) # (8, 100, 400) gru2 = gru_fw2.sg_concat(target=gru_bw2) # (16, 100, 800) # fc dense layer reshaped = gru2.sg_reshape(shape=[-1, gru2.get_shape().as_list()[-1]]) logits = reshaped.sg_dense(dim=3) # 1 for space 2 for non-space self.logits = logits.sg_reshape(shape=gru2.get_shape().as_list()[:-1] + [-1]) if mode == 'train': # cross entropy loss with logits ( for training set ) self.loss = self.logits.sg_ce(target=self.Y_batch, mask=True) # accuracy evaluation ( for validation set ) self.X_val_batch, self.Y_val_batch, self.num_batch = get_batch_data( 'val') self.acc = (self.logits.sg_reuse( input=self.X_val_batch).sg_accuracy(target=self.Y_val_batch, name='val'))
def __init__(self): # set log level to debug tf.sg_verbosity(10) # batch size self.batch_size = 1 # vocabulary size self.voca_size = sttwdata.voca_size # mfcc feature of audio self.x = tf.placeholder(dtype=tf.sg_floatx, shape=(self.batch_size, None, 20)) # encode audio feature self.logit = get_logit(self.x, voca_size=self.voca_size) # sequence length except zero-padding self.seq_len = tf.not_equal(self.x.sg_sum(axis=2), 0.).sg_int().sg_sum(axis=1) # run network self.session = tf.Session() tf.sg_init(self.session) self.saver = tf.train.Saver() self.saver.restore(self.session, tf.train.latest_checkpoint('asset/train'))
def init_model(): global x, y # set log level to debug tf.sg_verbosity(10) # # hyper parameters # batch_size = 1 # batch size # # inputs # # vocabulary size voca_size = data.voca_size # print(voca_size) # mfcc feature of audio x = tf.placeholder(dtype=tf.sg_floatx, shape=(batch_size, None, 20)) # sequence length except zero-padding seq_len = tf.not_equal(x.sg_sum(axis=2), 0.).sg_int().sg_sum(axis=1) # encode audio feature logit = get_logit(x, voca_size=voca_size) # ctc decoding decoded, _ = tf.nn.ctc_beam_search_decoder( logit.sg_transpose(perm=[1, 0, 2]), seq_len, merge_repeated=False) # to dense tensor y = tf.sparse_to_dense(decoded[0].indices, decoded[0].dense_shape, decoded[0].values) + 1
def sg_input(shape=None, dtype=sg_floatx, name=None): r"""Creates a placeholder. Args: shape: A tuple/list of integers. If an integers is given, it will turn to a list. dtype: A data type. Default is float32. name: A name for the placeholder. Returns: A wrapped placeholder `Tensor`. """ if shape is None: return tf.placeholder(dtype, shape=None, name=name) else: if not isinstance(shape, (list, tuple)): shape = [shape] return tf.placeholder(dtype, shape=[None] + list(shape), name=name)
def test(self): print 'Testing model {}: addnoise={}, rotate={}, var={}'.format( save_dir, addnoise, rotate, var) input_ph = tf.placeholder(shape=[batch_size, 28, 28, 1], dtype=tf.float32) label_ph = tf.placeholder(shape=[ batch_size, ], dtype=tf.int32) predict = self.forward(input_ph) acc = (predict.sg_softmax().sg_accuracy(target=label_ph, name='test')) sess = tf.Session() with tf.sg_queue_context(sess): tf.sg_init(sess) saver = tf.train.Saver() saver.restore(sess, tf.train.latest_checkpoint(save_dir)) total_accuracy = 0 for i in range(Mnist.test.num_batch): [image_array, label_array] = sess.run([Mnist.test.image, Mnist.test.label]) if addnoise: image_array[0, :, :, 0] = addnoisy(image_array[0, :, :, 0], var) if rotate: image_array[0, :, :, 0] = rotate_90(image_array[0, :, :, 0]) acc_value = sess.run([acc], feed_dict={ input_ph: image_array, label_ph: label_array })[0] total_accuracy += np.sum(acc_value) print 'Evaluation accuracy: {}'.format( float(total_accuracy) / (Mnist.test.num_batch * batch_size)) # close session sess.close()
def wrapper(**kwargs): r"""Manages arguments of `tf.sg_opt`. Args: **kwargs: source: A source queue list to enqueue dtypes: Data types of each tensor capacity: Queue capacity. Default is 32. num_threads: Number of threads. Default is 1. """ # default option opt = tf.sg_opt(kwargs) + tf.sg_opt( dtypes=[tf.sg_floatx], capacity=32, num_threads=1) # source queue list check assert opt.source is not None, 'source is mandatory.' if type(opt.source) is not list and type(opt.source) is not tuple: opt.source = [opt.source] if type(opt.dtypes) is not list and type(opt.dtypes) is not tuple: opt.dtypes = [opt.dtypes] assert len(opt.source) == len( opt.dtypes), 'Source and dtypes should have same length.' # enqueue function def enqueue_func(sess, op): # read data from source queue data = func(sess.run(opt.source)) # create feeder dict feed_dict = {} for ph, col in zip(placeholders, data): feed_dict[ph] = col # run session sess.run(op, feed_dict=feed_dict) # create place holder list placeholders = [] for dtype in opt.dtypes: placeholders.append(tf.placeholder(dtype=dtype)) # create FIFO queue queue = tf.FIFOQueue(opt.capacity, dtypes=opt.dtypes) # enqueue operation enqueue_op = queue.enqueue(placeholders) # create queue runner runner = _FuncQueueRunner(enqueue_func, queue, [enqueue_op] * opt.num_threads) # register to global collection tf.train.add_queue_runner(runner) # return de-queue operation return queue.dequeue()
def testIt(): data = raw positive = np.array(data.label_train) > 0 x = tf.placeholder(tf.float32, [None, 4096]) y = tf.placeholder(tf.float32) disc_real = discriminator(x) accuracy = tf.reduce_mean( tf.cast(tf.equal(tf.cast(disc_real > 0.5, "float"), y), tf.float32)) np.set_printoptions(precision=3, suppress=True) with tf.Session() as sess: sess.run( tf.group(tf.global_variables_initializer(), tf.sg_phase().assign(False))) # restore parameters tf.sg_restore(sess, tf.train.latest_checkpoint('asset/train/gan'), category=['generator', 'discriminator']) ans = sess.run(disc_real, feed_dict={x: np.array(data.test)}) print np.sum(ans > 0.5) np.save('dm_bird.npy', ans)
def __init__(self, mode="train"): ''' Args: mode: A string. Either "train" or "test" ''' self.char2idx, self.idx2char = load_char_vocab() self.word2idx, self.idx2word = load_word_vocab() if mode == "train": self.x, self.y, self.num_batch = get_batch_data() else: self.x = tf.placeholder(tf.int32, [None, Hyperparams.seqlen]) self.emb_x = tf.sg_emb(name='emb_x', voca_size=len(self.char2idx), dim=Hyperparams.embed_dim) self.enc = self.x.sg_lookup(emb=self.emb_x) with tf.sg_context(size=5, act='relu', bn=True): for _ in range(20): dim = self.enc.get_shape().as_list()[-1] self.enc += self.enc.sg_conv1d( dim=dim) # (64, 50, 300) float32 self.enc = self.enc.sg_conv1d(size=1, dim=len(self.word2idx), act='linear', bn=False) # (64, 50, 21293) float32 # self.logits = self.enc.sg_mean(dims=[1], keep_dims=False) # (64, 21293) float32 # Weighted Sum. Updated on Feb. 15, 2017. def make_weights(size): weights = tf.range(1, size + 1, dtype=tf.float32) weights *= 1. / ((1 + size) * size // 2) weights = tf.expand_dims(weights, 0) weights = tf.expand_dims(weights, -1) return weights self.weights = make_weights(Hyperparams.seqlen) # (1, 50, 1) self.enc *= self.weights # Broadcasting self.logits = self.enc.sg_sum(axis=[1], keep_dims=False) # (64, 21293) if mode == "train": self.ce = self.logits.sg_ce(target=self.y, mask=False, one_hot=False) self.istarget = tf.not_equal(self.y, tf.ones_like( self.y)).sg_float() # 1: Unkown self.reduced_loss = ((self.ce * self.istarget).sg_sum()) / ( self.istarget.sg_sum() + 1e-5) tf.sg_summary_loss(self.reduced_loss, "reduced_loss")
class Hp: batch_size = 51 # batch size hd = 400 # hidden dimension c_maxlen = 150 # Maximum sentence length w_maxlen = 25 char_vocab = u'''␀␂␃⁇N abcdefghijklmnopqrstuvwxyz''' char_vs = len(char_vocab) par_maxlen = [3] num_blocks = 3 # dilated blocks keep_prob = tf.placeholder(tf.float32) w_emb_size = 300 num_gpus = num_gpus rnn_hd = 300
def __init__(self, is_train=True): # inputs if is_train: self.X, self.Y, self.num_batch = get_batch_data( ) # (16, 9, 9, 1), (16, 9, 9) self.X_val, self.Y_val, _ = get_batch_data(is_train=False) else: self.X = tf.placeholder(tf.float32, [None, 9, 9, 1]) with tf.sg_context(size=3, act='relu', bn=True): self.logits = self.X.sg_identity() for _ in range(5): self.logits = (self.logits.sg_conv(dim=512)) self.logits = self.logits.sg_conv( dim=10, size=1, act='linear', bn=False) # (16, 9, 9, 10) float32 if is_train: self.ce = self.logits.sg_ce(target=self.Y, mask=False) # (16, 9, 9) dtype=float32 self.istarget = tf.equal( self.X.sg_squeeze(), tf.zeros_like(self.X.sg_squeeze()) ).sg_float() # zeros: 1, non-zeros: 0 (16, 9, 9) dtype=float32 self.loss = self.ce * self.istarget # (16, 9, 9) dtype=float32 self.reduced_loss = self.loss.sg_sum() / self.istarget.sg_sum() tf.sg_summary_loss(self.reduced_loss, "reduced_loss") # accuracy evaluation ( for train set ) self.preds = (self.logits.sg_argmax()).sg_int() self.hits = tf.equal(self.preds, self.Y).sg_float() self.acc_train = (self.hits * self.istarget).sg_sum() / self.istarget.sg_sum() # accuracy evaluation ( for validation set ) self.preds_ = (self.logits.sg_reuse( input=self.X_val).sg_argmax()).sg_int() self.hits_ = tf.equal(self.preds_, self.Y_val).sg_float() self.istarget_ = tf.equal(self.X_val.sg_squeeze(), tf.zeros_like( self.X_val.sg_squeeze())).sg_float() self.acc_val = (self.hits_ * self.istarget_).sg_sum() / self.istarget_.sg_sum()
def __init__(self, mode="train"): ''' Args: is_train: Boolean. If True, backprop is executed. ''' if mode == "train": self.x, self.y, self.num_batch = get_batch_data( ) # (64, 50) int64, (64, 50) int64, 1636 else: # test self.x = tf.placeholder(tf.int64, [None, Hyperparams.maxlen]) # make embedding matrix for input characters pnyn2idx, _, hanzi2idx, _ = load_vocab() self.emb_x = tf.sg_emb(name='emb_x', voca_size=len(pnyn2idx), dim=Hyperparams.embed_dim) self.enc = self.x.sg_lookup(emb=self.emb_x) with tf.sg_context(size=5, act='relu', bn=True): for _ in range(20): dim = self.enc.get_shape().as_list()[-1] self.enc += self.enc.sg_conv1d( dim=dim) # (64, 50, 300) float32 # final fully convolutional layer for softmax self.logits = self.enc.sg_conv1d(size=1, dim=len(hanzi2idx), act='linear', bn=False) # (64, 50, 5072) float32 if mode == "train": self.ce = self.logits.sg_ce(target=self.y, mask=True) # (64, 50) float32 self.istarget = tf.not_equal(self.y, tf.zeros_like( self.y)).sg_float() # (64, 50) float32 self.reduced_loss = self.ce.sg_sum() / self.istarget.sg_sum( ) # () float32 tf.sg_summary_loss(self.reduced_loss, "reduced_loss")
def __init__(self, is_train=True): ''' Args: is_train: Boolean. If True, backprop is executed. ''' if is_train: self.x, self.y = get_batch_data() # (16, 100), (16, 100) else: # self.x = tf.placeholder(tf.int32, [Hyperparams.batch_size, Hyperparams.maxlen]) self.x = tf.placeholder(tf.int32, [None, Hyperparams.maxlen]) # make embedding matrix for input characters hangul2idx, _, hanja2idx, _ = load_charmaps() self.emb_x = tf.sg_emb(name='emb_x', voca_size=len(hangul2idx), dim=Hyperparams.hidden_dim) # embed table lookup self.enc = self.x.sg_lookup(emb=self.emb_x).sg_float() # (16, 100, 200) # loop dilated conv block for i in range(2): self.enc = (self.enc .sg_res_block(size=5, rate=1) .sg_res_block(size=5, rate=2) .sg_res_block(size=5, rate=4) .sg_res_block(size=5, rate=8) .sg_res_block(size=5, rate=16)) # final fully convolutional layer for softmax self.logits = self.enc.sg_conv1d(size=1, dim=len(hanja2idx)) # (16, 100, 4543) if is_train: self.ce = self.logits.sg_ce(target=self.y, mask=True) # (16, 100) self.nonzeros = tf.not_equal(self.y, tf.zeros_like(self.y)).sg_float() # (16, 100) self.reduced_loss = self.ce.sg_sum() / self.nonzeros.sg_sum() # () tf.sg_summary_loss(self.reduced_loss, "reduced_loss")
def __init__(self, is_train=True): # inputs if is_train: self.x, self.y, self.num_batch = get_batch_data() self.x_val, self.y_val, _ = get_batch_data(is_train=False) else: self.x = tf.placeholder(tf.float32, [None, 9, 9, 1]) with tf.sg_context(size=3, act='relu', bn=True): self.logits = self.x.sg_identity() for _ in range(10): self.logits = (self.logits.sg_conv(dim=512)) self.logits = self.logits.sg_conv(dim=10, size=1, act='linear', bn=False) if is_train: self.ce = self.logits.sg_ce(target=self.y, mask=False) self.istarget = tf.equal(self.x.sg_squeeze(), tf.zeros_like( self.x.sg_squeeze())).sg_float() self.loss = self.ce * self.istarget self.reduced_loss = self.loss.sg_sum() / self.istarget.sg_sum() tf.sg_summary_loss(self.reduced_loss, "reduced_loss") # accuracy evaluation ( for validation set ) self.preds_ = (self.logits.sg_reuse( input=self.x_val).sg_argmax()).sg_int() self.hits_ = tf.equal(self.preds_, self.y_val).sg_float() self.istarget_ = tf.equal(self.x_val.sg_squeeze(), tf.zeros_like( self.x_val.sg_squeeze())).sg_float() self.acc = (self.hits_ * self.istarget_).sg_sum() / self.istarget_.sg_sum()
def __init__(self, mode="train"): # Inputs and Labels if mode == "train": self.x, self.y, self.num_batch = get_batch_data( ) # (16, 150) int32, (16, 150) int32, int self.y_src = tf.concat( [tf.zeros((Hp.batch_size, 1), tf.int32), self.y[:, :-1]], 1) # (16, 150) int32 else: # inference self.x = tf.placeholder(tf.int32, shape=(Hp.batch_size, Hp.maxlen)) self.y_src = tf.placeholder(tf.int32, shape=(Hp.batch_size, Hp.maxlen)) # Load vocabulary char2idx, idx2char = load_vocab() # Embedding emb_x = tf.sg_emb(name='emb_x', voca_size=len(char2idx), dim=Hp.hidden_units) # (179, 320) emb_y = tf.sg_emb(name='emb_y', voca_size=len(char2idx), dim=Hp.hidden_units) # (179, 320) X = self.x.sg_lookup(emb=emb_x) # (16, 150, 320) Y = self.y_src.sg_lookup(emb=emb_y) # (16, 150, 320) # Encoding conv = X.sg_quasi_conv1d(is_enc=True, size=6) # (16*3, 150, 320) pool = conv.sg_quasi_rnn(is_enc=True, att=False) # (16*4, 150, 320) H_zfo1 = pool[Hp.batch_size:] # (16*3, 15, 320) for decoding conv = pool.sg_quasi_conv1d(is_enc=True, size=2) # (16*3, 150, 320) pool = conv.sg_quasi_rnn(is_enc=True, att=False) # (16*4, 150, 320) H_zfo2 = pool[Hp.batch_size:] # (16*3, 150, 320) for decoding conv = pool.sg_quasi_conv1d(is_enc=True, size=2) # (16*3, 150, 320) pool = conv.sg_quasi_rnn(is_enc=True, att=False) # (16*4, 150, 320) H_zfo3 = pool[Hp.batch_size:] # (16*3, 150, 320) for decoding conv = pool.sg_quasi_conv1d(is_enc=True, size=2) # (16*3, 150, 320) pool = conv.sg_quasi_rnn(is_enc=True, att=False) # (16*4, 150, 320) H4 = pool[:Hp.batch_size] # (16, 150, 320) for decoding H_zfo4 = pool[Hp.batch_size:] # (16*3, 150, 320) for decoding # Decoding d_conv = (Y.sg_concat(target=H_zfo1, axis=0).sg_quasi_conv1d(is_enc=False, size=2)) d_pool = d_conv.sg_quasi_rnn(is_enc=False, att=False) # (16*4, 150, 320) d_conv = (d_pool.sg_concat(target=H_zfo2, axis=0).sg_quasi_conv1d(is_enc=False, size=2)) d_pool = d_conv.sg_quasi_rnn(is_enc=False, att=False) # (16*4, 150, 320) d_conv = (d_pool.sg_concat(target=H_zfo3, axis=0).sg_quasi_conv1d(is_enc=False, size=2)) d_pool = d_conv.sg_quasi_rnn(is_enc=False, att=False) # (16*4, 150, 320) d_conv = (d_pool.sg_concat(target=H_zfo4, axis=0).sg_quasi_conv1d(is_enc=False, size=2)) concat = H4.sg_concat(target=d_conv, axis=0) d_pool = concat.sg_quasi_rnn(is_enc=False, att=True) # (16, 150, 320) logits = d_pool.sg_conv1d(size=1, dim=len(char2idx), act="linear") # (16, 150, 179) if mode == 'train': # cross entropy loss with logits ( for training set ) loss = logits.sg_ce(target=self.y, mask=True) istarget = tf.not_equal(self.y, 0).sg_float() self.reduced_loss = (loss.sg_sum()) / (istarget.sg_sum() + 0.00001) tf.sg_summary_loss(self.reduced_loss, "reduced_loss") else: # inference self.preds = logits.sg_argmax()
def generate(): dev = '/cpu:0' with tf.device(dev): mydir = 'tfrc150char_wrd0704' files = [f for f in listdir(mydir) if isfile(join(mydir, f))] tfrecords_filename = [] tfrecords_filename = [join(mydir, 'short_infer3.tfrecords') ] #[join(mydir, f) for f in tfrecords_filename] tfrecords_filename_inf = [join(mydir, '11_3.tfrecords')] print(tfrecords_filename) filename_queue = tf.train.string_input_producer(tfrecords_filename, num_epochs=num_epochs, shuffle=True, capacity=1) infer_queue = tf.train.string_input_producer(tfrecords_filename_inf, num_epochs=num_epochs, shuffle=True, capacity=1) optim = tf.train.AdamOptimizer(learning_rate=0.0001, beta1=0.9, beta2=0.99) # Calculate the gradients for each model tower. tower_grads = [] reuse_vars = False with tf.variable_scope("dec_lstm") as scp: dec_cell = BasicLSTMCell2(Hp.w_emb_size, Hp.rnn_hd, state_is_tuple=True) with tf.variable_scope("contx_lstm") as scp: cell = BasicLSTMCell2(Hp.hd, Hp.rnn_hd, state_is_tuple=True) rnn_cell = tf.contrib.rnn.DropoutWrapper( cell, input_keep_prob=Hp.keep_prob, output_keep_prob=Hp.keep_prob) (words, chars) = read_and_decode(filename_queue, Hp.batch_size * Hp.num_gpus) words_splits = tf.split(axis=0, num_or_size_splits=Hp.num_gpus, value=words) chars_splits = tf.split(axis=0, num_or_size_splits=Hp.num_gpus, value=chars) word_emb = np.loadtxt("glove300d_0704.txt") Hp.word_vs = word_emb.shape[0] # -------------------------------------------------------------------------------- with tf.name_scope('%s_%d' % ("tower", 0)) as scope: rnn_state = tower_infer_enc(chars_splits[0], scope, rnn_cell, dec_cell, word_emb, out_reuse_vars=False, dev='/cpu:0') chars_pl = tf.placeholder(tf.int32, shape=(None, Hp.c_maxlen)) rnn_state_pl1 = [ tf.placeholder(tf.float32, shape=(None, Hp.rnn_hd)), tf.placeholder(tf.float32, shape=(None, Hp.rnn_hd)) ] rnn_state_pl = tf.contrib.rnn.LSTMStateTuple( rnn_state_pl1[0], rnn_state_pl1[1]) final_ids, rnn_state_dec = tower_infer_dec(chars_pl, scope, rnn_cell, dec_cell, word_emb, rnn_state_pl, out_reuse_vars=False, dev='/cpu:0') # -------------------------------------------------------------------------------- saver = tf.train.Saver(tf.trainable_variables()) session_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) session_config.gpu_options.per_process_gpu_memory_fraction = 0.94 session_config.gpu_options.allow_growth = False restore_dir = 'tnsrbrd/hin17d08m_1313g2' # lec30d07m_1634g2 lec04d07m_2006g2 lec28d07m_1221g2 lec31d07m_1548g2 csv_file = join(restore_dir, time.strftime("hin%dd%mm_%H%M.csv")) csv_f = open(csv_file, 'a') csv_writer = csv.writer(csv_f) with tf.Session(config=session_config) as sess: sess.run( tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())) tf.train.start_queue_runners(sess=sess) saver.restore(sess, tf.train.latest_checkpoint( join(restore_dir, 'last_chpt'))) # lec04d07m_2006g2 coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) for ep in range(num_epochs): tf.sg_set_infer(sess) rnn_state_val, w_txt, ch_txt = sess.run( [rnn_state, words_splits[0], chars_splits[0]], feed_dict={Hp.keep_prob: 1.0}) predictions = [] #[w_txt[:,2,:]] for idx in range(3): char_inpt = word2char_ids( ids_val) if idx != 0 else ch_txt[:, 2, :] ids_val, rnn_state_val = sess.run( [final_ids, rnn_state_dec], feed_dict={ Hp.keep_prob: 1.0, rnn_state_pl1[0]: rnn_state_val[0], rnn_state_pl1[1]: rnn_state_val[1], chars_pl: char_inpt }) temp = np.zeros((Hp.batch_size, Hp.w_maxlen)) for b in range(Hp.batch_size): stop_ind = np.where(ids_val[b] == 2)[0] if stop_ind.size > 0: stop_ind = stop_ind[0] ids_val[b, stop_ind + 1:] = ids_val[b, stop_ind + 1:] * 0 temp[:, :ids_val.shape[1]] = ids_val predictions.append(temp) # predictions are decode_sent x b x w_maxlen predictions = np.array(predictions) in_batches = [w_txt[b, :, :] for b in range(Hp.batch_size)] res_batches = [ predictions[:, b, :] for b in range(Hp.batch_size) ] for b in range(Hp.batch_size): in_paragraph = idxword2txt(in_batches[b]) print("\n INPUT SAMPLE \n") print(in_paragraph) res_paragraph = idxword2txt(res_batches[b]) print("\n RESULTS \n") print(res_paragraph) csv_writer.writerow([ " ".join(in_paragraph[:3]), " ".join(in_paragraph[3:]), " ".join(res_paragraph) ]) csv_f.close()
config.gpu_options.allow_growth = True #allocate dynamically # PARAMS # num_layers = 3 activation = tf.nn.relu batchsize = 8 w_dropout = 0.4 path_to_train_data = "__data/stage1_train" path_to_test_data = "__data/stage1_test" img_dat = util.crawl_path(path_to_train_data) graph = tf.Graph() with graph.as_default(): #batchsize*x_max*y_max*rgb(3) features = tf.placeholder(tf.float32, [None, None, None, 3]) labels = tf.placeholder(tf.int32, [None, None]) input_layer = tf.reshape(features, [-1, 2048, 2048, 3]) training = tf.placeholder(tf.bool) conv1 = tf.layers.conv2d(inputs=input_layer, filters=32, kernel_size=[11, 11], padding="same", activation=tf.nn.relu) print('conv1', conv1.get_shape()) pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[4, 4], strides=2) print('pool1', pool1.get_shape()) conv2 = tf.layers.conv2d(inputs=input_layer, filters=64, kernel_size=[5, 5],
res = (tensor.sg_dense(dim=1024, name='fc1').sg_dense( dim=7 * 7 * 128, name='fc2').sg_reshape(shape=(-1, 7, 7, 128)).sg_upconv( dim=64, name='conv1').sg_upconv(dim=1, act='sigmoid', bn=False, name='conv2')) return res # # inputs # # target_number target_num = tf.placeholder(dtype=tf.sg_intx, shape=batch_size) # target continuous variable # 1 target_cval_1 = tf.placeholder(dtype=tf.sg_floatx, shape=batch_size) # target continuous variable # 2 target_cval_2 = tf.placeholder(dtype=tf.sg_floatx, shape=batch_size) # category variables z = (tf.ones(batch_size, dtype=tf.sg_intx) * target_num).sg_one_hot(depth=cat_dim) # continuous variables z = z.sg_concat( target=[target_cval_1.sg_expand_dims(), target_cval_2.sg_expand_dims()]) # random seed = categorical variable + continuous variable + random normal
from model import * import data import sys # mengatur log level untuk debug tf.sg_verbosity(10) # hyper parameters batch_size = 1 # batch size # inputs # panjang kata voca_size = data.voca_size # menginput mfcc feature pada file audio x = tf.placeholder(dtype=tf.sg_floatx, shape=(batch_size, None, 20)) # panjang sequence kecuali zero-padding seq_len = tf.not_equal(x.sg_sum(axis=2), 0.).sg_int().sg_sum(axis=1) # encode audio feature logit = get_logit(x, voca_size=voca_size) # ctc decoding decoded, _ = tf.nn.ctc_beam_search_decoder(logit.sg_transpose(perm=[1, 0, 2]), seq_len, merge_repeated=False) # to dense tensor y = tf.sparse_to_dense(decoded[0].indices, decoded[0].dense_shape, decoded[0].values) + 1
Author: JJ Fu """ import sugartensor as tf import numpy as np import librosa import tensorflow as tfw from tensorflow.python.framework import graph_util from model import * import data batch_size = 1 # batch size voca_size = data.voca_size x = tf.placeholder(dtype=tf.sg_floatx, shape=(batch_size, None, 20)) # sequence length except zero-padding seq_len = tf.not_equal(x.sg_sum(axis=2), 0.).sg_int().sg_sum(axis=1) # encode audio feature logit = get_logit(x, voca_size) # ctc decoding decoded, _ = tf.nn.ctc_beam_search_decoder(logit.sg_transpose(perm=[1, 0, 2]), seq_len, merge_repeated=False) # to dense tensor y = tf.add(tf.sparse_to_dense(decoded[0].indices, decoded[0].dense_shape, decoded[0].values), 1, name="output") with tf.Session() as sess: tf.sg_init(sess) saver = tf.train.Saver() saver.restore(sess, tf.train.latest_checkpoint('asset/train')) graph = tf.get_default_graph()
# # hyper parameters # batch_size = 10 # # inputs # # ComTrans parallel corpus input tensor ( with QueueRunner ) data = ComTrans(batch_size=batch_size) # place holders x = tf.placeholder(dtype=tf.sg_intx, shape=(batch_size, data.max_len)) y_in = tf.placeholder(dtype=tf.sg_intx, shape=(batch_size, data.max_len)) # vocabulary size voca_size = data.voca_size # make embedding matrix for source and target emb_x = tf.sg_emb(name='emb_x', voca_size=voca_size, dim=latent_dim) emb_y = tf.sg_emb(name='emb_y', voca_size=voca_size, dim=latent_dim) # latent from embed table z_x = x.sg_lookup(emb=emb_x) z_y = y_in.sg_lookup(emb=emb_y) # encode graph ( atrous convolution ) enc = encode(z_x)
def __init__(self, x, y, num_batch, vocab_size, emb_dim, hidden_dim, max_ep=240, infer_shape=(1, 1), mode="train"): self.num_batch = num_batch self.emb_dim = emb_dim self.hidden_dim = hidden_dim self.vocab_size = vocab_size self.max_len_infer = 512 self.max_ep = max_ep # reuse = len([t for t in tf.global_variables() if t.name.startswith('gen')]) > 0 reuse = (mode == 'infer') if mode == "train": self.x = x self.y = y elif mode == "infer": self.x = tf.placeholder(tf.int32, shape=infer_shape) self.y = tf.placeholder(tf.int32, shape=infer_shape) with tf.variable_scope("gen_embs", reuse=reuse): self.emb_x = tf.get_variable("emb_x", [self.vocab_size, self.emb_dim]) self.emb_y = tf.get_variable("emb_y", [self.vocab_size, self.emb_dim]) self.X = tf.nn.embedding_lookup(self.emb_x, self.x) self.Y = tf.nn.embedding_lookup(self.emb_y, self.y) with tf.sg_context(name='gen', reuse=reuse): # self.emb_x = tf.Variable(tf.random_uniform([self.vocab_size, self.emb_dim], 0.0, 1.0), name="emb_x") # self.emb_y = tf.Variable(tf.random_uniform([self.vocab_size, self.emb_dim], 0.0, 1.0), name="emb_y") # self.emb_x = tf.sg_emb(name='emb_x', voca_size=self.vocab_size, dim=self.emb_dim) # (68,16) # self.emb_y = tf.sg_emb(name='emb_y', voca_size=self.vocab_size, dim=self.emb_dim) # (68,16) # self.X = self.x.sg_lookup(emb=self.emb_x) # (8,63,16) # self.Y = self.y.sg_lookup(emb=self.emb_y) # (8,63,16) if mode == "train": self.lstm_layer = self.X.sg_lstm(in_dim=self.emb_dim, dim=self.vocab_size, name="lstm") # (8, 63, 68) self.test = self.lstm_layer.sg_softmax(name="testtt") print "mazum??" print self.test elif mode == "infer": self.lstm_layer = self.X.sg_lstm(in_dim=self.emb_dim, dim=self.vocab_size, last_only=True, name="lstm") self.log_prob = tf.log(self.lstm_layer) # next_token: select by distribution probability, preds: select by argmax self.multinormed = tf.multinomial(self.log_prob, 1) self.next_token = tf.cast( tf.reshape(tf.multinomial(self.log_prob, 1), [1, infer_shape[0]]), tf.int32) self.preds = self.lstm_layer.sg_argmax() if mode == "train": self.loss = self.lstm_layer.sg_ce(target=self.y) self.istarget = tf.not_equal(self.y, 0).sg_float() self.reduced_loss = (self.loss.sg_sum()) / ( self.istarget.sg_sum() + 0.0000001) tf.sg_summary_loss(self.reduced_loss, "reduced_loss")
tf.sg_verbosity(10) # # hyper parameters # batch_size = 100 # batch size # # inputs # # target continuous variable target_cval = [] for _ in range(con_dim): target_cval.append(tf.placeholder(dtype=tf.sg_floatx, shape=batch_size)) # continuous variables z = target_cval[0].sg_expand_dims() for i in range(1, con_dim): z = z.sg_concat(target=target_cval[i].sg_expand_dims()) # random seed = continuous variable + random normal z = z.sg_concat(target=tf.random_normal((batch_size, rand_dim))) # generator gen = generator(z).sg_squeeze(axis=(2, 3)) # # run generator
def __init__(self, mode="train"): # Inputs and Labels if mode == "train": self.x, self.y, self.num_batch = get_batch_data( ) # (16, 150) int32, (16, 150) int32, int self.y_src = tf.concat( axis=1, values=[tf.zeros((Hp.bs, 1), tf.int32), self.y[:, :-1]]) # (16, 150) int32 else: # inference self.x = tf.placeholder(tf.int32, shape=(Hp.bs, Hp.maxlen)) self.y_src = tf.placeholder(tf.int32, shape=(Hp.bs, Hp.maxlen)) # Load vocabulary self.char2idx, self.idx2char = load_vocab() # Embedding self.emb_x = tf.sg_emb(name='emb_x', voca_size=len(self.char2idx), dim=Hp.hd) # (179, 320) self.emb_y = tf.sg_emb(name='emb_y', voca_size=len(self.char2idx), dim=Hp.hd) # (179, 320) self.X = self.x.sg_lookup(emb=self.emb_x) # (16, 150, 320) self.Y = self.y_src.sg_lookup(emb=self.emb_y) # (16, 150, 320) # Encoding self.conv = self.X.sg_quasi_conv1d(is_enc=True, size=6) # (16*4, 150, 320) self.pool = self.conv.sg_quasi_rnn(is_enc=True, att=False) # (16*4, 150, 320) self.H_zfo1 = self.pool[Hp.bs:] # (16*3, 15, 320) for decoding self.conv = self.pool.sg_quasi_conv1d(is_enc=True, size=2) # (16*4, 150, 320) self.pool = self.conv.sg_quasi_rnn(is_enc=True, att=False) # (16*4, 150, 320) self.H_zfo2 = self.pool[Hp.bs:] # (16*3, 150, 320) for decoding self.conv = self.pool.sg_quasi_conv1d(is_enc=True, size=2) # (16*4, 150, 320) self.pool = self.conv.sg_quasi_rnn(is_enc=True, att=False) # (16*4, 150, 320) self.H_zfo3 = self.pool[Hp.bs:] # (16*3, 150, 320) for decoding self.conv = self.pool.sg_quasi_conv1d(is_enc=True, size=2) # (16*4, 150, 320) self.pool = self.conv.sg_quasi_rnn(is_enc=True, att=False) # (16*4, 150, 320) self.H4 = self.pool[:Hp.bs] self.H_zfo4 = self.pool[Hp.bs:] # (16*3, 150, 320) for decoding # Decoding self.dec = self.Y.sg_concat(target=self.H_zfo1, dim=0) self.d_conv = self.dec.sg_quasi_conv1d(is_enc=False, size=2) self.d_pool = self.d_conv.sg_quasi_rnn(is_enc=False, att=False) # (16*4, 150, 320) self.d_conv = (self.d_pool.sg_concat( target=self.H_zfo2, dim=0).sg_quasi_conv1d(is_enc=False, size=2)) self.d_pool = self.d_conv.sg_quasi_rnn(is_enc=False, att=False) # (16*4, 150, 320) self.d_conv = (self.d_pool.sg_concat( target=self.H_zfo3, dim=0).sg_quasi_conv1d(is_enc=False, size=2)) self.d_pool = self.d_conv.sg_quasi_rnn(is_enc=False, att=False) # (16*4, 150, 320) self.d_conv = (self.d_pool.sg_concat( target=self.H_zfo4, dim=0).sg_quasi_conv1d(is_enc=False, size=2)) self.concat = self.H4.sg_concat(target=self.d_conv, dim=0) self.d_pool = self.concat.sg_quasi_rnn(is_enc=False, att=True) # (16*4, 150, 320) self.logits = self.d_pool.sg_conv1d(size=1, dim=len(self.char2idx), act="linear") # (16, 150, 179) self.preds = self.logits.sg_argmax() if mode == 'train': # cross entropy loss with logits ( for training set ) self.loss = self.logits.sg_ce(target=self.y, mask=True) self.istarget = tf.not_equal(self.y, 0).sg_float() self.reduced_loss = (self.loss.sg_sum()) / ( self.istarget.sg_sum() + 0.00001) tf.sg_summary_loss(self.reduced_loss, "reduced_loss")
#corpus = SpeechCorpus(batch_size=batch_size * tf.sg_gpus()) mfccs = [] # for mfcc_file in corpus.mfcc_file: # mfcc = np.load(mfcc_file, allow_pickle=False) # mfccs.append(mfcc.reshape((1, mfcc.shape[0], mfcc.shape[1])).transpose([0,2,1])) for mfcc_file in ["more_data/mfcc-one/rainbow.wav.npy"]: mfcc = np.load(mfcc_file, allow_pickle=False) mfccs.append( mfcc.reshape((1, mfcc.shape[0], mfcc.shape[1])).transpose([0, 2, 1])) # vocabulary size voca_size = data.voca_size # mfcc feature of audio x = tf.placeholder(dtype=tf.sg_floatx, shape=(batch_size, None, 20)) noise = tf.get_variable("noise", shape=(batch_size, mfccs[index].shape[1], 20), initializer=tf.zeros_initializer()) perturbed_input = x + noise # sequence length except zero-padding seq_len = tf.not_equal(x.sg_sum(axis=2), 0.).sg_int().sg_sum(axis=1) # encode audio feature logit = get_logit(perturbed_input, voca_size=voca_size) # ctc decoding decoded, _ = tf.nn.ctc_beam_search_decoder(logit.sg_transpose(perm=[1, 0, 2]),
def train_with_GP(self): input_ph = tf.placeholder(shape=[batch_size, 28, 28, 1], dtype=tf.float32) label_ph = tf.placeholder(shape=[ batch_size, ], dtype=tf.int32) predict = self.forward(input_ph) loss_tensor = tf.reduce_mean(predict.sg_ce(target=label_ph)) # use to update network parameters optim = tf.sg_optim(loss_tensor, optim='Adam', lr=1e-3) # use saver to save a new model saver = tf.train.Saver() sess = tf.Session() with tf.sg_queue_context(sess): # inital tf.sg_init(sess) # train by GP guilding for e in range(max_ep): previous_loss = None for i in range(Mnist.train.num_batch): [image_array, label_array ] = sess.run([Mnist.train.image, Mnist.train.label]) if (e == 0 or e == 1 ): # first and second epoch train no noisy image loss = sess.run([loss_tensor, optim], feed_dict={ input_ph: image_array, label_ph: label_array })[0] print 'Baseline loss = ', loss elif ( e == 2 ): # third epoch train with gp image and original image gpIn1 = np.squeeze(image_array) gpIn2 = np.zeros((28, 28)) image_gp = GP(gpIn1, gpIn2, seed=0.8) image_gp2 = image_gp[np.newaxis, ...] image_gp2 = image_gp2[..., np.newaxis] loss = sess.run([loss_tensor, optim], feed_dict={ input_ph: image_array, label_ph: label_array })[0] print 'GP without nosiy loss = ', loss loss = sess.run([loss_tensor, optim], feed_dict={ input_ph: image_gp2, label_ph: label_array })[0] print 'GP loss = ', loss else: # other epoch train with gp evolution gpIn1 = np.squeeze(image_array) gpIn2 = np.zeros((28, 28)) image_gp = GP(gpIn1, gpIn2, seed=random.random()) image_gp2 = image_gp[np.newaxis, ...] image_gp2 = image_gp2[..., np.newaxis] loss = sess.run([loss_tensor, optim], feed_dict={ input_ph: image_array, label_ph: label_array })[0] print 'GP without nosiy loss = ', loss loss = sess.run([loss_tensor, optim], feed_dict={ input_ph: image_gp2, label_ph: label_array })[0] print 'GP loss = ', loss if loss < previous_loss: for i in range(5): loss = sess.run([loss_tensor, optim], feed_dict={ input_ph: image_gp2, label_ph: label_array })[0] gpIn1 = image_gp2 image_gp2[0, :, :, 0] = GP(gpIn1[0, :, :, 0], gpIn2, seed=random.random()) print 'GP EV loss = ', loss previous_loss = loss saver.save(sess, os.path.join(save_dir, 'gp_model'), global_step=e) # close session sess.close()
def __init__(self): with tf.sg_context(name='generator'): self.x = tf.sg_initializer.he_uniform(name="x", shape=[1, 224, 224, 1]) # noise image self.y = tf.placeholder(dtype=tf.float32, shape=[1, 224, 224, 1]) # true target image with tf.sg_context(name='conv', act='relu'): self.x_conv1 = (self.x.sg_conv(dim=64).sg_conv().sg_pool() ) # (1, 112, 112, 64) self.x_conv2 = (self.x_conv1.sg_conv(dim=128).sg_conv().sg_pool() ) # (1, 56, 56, 128) self.x_conv3 = (self.x_conv2.sg_conv( dim=256).sg_conv().sg_conv().sg_conv().sg_pool() ) # (1, 28, 28, 256) self.x_conv4 = (self.x_conv3.sg_conv( dim=512).sg_conv().sg_conv().sg_conv().sg_pool() ) # (1, 14, 14, 512) # .sg_conv(dim=512) # .sg_conv() # .sg_conv() # .sg_conv() # .sg_pool()) self.y_conv1 = self.x_conv1.sg_reuse(input=self.y) self.y_conv2 = self.x_conv2.sg_reuse(input=self.y) self.y_conv3 = self.x_conv3.sg_reuse(input=self.y) self.y_conv4 = self.x_conv4.sg_reuse(input=self.y) # def get_gram_mat(tensor): ''' Arg: tensor: 4-D tensor. The first dimension must be 1. Returns: gram matrix. Read `https://en.wikipedia.org/wiki/Gramian_matrix` for details. 512 by 512. ''' assert tensor.get_shape( ).ndims == 4, "The tensor must be 4 dimensions." dim0, dim1, dim2, dim3 = tensor.get_shape().as_list() tensor = tensor.sg_reshape(shape=[dim0 * dim1 * dim2, dim3]) #(1*7*7, 512) # normalization: Why? Because the original value of gram mat. would be too huge. mean, variance = tf.nn.moments(tensor, [0, 1]) tensor = (tensor - mean) / tf.sqrt(variance + tf.sg_eps) tensor_t = tensor.sg_transpose(perm=[1, 0]) #(512, 1*7*7) gram_mat = tf.matmul(tensor_t, tensor) # (512, 512) return gram_mat # Loss: Add the loss of each layer self.mse = tf.squared_difference(get_gram_mat(self.x_conv1), get_gram_mat(self.y_conv1)).sg_mean() +\ tf.squared_difference(get_gram_mat(self.x_conv2), get_gram_mat(self.y_conv2)).sg_mean() +\ tf.squared_difference(get_gram_mat(self.x_conv3), get_gram_mat(self.y_conv3)).sg_mean() +\ tf.squared_difference(get_gram_mat(self.x_conv4), get_gram_mat(self.y_conv4)).sg_mean() self.train_gen = tf.sg_optim( self.mse, lr=0.0001, category='generator') # Note that we train only variable x.
# # hyper parameters # batch_size = 100 # batch size num_category = 10 # category variable number num_cont = 2 # continuous variable number num_dim = 30 # total latent dimension ( category + continuous + noise ) # # inputs # # target_number target_num = tf.placeholder(dtype=tf.sg_intx, shape=batch_size) # target continuous variable # 1 target_cval_1 = tf.placeholder(dtype=tf.sg_floatx, shape=batch_size) # target continuous variable # 2 target_cval_2 = tf.placeholder(dtype=tf.sg_floatx, shape=batch_size) # category variables z = (tf.ones(batch_size, dtype=tf.sg_intx) * target_num).sg_one_hot(depth=num_category) # continuous variables z = z.sg_concat(target=[target_cval_1.sg_expand_dims(), target_cval_2.sg_expand_dims()]) # random seed = categorical variable + continuous variable + random uniform z = z.sg_concat(target=tf.random_uniform((batch_size, num_dim-num_cont-num_category)))
x += np.random.normal(scale=noise_magnitude, size=x.shape) return x, lengths #################### with tf.device("/device:GPU:0"): print(time.strftime('[%H:%M:%S]'), 'Loading network functions... ') graph = tf.Graph() with graph.as_default(): def lstm_cell(): with tf.name_scope('cell'): return tf.contrib.rnn.LSTMCell(num_hidden, state_is_tuple=True) with tf.name_scope('inputLength'): seq_len = tf.placeholder(tf.int32, [None]) with tf.name_scope('input'): inputs = tf.placeholder(tf.float32, [None, None, num_mfccs * 2]) targets = tf.sparse_placeholder(tf.int32) # Stacking rnn cells with tf.name_scope('cellStack'): stack = tf.contrib.rnn.MultiRNNCell( [lstm_cell() for _ in range(num_layers)], state_is_tuple=True) outputs, _ = tf.nn.dynamic_rnn(stack, inputs, seq_len, dtype=tf.float32) shape = tf.shape(inputs) batch_s, TF_max_timesteps = shape[0], shape[1]
def __init__(self, mode="train"): # Inputs and Labels if mode == "train": self.x, self.y, self.num_batch = get_batch_data() # (16, 150) int32, (16, 150) int32, int self.y_src = tf.concat([tf.zeros((Hp.batch_size, 1), tf.int32), self.y[:, :-1]], 1) # (16, 150) int32 else: # inference self.x = tf.placeholder(tf.int32, shape=(Hp.batch_size, Hp.maxlen)) self.y_src = tf.placeholder(tf.int32, shape=(Hp.batch_size, Hp.maxlen)) # Load vocabulary char2idx, idx2char = load_vocab() # Embedding def embed(inputs, vocab_size, embed_size, variable_scope): ''' inputs = tf.expand_dims(tf.range(5), 0) => (1, 5) _embed(inputs, 5, 10) => (1, 5, 10) ''' with tf.variable_scope(variable_scope): lookup_table = tf.get_variable('lookup_table', dtype=tf.float32, shape=[vocab_size, embed_size], initializer=tf.truncated_normal_initializer()) return tf.nn.embedding_lookup(lookup_table, inputs) X = embed(self.x, vocab_size=len(char2idx), embed_size=Hp.hidden_units, variable_scope='X') # (179, 320) Y = embed(self.y_src, vocab_size=len(char2idx), embed_size=Hp.hidden_units, variable_scope='Y') # (179, 320) # Y = tf.concat((tf.zeros_like(Y[:, :1, :]), Y[:, :-1, :]), 1) # Encoding conv = X.sg_quasi_conv1d(is_enc=True, size=6) # (16*3, 150, 320) pool = conv.sg_quasi_rnn(is_enc=True, att=False) # (16*4, 150, 320) H_zfo1 = pool[Hp.batch_size:] # (16*3, 15, 320) for decoding conv = pool.sg_quasi_conv1d(is_enc=True, size=2) # (16*3, 150, 320) pool = conv.sg_quasi_rnn(is_enc=True, att=False) # (16*4, 150, 320) H_zfo2 = pool[Hp.batch_size:] # (16*3, 150, 320) for decoding conv = pool.sg_quasi_conv1d(is_enc=True, size=2) # (16*3, 150, 320) pool = conv.sg_quasi_rnn(is_enc=True, att=False) # (16*4, 150, 320) H_zfo3 = pool[Hp.batch_size:] # (16*3, 150, 320) for decoding conv = pool.sg_quasi_conv1d(is_enc=True, size=2) # (16*3, 150, 320) pool = conv.sg_quasi_rnn(is_enc=True, att=False) # (16*4, 150, 320) H4 = pool[:Hp.batch_size] # (16, 150, 320) for decoding H_zfo4 = pool[Hp.batch_size:] # (16*3, 150, 320) for decoding # Decoding d_conv = (Y.sg_concat(target=H_zfo1, axis=0) .sg_quasi_conv1d(is_enc=False, size=2)) d_pool = d_conv.sg_quasi_rnn(is_enc=False, att=False) # (16*4, 150, 320) d_conv = (d_pool.sg_concat(target=H_zfo2, axis=0) .sg_quasi_conv1d(is_enc=False, size=2)) d_pool = d_conv.sg_quasi_rnn(is_enc=False, att=False) # (16*4, 150, 320) d_conv = (d_pool.sg_concat(target=H_zfo3, axis=0) .sg_quasi_conv1d(is_enc=False, size=2)) d_pool = d_conv.sg_quasi_rnn(is_enc=False, att=False) # (16*4, 150, 320) d_conv = (d_pool.sg_concat(target=H_zfo4, axis=0) .sg_quasi_conv1d(is_enc=False, size=2)) concat = H4.sg_concat(target=d_conv, axis=0) d_pool = concat.sg_quasi_rnn(is_enc=False, att=True) # (16, 150, 320) logits = d_pool.sg_conv1d(size=1, dim=len(char2idx), act="linear") # (16, 150, 179) if mode=='train': # cross entropy loss with logits ( for training set ) self.loss = logits.sg_ce(target=self.y, mask=True) istarget = tf.not_equal(self.y, 0).sg_float() self.reduced_loss = (self.loss.sg_sum()) / (istarget.sg_sum() + 1e-8) tf.sg_summary_loss(self.reduced_loss, "reduced_loss") else: # inference self.preds = logits.sg_argmax()
np.random.shuffle(index) ret = index[:batchsize] index = index[:-batchsize].copy() return ret def t_get_indices(batchsize): index = np.arange(batchsize) np.random.shuffle(index) return index ## Training Loop sd = 1 / np.sqrt(num_features) with tf.name_scope('input'): X = tf.placeholder(tf.float32, [None, num_features], name="x_inp") Y = tf.placeholder(tf.float32, [None, num_classes], name="y_inp") W_1 = tf.Variable( tf.random_normal([num_features, n_hidden_units_one], mean=0, stddev=sd)) b_1 = tf.Variable(tf.random_normal([n_hidden_units_one], mean=0, stddev=sd)) h_1 = tf.nn.tanh(tf.matmul(X, W_1) + b_1) W_2 = tf.Variable( tf.random_normal([n_hidden_units_one, n_hidden_units_two], mean=0, stddev=sd)) b_2 = tf.Variable(tf.random_normal([n_hidden_units_two], mean=0, stddev=sd)) h_2 = tf.nn.tanh(tf.matmul(h_1, W_2) + b_2) W_3 = tf.Variable(