def ner_accuracy(tensor, opt): r"""Returns accuracy of predictions. Args: tensor: A `Tensor`. Probability distributions or unscaled prediction scores. opt: target: A 'Tensor`. Labels. Returns: A `Tensor` of the same shape as `tensor`. Each value will be 1 if correct else 0. For example, ``` tensor = [[20.1, 18, -4.2], [0.04, 21.1, 31.3]] target = [[0, 1]] tensor.sg_accuracy(target=target) => [[ 1. 0.]] ``` """ assert opt.target is not None, 'target is mandatory.' opt += tf.sg_opt(k=1) # # calc accuracy out = tf.identity(tf.equal(tensor.sg_argmax() + 1, tf.cast(opt.target, tf.int64)).sg_float(), name='acc') # out = tf.identity(tf.nn.in_top_k(tensor, opt.target, opt.k).sg_float(), name='acc') # masking padding if opt.mask: out += tf.equal(opt.target, tf.zeros_like(opt.target)).sg_float() return out
def sg_reverse_seq(tensor, opt): r"""Reverses variable length slices. Before applying the pure tensorflow function tf.reverse_sequence, this function calculates sequence lengths by counting non-zeros. For example, ``` tensor = [[1, 2, 3, 0, 0], [4, 5, 0, 0, 0]] tensor.sg_reverse_seq() => [[3 2 1 0 0] [5 4 0 0 0]] ``` Args: tensor: A 2-D `Tensor` (automatically given by chain). opt: dim: Dimension to reverse. Default is 1. name : If provided, it replaces current tensor's name. Returns: A `Tensor` with the same shape and type as `tensor`. """ # default sequence dimension opt += tf.sg_opt(dim=1) seq_len = tf.not_equal(tensor, tf.zeros_like(tensor)).sg_int().sg_sum(dims=opt.dim) return tf.reverse_sequence(tensor, seq_len, opt.dim, name=opt.name)
def __init__(self, is_train=True): # inputs if is_train: self.X, self.Y, self.num_batch = get_batch_data( ) # (16, 9, 9, 1), (16, 9, 9) self.X_val, self.Y_val, _ = get_batch_data(is_train=False) else: self.X = tf.placeholder(tf.float32, [None, 9, 9, 1]) with tf.sg_context(size=3, act='relu', bn=True): self.logits = self.X.sg_identity() for _ in range(5): self.logits = (self.logits.sg_conv(dim=512)) self.logits = self.logits.sg_conv( dim=10, size=1, act='linear', bn=False) # (16, 9, 9, 10) float32 if is_train: self.ce = self.logits.sg_ce(target=self.Y, mask=False) # (16, 9, 9) dtype=float32 self.istarget = tf.equal( self.X.sg_squeeze(), tf.zeros_like(self.X.sg_squeeze()) ).sg_float() # zeros: 1, non-zeros: 0 (16, 9, 9) dtype=float32 self.loss = self.ce * self.istarget # (16, 9, 9) dtype=float32 self.reduced_loss = self.loss.sg_sum() / self.istarget.sg_sum() tf.sg_summary_loss(self.reduced_loss, "reduced_loss") # accuracy evaluation ( for train set ) self.preds = (self.logits.sg_argmax()).sg_int() self.hits = tf.equal(self.preds, self.Y).sg_float() self.acc_train = (self.hits * self.istarget).sg_sum() / self.istarget.sg_sum() # accuracy evaluation ( for validation set ) self.preds_ = (self.logits.sg_reuse( input=self.X_val).sg_argmax()).sg_int() self.hits_ = tf.equal(self.preds_, self.Y_val).sg_float() self.istarget_ = tf.equal(self.X_val.sg_squeeze(), tf.zeros_like( self.X_val.sg_squeeze())).sg_float() self.acc_val = (self.hits_ * self.istarget_).sg_sum() / self.istarget_.sg_sum()
def sg_ce(tensor, opt): r"""Returns softmax cross entropy loss between `tensor` and `target`. Args: tensor: A `Tensor`. Logits. Unscaled log probabilities. opt: target: A `Tensor` with the same length in the first dimension as the `tensor`. Labels. one_hot: Boolean. Whether to treat the labels as one-hot encoding. Default is False. mask: Boolean. If True, zeros in the target will be excluded from the calculation. name: A `string`. A name to display in the tensor board web UI. Returns: A 1-D `Tensor` with the same shape as `tensor`. For example, ``` tensor = [[[2, -1, 3], [3, 1, -2]]] target = [[2, 1]] tensor.sg_ce(target=target) => [[ 0.32656264 2.13284516]] ``` For example, ``` tensor = [[2, -1, 3], [3, 1, -2]] target = [[0, 0, 1], [1, 0, 0]] tensor.sg_ce(target=target, one_hot=True) => [ 0.32656264 0.13284527] ``` """ opt += tf.sg_opt(one_hot=False) assert opt.target is not None, 'target is mandatory.' if opt.one_hot: out = tf.identity( tf.nn.softmax_cross_entropy_with_logits(labels=opt.target, logits=tensor), 'ce') else: out = tf.identity( tf.nn.sparse_softmax_cross_entropy_with_logits(labels=opt.target, logits=tensor), 'ce') # masking loss if opt.mask: out *= tf.not_equal(opt.target, tf.zeros_like(opt.target)).sg_float() # add summary tf.sg_summary_loss(out, name=opt.name) return out
def __init__(self, is_train=True): # inputs if is_train: self.x, self.y, self.num_batch = get_batch_data() self.x_val, self.y_val, _ = get_batch_data(is_train=False) else: self.x = tf.placeholder(tf.float32, [None, 9, 9, 1]) with tf.sg_context(size=3, act='relu', bn=True): self.logits = self.x.sg_identity() for _ in range(10): self.logits = (self.logits.sg_conv(dim=512)) self.logits = self.logits.sg_conv(dim=10, size=1, act='linear', bn=False) if is_train: self.ce = self.logits.sg_ce(target=self.y, mask=False) self.istarget = tf.equal(self.x.sg_squeeze(), tf.zeros_like( self.x.sg_squeeze())).sg_float() self.loss = self.ce * self.istarget self.reduced_loss = self.loss.sg_sum() / self.istarget.sg_sum() tf.sg_summary_loss(self.reduced_loss, "reduced_loss") # accuracy evaluation ( for validation set ) self.preds_ = (self.logits.sg_reuse( input=self.x_val).sg_argmax()).sg_int() self.hits_ = tf.equal(self.preds_, self.y_val).sg_float() self.istarget_ = tf.equal(self.x_val.sg_squeeze(), tf.zeros_like( self.x_val.sg_squeeze())).sg_float() self.acc = (self.hits_ * self.istarget_).sg_sum() / self.istarget_.sg_sum()
def sg_ce(tensor, opt): opt += tf.sg_opt(one_hot=False) assert opt.target is not None, 'target is mandatory.' if opt.one_hot: out = tf.identity(tf.nn.softmax_cross_entropy_with_logits(tensor, opt.target), 'ce') else: out = tf.identity(tf.nn.sparse_softmax_cross_entropy_with_logits(tensor, opt.target), 'ce') # masking loss if opt.mask: out *= tf.not_equal(opt.target, tf.zeros_like(opt.target)).sg_float() # add summary tf.sg_summary_loss(out) return out
def q_process(t1, t2): ''' Processes each training sample so that it fits in the queue. ''' # Lstrip zeros zeros = tf.equal(t1, tf.zeros_like(t1)).sg_int().sg_sum() t1 = t1[zeros:] t2 = t2[zeros:] # zero-PrePadding t1 = tf.concat([tf.zeros([Hyperparams.seqlen-1], tf.int32), t1], 0)# 49 zero-prepadding t2 = tf.concat([tf.zeros([Hyperparams.seqlen-1], tf.int32), t2], 0)# 49 zero-prepadding # radom crop stacked = tf.stack((t1, t2)) cropped = tf.random_crop(stacked, [2, Hyperparams.seqlen]) t1, t2 = cropped[0], cropped[1] t2 = t2[-1] return t1, t2
def __init__(self, mode="train"): ''' Args: is_train: Boolean. If True, backprop is executed. ''' if mode == "train": self.x, self.y, self.num_batch = get_batch_data( ) # (64, 50) int64, (64, 50) int64, 1636 else: # test self.x = tf.placeholder(tf.int64, [None, Hyperparams.maxlen]) # make embedding matrix for input characters pnyn2idx, _, hanzi2idx, _ = load_vocab() self.emb_x = tf.sg_emb(name='emb_x', voca_size=len(pnyn2idx), dim=Hyperparams.embed_dim) self.enc = self.x.sg_lookup(emb=self.emb_x) with tf.sg_context(size=5, act='relu', bn=True): for _ in range(20): dim = self.enc.get_shape().as_list()[-1] self.enc += self.enc.sg_conv1d( dim=dim) # (64, 50, 300) float32 # final fully convolutional layer for softmax self.logits = self.enc.sg_conv1d(size=1, dim=len(hanzi2idx), act='linear', bn=False) # (64, 50, 5072) float32 if mode == "train": self.ce = self.logits.sg_ce(target=self.y, mask=True) # (64, 50) float32 self.istarget = tf.not_equal(self.y, tf.zeros_like( self.y)).sg_float() # (64, 50) float32 self.reduced_loss = self.ce.sg_sum() / self.istarget.sg_sum( ) # () float32 tf.sg_summary_loss(self.reduced_loss, "reduced_loss")
def __init__(self, is_train=True): ''' Args: is_train: Boolean. If True, backprop is executed. ''' if is_train: self.x, self.y = get_batch_data() # (16, 100), (16, 100) else: # self.x = tf.placeholder(tf.int32, [Hyperparams.batch_size, Hyperparams.maxlen]) self.x = tf.placeholder(tf.int32, [None, Hyperparams.maxlen]) # make embedding matrix for input characters hangul2idx, _, hanja2idx, _ = load_charmaps() self.emb_x = tf.sg_emb(name='emb_x', voca_size=len(hangul2idx), dim=Hyperparams.hidden_dim) # embed table lookup self.enc = self.x.sg_lookup(emb=self.emb_x).sg_float() # (16, 100, 200) # loop dilated conv block for i in range(2): self.enc = (self.enc .sg_res_block(size=5, rate=1) .sg_res_block(size=5, rate=2) .sg_res_block(size=5, rate=4) .sg_res_block(size=5, rate=8) .sg_res_block(size=5, rate=16)) # final fully convolutional layer for softmax self.logits = self.enc.sg_conv1d(size=1, dim=len(hanja2idx)) # (16, 100, 4543) if is_train: self.ce = self.logits.sg_ce(target=self.y, mask=True) # (16, 100) self.nonzeros = tf.not_equal(self.y, tf.zeros_like(self.y)).sg_float() # (16, 100) self.reduced_loss = self.ce.sg_sum() / self.nonzeros.sg_sum() # () tf.sg_summary_loss(self.reduced_loss, "reduced_loss")
def sg_reverse_seq(tensor, opt): # default sequence dimension opt += tf.sg_opt(dim=1) seq_len = tf.not_equal(tensor, tf.zeros_like(tensor)).sg_int().sg_sum(dims=opt.dim) return tf.reverse_sequence(tensor, seq_len, opt.dim, name=opt.name)