def _decode(self): start_logits = tf.squeeze( conv(tf.concat([self.enc[1], self.enc[2]], axis=-1), 1, bias=False, name="start_pointer"), -1) end_logits = tf.squeeze( conv(tf.concat([self.enc[1], self.enc[3]], axis=-1), 1, bias=False, name="end_pointer"), -1) self.logits = [ mask_logits(start_logits, mask=self.c_mask), mask_logits(end_logits, mask=self.c_mask) ] self.logits1, self.logits2 = [l for l in self.logits] outer = tf.matmul(tf.expand_dims(tf.nn.softmax(self.logits1), axis=2), tf.expand_dims(tf.nn.softmax(self.logits2), axis=1)) outer = tf.matrix_band_part(outer, 0, self.max_a_len) self.yp1 = tf.argmax(tf.reduce_max(outer, axis=2), axis=1) self.yp2 = tf.argmax(tf.reduce_max(outer, axis=1), axis=1)
def _create_conv(self): self._create_input() arg_scope = tf.contrib.framework.arg_scope with arg_scope([conv], nl=tf.nn.relu, trainable=True, mode=self.mode, graph=self.graph): conv1 = conv(self.image, 7, 96, 'conv1') mean1, var1 = tf.nn.moments(conv1, [0,1,2]) conv1_bn = tf.nn.batch_normalization(conv1, mean1, var1, 0, 1, 1e-5) pool1 = max_pool(conv1_bn, 'pool1', padding='SAME') conv2 = conv(pool1, 5, 256, 'conv2') mean2, var2 = tf.nn.moments(conv2, [0,1,2]) conv2_bn = tf.nn.batch_normalization(conv2, mean2, var2, 0, 1, 1e-5) pool2 = max_pool(conv2_bn, 'pool2', padding='SAME') conv3 = conv(pool2, 3, 512, 'conv3', stride = 1) conv4 = conv(conv3, 3, 512, 'conv4', stride = 1) conv5 = conv(conv4, 3, 512, 'conv5', stride = 1) pool5 = max_pool(conv5, 'pool5', padding='SAME') self.layer['conv1'] = conv1 self.layer['conv2'] = conv2 self.layer['conv3'] = conv3 self.layer['conv4'] = conv4 self.layer['pool5'] = pool5 self.layer['conv_out'] = self.layer['conv5'] = conv5 return pool5
def _decode(self): # self.config.batch_size if not self.demo else 1, # self.max_p_len, # self.max_q_len, # self.config.max_ch_len, # self.config.hidden_size, # self.config.char_embed_size, # self.config.head_size N, PL, QL, CL, d, dc, nh = self._params() if self.config.use_position_attn: logits = tf.squeeze( conv(self._attention(self.enc[3], name="attn_logits"), 1, bias=True, name="logits", activation=None), -1) else: logits = tf.squeeze( conv(self.enc[3], 1, bias=True, name="logits", activation=None), -1) self.logits = tf.layers.dense( logits, self.max_a_len, use_bias=True, kernel_regularizer=tf.contrib.layers.l2_regularizer( self.config.weight_decay), activation=None, name='fully_connected') self.yp = tf.argmax(self.logits, axis=-1)
def _decode(self): N, PL, QL, CL, d, dc, nh = self._params() if self.config.use_position_attn: start_logits = tf.squeeze( conv(self._attention(tf.concat([self.enc[1], self.enc[2]], axis = -1), name="attn1"), 1, bias = False, name = "start_pointer"), -1) end_logits = tf.squeeze( conv(self._attention(tf.concat([self.enc[1], self.enc[3]], axis = -1), name="attn2"), 1, bias = False, name = "end_pointer"), -1) else: start_logits = tf.squeeze( conv(tf.concat([self.enc[1], self.enc[2]], axis = -1), 1, bias = False, name = "start_pointer"), -1) end_logits = tf.squeeze( conv(tf.concat([self.enc[1], self.enc[3]], axis = -1), 1, bias = False, name = "end_pointer"), -1) self.logits = [mask_logits(start_logits, mask = tf.reshape(self.c_mask, [N, -1])), mask_logits(end_logits, mask = tf.reshape(self.c_mask, [N, -1]))] self.logits1, self.logits2 = [l for l in self.logits] outer = tf.matmul(tf.expand_dims(tf.nn.softmax(self.logits1), axis=2), tf.expand_dims(tf.nn.softmax(self.logits2), axis=1)) outer = tf.matrix_band_part(outer, 0, self.max_a_len) self.yp1 = tf.argmax(tf.reduce_max(outer, axis=2), axis=1) self.yp2 = tf.argmax(tf.reduce_max(outer, axis=1), axis=1)
def pixelCNN(latents, num_iteration=5, depth=256, scope='pixel_cnn'): cres = latents cres_dim = cres.shape[-1] num_channel = depth with tf.variable_scope(scope, reuse=tf.AUTO_REUSE): for _ in range(num_iteration): c = layers.conv(cres, scope='conv1', filter_dims=[1, 1, num_channel], stride_dims=[1, 1], non_linear_fn=tf.nn.relu, bias=True) c = layers.conv(c, scope='conv2', filter_dims=[1, 3, num_channel], stride_dims=[1, 1], non_linear_fn=None, bias=True) padding = tf.constant([[0, 0], [1, 0], [0, 0], [0, 0]], name='padding') c = tf.pad(c, padding, name='pad') c = layers.conv(c, scope='conv3', filter_dims=[2, 1, num_channel], stride_dims=[1, 1], padding='VALID', non_linear_fn=tf.nn.relu, bias=True) c = layers.conv(c, scope='conv4', filter_dims=[1, 1, cres_dim], stride_dims=[1, 1], non_linear_fn=None, bias=True) cres = cres + c cres = tf.nn.relu(cres) return cres
def build(self): """Create the network graph.""" # 1st Layer: Conv (w ReLu) -> Lrn -> Pool conv1 = conv(self.x, 11, 11, 96, 4, 4, padding='VALID', name='conv1') norm1 = lrn(conv1, 2, 1e-05, 0.75, name='norm1') pool1 = max_pool(norm1, 3, 3, 2, 2, padding='VALID', name='pool1') # 2nd Layer: Conv (w ReLu) -> Lrn -> Pool with 2 groups conv2 = conv(pool1, 5, 5, 256, 1, 1, groups=2, name='conv2') norm2 = lrn(conv2, 2, 1e-05, 0.75, name='norm2') pool2 = max_pool(norm2, 3, 3, 2, 2, padding='VALID', name='pool2') # 3rd Layer: Conv (w ReLu) conv3 = conv(pool2, 3, 3, 384, 1, 1, name='conv3') # 4th Layer: Conv (w ReLu) splitted into two groups conv4 = conv(conv3, 3, 3, 384, 1, 1, groups=2, name='conv4') # 5th Layer: Conv (w ReLu) -> Pool splitted into two groups conv5 = conv(conv4, 3, 3, 256, 1, 1, groups=2, name='conv5') pool5 = max_pool(conv5, 3, 3, 2, 2, padding='VALID', name='pool5') # 6th Layer: Flatten -> FC (w ReLu) -> Dropout flattened = tf.reshape(pool5, [-1, 6 * 6 * 256]) fc6 = fc(flattened, 6 * 6 * 256, 4096, name='fc6') # 7th Layer: FC (w ReLu) -> Dropout fc7 = fc(fc6, 4096, 4096, name='fc7') # 8th Layer: FC and return unscaled activations self.fc8 = fc(fc7, 4096, self.num_classes, relu=False, name='fc8')
def graph_forward(): model = graph.Graph() model.add( layers.conv(layers.xaxier_initilizer, layers.zero_initilizer, 1, 1, 32, 4, 3, 3)) #model.add(layers.Relu()) model.add( layers.conv(layers.xaxier_initilizer, layers.zero_initilizer, 1, 2, 16, 32, 3, 3)) #model.add(layers.Relu()) model.add( layers.max_pooling(layers.xaxier_initilizer, layers.zero_initilizer, 0, 2, 2, 2)) model.add(layers.flatten()) model.add( layers.FullConn(layers.xaxier_initilizer, layers.zero_initilizer, (1024, 10))) #model.add(layers.Relu()) crit = layers.softmax_with_loss() y = np.array([1, 2, 3]) def foo(x): logits = model.forward(x) prob = crit.forward(logits) dy, loss = crit.backward(logits, y) dx = model.backward(x, dy) return loss, dx return foo
def clock_shufflenet(x,training, classes, update_3, update_2, last_frame_stage2,last_frame_stage3): """The whole network Args: x: the input batch training: whether we are in the process of training or inference classes: total number of classes update_3: True: update stage 3; False: use stage 3 output from last frame directly. When training always true update_2: True: update stage 2; False: use stage 2 output from last frame directly. When training always true last_frame_stage3: Stage three from last frame last_frame_stage2: Stafe two from last frame Returns: The output of the third stage """ stage1 = stage_1(x,training) stage2 = tf.case([(update_2,stage_2(stage1,training))],default=last_frame_stage2) stage3 = tf.case([(update_3,stage_3(stage2,training,classes))],default=last_frame_stage3) ########DECODER####### stage1_conv = layers.conv(stage1,"conv4",1,48,classes,1,training) stage2_conv = layers.conv(stage2,"conv3",1,96,classes,1,training) stage_2_and_3 = tf.add(stage2_conv,stage3) stage_2_and_3 = layers.up_sample_unit(stage_2_and_3,"us2",2,classes) stage_123 = tf.add(stage_2_and_3,stage1_conv) final_output = layers.up_sample_unit(stage_123,"us3",8,classes) return final_output, stage2, stage3
def context_transform(input_layer, scope='context_transform'): l = layers.conv(input_layer, scope=scope + '_1', filter_dims=[1, 1, target_dim], stride_dims=[1, 1], non_linear_fn=tf.nn.relu, bias=True) l = layers.conv(l, scope=scope + '_2', filter_dims=[1, 1, target_dim], stride_dims=[1, 1], non_linear_fn=None, bias=True) return l
def shallow_net_5x5(x): net = L.conv(x, name="conv_sn5x5_1", kh=5, kw=5, n_out=24) net = L.pool(net, name="pool_sn5x5_1", kh=2, kw=2, dw=2, dh=2) net = L.conv(net, name="conv_sn5x5_2", kw=3, kh=3, n_out=48) net = L.pool(net, name="pool_sn5x5_2", kh=2, kw=2, dw=2, dh=2) net = L.conv(net, name="conv_sn5x5_3", kw=3, kh=3, n_out=24) net = L.conv(net, name="conv_sn5x5_4", kw=3, kh=3, n_out=12) return net
def shallow_net_9x9(x): net = L.conv(x, name="conv_sn9x9_1", kh=9, kw=9, n_out=16) net = L.pool(net, name="pool_sn9x9_1", kh=2, kw=2, dw=2, dh=2) net = L.conv(net, name="conv_sn9x9_2", kw=7, kh=7, n_out=32) net = L.pool(net, name="pool_sn9x9_2", kh=2, kw=2, dw=2, dh=2) net = L.conv(net, name="conv_sn9x9_3", kw=7, kh=7, n_out=16) net = L.conv(net, name="conv_sn9x9_4", kw=7, kh=7, n_out=8) return net
def shallow_net_7x7(x): net = L.conv(x, name="conv_sn7x7_1", kh=7, kw=7, n_out=20) net = L.pool(net, name="pool_sn7x7_1", kh=2, kw=2, dw=2, dh=2) net = L.conv(net, name="conv_sn7x7_2", kw=5, kh=5, n_out=40) net = L.pool(net, name="pool_sn7x7_2", kh=2, kw=2, dw=2, dh=2) net = L.conv(net, name="conv_sn7x7_3", kw=5, kh=5, n_out=20) net = L.conv(net, name="conv_sn7x7_4", kw=5, kh=5, n_out=10) return net
def __build_net(self): """ Introduction ------------ 构建ONet模型结构 """ with tf.variable_scope('onet'): self.input = tf.placeholder(shape=[None, 48, 48, 3], dtype=tf.float32, name='input_data') layer = conv('conv1', self.input, kernel_size=(3, 3), channels_output=32, stride=(1, 1), padding='VALID', relu=False) layer = prelu('prelu1', layer) layer = max_pool('pool1', layer, kernel_size=(3, 3), stride=(2, 2)) layer = conv('conv2', layer, kernel_size=(3, 3), channels_output=64, stride=(1, 1), padding='VALID', relu=False) layer = prelu('prelu2', layer) layer = max_pool('pool2', layer, kernel_size=(3, 3), stride=(2, 2), padding='VALID') layer = conv('conv3', layer, kernel_size=(3, 3), channels_output=64, stride=(1, 1), padding='VALID', relu=False) layer = prelu('prelu3', layer) layer = max_pool('pool3', layer, kernel_size=(2, 2), stride=(2, 2)) layer = conv('conv4', layer, kernel_size=(2, 2), channels_output=128, stride=(1, 1), padding='VALID', relu=False) layer = prelu('prelu4', layer) layer = fc('fc1', layer, channels_output=256, relu=False) layer = prelu('prelu5', layer) fc2 = fc('fc2-1', layer, channels_output=2, relu=False) self.prob = tf.nn.softmax(fc2, axis=1, name='prob') self.loc = fc('fc2-2', layer, channels_output=4, relu=False)
def forward(self): config = self.config N, PL, QL, CL, d, dc, nh = config.batch_size if not self.demo else 1, \ self.c_maxlen, \ self.q_maxlen, \ config.char_limit, \ config.hidden, \ config.char_dim, \ config.num_heads with tf.variable_scope('Input_Embedding_Layer', regularizer=regularizer): # ******************** char embedding ********************* # [batch_size, seq_len, word_len] -> [batch_size x seq_len, word_len, char_dim] ch_emb = tf.reshape(tf.nn.embedding_lookup(self.char_mat, self.ch), shape=[N * PL, CL, dc]) qh_emb = tf.reshape(tf.nn.embedding_lookup(self.char_mat, self.qh), shape=[N * QL, CL, dc]) ch_emb = tf.nn.dropout(ch_emb, keep_prob=1.0 - 0.5 * self.dropout) qh_emb = tf.nn.dropout(qh_emb, keep_prob=1.0 - 0.5 * self.dropout) # BiDAF style conv-highway encoder, share weights # [N * PL/QL, CL, d] ch_emb = conv(ch_emb, d, bias=True, activation=tf.nn.relu, kernel_size=5, name='char_conv', reuse=None) qh_emb = conv(qh_emb, d, bias=True, activation=tf.nn.relu, kernel_size=5, name='char_conv', reuse=True) # [N * CL/QL, d], reduce max along CL ch_emb = tf.reduce_max(ch_emb, axis=1) qh_emb = tf.reduce_max(qh_emb, axis=1) # [N, PL/QL, d] ch_emb = tf.reshape(ch_emb, shape=[N, PL, ch_emb.shape[-1]]) qh_emb = tf.reshape(qh_emb, shape=[N, QL, ch_emb.shape[-1]]) # *********************** Word embedding ************************ # [N, PL/QL, dw] c_emb = tf.nn.dropout(tf.nn.embedding_lookup(self.word_mat, self.c), keep_prob=1.0 - self.dropout) q_emb = tf.nn.dropout(tf.nn.embedding_lookup(self.word_mat, self.q), keep_prob=1.0 - self.dropout) # Concat char embedding and word embedding # [N, PL/QL, dw + d] c_emb = tf.concat([c_emb, ch_emb], axis=2) q_emb = tf.concat([q_emb, qh_emb], axis=2) # share weights c_emb = highway(c_emb, size=d, scope='highway', dropout=self.dropout, reuse=None) q_emb = highway(q_emb, size=d, scope='highway', dropout=self.dropout, reuse=True) print('highway, q_emb.shape: {}'.format(q_emb.shape)) print('highway, c_emb.shape: {}'.format(c_emb.shape)) """ *************************************Encoer ****************************************""" with tf.variable_scope('Encoder_Layer', regularizer=regularizer):
def encoder(input): # Create a conv network with 3 conv layers and 1 FC layer # Conv 1: filter: [3, 3, 1], stride: [2, 2], relu conv1 = layers.conv(input, name = 'conv1', filter_dims=[3,3,1], stride_dims=[2,2]) # Conv 2: filter: [3, 3, 8], stride: [2, 2], relu conv2 = layers.conv(conv1, name = 'conv2', filter_dims=[3,3,8], stride_dims=[2,2]) # Conv 3: filter: [3, 3, 8], stride: [2, 2], relu conv3 = layers.conv(conv2, name = 'conv3', filter_dims=[3,3,8], stride_dims=[2,2]) # FC: output_dim: 100, no non-linearity fc = layers.fc(conv3, name='fc', out_dim = 100) return fc raise NotImplementedError
def discriminator(x, y, is_training=True, update_batch_stats=True, act_fn=L.lrelu, bn=FLAGS.dis_bn, reuse=True): with tf.variable_scope('discriminator', reuse=reuse): if FLAGS.method == 'cgan': h = L.fc(y, y_dim, X_dim * X_dim, seed=rng.randint(123456), name='fc_y') h = tf.reshape(h, [-1, X_dim, X_dim, 1]) h = tf.concat((x, h), axis=3) h = L.conv(h, 3, 1, num_channels + 1, 32, name="conv1") else: h = L.conv(x, 3, 1, num_channels, 32, name="conv1") h = act_fn(h) # 64x64 -> 32x32 h = L.conv( h, 4, 2, 32, 64, name="conv2", ) h = L.bn(h, 64, is_training=is_training, update_batch_stats=update_batch_stats, use_gamma=False, name='bn1') if bn else h h = act_fn(h) # 32x32 -> 16x16 h = L.conv(h, 4, 2, 64, 128, name="conv3") h = L.bn(h, 128, is_training=is_training, update_batch_stats=update_batch_stats, use_gamma=False, name='bn2') if bn else h h = act_fn(h) h = L.conv(h, X_dim / 4, 1, 128, 1, name="conv5", padding="VALID") logits = tf.reshape(h, [-1, 1]) return logits
def _decode(self): """ Employs Pointer Network to get the the probs of each position to be the start or end of the predicted answer. Note that we concat the fuse_p_encodes for the passages in the same document. And since the encodes of queries in the same document is same, we select the first one. """ N, PL, QL, CL, d, dc, nh = self._params() if self.use_position_attn: start_logits = tf.squeeze( conv(self._attention(tf.concat([self.enc[1], self.enc[2]], axis=-1), name="attn1"), 1, bias=False, name="start_pointer"), -1) end_logits = tf.squeeze( conv(self._attention(tf.concat([self.enc[1], self.enc[3]], axis=-1), name="attn2"), 1, bias=False, name="end_pointer"), -1) else: start_logits = tf.squeeze( conv(self.mul_anttion_p, 1, bias=False, name="start_pointer"), -1) end_logits = tf.squeeze( conv(self.mul_anttion_p, 1, bias=False, name="end_pointer"), -1) start_logits = tf.reshape(start_logits, [N, -1]) self.sl = start_logits end_logits = tf.reshape(end_logits, [N, -1]) self.el = end_logits self.logits = [ mask_logits(start_logits, mask=tf.reshape(self.c_mask, [N, -1])), mask_logits(end_logits, mask=tf.reshape(self.c_mask, [N, -1])) ] self.logits1, self.logits2 = [l for l in self.logits] outer = tf.matmul(tf.expand_dims(tf.nn.softmax(self.logits1), axis=2), tf.expand_dims(tf.nn.softmax(self.logits2), axis=1)) outer = tf.matrix_band_part(outer, 0, self.max_a_len) self.yp1 = tf.argmax(tf.reduce_max(outer, axis=2), axis=1) self.yp2 = tf.argmax(tf.reduce_max(outer, axis=1), axis=1)
def latent_discriminator(input_data, activation='swish', scope='ldiscriminator', reuse=False, bn_phaze=False): with tf.variable_scope(scope, reuse=tf.AUTO_REUSE): #if reuse: # tf.get_variable_scope().reuse_variables() if activation == 'swish': act_func = util.swish elif activation == 'relu': act_func = tf.nn.relu elif activation == 'tanh': act_func = tf.nn.tanh else: act_func = tf.nn.sigmoid l = tf.reshape(input_data, shape=[-1, 4, 4, 8]) l = layers.conv(l, scope='conv1', filter_dims=[3, 3, g_dense_block_depth/2], stride_dims=[1, 1], non_linear_fn=None, bias=False) l = add_residual_dense_block(l, filter_dims=[3, 3, g_dense_block_depth/2], num_layers=3, act_func=act_func, bn_phaze=bn_phaze, scope='block_0') l = add_residual_dense_block(l, filter_dims=[3, 3, g_dense_block_depth/2], num_layers=3, act_func=act_func, bn_phaze=bn_phaze, scope='block_1') l = add_residual_dense_block(l, filter_dims=[3, 3, g_dense_block_depth/2], num_layers=3, act_func=act_func, bn_phaze=bn_phaze, scope='block_2') l = layers.global_avg_pool(l, representation_dim) dc_final_layer = l dc_output = layers.fc(dc_final_layer, scope='g_enc_z_fc', out_dim=1, non_linear_fn=None) return dc_final_layer, dc_output, tf.sigmoid(dc_output)
def _prediction(self): self.batch_size = self.inputs['data'].get_shape().as_list()[0] current_layer = self.inputs['data'] n_filters = 20 ksize = 8 depth = 7 for i in range(depth): current_layer = layers.conv(inputs=current_layer, n_filters=n_filters, ksize=ksize, stride=2, scope='conv{}'.format(i + 1)) tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, current_layer) self.layers['conv{}'.format(i + 1)] = current_layer bs, width, _ = current_layer.get_shape().as_list() print(bs, width, _) bs, width, _ = current_layer.get_shape().as_list() print(bs, width, _) current_layer = tf.reshape(current_layer, [bs, width * n_filters]) current_layer = self.fcn(current_layer, 2, scope='logits') #current_layer = tf.nn.softmax(current_layer) self.layers['logits'] = current_layer tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, current_layer) self.layers['class_prob'] = tf.nn.softmax(current_layer, name='class_prob') self.layers['class_prediction'] = tf.argmax(self.layers['class_prob'], 1, name='class_pred') tf.contrib.layers.apply_regularization( tf.contrib.layers.l2_regularizer(1e-3), weights_list=tf.get_collection(tf.GraphKeys.WEIGHTS))
def test_conv_shape(self): with self.test_session() as sess: x = tf.zeros((32,227,227,3), dtype=tf.float32) expected_conv_out = tf.zeros((32,55,55,96),dtype=tf.float32, name='expectedout') actual_conv_out, weights, biases = layers.conv(x, 11, 11, 96, 4, 4, name='convtest', padding='VALID') sess.run(tf.initializers.variables([weights, biases])) self.assertAllEqual(tf.shape(actual_conv_out), tf.shape(expected_conv_out))
def task(x, activation='relu', output_dim=256, scope='task_network', norm='layer', b_train=False): with tf.variable_scope(scope, reuse=tf.AUTO_REUSE): if activation == 'swish': act_func = util.swish elif activation == 'relu': act_func = tf.nn.relu elif activation == 'lrelu': act_func = tf.nn.leaky_relu else: act_func = tf.nn.sigmoid print('Task Layer1: ' + str(x.get_shape().as_list())) block_depth = dense_block_depth l = x l = layers.conv(l, scope='conv1', filter_dims=[3, 3, block_depth], stride_dims=[1, 1], non_linear_fn=None, bias=False, dilation=[1, 1, 1, 1]) if norm == 'layer': l = layers.layer_norm(l, scope='ln1') elif norm == 'batch': l = layers.batch_norm_conv(l, b_train=b_train, scope='bn1') l = act_func(l) for i in range(15): l = layers.add_residual_block(l, filter_dims=[3, 3, block_depth], num_layers=2, act_func=act_func, norm=norm, b_train=b_train, scope='block1_' + str(i)) latent = layers.global_avg_pool(l, output_length=output_dim) return latent
def __build_net(self): """ Introduction 构建mtcnn模型级联第一层 """ with tf.variable_scope('pnet'): self.input = tf.placeholder(name='input_data', shape=[None, None, None, 3], dtype=tf.float32) layer = conv('conv1', self.input, kernel_size=(3, 3), channels_output=10, stride=(1, 1), padding='VALID', relu=False) layer = prelu('prelu1', layer) layer = max_pool('pool1', layer, kernel_size=[2, 2], stride=(2, 2)) layer = conv('conv2', layer, kernel_size=(3, 3), channels_output=16, stride=(1, 1), padding='VALID', relu=False) layer = prelu('prelu2', layer) layer = conv('conv3', layer, kernel_size=(3, 3), channels_output=32, stride=(1, 1), padding='VALID', relu=False) layer = prelu('prelu3', layer) conv4_1 = conv('conv4-1', layer, kernel_size=(1, 1), channels_output=2, stride=(1, 1), relu=False) self.prob = tf.nn.softmax(conv4_1, axis=3, name='prob') self.loc = conv('conv4-2', layer, kernel_size=(1, 1), channels_output=4, stride=(1, 1), relu=False)
def build_graph(self): self.iterator = tf.data.Iterator.from_structure( (tf.float32, tf.int32), (tf.TensorShape([None, 227, 227, 3]), tf.TensorShape([None])) ) self.inputs, self.labels = self.iterator.get_next() self.conv1 = layers.conv(self.inputs, [11, 11], 96, [4, 4], padding='VALID', name='conv1', mask=True) norm1 = layers.lrn(self.conv1, 2, 1e-05, 0.75, name='norm1') pool1 = layers.max_pool(norm1, [3, 3], [2, 2], padding='VALID', name='pool1') self.conv2 = layers.conv(pool1, [5, 5], 256, [1, 1], groups=2, name='conv2', mask=True) norm2 = layers.lrn(self.conv2, 2, 1e-05, 0.75, name='norm2') pool2 = layers.max_pool(norm2, [3, 3], [2, 2], padding='VALID', name='pool2') self.conv3 = layers.conv(pool2, [3, 3], 384, [1, 1], name='conv3', mask=True) self.conv4 = layers.conv(self.conv3, [3, 3], 384, [1, 1], groups=2, name='conv4', mask=True) self.conv5 = layers.conv(self.conv4, [3, 3], 256, [1, 1], groups=2, name='conv5', mask=True) pool5 = layers.max_pool(self.conv5, [3, 3], [2, 2], padding='VALID', name='pool5') self.keep_prob = tf.get_variable('keep_prob', shape=(), trainable=False) flattened = tf.reshape(pool5, [-1, 6 * 6 * 256]) fc6 = layers.fc(flattened, 4096, name='fc6') dropout6 = layers.dropout(fc6, self.keep_prob) fc7 = layers.fc(dropout6, 4096, name='fc7') dropout7 = layers.dropout(fc7, self.keep_prob) self.logits = layers.fc(dropout7, self.num_classes, relu=False, name='fc8') self.probs_op = tf.nn.softmax(self.logits) self.pred_op = tf.argmax(input=self.logits, axis=1) corrects_op = tf.equal(tf.cast(self.pred_op, tf.int32), self.labels) self.acc_op = tf.reduce_mean(tf.cast(corrects_op, tf.float32))
def logit(h, is_training=True, update_batch_stats=True, stochastic=True, seed=1234, dropout_mask=None, return_mask=False, h_before_dropout=None): rng = np.random.RandomState(seed) if h_before_dropout is None: h = L.conv(h, ksize=3, stride=1, f_in=3, f_out=128, seed=rng.randint(123456), name='c1') h = L.lrelu(L.bn(h, 128, is_training=is_training, update_batch_stats=update_batch_stats, name='b1'), FLAGS.lrelu_a) h = L.conv(h, ksize=3, stride=1, f_in=128, f_out=128, seed=rng.randint(123456), name='c2') h = L.lrelu(L.bn(h, 128, is_training=is_training, update_batch_stats=update_batch_stats, name='b2'), FLAGS.lrelu_a) h = L.conv(h, ksize=3, stride=1, f_in=128, f_out=128, seed=rng.randint(123456), name='c3') h = L.lrelu(L.bn(h, 128, is_training=is_training, update_batch_stats=update_batch_stats, name='b3'), FLAGS.lrelu_a) h = L.max_pool(h, ksize=2, stride=2) if stochastic: h = tf.nn.dropout(h, keep_prob=FLAGS.keep_prob_hidden) h = L.conv(h, ksize=3, stride=1, f_in=128, f_out=256, seed=rng.randint(123456), name='c4') h = L.lrelu(L.bn(h, 256, is_training=is_training, update_batch_stats=update_batch_stats, name='b4'), FLAGS.lrelu_a) h = L.conv(h, ksize=3, stride=1, f_in=256, f_out=256, seed=rng.randint(123456), name='c5') h = L.lrelu(L.bn(h, 256, is_training=is_training, update_batch_stats=update_batch_stats, name='b5'), FLAGS.lrelu_a) h = L.conv(h, ksize=3, stride=1, f_in=256, f_out=256, seed=rng.randint(123456), name='c6') h = L.lrelu(L.bn(h, 256, is_training=is_training, update_batch_stats=update_batch_stats, name='b6'), FLAGS.lrelu_a) h_before_dropout = L.max_pool(h, ksize=2, stride=2) # Making it possible to change or return a dropout mask if stochastic: if dropout_mask is None: dropout_mask = tf.cast( tf.greater_equal(tf.random_uniform(tf.shape(h_before_dropout), 0, 1, seed=rng.randint(123456)), 1.0 - FLAGS.keep_prob_hidden), tf.float32) else: dropout_mask = tf.reshape(dropout_mask, tf.shape(h_before_dropout)) h = tf.multiply(h_before_dropout, dropout_mask) h = (1.0 / FLAGS.keep_prob_hidden) * h else: h = h_before_dropout h = L.conv(h, ksize=3, stride=1, f_in=256, f_out=512, seed=rng.randint(123456), padding="VALID", name='c7') h = L.lrelu(L.bn(h, 512, is_training=is_training, update_batch_stats=update_batch_stats, name='b7'), FLAGS.lrelu_a) h = L.conv(h, ksize=1, stride=1, f_in=512, f_out=256, seed=rng.randint(123456), name='c8') h = L.lrelu(L.bn(h, 256, is_training=is_training, update_batch_stats=update_batch_stats, name='b8'), FLAGS.lrelu_a) h = L.conv(h, ksize=1, stride=1, f_in=256, f_out=128, seed=rng.randint(123456), name='c9') h = L.lrelu(L.bn(h, 128, is_training=is_training, update_batch_stats=update_batch_stats, name='b9'), FLAGS.lrelu_a) h = tf.reduce_mean(h, reduction_indices=[1, 2]) # Global average pooling h = L.fc(h, 128, 10, seed=rng.randint(123456), name='fc') if FLAGS.top_bn: h = L.bn(h, 10, is_training=is_training, update_batch_stats=update_batch_stats, name='bfc') if return_mask: return h, tf.reshape(dropout_mask, [-1, 8*8*256]), h_before_dropout else: return h
def discriminator(input_data, activation='swish', scope='discriminator', reuse=False, bn_phaze=False): with tf.variable_scope(scope, reuse=tf.AUTO_REUSE): #if reuse: # tf.get_variable_scope().reuse_variables() if activation == 'swish': act_func = util.swish elif activation == 'relu': act_func = tf.nn.relu elif activation == 'tanh': act_func = tf.nn.tanh else: act_func = tf.nn.sigmoid l = layers.conv(input_data, scope='conv1', filter_dims=[3, 3, g_dense_block_depth/2], stride_dims=[1, 1], non_linear_fn=None, bias=False) l = add_residual_dense_block(l, filter_dims=[3, 3, g_dense_block_depth/2], num_layers=3, act_func=act_func, bn_phaze=bn_phaze, scope='block_0') l = tf.nn.avg_pool(l, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') l = add_residual_dense_block(l, filter_dims=[3, 3, g_dense_block_depth/2], num_layers=3, act_func=act_func, bn_phaze=bn_phaze, scope='block_1') l = tf.nn.avg_pool(l, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') l = add_residual_dense_block(l, filter_dims=[3, 3, g_dense_block_depth/2], num_layers=3, act_func=act_func, bn_phaze=bn_phaze, scope='block_2') #l = tf.nn.avg_pool(l, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') #l = add_residual_dense_block(l, filter_dims=[3, 3, g_dense_block_depth/2], num_layers=3, # act_func=act_func, bn_phaze=bn_phaze, scope='block_3') #l = tf.nn.avg_pool(l, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') #l = add_residual_dense_block(l, filter_dims=[3, 3, g_dense_block_depth/2], num_layers=3, # act_func=act_func, bn_phaze=bn_phaze, scope='block_4') #l = add_residual_dense_block(l, filter_dims=[3, 3, g_dense_block_depth/2], num_layers=3, # act_func=act_func, bn_phaze=bn_phaze, scope='block_5') #l = tf.nn.avg_pool(l, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') #l = add_residual_dense_block(l, filter_dims=[3, 3, g_dense_block_depth], num_layers=3, # act_func=act_func, bn_phaze=bn_phaze, scope='block_6') #l = add_residual_dense_block(l, filter_dims=[3, 3, g_dense_block_depth], num_layers=3, # act_func=act_func, bn_phaze=bn_phaze, scope='block_7') # dc_final_layer = batch_norm_conv(last_dense_layer, b_train=bn_phaze, scope='last_dense_layer') l = layers.global_avg_pool(l, representation_dim) dc_final_layer = l dc_output = layers.fc(dc_final_layer, scope='g_enc_z_fc', out_dim=1, non_linear_fn=None) return dc_final_layer, dc_output, tf.sigmoid(dc_output)
def _guide(cls, input_tensor, params, is_training): n_guide_feats = params['guide_complexity'] guidemap = conv(input_tensor, n_guide_feats, 1, batch_norm=True, is_training=is_training, scope='conv1') guidemap = conv(guidemap, 1, 1, activation_fn=tf.nn.sigmoid, scope='conv2') guidemap = tf.squeeze(guidemap, squeeze_dims=[ 3, ]) return guidemap
def logit(x, dropout_mask=None, is_training=True, update_batch_stats=True, stochastic=True, seed=1234): rng = numpy.random.RandomState(seed) h = L.gl(x, std=FLAGS.sigma) h = L.conv(h, ksize=3, stride=1, f_in=3, f_out=layer_sizes[0], seed=rng.randint(123456), name='c1') h = L.lrelu(bn(h, layer_sizes[0], is_training=is_training, update_batch_stats=update_batch_stats, name='b1'), FLAGS.lrelu_a) h = L.conv(h, ksize=3, stride=1, f_in=layer_sizes[0], f_out=layer_sizes[0], seed=rng.randint(123456), name='c2') h = L.lrelu(bn(h, layer_sizes[0], is_training=is_training, update_batch_stats=update_batch_stats, name='b2'), FLAGS.lrelu_a) h = L.conv(h, ksize=3, stride=1, f_in=layer_sizes[0], f_out=layer_sizes[0], seed=rng.randint(123456), name='c3') h = L.lrelu(bn(h, layer_sizes[0], is_training=is_training, update_batch_stats=update_batch_stats, name='b3'), FLAGS.lrelu_a) h = L.max_pool(h, ksize=2, stride=2) h = tf.nn.dropout(h, keep_prob=0.5, seed=rng.randint(123456)) if stochastic else h h = L.conv(h, ksize=3, stride=1, f_in=layer_sizes[0], f_out=layer_sizes[1], seed=rng.randint(123456), name='c4') h = L.lrelu(bn(h, layer_sizes[1], is_training=is_training, update_batch_stats=update_batch_stats, name='b4'), FLAGS.lrelu_a) h = L.conv(h, ksize=3, stride=1, f_in=layer_sizes[1], f_out=layer_sizes[1], seed=rng.randint(123456), name='c5') h = L.lrelu(bn(h, layer_sizes[1], is_training=is_training, update_batch_stats=update_batch_stats, name='b5'), FLAGS.lrelu_a) h = L.conv(h, ksize=3, stride=1, f_in=layer_sizes[1], f_out=layer_sizes[1], seed=rng.randint(123456), name='c6') h = L.lrelu(bn(h, layer_sizes[1], is_training=is_training, update_batch_stats=update_batch_stats, name='b6'), FLAGS.lrelu_a) h = L.max_pool(h, ksize=2, stride=2) h = tf.nn.dropout(h, keep_prob=0.5, seed=rng.randint(123456)) if stochastic else h h = L.conv(h, ksize=3, stride=1, f_in=layer_sizes[1], f_out=layer_sizes[2], seed=rng.randint(123456), padding="VALID", name='c7') h = L.lrelu(bn(h, layer_sizes[2], is_training=is_training, update_batch_stats=update_batch_stats, name='b7'), FLAGS.lrelu_a) h = L.conv(h, ksize=1, stride=1, f_in=layer_sizes[2], f_out=layer_sizes[3], seed=rng.randint(123456), name='c8') h = L.lrelu(bn(h, layer_sizes[3], is_training=is_training, update_batch_stats=update_batch_stats, name='b8'), FLAGS.lrelu_a) h = L.conv(h, ksize=1, stride=1, f_in=layer_sizes[3], f_out=layer_sizes[4], seed=rng.randint(123456), name='c9') h = L.lrelu(bn(h, layer_sizes[4], is_training=is_training, update_batch_stats=update_batch_stats, name='b9'), FLAGS.lrelu_a) h = tf.reduce_mean(h, reduction_indices=[1, 2]) # Global average pooling # dropout with mask if dropout_mask is None: # Base dropout mask is 1 (Fully Connected) dropout_mask = tf.ones_like(h) h = h*dropout_mask h = L.fc(h, layer_sizes[4], 10, seed=rng.randint(123456), name='fc') if FLAGS.top_bn: h = bn(h, 10, is_training=is_training, update_batch_stats=update_batch_stats, name='bfc') return h, dropout_mask
def residual(name, l, in_channel, out_channel, stride): """Residual function. Args: name: Scope name of this function. l: Output of previous layer. in_channel: # of channels of l. out_channel: # of channels of each output feature. stride: Stride of the first convolution in residual function. """ with tf.variable_scope(name): sc = l if stride == 1 else shortcut(l, in_channel, out_channel) l = layers.conv('conv_0', l, out_channel, stride=stride) l = layers.batchnorm('bn_0', l, is_train) l = tf.nn.relu(l) l = layers.conv('conv_1', l, out_channel, stride=1) l = layers.batchnorm('bn_1', l, is_train) l = tf.nn.relu(l + sc) return l
def __init__(self): self.lr = 0.01 # conv net self.c1 = conv(1, 6, kernel=5, learning_rate=self.lr) self.relu1 = relu() self.s2 = max_pool(kernel=2, stride=2) self.c3 = conv(6, 16, kernel=5, learning_rate=self.lr) self.relu3 = relu() self.s4 = max_pool(kernel=2, stride=2) self.c5 = conv(16, 120, kernel=4, learning_rate=self.lr) self.relu5 = relu() # fc net self.f6 = fc(120, 84, learning_rate=self.lr) self.relu6 = relu() self.f7 = fc(84, 10) self.sig7 = softmax() # record the shape between the conv net and fc net self.conv_out_shape = None
def getConvLayers(self, input): conv = layers.conv(input, reuse=self.reuse, name='conv') genr64 = layers.genr(conv, 128, reuse=self.reuse, name='genr64') genr128 = layers.genr(genr64, 256, reuse=self.reuse, name='genr128') if not self.reuse: self.reuse = tf.AUTO_REUSE return conv, genr64, genr128
def forward(self): config = self.config N, PL, QL, CL, d, dc, nh = config.batch_size if not self.demo else 1, self.c_maxlen, self.q_maxlen, config.char_limit, config.hidden, config.char_dim, config.num_heads with tf.variable_scope("Input_Embedding_Layer"): ch_emb = tf.reshape(tf.nn.embedding_lookup( self.char_mat, self.ch), [N * PL, CL, dc]) qh_emb = tf.reshape(tf.nn.embedding_lookup( self.char_mat, self.qh), [N * QL, CL, dc]) ch_emb = tf.nn.dropout(ch_emb, 1.0 - 0.5 * self.dropout) qh_emb = tf.nn.dropout(qh_emb, 1.0 - 0.5 * self.dropout) # Bidaf style conv-highway encoder ch_emb = conv(ch_emb, d, bias = True, activation = tf.nn.relu, kernel_size = 5, name = "char_conv", reuse = None) qh_emb = conv(qh_emb, d, bias = True, activation = tf.nn.relu, kernel_size = 5, name = "char_conv", reuse = True) ch_emb = tf.reduce_max(ch_emb, axis = 1) qh_emb = tf.reduce_max(qh_emb, axis = 1) ch_emb = tf.reshape(ch_emb, [N, PL, ch_emb.shape[-1]]) qh_emb = tf.reshape(qh_emb, [N, QL, ch_emb.shape[-1]]) c_emb = tf.nn.dropout(tf.nn.embedding_lookup(self.word_mat, self.c), 1.0 - self.dropout) q_emb = tf.nn.dropout(tf.nn.embedding_lookup(self.word_mat, self.q), 1.0 - self.dropout) c_emb = tf.concat([c_emb, ch_emb], axis=2) q_emb = tf.concat([q_emb, qh_emb], axis=2) c_emb = highway(c_emb, size = d, scope = "highway", dropout = self.dropout, reuse = None) q_emb = highway(q_emb, size = d, scope = "highway", dropout = self.dropout, reuse = True) with tf.variable_scope("Embedding_Encoder_Layer"): c = residual_block(c_emb, num_blocks = 1, num_conv_layers = 4, kernel_size = 7, mask = self.c_mask, num_filters = d, num_heads = nh, seq_len = self.c_len, scope = "Encoder_Residual_Block", bias = False, dropout = self.dropout) q = residual_block(q_emb, num_blocks = 1, num_conv_layers = 4, kernel_size = 7, mask = self.q_mask, num_filters = d, num_heads = nh, seq_len = self.q_len, scope = "Encoder_Residual_Block", reuse = True, # Share the weights between passage and question bias = False, dropout = self.dropout) with tf.variable_scope("Context_to_Query_Attention_Layer"): # C = tf.tile(tf.expand_dims(c,2),[1,1,self.q_maxlen,1]) # Q = tf.tile(tf.expand_dims(q,1),[1,self.c_maxlen,1,1]) # S = trilinear([C, Q, C*Q], input_keep_prob = 1.0 - self.dropout) S = optimized_trilinear_for_attention([c, q], self.c_maxlen, self.q_maxlen, input_keep_prob = 1.0 - self.dropout) mask_q = tf.expand_dims(self.q_mask, 1) S_ = tf.nn.softmax(mask_logits(S, mask = mask_q)) mask_c = tf.expand_dims(self.c_mask, 2) S_T = tf.transpose(tf.nn.softmax(mask_logits(S, mask = mask_c), dim = 1),(0,2,1)) self.c2q = tf.matmul(S_, q) self.q2c = tf.matmul(tf.matmul(S_, S_T), c) attention_outputs = [c, self.c2q, c * self.c2q, c * self.q2c] with tf.variable_scope("Model_Encoder_Layer"): inputs = tf.concat(attention_outputs, axis = -1) self.enc = [conv(inputs, d, name = "input_projection")] for i in range(3): if i % 2 == 0: # dropout every 2 blocks self.enc[i] = tf.nn.dropout(self.enc[i], 1.0 - self.dropout) self.enc.append( residual_block(self.enc[i], num_blocks = 7, num_conv_layers = 2, kernel_size = 5, mask = self.c_mask, num_filters = d, num_heads = nh, seq_len = self.c_len, scope = "Model_Encoder", bias = False, reuse = True if i > 0 else None, dropout = self.dropout) ) with tf.variable_scope("Output_Layer"): start_logits = tf.squeeze(conv(tf.concat([self.enc[1], self.enc[2]],axis = -1),1, bias = False, name = "start_pointer"),-1) end_logits = tf.squeeze(conv(tf.concat([self.enc[1], self.enc[3]],axis = -1),1, bias = False, name = "end_pointer"), -1) self.logits = [mask_logits(start_logits, mask = self.c_mask), mask_logits(end_logits, mask = self.c_mask)] logits1, logits2 = [l for l in self.logits] outer = tf.matmul(tf.expand_dims(tf.nn.softmax(logits1), axis=2), tf.expand_dims(tf.nn.softmax(logits2), axis=1)) outer = tf.matrix_band_part(outer, 0, config.ans_limit) self.yp1 = tf.argmax(tf.reduce_max(outer, axis=2), axis=1) self.yp2 = tf.argmax(tf.reduce_max(outer, axis=1), axis=1) losses = tf.nn.softmax_cross_entropy_with_logits( logits=logits1, labels=self.y1) losses2 = tf.nn.softmax_cross_entropy_with_logits( logits=logits2, labels=self.y2) self.loss = tf.reduce_mean(losses + losses2) if config.l2_norm is not None: variables = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES) l2_loss = tf.contrib.layers.apply_regularization(regularizer, variables) self.loss += l2_loss if config.decay is not None: self.var_ema = tf.train.ExponentialMovingAverage(config.decay) ema_op = self.var_ema.apply(tf.trainable_variables()) with tf.control_dependencies([ema_op]): self.loss = tf.identity(self.loss) self.assign_vars = [] for var in tf.global_variables(): v = self.var_ema.average(var) if v: self.assign_vars.append(tf.assign(var,v))