def __init__(self): self.input = tf.placeholder(tf.float32, shape=(9)) self.targetQ = tf.placeholder(tf.float32, shape=(5)) features = [tf.reshape(self.input, [-1])] regularizer = layers.l2_regularizer(0.01) # Structure features = layers.fully_connected(features, 100, weights_regularizer=regularizer) features = layers.bias_add(features, regularizer=regularizer) features = layers.fully_connected(features, 100, weights_regularizer=regularizer) features = layers.bias_add(features, regularizer=regularizer) features = layers.fully_connected(features, 5, weights_regularizer=regularizer) self.predict = features[0] self.loss = tf.reduce_sum(tf.square(self.targetQ - self.predict)) trainer = tf.train.AdamOptimizer() self.train = trainer.minimize(self.loss)
def conv1D(x, inp, out, kernel, stride, name): W = tf.get_variable(name + "W", shape=[kernel, inp, out], initializer=tf.contrib.layers.xavier_initializer()) x = tf.nn.conv1d(x, W, stride, 'SAME') x = layers.bias_add(x) return x
def darkconv(*args, **kwargs): scope = kwargs.pop('scope', None) onlyconv = kwargs.pop('onlyconv', False) with tf.variable_scope(scope): conv_kwargs = { 'padding': 'SAME', 'activation_fn': None, 'weights_initializer': variance_scaling_initializer(1.53846), 'weights_regularizer': l2(5e-4), 'biases_initializer': None, 'scope': 'conv' } if onlyconv: conv_kwargs.pop('biases_initializer') with arg_scope([conv2d], **conv_kwargs): x = conv2d(*args, **kwargs) if onlyconv: return x x = batch_norm(x, decay=0.99, center=False, scale=True, epsilon=1e-5, scope='bn') x = bias_add(x, scope='bias') x = leaky_relu(x, alpha=0.1, name='lrelu') return x
def conv1D(x, inp, out, kernel, stride, dilation_rate, name, use_bias=True, activation=None, batch_norm=False): try: with tf.variable_scope("conv"): W = tf.get_variable( name + "W", shape=[kernel, inp, out], initializer=tf.contrib.layers.xavier_initializer()) except: with tf.variable_scope("conv", reuse=True): W = tf.get_variable( name + "W", shape=[kernel, inp, out], initializer=tf.contrib.layers.xavier_initializer()) x = tf.nn.convolution(input=x, filter=W, strides=(stride, ), dilation_rate=(dilation_rate, ), padding="SAME", data_format="NWC") if use_bias: x = layers.bias_add(x) if batch_norm: x = layers.batch_norm(x) if activation is not None: x = activation(x) return x
def conv(X, do_activation_first=True): if do_activation_first: X = activation(X) X = conv2d(X, num_kernels, kernel_size, activation_fn=None, normalizer_fn=conv_normalization) X = tf.nn.dropout(X, keep_prob) X = bias_add(X) return X
def __init__(self): self.input = tf.placeholder(tf.float32, shape=(210, 160, 3)) self.targetQ = tf.placeholder(tf.float32, shape=(4)) features = [tf.reshape(self.input, [-1])] features = features[0:210 * 160] regularizer = layers.l2_regularizer(0.01) # Structure features = layers.bias_add(features, regularizer=regularizer) features = layers.fully_connected(features, 550, weights_regularizer=regularizer) features = layers.bias_add(features, regularizer=regularizer) features = layers.dropout(features) #features = layers.fully_connected(features, 1520, weights_regularizer=regularizer) #features = layers.fully_connected(features, 2010) #features = layers.bias_add(features, regularizer=regularizer) #features = layers.fully_connected(features, 700, weights_regularizer=regularizer) features = layers.bias_add(features, regularizer=regularizer) features = layers.fully_connected(features, 200, weights_regularizer=regularizer) features = layers.bias_add(features, regularizer=regularizer) features = layers.fully_connected(features, 100, weights_regularizer=regularizer) features = layers.bias_add(features, regularizer=regularizer) features = layers.fully_connected(features, 4, weights_regularizer=regularizer) self.predict = features[0] self.loss = tf.reduce_sum(tf.square(self.targetQ - self.predict)) trainer = tf.train.AdamOptimizer() self.train = trainer.minimize(self.loss)
def DBlock(self, last, channels, resblocks=1, kernel=[3, 3], stride=[2, 2], biases=True, format=DATA_FORMAT, activation=ACTIVATION, normalizer=None, regularizer=None, collections=None): # residual blocks for i in range(resblocks): with tf.variable_scope('ResBlock_{}'.format(i)): last = self.ResBlock(last, self.res_kernel, biases=False, format=format, activation=self.activation_res, normalizer=normalizer, regularizer=regularizer, collections=collections) # pre-activation if activation: last = activation(last) # up-convolution if stride[-1] > 1: last = layers.upscale2d_conv2d(last, channels, kernel[-1], format=format) # last = layers.blur2d(last, format=format) else: last = layers.conv2d(last, channels, kernel[-1], format=format) # bias if biases: last = slim.bias_add(last, variables_collections=collections, data_format=format) return last
X = conv(X, num_kernels) + conv(bn(conv(X, num_kernels)), num_kernels) X = tf.nn.dropout(X, drop_keep_prob, name='dropout') with tf.name_scope("residual_convolution") as node: X += conv(relu(bn(conv(relu(bn(X)), num_kernels))), num_kernels) X = tf.nn.dropout(X, drop_keep_prob, name='dropout') X = pool(X) with tf.name_scope("normalization") as node: X = bn(X) #X = tf.reduce_mean(X, [1, 2]) with tf.name_scope("flattening") as node: X = flatten(X) with tf.name_scope("hidden") as scope: with tf.name_scope('bias') as node: X = bias_add(X) with tf.name_scope('weights') as node: X = linear(X, num_hidden, activation_fn=relu, normalizer_fn=batch_norm) with tf.name_scope("output-Layer") as scope: X = linear(X, num_labels, activation_fn=None) Y = X ####################################################### with tf.name_scope('training'): with tf.name_scope('cross_entropy'): with tf.name_scope('total'): loss = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(Y, labels)) tf.summary.scalar('loss', loss) with tf.name_scope('accuracy'): with tf.name_scope('correct_prediction'):
def __init__(self, data, training=False): self.data = data self.initializer = tf.orthogonal_initializer() q_mask = make_mask(self.data.ql, 25) # (1, L_q, E) s_mask = make_mask(self.data.sl, 29) # (N, L_s, E) a_mask = make_mask(self.data.al, 34) # (5, L_a, E) ques_shape = tf.shape(q_mask) subt_shape = tf.shape(s_mask) ans_shape = tf.shape(a_mask) with tf.variable_scope('Embedding'): self.embedding = tf.get_variable('embedding_matrix', initializer=np.load( _mp.embedding_file), trainable=False) self.ques = tf.nn.embedding_lookup(self.embedding, self.data.ques) # (1, L_q, E) self.ans = tf.nn.embedding_lookup(self.embedding, self.data.ans) # (5, L_a, E) self.subt = tf.nn.embedding_lookup(self.embedding, self.data.subt) # (N, L_s, E) # self.ques = dropout(self.ques, training=training) # (1, L_q, E) # self.ans = dropout(self.ans, training=training) # (5, L_a, E) # self.subt = dropout(self.subt, training=training) # (N, L_s, E) with tf.variable_scope('Embedding_Linear'): # (1, L_q, E_t) self.ques_embedding = unit_norm( mask_dense(self.ques, q_mask, reuse=False)) # (5, L_a, E_t) self.ans_embedding = unit_norm(mask_dense(self.ans, a_mask)) # (N, L_s, E_t) self.subt_embedding = unit_norm(mask_dense(self.subt, s_mask)) with tf.variable_scope('Language_Encode'): mask = tf.expand_dims(tf.sequence_mask(self.data.ql, 25), axis=-1) # (1, E_t) self.ques_enc = unit_norm( conv_encode(self.ques_embedding, mask, reuse=False) + dilated_conv_encode(self.ques_embedding, mask, reuse=False), dim=1) mask = tf.expand_dims(tf.sequence_mask(self.data.al, 34), axis=-1) # (5, E_t) self.ans_enc = unit_norm( conv_encode(self.ans_embedding, mask) + dilated_conv_encode(self.ans_embedding, mask), dim=1) mask = tf.expand_dims(tf.sequence_mask(self.data.sl, 29), axis=-1) # (N, E_t) self.subt_enc = unit_norm( conv_encode(self.subt_embedding, mask) + dilated_conv_encode(self.subt_embedding, mask), dim=1) with tf.variable_scope('Temporal_Attention'): # (N, E_t) self.temp_attn = dense(self.ques_enc, use_bias=False, activation=None, reuse=False) + \ dense(self.subt_enc, use_bias=False, activation=None, reuse=False) self.temp_attn = dense(layers.bias_add(self.temp_attn, tf.nn.tanh), units=1, use_bias=False, activation=None, reuse=False) amount = tf.squeeze( dense(self.ques_enc, 1, tf.nn.sigmoid, reuse=False)) nth = nn.nth_element( tf.transpose(self.temp_attn), tf.cast(tf.cast(subt_shape[0], tf.float32) * amount, tf.int32), True) # (N, 1) attn_mask = tf.cast( tf.squeeze(tf.greater_equal(self.temp_attn, nth), axis=1), tf.int32) _, self.subt_enc = tf.dynamic_partition(self.subt_enc, attn_mask, 2) _, self.temp_attn = tf.dynamic_partition(self.temp_attn, attn_mask, 2) self.subt_enc = self.subt_enc * self.temp_attn self.summarize = unit_norm(tf.reduce_mean(self.subt_enc, axis=0, keepdims=True), dim=1) # (1, 4 * E_t) # gamma = tf.get_variable('gamma', [1, 1], initializer=tf.zeros_initializer) # self.ans_vec = self.summarize * tf.nn.sigmoid(gamma) + \ # tf.squeeze(self.ques_enc, axis=0) * (1 - tf.nn.sigmoid(gamma)) self.ans_vec = unit_norm(self.summarize + self.ques_enc, dim=1) # (1, 4 * E_t) self.output = tf.matmul(self.ans_vec, self.ans_enc, transpose_b=True) # (1, 5)
def quantizable_separable_conv2d(inputs, num_outputs, kernel_size, is_quantized=True, depth_multiplier=1, stride=1, activation_fn=tf.nn.relu6, normalizer_fn=None, scope=None): """Quantization friendly backward compatible separable conv2d. This op has the same API is separable_conv2d. The main difference is that an additional BiasAdd is manually inserted after the depthwise conv, such that the depthwise bias will not have name conflict with pointwise bias. The motivation of this op is that quantization script need BiasAdd in order to recognize the op, in which a native call to separable_conv2d do not create for the depthwise conv. Args: inputs: A tensor of size [batch_size, height, width, channels]. num_outputs: The number of pointwise convolution output filters. If is None, then we skip the pointwise convolution stage. kernel_size: A list of length 2: [kernel_height, kernel_width] of the filters. Can be an int if both values are the same. is_quantized: flag to enable/disable quantization. depth_multiplier: The number of depthwise convolution output channels for each input channel. The total number of depthwise convolution output channels will be equal to num_filters_in * depth_multiplier. stride: A list of length 2: [stride_height, stride_width], specifying the depthwise convolution stride. Can be an int if both strides are the same. activation_fn: Activation function. The default value is a ReLU function. Explicitly set it to None to skip it and maintain a linear activation. normalizer_fn: Normalization function to use instead of biases. scope: Optional scope for variable_scope. Returns: Tensor resulting from concatenation of input tensors """ if is_quantized: outputs = contrib_layers.separable_conv2d( inputs, None, kernel_size, depth_multiplier=depth_multiplier, stride=1, activation_fn=None, normalizer_fn=None, biases_initializer=None, scope=scope) outputs = contrib_layers.bias_add(outputs, trainable=True, scope='%s_bias' % scope) outputs = contrib_layers.conv2d(outputs, num_outputs, [1, 1], activation_fn=activation_fn, stride=stride, normalizer_fn=normalizer_fn, scope=scope) else: outputs = contrib_layers.separable_conv2d( inputs, num_outputs, kernel_size, depth_multiplier=depth_multiplier, stride=stride, activation_fn=activation_fn, normalizer_fn=normalizer_fn, scope=scope) return outputs
def __init__(self, data, training=False): self.data = data self.initializer = layers.xavier_initializer() with tf.variable_scope('Embedding_Linear'): # (1, L_q, E_t) self.ques = l2_norm( tf.layers.dense(self.data.ques, hp['emb_dim'], activation=tf.nn.relu, kernel_initializer=self.initializer, bias_initializer=self.initializer)) # (5, L_a, E_t) self.ans = l2_norm( tf.layers.dense(self.data.ans, hp['emb_dim'], activation=tf.nn.relu, kernel_initializer=self.initializer, bias_initializer=self.initializer, reuse=True)) # (N, L_s, E_t) self.subt = l2_norm( tf.layers.dense(self.data.subt, hp['emb_dim'], activation=tf.nn.relu, kernel_initializer=self.initializer, bias_initializer=self.initializer, reuse=True)) # self.ques = self.data.ques # self.ans = self.data.ans # self.subt = self.data.subt with tf.variable_scope('Temporal_Attention'): # (N, E_t) self.temp_attn = tf.layers.dense(self.ques, hp['emb_dim'], use_bias=False, activation=None, kernel_initializer=self.initializer) + \ tf.layers.dense(self.subt, hp['emb_dim'], use_bias=False, activation=None, kernel_initializer=self.initializer) self.temp_attn = tf.layers.dense( layers.bias_add(self.temp_attn, tf.nn.tanh, initializer=self.initializer), units=1, use_bias=False, activation=None, kernel_initializer=self.initializer) self.temp_attn = tf.nn.softmax(self.temp_attn, axis=1) # amount = tf.squeeze(dense(self.ques_enc, 1, tf.nn.sigmoid, reuse=False)) # nth = nn.nth_element(tf.transpose(self.temp_attn), # tf.cast(tf.cast(subt_shape[0], tf.float32) * amount, tf.int32), True) # # (N, 1) # attn_mask = tf.cast(tf.squeeze(tf.greater_equal(self.temp_attn, nth), axis=1), tf.int32) # _, self.subt_enc = tf.dynamic_partition(self.subt_enc, attn_mask, 2) # _, self.temp_attn = tf.dynamic_partition(self.temp_attn, attn_mask, 2) self.subt_temp = self.subt * self.temp_attn self.summarize = l2_norm( tf.reduce_sum(self.subt_temp, axis=0, keepdims=True)) # (1, E_t) # gamma = tf.get_variable('gamma', [1, 1], initializer=tf.zeros_initializer) # self.ans_vec = self.summarize * tf.nn.sigmoid(gamma) + \ # tf.squeeze(self.ques_enc, axis=0) * (1 - tf.nn.sigmoid(gamma)) self.ans_vec = l2_norm(self.summarize + self.ques) # (1, E_t) self.output = tf.matmul(self.ans_vec, self.ans, transpose_b=True) # (1, 5)