def build_shake_shake_model(images, num_classes, hparams, is_training): """Builds the Shake-Shake model. Build the Shake-Shake model from https://arxiv.org/abs/1705.07485. Args: images: Tensor of images that will be fed into the Wide ResNet Model. num_classes: Number of classed that the model needs to predict. hparams: tf.HParams object that contains additional hparams needed to construct the model. In this case it is the `shake_shake_widen_factor` that is used to determine how many filters the model has. is_training: Is the model training or not. Returns: The logits of the Shake-Shake model. """ depth = 26 k = hparams.shake_shake_widen_factor # The widen factor n = int((depth - 2) / 6) x = images x = ops.conv2d(x, 16, 3, scope='init_conv') x = ops.batch_norm(x, scope='init_bn') with tf.variable_scope('L1'): x = _shake_shake_layer(x, 16 * k, n, 1, is_training) with tf.variable_scope('L2'): x = _shake_shake_layer(x, 32 * k, n, 2, is_training) with tf.variable_scope('L3'): x = _shake_shake_layer(x, 64 * k, n, 2, is_training) x = tf.nn.relu(x) x = ops.global_avg_pool(x) # Fully connected logits = ops.fc(x, num_classes) return logits
def context_embedding(self, inputs=None, if_reuse=None): template = fc(inputs, self.ef_dim, 'd_embedd/fc', activation_fn=tf.nn.leaky_relu, reuse=if_reuse) return template
def generate_condition(self, c_var): conditions = fc(c_var, self.ef_dim * 2, 'gen_cond/fc', activation_fn=tf.nn.leaky_relu) mean = conditions[:, :self.ef_dim] log_sigma = conditions[:, self.ef_dim:] return [mean, log_sigma]
def build_wrn_model(images, num_classes, wrn_size): """Builds the WRN model. Build the Wide ResNet model from https://arxiv.org/abs/1605.07146. Args: images: Tensor of images that will be fed into the Wide ResNet Model. num_classes: Number of classed that the model needs to predict. wrn_size: Parameter that scales the number of filters in the Wide ResNet model. Returns: The logits of the Wide ResNet model. """ kernel_size = wrn_size filter_size = 3 num_blocks_per_resnet = 4 filters = [ min(kernel_size, 16), kernel_size, kernel_size * 2, kernel_size * 4 ] strides = [1, 2, 2] # stride for each resblock # Run the first conv with tf.variable_scope('init'): x = images output_filters = filters[0] x = ops.conv2d(x, output_filters, filter_size, scope='init_conv') first_x = x # Res from the beginning orig_x = x # Res from previous block for block_num in range(1, 4): with tf.variable_scope('unit_{}_0'.format(block_num)): activate_before_residual = True if block_num == 1 else False x = residual_block( x, filters[block_num - 1], filters[block_num], strides[block_num - 1], activate_before_residual=activate_before_residual) for i in range(1, num_blocks_per_resnet): with tf.variable_scope('unit_{}_{}'.format(block_num, i)): x = residual_block( x, filters[block_num], filters[block_num], 1, activate_before_residual=False) x, orig_x = _res_add(filters[block_num - 1], filters[block_num], strides[block_num - 1], x, orig_x) final_stride_val = np.prod(strides) x, _ = _res_add(filters[0], filters[3], final_stride_val, x, first_x) with tf.variable_scope('unit_last'): x = ops.batch_norm(x, scope='final_bn') x = tf.nn.relu(x) x = ops.global_avg_pool(x) logits = ops.fc(x, num_classes) return logits
def build_wrn_model(images, num_classes, wrn_size): """Builds the WRN model. Build the Wide ResNet model from https://arxiv.org/abs/1605.07146. Args: images: Tensor of images that will be fed into the Wide ResNet Model. num_classes: Number of classed that the model needs to predict. wrn_size: Parameter that scales the number of filters in the Wide ResNet model. Returns: The logits of the Wide ResNet model. """ kernel_size = wrn_size filter_size = 3 num_blocks_per_resnet = 4 filters = [ min(kernel_size, 16), kernel_size, kernel_size * 2, kernel_size * 4 ] strides = [1, 2, 2] # stride for each resblock # Run the first conv with tf.variable_scope('init'): x = images output_filters = filters[0] x = ops.conv2d(x, output_filters, filter_size, scope='init_conv') first_x = x # Res from the beginning orig_x = x # Res from previous block for block_num in range(1, 4): with tf.variable_scope('unit_{}_0'.format(block_num)): activate_before_residual = True if block_num == 1 else False x = residual_block( x, filters[block_num - 1], filters[block_num], strides[block_num - 1], activate_before_residual=activate_before_residual) for i in range(1, num_blocks_per_resnet): with tf.variable_scope('unit_{}_{}'.format(block_num, i)): x = residual_block(x, filters[block_num], filters[block_num], 1, activate_before_residual=False) x, orig_x = _res_add(filters[block_num - 1], filters[block_num], strides[block_num - 1], x, orig_x) final_stride_val = np.prod(strides) x, _ = _res_add(filters[0], filters[3], final_stride_val, x, first_x) with tf.variable_scope('unit_last'): x = ops.batch_norm(x, scope='final_bn') x = tf.nn.relu(x) x = ops.global_avg_pool(x) logits = ops.fc(x, num_classes) return logits
def build_wrn_model(images, num_classes, hparams): """Builds the WRN model. Build the Wide ResNet model from https://arxiv.org/abs/1605.07146. Args: images: Tensor of images that will be fed into the Wide ResNet Model. num_classes: Number of classed that the model needs to predict. hparams: hparams. Returns: The logits of the Wide ResNet model. """ kernel_size = hparams.wrn_size filter_size = 3 num_blocks_per_resnet = 4 filters = [ min(kernel_size, 16), kernel_size, kernel_size * 2, kernel_size * 4 ] strides = [1, 2, 2] # stride for each resblock # Run the first conv with tf.variable_scope('init'): x = images output_filters = filters[0] x = ops.conv2d(x, output_filters, filter_size, scope='init_conv', position='input') layer = 0 for block_num in range(1, 4): with tf.variable_scope('unit_{}_0'.format(block_num)): x = residual_block(x, filters[block_num - 1], filters[block_num], strides[block_num - 1], hparams=hparams, layer=layer) layer += 1 for i in range(1, num_blocks_per_resnet): with tf.variable_scope('unit_{}_{}'.format(block_num, i)): x = residual_block(x, filters[block_num], filters[block_num], 1, hparams=hparams, layer=layer) layer += 1 with tf.variable_scope('unit_last'): x = ops.maybe_normalize(x, scope='final_bn') x = activation_fn(x, hparams=hparams, layer=layer) hiddens = ops.global_avg_pool(x) logits = ops.fc(hiddens, num_classes) return logits, hiddens
def generator_simple(self, z_var, training=True): output_tensor = fc(z_var, self.s16 * self.s16 * self.gf_dim * 8, 'g_simple_OT/fc') output_tensor = reshape(output_tensor, [-1, self.s16, self.s16, self.gf_dim * 8], name='g_simple_OT/reshape') output_tensor = conv_batch_normalization(output_tensor, 'g_simple_OT/batch_norm', activation_fn=tf.nn.relu, is_training=training) output_tensor = Deconv2d(output_tensor, [0, self.s8, self.s8, self.gf_dim * 4], name='g_simple_OT/deconv2d', k_h=4, k_w=4) output_tensor = conv_batch_normalization(output_tensor, 'g_simple_OT/batch_norm2', activation_fn=tf.nn.relu, is_training=training) output_tensor = Deconv2d(output_tensor, [0, self.s4, self.s4, self.gf_dim * 2], name='g_simple_OT/deconv2d2', k_h=4, k_w=4) output_tensor = conv_batch_normalization(output_tensor, 'g_simple_OT/batch_norm3', activation_fn=tf.nn.relu, is_training=training) output_tensor = Deconv2d(output_tensor, [0, self.s2, self.s2, self.gf_dim], name='g_simple_OT/deconv2d3', k_h=4, k_w=4) output_tensor = conv_batch_normalization(output_tensor, 'g_simple_OT/batch_norm4', activation_fn=tf.nn.relu, is_training=training) output_tensor = Deconv2d(output_tensor, [0] + list(self.image_shape), name='g_simple_OT/deconv2d4', k_h=4, k_w=4, activation_fn=tf.nn.tanh) return output_tensor
def build_architecture(self): filter_size = 3 strides = [1, 2, 2] filters = [16, 16, 32, 64] if self.model_name == 'resnet_8': num_blocks_per_resnet = [1, 1, 1] elif self.model_name == 'resnet_14': num_blocks_per_resnet = [2, 2, 2] elif self.model_name == 'resnet_20': num_blocks_per_resnet = [3, 3, 3] elif self.model_name == 'resnet_26': num_blocks_per_resnet = [4, 4, 4] # First Convolutional Network with tf.variable_scope('init'): x = self.X output_filters = filters[0] x = ops.conv2d(x, output_filters, 3, 1, scope='init_conv', weight_decay=self.weight_decay) x = ops.batch_norm(x, 0.9, scope='init_bn') #x = tf.contrib.layers.batch_norm(x, decay=0.9, scope='init_bn', epsilon=0.00005, is_training=True) x = tf.nn.relu(x) for block_num in range(1, 4): with tf.variable_scope('unit_{}_0'.format(block_num)): x = no_relu_residual_block(x, filters[block_num - 1], filters[block_num], strides[block_num - 1], self.weight_decay) for i in range(1, num_blocks_per_resnet[block_num - 1]): with tf.variable_scope('unit_{}_{}'.format(block_num, i)): x = no_relu_residual_block(x, filters[block_num], filters[block_num], 1, self.weight_decay) print("Building.....") with tf.variable_scope('unit_last'): x = ops.global_avg_pool(x) self.orig_logits = ops.fc(x, self.num_classes, self.weight_decay)
def build_wrn_model(input_data, num_classes, wrn_size): kernel_size = 3 features = [min(wrn_size, 16), wrn_size, wrn_size * 2, wrn_size * 4] strides = [1, 2, 2] # stride for each resblock # create the first convolutional layer with tf.variable_scope('init'): x = ops.conv2d(input_data, features[0], kernel_size, scope='init_conv') first_x = x orig_x = x # create 2nd, 3rd, 4th resnet layers, two convs per res_block, four blocks per res layer. n=(28-4)/6 for num_res_layer in range(1, 4): with tf.variable_scope('unit_{}_0'.format(num_res_layer)): activate_before_res = True if num_res_layer == 1 else False x = res_block(orig_x, features[num_res_layer - 1], features[num_res_layer], stride=strides[num_res_layer - 1], activate_before_res=activate_before_res) for num_block in range(1, 4): with tf.variable_scope('unit_{}_{}'.format( num_res_layer, num_block)): x = res_block(x, features[num_res_layer], features[num_res_layer], 1, activate_before_res=False) x, orig_x = res_add(x, orig_x, features[num_res_layer - 1], features[num_res_layer], strides[num_res_layer - 1]) final_stride = np.prod(strides) x, _ = res_add(x, first_x, features[0], features[3], final_stride) with tf.variable_scope('unit_last'): x = ops.batch_norm(x, scope='final_bn') x = tf.nn.relu(x) x = ops.global_avg_pool(x) logits = ops.fc(x, num_classes) return logits
def generator(self, z_var, training=True): node1_0 = fc(z_var, self.s16 * self.s16 * self.gf_dim * 8, 'g_n1.0/fc') node1_0 = fc_batch_normalization(node1_0, 'g_n1.0/batch_norm') node1_0 = reshape(node1_0, [-1, self.s16, self.s16, self.gf_dim * 8], name='g_n1.0/reshape') node1_1 = Conv2d(node1_0, 1, 1, self.gf_dim * 2, 1, 1, name='g_n1.1/conv2d') node1_1 = conv_batch_normalization(node1_1, 'g_n1.1/batch_norm_1', activation_fn=tf.nn.relu, is_training=training) node1_1 = Conv2d(node1_1, 3, 3, self.gf_dim * 2, 1, 1, name='g_n1.1/conv2d2') node1_1 = conv_batch_normalization(node1_1, 'g_n1.1/batch_norm_2', activation_fn=tf.nn.relu, is_training=training) node1_1 = Conv2d(node1_1, 3, 3, self.gf_dim * 8, 1, 1, name='g_n1.1/conv2d3') node1_1 = conv_batch_normalization(node1_1, 'g_n1.1/batch_norm_3', activation_fn=tf.nn.relu, is_training=training) node1 = add([node1_0, node1_1], name='g_n1_res/add') node1_output = tf.nn.relu(node1) node2_0 = UpSample(node1_output, size=[self.s8, self.s8], method=1, align_corners=False, name='g_n2.0/upsample') node2_0 = Conv2d(node2_0, 3, 3, self.gf_dim * 4, 1, 1, name='g_n2.0/conv2d') node2_0 = conv_batch_normalization(node2_0, 'g_n2.0/batch_norm', is_training=training) node2_1 = Conv2d(node2_0, 1, 1, self.gf_dim * 1, 1, 1, name='g_n2.1/conv2d') node2_1 = conv_batch_normalization(node2_1, 'g_n2.1/batch_norm', activation_fn=tf.nn.relu, is_training=training) node2_1 = Conv2d(node2_1, 3, 3, self.gf_dim * 1, 1, 1, name='g_n2.1/conv2d2') node2_1 = conv_batch_normalization(node2_1, 'g_n2.1/batch_norm2', activation_fn=tf.nn.relu, is_training=training) node2_1 = Conv2d(node2_1, 3, 3, self.gf_dim * 4, 1, 1, name='g_n2.1/conv2d3') node2_1 = conv_batch_normalization(node2_1, 'g_n2.1/batch_norm3', is_training=training) node2 = add([node2_0, node2_1], name='g_n2_res/add') node2_output = tf.nn.relu(node2) output_tensor = UpSample(node2_output, size=[self.s4, self.s4], method=1, align_corners=False, name='g_OT/upsample') output_tensor = Conv2d(output_tensor, 3, 3, self.gf_dim * 2, 1, 1, name='g_OT/conv2d') output_tensor = conv_batch_normalization(output_tensor, 'g_OT/batch_norm', activation_fn=tf.nn.relu, is_training=training) output_tensor = UpSample(output_tensor, size=[self.s2, self.s2], method=1, align_corners=False, name='g_OT/upsample2') output_tensor = Conv2d(output_tensor, 3, 3, self.gf_dim, 1, 1, name='g_OT/conv2d2') output_tensor = conv_batch_normalization(output_tensor, 'g_OT/batch_norm2', activation_fn=tf.nn.relu, is_training=training) output_tensor = UpSample(output_tensor, size=[self.s, self.s], method=1, align_corners=False, name='g_OT/upsample3') output_tensor = Conv2d(output_tensor, 3, 3, 3, 1, 1, activation_fn=tf.nn.tanh, name='g_OT/conv2d3') return output_tensor
def build_shake_drop_model(images, num_classes, is_training): """Builds the PyramidNet Shake-Drop model. Build the PyramidNet Shake-Drop model from https://arxiv.org/abs/1802.02375. Args: images: Tensor of images that will be fed into the Wide ResNet Model. num_classes: Number of classed that the model needs to predict. is_training: Is the model training or not. Returns: The logits of the PyramidNet Shake-Drop model. """ # ShakeDrop Hparams p_l = 0.5 alpha_shake = [-1, 1] beta_shake = [0, 1] # PyramidNet Hparams alpha = 200 depth = 272 # This is for the bottleneck architecture specifically n = int((depth - 2) / 9) start_channel = 16 add_channel = alpha / (3 * n) # Building the models x = images x = ops.conv2d(x, 16, 3, scope='init_conv') x = ops.batch_norm(x, scope='init_bn') layer_num = 1 total_layers = n * 3 start_channel += add_channel prob = calc_prob(layer_num, total_layers, p_l) x = bottleneck_layer( x, round_int(start_channel), 1, prob, is_training, alpha_shake, beta_shake) layer_num += 1 for _ in range(1, n): start_channel += add_channel prob = calc_prob(layer_num, total_layers, p_l) x = bottleneck_layer( x, round_int(start_channel), 1, prob, is_training, alpha_shake, beta_shake) layer_num += 1 start_channel += add_channel prob = calc_prob(layer_num, total_layers, p_l) x = bottleneck_layer( x, round_int(start_channel), 2, prob, is_training, alpha_shake, beta_shake) layer_num += 1 for _ in range(1, n): start_channel += add_channel prob = calc_prob(layer_num, total_layers, p_l) x = bottleneck_layer( x, round_int(start_channel), 1, prob, is_training, alpha_shake, beta_shake) layer_num += 1 start_channel += add_channel prob = calc_prob(layer_num, total_layers, p_l) x = bottleneck_layer( x, round_int(start_channel), 2, prob, is_training, alpha_shake, beta_shake) layer_num += 1 for _ in range(1, n): start_channel += add_channel prob = calc_prob(layer_num, total_layers, p_l) x = bottleneck_layer( x, round_int(start_channel), 1, prob, is_training, alpha_shake, beta_shake) layer_num += 1 assert layer_num - 1 == total_layers x = ops.batch_norm(x, scope='final_bn') x = tf.nn.relu(x) x = ops.global_avg_pool(x) # Fully connected logits = ops.fc(x, num_classes) return logits