def build_model(data_tensor, reuse, training): """Create the hgru from Learning long-range...""" with tf.variable_scope('cnn', reuse=reuse): with tf.variable_scope('input', reuse=reuse): conv_aux = { 'pretrained': os.path.join('weights', 'gabors_for_contours_7.npy'), 'pretrained_key': 's1', 'nonlinearity': 'square' } x = conv.conv_layer(bottom=data_tensor, name='gabor_input', stride=[1, 1, 1, 1], padding='SAME', trainable=training, use_bias=True, aux=conv_aux) layer_hgru = hgru.hGRU('hgru_1', x_shape=x.get_shape().as_list(), timesteps=8, h_ext=15, strides=[1, 1, 1, 1], padding='SAME', aux={ 'lesion_alpha': True, 'lesion_omega': True }, train=training) h2 = layer_hgru.build(x) h2 = normalization.batch( bottom=h2, reuse=reuse, # renorm=True, name='hgru_bn', training=training) with tf.variable_scope('readout_1', reuse=reuse): activity = conv.conv_layer(bottom=h2, name='pre_readout_conv', num_filters=2, kernel_size=1, trainable=training, use_bias=False) pool_aux = {'pool_type': 'max'} activity = pooling.global_pool(bottom=activity, name='pre_readout_pool', aux=pool_aux) activity = normalization.batch( bottom=activity, reuse=reuse, # renorm=True, name='readout_1_bn', training=training) with tf.variable_scope('readout_2', reuse=reuse): activity = tf.layers.flatten(activity, name='flat_readout') activity = tf.layers.dense(inputs=activity, units=2) return activity, h2
def build_model(data_tensor, reuse, training): """Create the gru from Learning long-range...""" with tf.variable_scope('cnn', reuse=reuse): with tf.variable_scope('input', reuse=reuse): conv_aux = { 'pretrained': os.path.join('weights', 'gabors_for_contours_7.npy'), 'pretrained_key': 's1', 'nonlinearity': 'square' } x = conv.conv_layer(bottom=data_tensor, name='gabor_input', stride=[1, 1, 1, 1], padding='SAME', trainable=training, use_bias=True, aux=conv_aux) activity = conv.conv_layer(bottom=x, name='c1', num_filters=9, kernel_size=20, trainable=training, use_bias=False) activity = normalization.batch(bottom=activity, name='c1_bn', training=training) activity = tf.nn.relu(activity) with tf.variable_scope('readout_1', reuse=reuse): activity = conv.conv_layer(bottom=activity, name='pre_readout_conv', num_filters=2, kernel_size=1, trainable=training, use_bias=False) pool_aux = {'pool_type': 'max'} activity = pooling.global_pool(bottom=activity, name='pre_readout_pool', aux=pool_aux) activity = normalization.batch(bottom=activity, name='readout_1_bn', training=training) with tf.variable_scope('readout_2', reuse=reuse): activity = tf.layers.flatten(activity, name='flat_readout') activity = tf.layers.dense(inputs=activity, units=2) extra_activities = {'activity': activity} return activity, extra_activities
def full(self, i0, caps_match, caps2_norm, labels, labels_net_change): """ DEPRECIATED """ labels_new = helpers.label_backward( labels, labels, caps_match, self.labels_filter, self.decay_const, self.b, labels_gate_scale=self.labels_gate_scale, labels_gate_bias=self.labels_gate_bias, caps1_norm=caps2_norm, labels_update_mode=self.labels_update_mode, sensitivity_const=self.sensitivity_const, pushpull_scale=self.pushpull_scale, pushpull_bias=self.pushpull_bias, cap_labels=self.cap_labels, fixed_labels_mask=self.fixed_labels_mask) if self.bn_per_time: print('horizontal: using batchnorm per ts') labels_new = normalization.batch(bottom=labels_new, renorm=True, name='in_model_bn_' + str(i0), training=True) # Iterate loop labels_net_change += tf.sqrt(tf.square(labels_new - labels)) i0 += 1 return i0, caps_match, caps2_norm, labels_new, labels_net_change
def up_block(layer_name, bottom, skip_activity, reuse, kernel_size, num_filters, training, stride=[2, 2, 1], use_bias=False): """Forward block for seung model.""" with tf.variable_scope('%s_block' % layer_name, reuse=reuse): with tf.variable_scope('%s_layer_1' % layer_name, reuse=reuse): x = conv3d_transpose_layer(bottom=bottom, name='%s_1' % layer_name, stride=stride, padding='SAME', num_filters=num_filters, kernel_size=kernel_size, trainable=training, use_bias=use_bias) x = x + skip_activity # Rethink if this is valid x = normalization.batch(bottom=x, name='%s_bn_1' % layer_name, training=training) x = tf.nn.elu(x) return x
def up_block(inputs, skip, up_filters, name, training, reuse, up_kernels=4, up_strides=(2, 2), up_padding='same', nl=tf.nn.relu): """Do a unet upsample.""" upact = tf.layers.conv2d_transpose(inputs=inputs, filters=up_filters, kernel_size=up_kernels, strides=up_strides, name='%s_up' % name, padding=up_padding) upact = nl(upact) upact = normalization.batch(bottom=upact, name='%s_bn' % name, training=training, reuse=reuse) return conv_block(x=upact + skip, filters=up_filters, name='%s_skipped' % name, training=training, reuse=reuse)
def up_layer(layer_name, bottom, reuse, kernel_size, num_filters, training, trainable=None, stride=[2, 2], padding='same', kernel_initializer=None, renorm=False, use_bias=True): """Wrapper for transpose convolutions.""" if trainable is None: trainable = training with tf.variable_scope('%s_block' % layer_name, reuse=reuse): with tf.variable_scope('%s_layer' % layer_name, reuse=reuse): x = tf.layers.conv2d_transpose( inputs=bottom, filters=num_filters, kernel_size=kernel_size, name=layer_name, strides=stride, padding=padding, kernel_initializer=kernel_initializer, trainable=training, use_bias=use_bias) x = tf.nn.elu(x) x = normalization.batch(bottom=x, name='%s_bn' % layer_name, renorm=renorm, training=training) return x
def conv_block(x, filters, kernel_size=3, strides=(1, 1), padding='same', kernel_initializer=tf.initializers.variance_scaling, data_format='channels_last', activation=tf.nn.relu, name=None, training=True, reuse=False, batchnorm=True, pool=True): """VGG conv block.""" assert name is not None, 'Give the conv block a name.' activity = tf.layers.conv2d(inputs=x, filters=filters, kernel_size=kernel_size, strides=strides, padding=padding, data_format=data_format, activation=activation, name='%s_conv' % name, reuse=reuse) if batchnorm: activity = normalization.batch(bottom=activity, name='%s_bn' % name, training=training, reuse=reuse) if pool: return pooling.max_pool(bottom=activity, name='%s_pool' % name) else: return activity
def build_model(data_tensor, reuse, training, output_shape): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[0] with tf.variable_scope('cnn', reuse=reuse): # Add input so_filters = np.load( os.path.join('weights', 'so_filters.npy')).squeeze().reshape( 11, 11, 3, 8 * 4)[..., 0:32:4] so_filter_tensor = tf.get_variable( name='so_filters', initializer=so_filters, trainable=training) so_bias = tf.get_variable( name='so_bias', initializer=tf.zeros(so_filters.shape[-1]), trainable=training) in_emb = tf.nn.conv2d( input=data_tensor, filter=so_filter_tensor, strides=[1, 1, 1, 1], padding='SAME', name='so') in_emb = tf.nn.bias_add(in_emb, so_bias) in_emb = in_emb ** 2 layer_hgru = hgru.hGRU( 'hgru_1', x_shape=in_emb.get_shape().as_list(), timesteps=12, h_ext=1, crf=15, strides=[1, 1, 1, 1], padding='SAME', aux={ 'reuse': False, 'constrain': False, 'nonnegative': True, 'while_loop': False, 'hidden_states': 'gru', 'horizontal_dilations': [1, 1, 1, 1] }, train=training) h2 = layer_hgru.build(in_emb) h2 = normalization.batch( bottom=h2, renorm=False, name='hgru_bn', training=training) activity = conv.readout_layer( activity=h2, reuse=reuse, training=training, pool_type='max', # 'select', output_shape=output_shape, features=so_filters.shape[-1]) extra_activities = { 'activity': h2 } return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[0] with tf.variable_scope('cnn', reuse=reuse): # Add input in_emb = conv.skinny_input_layer( X=data_tensor, reuse=reuse, training=training, features=24, conv_activation=tf.nn.elu, conv_kernel_size=7, pool=False, name='l0') layer_hgru = feedback_hgru.hGRU( layer_name='hgru_1', x_shape=in_emb.get_shape().as_list(), timesteps=8, h_ext=[[9, 9], [5, 5], [1, 1]], strides=[1, 1, 1, 1], pool_strides=[4, 4], padding='SAME', aux={ 'symmetric_weights': True, 'dilations': [[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]], 'batch_norm': True, 'while_loop': False, 'nonnegative': True, 'residual': True, 'final_bn': False, 'renorm': False, 'lesion_alpha': True, 'lesion_omega': True, 'pooling_kernel': [4, 4], 'intermediate_ff': [24, 24], # + filters, 'intermediate_ks': [[5, 5], [5, 5]]}, train=training) h2 = layer_hgru.build(in_emb) nh2 = normalization.batch( bottom=h2, name='hgru_bn', fused=True, renorm=True, training=training) activity = conv.conv_layer( bottom=nh2, name='pre_readout_conv', num_filters=output_shape['output'], kernel_size=1, trainable=training, use_bias=True) extra_activities = { # 'activity': h2 } return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[0] with tf.variable_scope('cnn', reuse=reuse): # Add input in_emb = conv.skinny_input_layer( X=data_tensor, reuse=reuse, training=training, features=24, conv_activation=tf.nn.elu, conv_kernel_size=7, pool=False, name='l0') layer_hgru = hgru.hGRU( 'fgru', x_shape=in_emb.get_shape().as_list(), timesteps=8, h_ext=[{'h1': [9, 9]}, {'h2': [5, 5]}, {'fb1': [1, 1]}], strides=[1, 1, 1, 1], hgru_ids=[{'h1': 24}, {'h2': 24}, {'fb1': 24}], hgru_idx=[{'h1': 0}, {'h2': 1}, {'fb1': 2}], padding='SAME', aux={ 'readout': 'fb', 'intermediate_ff': [24, 24], 'intermediate_ks': [[5, 5], [5, 5]], 'intermediate_repeats': [2, 2], 'while_loop': False, 'skip': False, 'symmetric_weights': True, 'use_homunculus': False, 'include_pooling': True }, pool_strides=[4, 4], pooling_kernel=[4, 4], train=training) h2 = layer_hgru.build(in_emb) nh2 = normalization.batch( bottom=h2, name='hgru_bn', fused=True, renorm=True, training=training) activity = conv.conv_layer( bottom=nh2, name='pre_readout_conv', num_filters=output_shape['output'], kernel_size=1, trainable=training, use_bias=True) extra_activities = { # 'activity': h2 } return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[0] with tf.variable_scope('cnn', reuse=reuse): # Add input in_emb = conv.input_layer( X=data_tensor, reuse=reuse, training=training, features=20, conv_activation=[tf.nn.relu, None], conv_kernel_size=7, pool_kernel_size=[1, 2, 2, 1], pool_kernel_strides=[1, 2, 2, 1], pool=True, name='l0') layer_hgru = hgru.hGRU( 'fgru', x_shape=in_emb.get_shape().as_list(), timesteps=1, h_ext=[{'h1': [15, 15]}, {'h2': [1, 1]}, {'fb1': [1, 1]}], strides=[1, 1, 1, 1], hgru_ids=[{'h1': 20}, {'h2': 128}, {'fb1': 20}], hgru_idx=[{'h1': 0}, {'h2': 1}, {'fb1': 2}], padding='SAME', aux={ 'readout': 'fb', 'intermediate_ff': [32, 128], 'intermediate_ks': [[3, 3], [3, 3]], 'intermediate_repeats': [3, 3], 'while_loop': False, 'skip': False, 'force_horizontal': True, 'symmetric_weights': True, 'include_pooling': True }, pool_strides=[2, 2], pooling_kernel=[2, 2], train=training) h2 = layer_hgru.build(in_emb) h2 = normalization.batch( bottom=h2, renorm=False, name='hgru_bn', training=training) activity = conv.readout_layer( activity=h2, reuse=reuse, training=training, output_shape=output_shape) extra_activities = { 'activity': h2 } return activity, extra_activities
def build(self, caps, labels): self.prepare_tensors() i0 = tf.constant(0) labels_change = tf.zeros_like(labels) # Custom iterations caps_intermediate1, caps_match, caps1_norm, caps2_norm = \ helpers.get_caps_match( caps, caps, self.caps_filter, self.tolerance_const, self.c, self.b, self.eps) if self.use_while_loop: elms = [i0, caps_match, caps2_norm, labels, labels_change] returned = tf.while_loop(self.condition, self.full, loop_vars=elms, back_prop=True, swap_memory=False) i0, caps_match, caps2_norm, labels, labels_change = returned else: labels_old = labels for t in range(self.timesteps): labels = helpers.label_backward( labels_old, labels_old, caps_match, self.labels_filter, self.decay_const, self.b, labels_gate_scale=self.labels_gate_scale, labels_gate_bias=self.labels_gate_bias, caps1_norm=caps2_norm, labels_update_mode=self.labels_update_mode, sensitivity_const=self.sensitivity_const, pushpull_scale=self.pushpull_scale, pushpull_bias=self.pushpull_bias, cap_labels=self.cap_labels, fixed_labels_mask=self.fixed_labels_mask) if self.bn_per_time: print('horizontal: using batchnorm per ts') labels = normalization.batch(bottom=labels, renorm=True, name='in_model_bn_' + str(i0), training=True) # Iterate loop labels_change += tf.sqrt(tf.square(labels - labels_old)) labels_old = labels return caps, labels, labels_change
def build_model(data_tensor, reuse, training, output_shape): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[0] with tf.variable_scope('cnn', reuse=reuse): # Add input so_filters = np.load( '/media/data_cifs/clicktionary/pretrained_weights/alexnet.npy' ).item() so_filter_tensor = tf.get_variable(name='so_filters', initializer=so_filters['conv1'][0], trainable=training) so_bias = tf.get_variable(name='so_bias', initializer=so_filters['conv1'][1], trainable=training) in_emb = tf.nn.conv2d(input=data_tensor, filter=so_filter_tensor, strides=[1, 1, 1, 1], padding='SAME', name='so') in_emb = tf.nn.bias_add(in_emb, so_bias) # in_emb = in_emb ** 2 layer_hgru = hgru.hGRU('hgru_1', x_shape=in_emb.get_shape().as_list(), timesteps=8, h_ext=11, strides=[1, 1, 1, 1], padding='SAME', aux={ 'reuse': False, 'constrain': False, 'nonnegative': True, 'while_loop': False, 'horizontal_dilations': [1, 2, 2, 1] }, train=training) h2 = layer_hgru.build(in_emb) h2 = normalization.batch(bottom=h2, renorm=False, name='hgru_bn', training=training) activity = conv.readout_layer( activity=h2, reuse=reuse, training=training, pool_type='max', # 'select', output_shape=output_shape, features=output_shape) extra_activities = {'activity': h2} return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[0] alpha = 5. beta = 1. bs = data_tensor.get_shape().as_list()[0] dist = tf.distributions.Beta(alpha, beta) rand = dist.sample([]) sel = tf.cast(tf.round(tf.random_uniform([], minval=1, maxval=bs - 1)), tf.int32) with tf.variable_scope('cnn', reuse=reuse): with tf.variable_scope('input', reuse=reuse): o_x = tf.layers.conv2d(inputs=data_tensor, filters=24, kernel_size=11, name='l0', strides=(1, 1), padding='same', activation=tf.nn.relu, trainable=training, use_bias=True) x = rand * o_x + ((1 - rand) * tf.roll(o_x, sel, 0)) layer_hgru = hgru.hGRU('hgru_1', x_shape=x.get_shape().as_list(), timesteps=8, h_ext=15, strides=[1, 1, 1, 1], padding='SAME', aux={ 'reuse': False, 'constrain': False }, train=training) h2, inh = layer_hgru.build(x) with tf.variable_scope('ro', reuse=reuse): h2 = normalization.batch(bottom=h2, renorm=False, name='hgru_bn', training=training) activity = conv.readout_layer(activity=h2, reuse=reuse, training=training, output_shape=output_shape) extra_activities = {'activity': h2} return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[0] reduction_filters = 16 with tf.variable_scope('cnn', reuse=reuse): # Add input so_filters = np.load(os.path.join('weights', 'so_filters.npy')).squeeze().reshape( 11, 11, 3, 8 * 4) in_emb = tf.nn.conv2d(input=data_tensor, filter=so_filters, strides=[1, 1, 1, 1], padding='SAME', name='so') in_emb = tf.layers.conv2d(inputs=in_emb, filters=reduction_filters, kernel_size=(1, 1)) in_emb = in_emb**2 layer_hgru = hgru.hGRU('hgru_1', x_shape=in_emb.get_shape().as_list(), timesteps=8, h_ext=11, strides=[1, 1, 1, 1], padding='SAME', aux={ 'reuse': False, 'constrain': False, 'nonnegative': True, 'while_loop': False }, train=training) h2 = layer_hgru.build(in_emb) h2 = normalization.batch(bottom=h2, renorm=False, name='hgru_bn', training=training) activity = conv.readout_layer( activity=h2, reuse=reuse, training=training, pool_type='max', # 'select', output_shape=output_shape, features=reduction_filters) extra_activities = {'activity': h2} return activity, extra_activities
def up_block( layer_name, bottom, skip_activity, reuse, kernel_size, num_filters, training, trainable=None, stride=[2, 2], padding='same', kernel_initializer=None, normalization_type='batch_norm', data_format='channels_last', renorm=False, use_bias=False): """Forward block for seung model.""" if trainable is None: trainable = training with tf.variable_scope('%s_block' % layer_name, reuse=reuse): with tf.variable_scope('%s_layer_1' % layer_name, reuse=reuse): x = tf.layers.conv2d_transpose( inputs=bottom, filters=num_filters, kernel_size=kernel_size, name='%s_1' % layer_name, strides=stride, padding=padding, kernel_initializer=kernel_initializer, trainable=trainable, use_bias=use_bias) x = x + skip_activity # Rethink if this is valid if normalization_type is 'batch_norm': x = normalization.batch( bottom=x, name='%s_bn_1' % layer_name, data_format=data_format, renorm=renorm, trainable=trainable, training=training) else: x = normalization.instance( bottom=x, training=training) x = tf.nn.elu(x) return x
def build_model(data_tensor, reuse, training, output_shape): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[0] with tf.variable_scope('cnn', reuse=reuse): with tf.variable_scope('input', reuse=reuse): x = tf.layers.conv2d(inputs=data_tensor, filters=24, kernel_size=11, name='l0', strides=(1, 1), padding='same', activation=tf.nn.relu, trainable=training, use_bias=True) layer_hgru = hgru.hGRU('hgru_1', x_shape=x.get_shape().as_list(), timesteps=8, h_ext=15, strides=[1, 1, 1, 1], padding='SAME', aux={ 'reuse': False, 'constrain': False }, train=training) h2 = layer_hgru.build(x) h2 = normalization.batch(bottom=h2, renorm=True, name='hgru_bn', training=training) with tf.variable_scope('readout_1', reuse=reuse): activity = conv.conv_layer(bottom=h2, name='readout_conv', num_filters=1, kernel_size=1, trainable=training, use_bias=True) extra_activities = {'activity': h2} return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[0] with tf.variable_scope('cnn', reuse=reuse): # Add input in_emb = conv.input_layer( X=data_tensor, reuse=reuse, training=training, features=20, conv_activation=[tf.nn.relu, None], conv_kernel_size=7, pool_kernel_size=[1, 2, 2, 1], pool_kernel_strides=[1, 2, 2, 1], pool=True, name='l0') layer_hgru = hgru.hGRU( 'hgru_1', x_shape=in_emb.get_shape().as_list(), timesteps=8, h_ext=15, strides=[1, 1, 1, 1], padding='SAME', aux={'reuse': False, 'constrain': False}, train=training) h2 = layer_hgru.build(in_emb) h2 = normalization.batch( bottom=h2, renorm=False, name='hgru_bn', training=training) activity = conv.readout_layer( activity=h2, reuse=reuse, training=training, output_shape=output_shape) extra_activities = { 'activity': h2 } return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[0] with tf.variable_scope('cnn', reuse=reuse): # Add input conv_aux = { 'pretrained': os.path.join('weights', 'gabors_for_contours_11.npy'), 'pretrained_key': 's1', 'nonlinearity': 'square' } activity = conv.conv_layer(bottom=data_tensor, name='gabor_input', stride=[1, 1, 1, 1], padding='SAME', trainable=training, use_bias=True, aux=conv_aux) layer_hgru = hgru.hGRU('hgru_1', x_shape=activity.get_shape().as_list(), timesteps=8, h_ext=15, strides=[1, 1, 1, 1], padding='SAME', aux={ 'reuse': False, 'constrain': False }, train=training) h2 = layer_hgru.build(activity) h2 = normalization.batch(bottom=h2, renorm=False, name='hgru_bn', training=training) activity = conv.readout_layer(activity=h2, reuse=reuse, training=training, output_shape=output_shape) extra_activities = {'activity': h2} return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[0] with tf.variable_scope('cnn', reuse=reuse): # Add input in_emb = conv.skinny_input_layer(X=data_tensor, reuse=reuse, training=training, features=24, conv_activation=tf.nn.elu, conv_kernel_size=7, pool=False, name='l0') layer_hgru = hgru.hGRU('hgru_1', x_shape=in_emb.get_shape().as_list(), timesteps=8, h_ext=9, strides=[1, 1, 1, 1], padding='SAME', aux={ 'reuse': False, 'constrain': False, 'nonnegative': True }, train=training) h2 = layer_hgru.build(in_emb) h2 = normalization.batch(bottom=h2, renorm=True, name='hgru_bn', training=training) activity = conv.conv_layer(bottom=h2, name='pre_readout_conv', num_filters=output_shape['output'], kernel_size=1, trainable=training, use_bias=True) extra_activities = {} return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[0] alpha = 5. beta = 1. bs = data_tensor.get_shape().as_list()[0] dist = tf.distributions.Beta(alpha, beta) rand = dist.sample([]) sel = tf.cast(tf.round(tf.random_uniform([], minval=1, maxval=bs - 1)), tf.int32) with tf.variable_scope('cnn', reuse=reuse): with tf.variable_scope('input', reuse=reuse): o_x = tf.layers.conv2d(inputs=data_tensor, filters=24, kernel_size=11, name='l0', strides=(1, 1), padding='same', activation=tf.nn.relu, trainable=training, use_bias=True) # x = rand * o_x + ((1 - rand) * tf.roll(o_x, sel, 0)) layer_hgru = hgru.hGRU('hgru_1', x_shape=o_x.get_shape().as_list(), timesteps=8, h_ext=15, strides=[1, 1, 1, 1], padding='SAME', aux={ 'reuse': False, 'constrain': False }, train=training) h2, inh = layer_hgru.build(o_x) with tf.variable_scope('ss_readout', reuse=reuse): o_x = normalization.batch(bottom=o_x, renorm=False, name='hgru_x', training=training) inh = normalization.batch(bottom=inh, renorm=False, name='hgru_inh', training=training) layers = [[o_x, inh]] ds_list = [] for idx, acts in enumerate(zip(layers)): (xs, inhs) = acts[0] xs = tf.reshape(xs, [bs, -1]) inhs = tf.reshape(inhs, [bs, -1]) # Augmentation rolled_xs = [] for r in range(1, bs - 1): rolled_xs += [tf.roll(xs, r, 0)] xs = tf.concat([xs] + rolled_xs, 0) inhs = tf.tile(inhs, [bs - 1, 1]) # CPC distances ds = tf.reduce_sum((xs * inhs), axis=-1) ds = tf.reshape(ds, [bs - 1, -1]) ds_list += [ds] ds_list = tf.transpose(tf.concat(ds_list, -1)) return ds_list, {}
def build_model(data_tensor, reuse, training): """Create the hgru from Learning long-range...""" with tf.variable_scope('cnn', reuse=reuse): with tf.variable_scope('input', reuse=reuse): x = tf.layers.conv2d(inputs=data_tensor, filters=24, kernel_size=3, padding='same', trainable=training, name='c1_1', use_bias=True) x = tf.layers.conv2d(inputs=x, filters=24, kernel_size=3, padding='same', trainable=training, name='c1_2', use_bias=True) x = tf.layers.conv2d(inputs=x, filters=24, kernel_size=3, padding='same', trainable=training, name='c1_3', use_bias=True) with tf.variable_scope('hGRU', reuse=reuse): layer_hgru = hgru.hGRU('hgru', x_shape=x.get_shape().as_list(), timesteps=6, h_ext=7, strides=[1, 1, 1, 1], padding='SAME', aux={ 'readout': 'fb', 'pooling_kernel': [1, 4, 4, 1], 'intermediate_ff': [24, 24, 24], 'intermediate_ks': [[3, 3], [3, 3], [3, 3]], }, pool_strides=[1, 4, 4, 1], train=training) x = layer_hgru.build(x) x = normalization.batch(bottom=x, name='hgru_bn', fused=True, training=training) fb = tf.identity(x) with tf.variable_scope('readout_1', reuse=reuse): x = conv.conv_layer(bottom=x, name='pre_readout_conv', num_filters=2, kernel_size=1, trainable=training, use_bias=True) pool_aux = {'pool_type': 'max'} x = pooling.global_pool(bottom=x, name='pre_readout_pool', aux=pool_aux) x = normalization.batch(bottom=x, name='hgru_bn', training=training) with tf.variable_scope('readout_2', reuse=reuse): x = tf.layers.flatten(x, name='flat_readout') x = tf.layers.dense(inputs=x, units=2) extra_activities = {'activity': fb} return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[0] elif isinstance(output_shape, dict): output_shape = output_shape['output'] with tf.variable_scope('cnn', reuse=reuse): # # Concatenate standard deviation # _, var = tf.nn.moments(data_tensor, axes=[3]) # std = tf.expand_dims(tf.sqrt(var), axis=-1) # data_tensor = tf.concat([data_tensor, std], axis=-1) # Add input in_emb = conv.input_layer(X=data_tensor, reuse=reuse, training=training, features=[12, 12], conv_activation=[tf.nn.relu, tf.nn.relu], conv_kernel_size=7, pool_kernel_size=[1, 4, 4, 1], pool_kernel_strides=[1, 2, 2, 1], pool=True, name='l0') # Run fGRU hgru_kernels = OrderedDict() hgru_kernels['h1'] = [15, 15] # height/width hgru_kernels['h2'] = [1, 1] hgru_kernels['fb1'] = [1, 1] hgru_features = OrderedDict() hgru_features['h1'] = [12, 12] # Fan-in/fan-out, I and E (match fb1) hgru_features['h2'] = [48, 48] hgru_features['fb1'] = [12, 12] # (match h1) # hgru_features['fb1'] = [24, 12] # (match h1 unless squeeze_fb) intermediate_ff = [24, 48] # Last feature must match h2 intermediate_ks = [[3, 3], [3, 3]] intermediate_repeats = [1, 1] # Repeat each interm this many times layer_hgru = hgru.hGRU( 'fgru', x_shape=in_emb.get_shape().as_list(), timesteps=8, strides=[1, 1, 1, 1], hgru_features=hgru_features, hgru_kernels=hgru_kernels, intermediate_ff=intermediate_ff, intermediate_ks=intermediate_ks, intermediate_repeats=intermediate_repeats, padding='SAME', aux={ 'readout': 'fb', 'squeeze_fb': False, # Compress Inh-hat with a 1x1 conv 'td_gate': True, # Add top-down activity to the in-gate 'attention': 'gala', 'attention_layers': 2, 'td_cell_state': False, 'excite_se': False, # Add S/E in the excitation stage 'residual': True, # intermediate resid connections 'while_loop': False, 'skip': True, 'time_skips': False, 'symmetric_weights': False, 'timestep_output': False, 'include_pooling': True }, pool_strides=[2, 2], pooling_kernel=[2, 2], train=training) h2 = layer_hgru.build(in_emb) if isinstance(h2, list): all_activities = [] for idx, it_h2 in enumerate(h2): it_h2 = normalization.batch(bottom=it_h2, renorm=False, name='hgru_bn_%s' % idx, training=training) activity = conv.readout_layer(activity=it_h2, reuse=reuse, training=training, var_scope='readout_%s' % idx, output_shape=output_shape) all_activities += [activity] h2 = all_activities[-1] activity = all_activities else: h2 = normalization.batch(bottom=h2, renorm=False, name='hgru_bn', training=training) activity = conv.readout_layer(activity=h2, reuse=reuse, training=training, output_shape=output_shape) extra_activities = {'activity': h2} return activity, extra_activities
def build_model(data_tensor, reuse, training): """Create the hgru from Learning long-range...""" down_pool_kernel = [1, 2, 2, 1] down_pool_strides = [1, 2, 2, 1] down_pool_padding = 'SAME' with tf.variable_scope('cnn', reuse=reuse): # Unclear if we should include l0 in the down/upsample cascade with tf.variable_scope('g1', reuse=reuse): # Downsample act11 = conv_block(x=data_tensor, name='l1_1', filters=64, training=training, reuse=reuse, pool=False) act12 = conv_block(x=act11, name='l1_2', filters=64, training=training, reuse=reuse, pool=False) poolact12, poolact12inds = tf.nn.max_pool_with_argmax( input=act12, ksize=down_pool_kernel, strides=down_pool_strides, padding=down_pool_padding, name='l1_2_pool') with tf.variable_scope('g2', reuse=reuse): # Downsample act21 = conv_block(x=act12, name='l2_1', filters=128, training=training, reuse=reuse, pool=False) act22 = conv_block(x=act21, filters=128, name='l2_2', training=training, reuse=reuse, pool=False) poolact22, poolact22inds = tf.nn.max_pool_with_argmax( input=act22, ksize=down_pool_kernel, strides=down_pool_strides, padding=down_pool_padding, name='l2_2_pool') with tf.variable_scope('g3', reuse=reuse): # Downsample act31 = conv_block(x=poolact22, name='l3_1', filters=256, training=training, reuse=reuse, pool=False) act32 = conv_block(x=act31, filters=256, name='l3_2', training=training, reuse=reuse, pool=False) act33 = conv_block(x=act32, filters=256, name='l3_3', training=training, reuse=reuse, pool=False) poolact33, poolact33inds = tf.nn.max_pool_with_argmax( input=act33, ksize=down_pool_kernel, strides=down_pool_strides, padding=down_pool_padding, name='l3_3_pool') with tf.variable_scope('g4', reuse=reuse): # Downsample act41 = conv_block(x=poolact33, name='l4_1', filters=512, training=training, reuse=reuse, pool=False) act42 = conv_block(x=act41, filters=512, name='l4_2', training=training, reuse=reuse, pool=False) act43 = conv_block(x=act42, filters=512, name='l4_3', training=training, reuse=reuse, pool=False) poolact43, poolact43inds = tf.nn.max_pool_with_argmax( input=act43, ksize=down_pool_kernel, strides=down_pool_strides, padding=down_pool_padding, name='l4_3_pool') with tf.variable_scope('g5', reuse=reuse): # Downsample act51 = conv_block(x=poolact43, name='l5_1', filters=512, training=training, reuse=reuse, pool=False) act52 = conv_block(x=act51, filters=512, name='l5_2', training=training, reuse=reuse, pool=False) act53 = conv_block(x=act52, filters=512, name='l5_3', training=training, reuse=reuse, pool=False) poolact53, poolact53inds = tf.nn.max_pool_with_argmax( input=act53, ksize=down_pool_kernel, strides=down_pool_strides, padding=down_pool_padding, name='l5_3_pool') with tf.variable_scope('g5_up', reuse=reuse): upact5 = pooling.unpool_with_argmax_layer(bottom=poolact53, ind=poolact53inds, filter_size=[3, 3], name='l5_unpool') uact53 = conv_block(x=upact5, name='ul5_3', filters=512, training=training, reuse=reuse, pool=False) uact52 = conv_block(x=uact53, filters=512, name='ul5_2', training=training, reuse=reuse, pool=False) uact51 = conv_block(x=uact52, filters=512, name='ul5_1', training=training, reuse=reuse, pool=False) with tf.variable_scope('g4_up', reuse=reuse): upact4 = pooling.unpool_with_argmax_layer(bottom=uact51, ind=poolact43inds, filter_size=[3, 3], name='l4_unpool') uact43 = conv_block(x=upact4, name='ul4_3', filters=512, training=training, reuse=reuse, pool=False) uact42 = conv_block(x=uact43, filters=512, name='ul4_2', training=training, reuse=reuse, pool=False) uact41 = conv_block(x=uact42, filters=256, name='ul4_1', training=training, reuse=reuse, pool=False) with tf.variable_scope('g3_up', reuse=reuse): upact3 = pooling.unpool_with_argmax_layer(bottom=uact41, ind=poolact33inds, filter_size=[3, 3], name='l3_unpool') uact33 = conv_block(x=upact3, name='ul3_3', filters=256, training=training, reuse=reuse, pool=False) uact32 = conv_block(x=uact33, filters=256, name='ul3_2', training=training, reuse=reuse, pool=False) uact31 = conv_block(x=uact32, filters=128, name='ul3_1', training=training, reuse=reuse, pool=False) with tf.variable_scope('g2_up', reuse=reuse): upact2 = pooling.unpool_with_argmax_layer(bottom=uact31, ind=poolact22inds, filter_size=[3, 3], name='l2_unpool') uact22 = conv_block(x=upact2, name='ul2_2', filters=128, training=training, reuse=reuse, pool=False) uact21 = conv_block(x=uact22, name='ul2_1', filters=64, training=training, reuse=reuse, pool=False) with tf.variable_scope('g1_up', reuse=reuse): upact1 = pooling.unpool_with_argmax_layer(bottom=uact21, ind=poolact12inds, filter_size=[3, 3], name='l1_unpool') uact12 = conv_block(x=upact1, name='ul1_2', filters=64, training=training, reuse=reuse, pool=False) uact11 = conv_block(x=uact12, name='ul1_1', filters=64, training=training, reuse=reuse, pool=False) with tf.variable_scope('readout_1', reuse=reuse): activity = conv.conv_layer(bottom=uact11, name='pre_readout_conv', num_filters=2, kernel_size=1, trainable=training, use_bias=False) pool_aux = {'pool_type': 'max'} activity = pooling.global_pool(bottom=activity, name='pre_readout_pool', aux=pool_aux) activity = normalization.batch(bottom=activity, renorm=True, name='readout_1_bn', training=training) with tf.variable_scope('readout_2', reuse=reuse): activity = tf.layers.flatten(activity, name='flat_readout') activity = tf.layers.dense(inputs=activity, units=2) extra_activities = {'activity': activity} return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[0] data_format = 'channels_last' conv_kernel = [ [3, 3], [3, 3], [3, 3], ] up_kernel = [2, 2] filters = [28, 36, 48, 64, 80] with tf.variable_scope('cnn', reuse=reuse): # Unclear if we should include l0 in the down/upsample cascade with tf.variable_scope('in_embedding', reuse=reuse): in_emb = tf.layers.conv2d(inputs=data_tensor, filters=filters[0], kernel_size=5, name='l0', strides=(1, 1), padding='same', activation=tf.nn.elu, data_format=data_format, trainable=training, use_bias=True) # Downsample l1 = conv.down_block(layer_name='l1', bottom=in_emb, kernel_size=conv_kernel, num_filters=filters[1], training=training, reuse=reuse) l2 = conv.down_block(layer_name='l2', bottom=l1, kernel_size=conv_kernel, num_filters=filters[2], training=training, reuse=reuse) l3 = conv.down_block(layer_name='l3', bottom=l2, kernel_size=conv_kernel, num_filters=filters[3], training=training, reuse=reuse) l4 = conv.down_block(layer_name='l4', bottom=l3, kernel_size=conv_kernel, num_filters=filters[4], training=training, reuse=reuse) # Upsample ul3 = conv.up_block(layer_name='ul3', bottom=l4, skip_activity=l3, kernel_size=up_kernel, num_filters=filters[3], training=training, reuse=reuse) ul3 = conv.down_block(layer_name='ul3_d', bottom=ul3, kernel_size=conv_kernel, num_filters=filters[3], training=training, reuse=reuse, include_pool=False) ul2 = conv.up_block(layer_name='ul2', bottom=ul3, skip_activity=l2, kernel_size=up_kernel, num_filters=filters[2], training=training, reuse=reuse) ul2 = conv.down_block(layer_name='ul2_d', bottom=ul2, kernel_size=conv_kernel, num_filters=filters[2], training=training, reuse=reuse, include_pool=False) ul1 = conv.up_block(layer_name='ul1', bottom=ul2, skip_activity=l1, kernel_size=up_kernel, num_filters=filters[1], training=training, reuse=reuse) ul1 = conv.down_block(layer_name='ul1_d', bottom=ul1, kernel_size=conv_kernel, num_filters=filters[1], training=training, reuse=reuse, include_pool=False) ul0 = conv.up_block(layer_name='ul0', bottom=ul1, skip_activity=in_emb, kernel_size=up_kernel, num_filters=filters[0], training=training, reuse=reuse) with tf.variable_scope('readout_1', reuse=reuse): activity = conv.conv_layer(bottom=ul0, name='pre_readout_conv', num_filters=2, kernel_size=1, trainable=training, use_bias=False) pool_aux = {'pool_type': 'max'} activity = pooling.global_pool(bottom=activity, name='pre_readout_pool', aux=pool_aux) activity = normalization.batch(bottom=activity, renorm=True, name='readout_1_bn', training=training) with tf.variable_scope('readout_2', reuse=reuse): activity = tf.layers.flatten(activity, name='flat_readout') activity = tf.layers.dense(inputs=activity, units=output_shape) extra_activities = {'l4': l4} return activity, extra_activities
def down_block(layer_name, bottom, reuse, kernel_size, num_filters, training, use_bias=False, include_pool=True): """Forward block for seung model.""" with tf.variable_scope('%s_block' % layer_name, reuse=reuse): with tf.variable_scope('%s_layer_1' % layer_name, reuse=reuse): #with bfloat16.bfloat16_scope(): print(bottom.dtype) x = conv3d_layer(bottom=bottom, name='%s_1' % layer_name, stride=[1, 1, 1], padding='SAME', kernel_size=kernel_size[0], num_filters=num_filters, trainable=training, use_bias=use_bias) print(x) x = tf.layers.batch_normalization(inputs=x, name='%s_bn_1' % layer_name, scale=True, center=True, training=training, fused=True, axis=len(x.shape) - 1) print(x.dtype) # x = normalization.batch( # bottom=x, # name='%s_bn_1' % layer_name, # training=training) # print(x.dtype) x = tf.nn.elu(x) print(x.dtype) skip = tf.identity(x) print(skip.dtype) with tf.variable_scope('%s_layer_2' % layer_name, reuse=reuse): x = conv3d_layer(bottom=x, name='%s_2' % layer_name, stride=[1, 1, 1], padding='SAME', kernel_size=kernel_size[1], num_filters=num_filters, trainable=training, use_bias=use_bias) x = normalization.batch(bottom=x, name='%s_bn_2' % layer_name, training=training) x = tf.nn.elu(x) with tf.variable_scope('%s_layer_3' % layer_name, reuse=reuse): x = conv3d_layer(bottom=x, name='%s_3' % layer_name, stride=[1, 1, 1], padding='SAME', kernel_size=kernel_size[2], num_filters=num_filters, trainable=training, use_bias=use_bias) x = tf.nn.elu(x) x = x + skip x = normalization.batch(bottom=x, name='%s_bn_3' % layer_name, training=training) if include_pool: with tf.variable_scope('%s_pool' % layer_name, reuse=reuse): x = pooling.max_pool3d( bottom=x, name='%s_pool' % layer_name, # TODO: swap back k=[2, 2, 1], s=[2, 2, 1]) return x
def build_model( data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] # output_shape = 2 # norm_moments_training = training # Force instance norm normalization_type = 'no_param_batch_norm_original' # normalization_type = 'ada_batch_norm' # normalization_type = 'no_param_instance_norm' output_normalization_type = 'batch_norm_original_renorm' # output_normalization_type = 'instance_norm' data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Prepare gammanet structure ( compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts) = v2_big_working() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model with tf.variable_scope('vgg', reuse=reuse): aux = get_aux() vgg = vgg16.Vgg16( vgg16_npy_path='/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy', reuse=reuse, aux=aux, train=training, timesteps=8, fgru_normalization_type=normalization_type, ff_normalization_type=normalization_type) vgg(rgb=data_tensor, constructor=gammanet_constructor) # activity = vgg.fgru_0 with tf.variable_scope('fgru', reuse=reuse): # Get side weights activity = normalization.batch( bottom=vgg.fgru_0, # activity, renorm=True, name='readout_0_bn', training=training) # activity = normalization.group( # bottom=vgg.fgru_0) # , # activity, # # name='readout_0_bn') prepool_activity = tf.layers.conv2d( inputs=activity, filters=output_shape, # int(activity.get_shape()[-1]), kernel_size=1, name='pre_readout_conv0', strides=(1, 1), padding='same', activation=tf.nn.elu, trainable=training, use_bias=True) """ prepool_activity = normalization.batch( bottom=prepool_activity, renorm=True, name='readout_1_bn', training=False) # training) """ """ prepool_activity = tf.layers.conv2d( inputs=prepool_activity, filters=output_shape, kernel_size=1, name='pre_readout_conv1', strides=(1, 1), padding='same', activation=None, trainable=training, use_bias=True) """ out_activity = tf.reduce_max(prepool_activity, reduction_indices=[1, 2]) activity = tf.layers.dense( inputs=out_activity, units=output_shape) if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) # extra_activities = {'orientations': orientations, 'model_output_y': vgg.fgru_0, 'model_output_x': vgg.fgru_0_init} extra_activities = {'x': vgg.fgru_0} if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) # return [activity, h_deep], extra_activities return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[0] elif isinstance(output_shape, dict): nhot_shape = output_shape['aux'] output_shape = output_shape['output'] use_aux = True with tf.variable_scope('cnn', reuse=reuse): # Unclear if we should include l0 in the down/upsample cascade with tf.variable_scope('g1', reuse=reuse): # Downsample act11 = conv_block( x=data_tensor, name='l1_1', filters=64, training=training, reuse=reuse, pool=False) act12 = conv_block( x=act11, name='l1_2', filters=64, training=training, reuse=reuse, pool=False) poolact12 = pooling.max_pool( bottom=act12, name='l1_2_pool') with tf.variable_scope('g2', reuse=reuse): # Downsample act21 = conv_block( x=poolact12, name='l2_1', filters=128, training=training, reuse=reuse, pool=False) act22 = conv_block( x=act21, filters=128, name='l2_2', training=training, reuse=reuse, pool=False) poolact22 = pooling.max_pool( bottom=act22, name='l2_2_pool') with tf.variable_scope('g3', reuse=reuse): # Downsample act31 = conv_block( x=poolact22, name='l3_1', filters=256, training=training, reuse=reuse, pool=False) act32 = conv_block( x=act31, filters=256, name='l3_2', training=training, reuse=reuse, pool=False) act33 = conv_block( x=act32, filters=256, name='l3_3', training=training, reuse=reuse, pool=False) poolact33 = pooling.max_pool( bottom=act33, name='l3_3_pool') with tf.variable_scope('g4', reuse=reuse): # Downsample act41 = conv_block( x=poolact33, name='l4_1', filters=512, training=training, reuse=reuse, pool=False) act42 = conv_block( x=act41, filters=512, name='l4_2', training=training, reuse=reuse, pool=False) act43 = conv_block( x=act42, filters=512, name='l4_3', training=training, reuse=reuse, pool=False) poolact43 = pooling.max_pool( bottom=act43, name='l4_3_pool') with tf.variable_scope('g5', reuse=reuse): # Downsample act51 = conv_block( x=poolact43, name='l5_1', filters=512, training=training, reuse=reuse, pool=False) act52 = conv_block( x=act51, filters=512, name='l5_2', training=training, reuse=reuse, pool=False) act53 = conv_block( x=act52, filters=512, name='l5_3', training=training, reuse=reuse, pool=False) poolact53 = pooling.max_pool( bottom=act53, name='l5_3_pool') with tf.variable_scope('g5_skip', reuse=reuse): upact5 = up_block( inputs=poolact53, skip=act53, up_filters=512, name='ul5', training=training, reuse=reuse) with tf.variable_scope('g4_skip', reuse=reuse): upact4 = up_block( inputs=upact5, skip=act43, up_filters=512, name='ul4', training=training, reuse=reuse) with tf.variable_scope('g3_skip', reuse=reuse): upact3 = up_block( inputs=upact4, skip=act33, up_filters=256, name='ul3', training=training, reuse=reuse) with tf.variable_scope('g2_skip', reuse=reuse): upact2 = up_block( inputs=upact3, skip=act22, up_filters=128, name='ul2', training=training, reuse=reuse) with tf.variable_scope('g1_skip', reuse=reuse): upact1 = up_block( inputs=upact2, skip=act12, up_filters=64, name='ul1', training=training, reuse=reuse) with tf.variable_scope('readout_1', reuse=reuse): activity = conv.conv_layer( bottom=upact1, name='pre_readout_conv', num_filters=2, kernel_size=1, trainable=training, use_bias=False) pool_aux = {'pool_type': 'max'} activity = pooling.global_pool( bottom=activity, name='pre_readout_pool', aux=pool_aux) activity = normalization.batch( bottom=activity, renorm=True, name='readout_1_bn', training=training) with tf.variable_scope('readout_2', reuse=reuse): pre_activity = tf.layers.flatten( activity, name='flat_readout') activity = tf.layers.dense( inputs=pre_activity, units=output_shape) if use_aux: nhot = tf.layers.dense(inputs=pre_activity, units=nhot_shape) else: nhot = tf.constant(0.) extra_activities = { 'activity': activity, 'nhot': nhot } return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] output_shape = 1 # norm_moments_training = training # Force instance norm # normalization_type = 'no_param_batch_norm_original' normalization_type = 'no_param_instance_norm' # output_normalization_type = 'batch_norm_original_renorm' output_normalization_type = 'instance_norm' # data_tensor = tf.concat((data_tensor, data_tensor, data_tensor), -1) # Expand to 3D data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Prepare gammanet structure (compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts) = v2_big_working() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model with tf.variable_scope('vgg', reuse=reuse): aux = get_aux() vgg = vgg16.Vgg16( vgg16_npy_path= '/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy', reuse=reuse, aux=aux, train=False, # training, timesteps=8, fgru_normalization_type=normalization_type, ff_normalization_type=normalization_type) vgg(rgb=data_tensor, constructor=gammanet_constructor) # activity = vgg.fgru_0 with tf.variable_scope('fgru', reuse=reuse): # Get side weights h2_rem = [vgg.fgru_0, vgg.fgru_1, vgg.fgru_2] activities = [] for idx, h in enumerate(h2_rem): # res = normalization.apply_normalization( # activity=h, # name='output_norm1_%s' % idx, # normalization_type=output_normalization_type, # data_format=data_format, # training=False, # trainable=False, # reuse=reuse) activities.append(aux['image_resize']( h, vgg.fgru_0.get_shape().as_list()[1:3], align_corners=True)) activity = tf.concat(activities, -1) # Then read out # activity = normalization.apply_normalization( # activity=res, # name='output_norm1', # normalization_type=output_normalization_type, # data_format=data_format, # training=training, # trainable=training, # reuse=reuse) with tf.variable_scope('readout', reuse=reuse): activity = normalization.batch(bottom=activity, renorm=False, name='hgru_bn', training=training) activity = conv.readout_layer(activity=activity, reuse=reuse, training=training, output_shape=output_shape) if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) # extra_activities = {'orientations': orientations, 'model_output_y': vgg.fgru_0, 'model_output_x': vgg.fgru_0_init} extra_activities = {} if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) # return [activity, h_deep], extra_activities return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[0] elif isinstance(output_shape, dict): output_shape = output_shape['output'] with tf.variable_scope('cnn', reuse=reuse): normalization_type = 'ada_batch_norm' # 'instance_norm' # # Concatenate standard deviation # _, var = tf.nn.moments(data_tensor, axes=[3]) # std = tf.expand_dims(tf.sqrt(var), axis=-1) # data_tensor = tf.concat([data_tensor, std], axis=-1) # Add input in_emb = conv.skinny_input_layer(X=data_tensor, reuse=reuse, training=training, features=16, conv_activation=tf.nn.relu, conv_kernel_size=7, pool=False, name='l0') # Run fGRU hgru_kernels = OrderedDict() hgru_kernels['h1'] = [15, 15] # height/width hgru_kernels['h2'] = [5, 5] hgru_kernels['fb1'] = [1, 1] hgru_features = OrderedDict() hgru_features['h1'] = [16, 16] # Fan-in/fan-out, I and E (match fb1) hgru_features['h2'] = [48, 48] hgru_features['fb1'] = [16, 16] # (match h1) # hgru_features['fb1'] = [24, 12] # (match h1 unless squeeze_fb) intermediate_ff = [24, 24, 48] # Last feature must match h2 intermediate_ks = [[5, 5], [5, 5], [5, 5]] intermediate_repeats = [1, 1, 1] # Repeat each interm this many times layer_hgru = hgru.hGRU( 'fgru', x_shape=in_emb.get_shape().as_list(), timesteps=8, strides=[1, 1, 1, 1], hgru_features=hgru_features, hgru_kernels=hgru_kernels, intermediate_ff=intermediate_ff, intermediate_ks=intermediate_ks, intermediate_repeats=intermediate_repeats, padding='SAME', aux={ 'readout': 'fb', 'squeeze_fb': False, # Compress Inh-hat with a 1x1 conv 'td_gate': True, # Add top-down activity to the in-gate 'attention': 'gala', # 'gala', 'attention_layers': 2, 'upsample_convs': True, 'td_cell_state': True, 'normalization_type': normalization_type, 'excite_se': False, # Add S/E in the excitation stage 'residual': True, # intermediate resid connections 'while_loop': False, 'skip': True, 'time_skips': False, 'force_horizontal': False, 'symmetric_weights': True, 'timestep_output': False, 'bilinear_init': True, 'include_pooling': True }, pool_strides=[2, 2], pooling_kernel=[4, 4], up_kernel=[4, 4], train=training) h2 = layer_hgru.build(in_emb) if normalization_type is 'batch_norm': h2 = normalization.batch(bottom=h2, renorm=False, name='hgru_bn', training=training) elif normalization_type is 'instance_norm': h2 = normalization.instance(bottom=h2, training=training) elif normalization_type is 'ada_batch_norm': h2 = normalization.batch(bottom=h2, renorm=False, name='hgru_bn', training=True) else: raise NotImplementedError(normalization_type) fc = tf.layers.conv2d(inputs=h2, filters=output_shape, kernel_size=1, name='fc') # activity = tf.reduce_mean(fc, reduction_indices=[1, 2]) activity = tf.reduce_max(fc, reduction_indices=[1, 2]) extra_activities = {'activity': h2} return activity, extra_activities