def build_model(data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] normalization_type = 'no_param_instance_norm' output_normalization_type = 'instance_norm' ff_kernel_size = (3, 3) # ff_nl = tf.nn.relu ff_nl = tf.nn.elu data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Prepare gammanet structure (compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts) = spec() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model with tf.variable_scope('gammanet', reuse=reuse): activity = tf.layers.conv2d( inputs=data_tensor, filters=gammanet_constructor[0]['features'], kernel_size=ff_kernel_size, padding='same', data_format=long_data_format, name='l0_0', activation=ff_nl, use_bias=True, reuse=reuse) activity = tf.layers.conv2d( inputs=activity, filters=gammanet_constructor[0]['features'], kernel_size=ff_kernel_size, padding='same', data_format=long_data_format, name='l0_1', activation=ff_nl, use_bias=True, reuse=reuse) activity = normalization.apply_normalization( activity=activity, name='input_norm', normalization_type=output_normalization_type, data_format=data_format, training=training, trainable=training, reuse=reuse) # Shift conv-layers to the front # Run fGRU gn = gammanet.GN( layer_name='fgru', gammanet_constructor=gammanet_constructor, data_format=data_format, reuse=reuse, timesteps=8, fgru_connectivity='', # 'all_to_all', additional_readouts=additional_readouts, fgru_normalization_type=normalization_type, ff_normalization_type=normalization_type, horizontal_padding='SAME', ff_padding='SAME', ff_nl=ff_nl, recurrent_ff=True, horizontal_kernel_initializer=tf.initializers.orthogonal(), kernel_initializer=tf.initializers.orthogonal(), gate_initializer=tf.initializers.orthogonal(), aux=aux, strides=[1, 1, 1, 1], pool_strides=[2, 2], pool_kernel=[2, 2], up_kernel=[4, 4], train=training) h2, h_deep = gn(X=activity) with tf.variable_scope('cv_readout', reuse=reuse): # Apply one more normalization activity = normalization.apply_normalization( activity=h2, name='output_norm', normalization_type=output_normalization_type, data_format=data_format, training=training, trainable=training, reuse=reuse) activity = tf.layers.conv2d( inputs=activity, filters=output_shape, kernel_size=(5, 5), # (1, 1), padding='same', data_format=long_data_format, name='readout_conv_2', activation=None, use_bias=True, reuse=reuse) if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) extra_activities = { 'activity_low': h2, 'activity_high': h_deep, } if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] input_training = training readout_training = training # False norm_moments_training = training # Force instance norm norm_params_training = training fgru_kernel_training = training fgru_param_training = training ff_gate_training = training fgru_gate_training = training remaining_param_training = training # normalization_type = 'no_param_batch_norm_original_renorm' # output_normalization_type = 'batch_norm_original_renorm' normalization_type = 'no_param_batch_norm_original' output_normalization_type = 'batch_norm_original' # normalization_type = 'no_param_instance_norm' # output_normalization_type = 'instance_norm' ff_kernel_size = (3, 3) ff_nl = tf.nn.elu data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Prepare gammanet structure (compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts) = v2_big_working() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model with tf.variable_scope('gammanet', reuse=reuse): activity_1 = tf.layers.conv2d( inputs=data_tensor, filters=gammanet_constructor[0]['features'], kernel_size=ff_kernel_size, padding='same', data_format=long_data_format, name='l0_0', activation=ff_nl, use_bias=True, trainable=input_training, reuse=reuse) activity_2 = tf.layers.conv2d( inputs=activity_1, filters=gammanet_constructor[0]['features'], kernel_size=ff_kernel_size, padding='same', data_format=long_data_format, name='l0_1', activation=ff_nl, use_bias=True, trainable=input_training, reuse=reuse) # activity = normalization.apply_normalization( # activity=activity_2, # name='input_norm_instance', # normalization_type=normalization_type, # output_normalization_type, # data_format=data_format, # training=norm_moments_training, # trainable=input_training, # reuse=reuse) # with tf.variable_scope('contour_readout', reuse=reuse): # beta = tf.get_variable( # name='hello_instance_beta', # shape=[1, 1, 1, gammanet_constructor[0]['features']], # initializer=tf.initializers.zeros, # trainable=input_training) # gamma = tf.get_variable( # name='helo_instance_gamma', # shape=[1, 1, 1, gammanet_constructor[0]['features']], # initializer=tf.initializers.ones, # trainable=input_training) # activity = beta + gamma * activity_2 activity = normalization.apply_normalization( activity=activity_2, name='input_norm', normalization_type=output_normalization_type, data_format=data_format, training=training, trainable=training, reuse=reuse) with tf.variable_scope('gammanet', reuse=reuse): # Shift conv-layers to the front # Run fGRU gn = gammanet.GN( layer_name='fgru', gammanet_constructor=gammanet_constructor, data_format=data_format, reuse=reuse, timesteps=8, train=remaining_param_training, train_fgru_gate=fgru_gate_training, train_ff_gate=ff_gate_training, train_norm_moments=norm_moments_training, train_norm_params=norm_params_training, train_fgru_kernels=fgru_kernel_training, train_fgru_params=fgru_param_training, fgru_connectivity='', # 'all_to_all', additional_readouts=additional_readouts, fgru_normalization_type=normalization_type, ff_normalization_type=normalization_type, horizontal_padding='SAME', ff_padding='SAME', ff_nl=ff_nl, recurrent_ff=True, horizontal_kernel_initializer=tf.initializers.orthogonal(), kernel_initializer=tf.initializers.orthogonal(), gate_initializer=tf.initializers.orthogonal(), # horizontal_kernel_initializer=tf.contrib.layers.xavier_initializer(), # kernel_initializer=tf.contrib.layers.xavier_initializer(), # gate_initializer=tf.contrib.layers.xavier_initializer(), aux=aux, strides=[1, 1, 1, 1], pool_strides=[2, 2], pool_kernel=[2, 2], up_kernel=[4, 4]) layers = gn(X=activity) h2 = layers[0] layers = layers[1:] # h2 = h2[0] # h2 = gn(X=activity) with tf.variable_scope('contour_readout', reuse=reuse): activity = normalization.apply_normalization( activity=h2, # activity, name='output_norm', normalization_type=normalization_type, data_format=data_format, training=training, trainable=training, reuse=reuse) beta = tf.get_variable(name='hello_instance_beta', shape=[1, 1, 1, h2.get_shape().as_list()[-1]], initializer=tf.initializers.zeros, trainable=input_training) gamma = tf.get_variable(name='helo_instance_gamma', shape=[1, 1, 1, h2.get_shape().as_list()[-1]], initializer=tf.initializers.constant(0.1), trainable=input_training) activity = beta + gamma * activity activity = tf.layers.conv2d( inputs=activity, filters=512, # gammanet_constructor[0]['features'], kernel_size=ff_kernel_size, padding='same', data_format=long_data_format, name='readout_conv', activation=ff_nl, use_bias=True, reuse=reuse) activity = normalization.apply_normalization( activity=activity, # activity, name='output_norm_1', normalization_type=output_normalization_type, data_format=data_format, training=training, trainable=training, reuse=reuse) activity = tf.layers.conv2d( inputs=activity, filters=128, # gammanet_constructor[0]['features'], kernel_size=ff_kernel_size, padding='same', data_format=long_data_format, name='readout_conv_1', activation=ff_nl, use_bias=True, reuse=reuse) # activity = normalization.apply_normalization( # activity=activity, # name='output_norm_2', # normalization_type=output_normalization_type, # data_format=data_format, # training=training, # trainable=training, # reuse=reuse) activity = tf.layers.conv2d(inputs=activity, filters=output_shape, kernel_size=ff_kernel_size, padding='same', data_format=long_data_format, name='readout_conv_2', activation=None, use_bias=True, reuse=reuse) if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) extra_activities = {} if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) # return [activity, h_deep], extra_activities return activity, extra_activities
def build_model( data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] output_shape = 2 # norm_moments_training = training # Force instance norm # normalization_type = 'no_param_batch_norm_original' normalization_type = 'no_param_instance_norm' # output_normalization_type = 'batch_norm_original_renorm' output_normalization_type = 'instance_norm' data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Prepare gammanet structure ( compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts) = v2_big_working() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model with tf.variable_scope('vgg', reuse=reuse): vgg = vgg16.Vgg16( vgg16_npy_path='/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy') vgg(rgb=data_tensor, train=False, ff_reuse=reuse) # activity = vgg.fgru_0 with tf.variable_scope('fgru', reuse=reuse): # Get side weights h2_rem = [ vgg.fgru_0, vgg.fgru_1, vgg.fgru_2, vgg.fgru_3] h2s = [] for idx, h in enumerate(h2_rem): res = aux['image_resize']( h, data_tensor.get_shape().as_list()[1:3], align_corners=True) h2s += [res] res = normalization.apply_normalization( activity=tf.concat(h2s, -1), name='output_norm1', normalization_type=output_normalization_type, data_format=data_format, training=False, trainable=False, reuse=reuse) activity = tf.layers.conv2d( inputs=res, filters=gammanet_constructor[0]['features'], kernel_size=(1, 1), padding='same', data_format=long_data_format, name='readout_l0_1', activation=tf.nn.relu, use_bias=True, trainable=training, reuse=reuse) orientations = tf.layers.conv2d( inputs=activity, filters=output_shape, kernel_size=(1, 1), padding='same', data_format=long_data_format, name='readout_l0_2', activation=None, use_bias=True, trainable=training, reuse=reuse) _, h, w, _ = orientations.get_shape().as_list() activity = orientations[:, h // 2, w // 2, :] if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) extra_activities = {'orientations': orientations} if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) # return [activity, h_deep], extra_activities return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] normalization_type = 'no_param_instance_norm' output_normalization_type = 'instance_norm' # normalization_type = 'no_param_batch_norm' # output_normalization_type = 'batch_norm' ff_kernel_size = (3, 3) ff_nl = tf.nn.relu data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Prepare gammanet structure (compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts) = v2_big_working() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model fix_training = False with tf.variable_scope('gammanet', reuse=reuse): activity = tf.layers.conv2d( inputs=data_tensor, filters=gammanet_constructor[0]['features'], kernel_size=ff_kernel_size, padding='same', data_format=long_data_format, name='l0_0', activation=ff_nl, use_bias=True, # trainable=training, trainable=fix_training, reuse=reuse) activity = tf.layers.conv2d( inputs=activity, filters=gammanet_constructor[0]['features'], kernel_size=ff_kernel_size, padding='same', data_format=long_data_format, name='l0_1', activation=None, use_bias=True, # trainable=training, trainable=fix_training, reuse=reuse) activity = normalization.apply_normalization( activity=activity, name='input_norm', normalization_type=output_normalization_type, data_format=data_format, # training=training, training=fix_training, reuse=reuse) # + pre_activity activity = ff_nl(activity) # # Downsample # activity = conv.down_block( # layer_name='l1', # bottom=activity, # kernel_size=3 * [[3, 3]], # num_filters=gammanet_constructor[0]['features'], # normalization_type=output_normalization_type, # training=training, # activation=ff_nl, # include_pool=False, # reuse=reuse) # Shift conv-layers to the front # Run fGRU gn = gammanet.GN( layer_name='fgru', gammanet_constructor=gammanet_constructor, data_format=data_format, reuse=reuse, timesteps=8, fgru_connectivity='', # 'all_to_all', additional_readouts=additional_readouts, fgru_normalization_type=normalization_type, ff_normalization_type=normalization_type, horizontal_padding='SAME', ff_padding='SAME', ff_nl=ff_nl, recurrent_ff=True, horizontal_kernel_initializer=tf.initializers.orthogonal(), kernel_initializer=tf.initializers.orthogonal(), gate_initializer=tf.initializers.orthogonal(), # horizontal_kernel_initializer=tf.contrib.layers.xavier_initializer(), # gate_initializer=tf.contrib.layers.xavier_initializer(), # gate_initializer=tf.contrib.layers.xavier_initializer(), aux=aux, strides=[1, 1, 1, 1], pool_strides=[2, 2], pool_kernel=[2, 2], up_kernel=[4, 4], train=fix_training) h2, h_deep = gn(X=activity) with tf.variable_scope('gratings_readout', reuse=reuse): h2 = normalization.apply_normalization( activity=h2, name='output_norm', normalization_type=output_normalization_type, data_format=data_format, training=training, reuse=reuse) # + pre_activity # Then read out activity = tf.layers.conv2d( inputs=h2, filters=gammanet_constructor[0]['features'], kernel_size=(1, 1), padding='same', data_format=long_data_format, name='readout_l0_1', activation=ff_nl, use_bias=True, trainable=training, reuse=reuse) activity = tf.layers.conv2d(inputs=activity, filters=output_shape, kernel_size=(1, 1), padding='same', data_format=long_data_format, name='readout_l0_2', activation=None, use_bias=True, trainable=training, reuse=reuse) _, h, w, _ = activity.get_shape().as_list() activity = activity[:, h // 2, w // 2, :] if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) extra_activities = { 'activity_low': h2, 'activity_high': h_deep, } if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) # return [activity, h_deep], extra_activities return activity, extra_activities
def build(self, i0, extra_convs=True): # Convert RGB to BGR with tf.variable_scope('fgru'): error_horizontal_0, fgru_activity = self.fgru_ops( # h^(1), h^(2) ff_drive=self.conv2_2, h2=self.fgru_0, layer_id=0, i0=i0) self.fgru_0 = fgru_activity # + self.conv2_2 self.pool2 = self.max_pool(self.fgru_0, 'pool2') self.conv3_1 = self.conv_layer(self.pool2, "conv3_1") self.conv3_2 = self.conv_layer(self.conv3_1, "conv3_2") self.conv3_3 = self.conv_layer(self.conv3_2, "conv3_3") if i0 == 0: self.fgru_1 = self.conv3_3 with tf.variable_scope('fgru'): error, fgru_activity = self.fgru_ops( # h^(1), h^(2) ff_drive=self.conv3_3, h2=self.fgru_1, layer_id=1, i0=i0) self.fgru_1 = fgru_activity # + self.conv2_2 self.pool3 = self.max_pool(self.fgru_1, 'pool3') self.conv4_1 = self.conv_layer(self.pool3, "conv4_1") self.conv4_2 = self.conv_layer(self.conv4_1, "conv4_2") self.conv4_3 = self.conv_layer(self.conv4_2, "conv4_3") if i0 == 0: self.fgru_2 = self.conv4_3 with tf.variable_scope('fgru'): error, fgru_activity = self.fgru_ops( # h^(1), h^(2) ff_drive=self.conv4_3, h2=self.fgru_2, layer_id=2, i0=i0) self.fgru_2 = fgru_activity # + self.conv3_3 self.pool4 = self.max_pool(self.fgru_2, 'pool4') self.conv5_1 = self.conv_layer(self.pool4, "conv5_1") self.conv5_2 = self.conv_layer(self.conv5_1, "conv5_2") self.conv5_3 = self.conv_layer(self.conv5_2, "conv5_3") if i0 == 0: self.fgru_3 = self.conv5_3 with tf.variable_scope('fgru'): error, fgru_activity = self.fgru_ops( # h^(1), h^(2) ff_drive=self.conv5_3, h2=self.fgru_3, layer_id=3, i0=i0) self.fgru_3 = fgru_activity # + self.conv5_3 # Resize and conv with tf.variable_scope('fgru'): fgru_2_td = normalization.apply_normalization( activity=self.fgru_3, name='td_norm2_%s' % i0, # normalization_type='instance_norm', normalization_type=self.fgru_normalization_type, data_format=self.data_format, training=self.train, trainable=self.train, reuse=self.reuse) fgru_2_td = self.conv_layer( fgru_2_td, '4_to_3', learned=True, apply_relu=True, shape=[ 1, 1, fgru_2_td.get_shape().as_list()[-1], self.fgru_2.get_shape().as_list()[-1] // 64 ]) fgru_2_td = normalization.apply_normalization( activity=fgru_2_td, name='td_norm2_1%s' % i0, # normalization_type='instance_norm', normalization_type=self.fgru_normalization_type, data_format=self.data_format, training=self.train, trainable=self.train, reuse=self.reuse) if extra_convs: fgru_2_td = self.conv_layer( fgru_2_td, '4_to_3_2', learned=True, shape=[ 1, 1, self.fgru_2.get_shape().as_list()[-1] // 64, self.fgru_2.get_shape().as_list()[-1] ]) fgru_2_td = self.image_resize( fgru_2_td, self.fgru_2.get_shape().as_list()[1:3], align_corners=True) error, fgru_activity = self.fgru_ops( # h^(1), h^(2) ff_drive=self.fgru_2, h2=fgru_2_td, layer_id=4, i0=i0) self.fgru_2 += fgru_activity if i0 == self.last_timestep and self.downsampled: return self.fgru_2 # Resize and conv with tf.variable_scope('fgru'): fgru_1_td = normalization.apply_normalization( activity=self.fgru_2, name='td_norm1_%s' % i0, # normalization_type='instance_norm', normalization_type=self.fgru_normalization_type, data_format=self.data_format, training=self.train, trainable=self.train, reuse=self.reuse) fgru_1_td = self.conv_layer( fgru_1_td, '3_to_2', learned=True, apply_relu=True, shape=[ 1, 1, fgru_1_td.get_shape().as_list()[-1], self.fgru_1.get_shape().as_list()[-1] // 16 ]) fgru_1_td = normalization.apply_normalization( activity=fgru_1_td, name='td_norm1_1%s' % i0, # normalization_type='instance_norm', normalization_type=self.fgru_normalization_type, data_format=self.data_format, training=self.train, trainable=self.train, reuse=self.reuse) if extra_convs: fgru_1_td = self.conv_layer( fgru_1_td, '3_to_2_2', learned=True, shape=[ 1, 1, self.fgru_1.get_shape().as_list()[-1] // 16, self.fgru_1.get_shape().as_list()[-1] ]) fgru_1_td = self.image_resize( fgru_1_td, self.fgru_1.get_shape().as_list()[1:3], align_corners=True) error, fgru_activity = self.fgru_ops( # h^(1), h^(2) ff_drive=self.fgru_1, h2=fgru_1_td, layer_id=5, i0=i0) # self.fgru_0 = fgru_activity self.fgru_1 += fgru_activity # Resize and conv with tf.variable_scope('fgru'): fgru_0_td = normalization.apply_normalization( activity=self.fgru_1, name='td_norm0_%s' % i0, # normalization_type='instance_norm', normalization_type=self.fgru_normalization_type, data_format=self.data_format, training=self.train, trainable=self.train, reuse=self.reuse) fgru_0_td = self.conv_layer( fgru_0_td, '2_to_1', learned=True, apply_relu=True, shape=[ 1, 1, fgru_0_td.get_shape().as_list()[-1], self.fgru_0.get_shape().as_list()[-1] // 4 ]) fgru_0_td = normalization.apply_normalization( activity=fgru_0_td, name='td_norm0_1%s' % i0, # normalization_type='instance_norm', normalization_type=self.fgru_normalization_type, data_format=self.data_format, training=self.train, trainable=self.train, reuse=self.reuse) if extra_convs: fgru_0_td = self.conv_layer( fgru_0_td, '2_to_1_2', learned=True, shape=[ 1, 1, self.fgru_0.get_shape().as_list()[-1] // 4, self.fgru_0.get_shape().as_list()[-1] ]) fgru_0_td = self.image_resize( fgru_0_td, self.fgru_0.get_shape().as_list()[1:3], align_corners=True) error_td_0, fgru_activity = self.fgru_ops( # h^(1), h^(2) ff_drive=self.fgru_0, h2=fgru_0_td, layer_id=6, i0=i0) self.fgru_0 = self.fgru_0 + fgru_activity
def build_model(data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] # norm_moments_training = training # Force instance norm # normalization_type = 'no_param_batch_norm_original' normalization_type = 'no_param_instance_norm' # output_normalization_type = 'batch_norm_original_renorm' output_normalization_type = 'instance_norm' data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Prepare gammanet structure (compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts) = v2_big_working() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model with tf.variable_scope('vgg', reuse=reuse): aux = get_aux() vgg = vgg16.Vgg16( vgg16_npy_path= '/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy', reuse=reuse, aux=aux, train=training, timesteps=8, fgru_normalization_type=normalization_type, ff_normalization_type=normalization_type) vgg(rgb=data_tensor, constructor=gammanet_constructor) # activity = vgg.fgru_0 with tf.variable_scope('fgru', reuse=reuse): # Get side weights h2_rem = [vgg.fgru_0] for idx, h in enumerate(h2_rem): res = normalization.apply_normalization( activity=h, name='output_norm1_%s' % idx, normalization_type=output_normalization_type, data_format=data_format, training=training, trainable=training, reuse=reuse) res = aux['image_resize'](res, data_tensor.get_shape().as_list()[1:3], align_corners=True) activity = tf.layers.conv2d(res, filters=res.get_shape().as_list()[-1], kernel_size=(3, 3), padding='same', data_format=long_data_format, name='out', activation=tf.nn.relu, trainable=training, use_bias=True, reuse=reuse) activity = tf.layers.conv2d( activity, filters=output_shape, kernel_size=(1, 1), padding='same', data_format=long_data_format, name='out2', # activation=tf.nn.relu, activation=None, trainable=training, use_bias=True, reuse=reuse) if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) extra_activities = {} # idx: v for idx, v in enumerate(hs_0)} if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) # return [activity, h_deep], extra_activities return activity, extra_activities
def build(self, i0, extra_convs=True): # Convert RGB to BGR with tf.variable_scope('fgru'): _, fgru_activity, self.inh_0 = self.alt_fgru_ops( # h^(1), h^(2) ff_drive=self.conv2_2, h2=self.fgru_0, layer_id=0, i0=i0) self.x_0 = self.conv2_2 self.fgru_0 = fgru_activity # + self.conv2_2 self.pool2 = self.max_pool(self.fgru_0, 'pool2') self.conv3_1 = self.conv_layer(self.pool2, "conv3_1") self.conv3_2 = self.conv_layer(self.conv3_1, "conv3_2") self.conv3_3 = self.conv_layer(self.conv3_2, "conv3_3") if i0 == 0: self.fgru_1 = self.conv3_3 with tf.variable_scope('fgru'): _, fgru_activity, self.inh_1 = self.alt_fgru_ops( # h^(1), h^(2) ff_drive=self.conv3_3, h2=self.fgru_1, layer_id=1, i0=i0) self.x_1 = self.conv3_3 self.fgru_1 = fgru_activity # + self.conv2_2 self.pool3 = self.max_pool(self.fgru_1, 'pool3') self.conv4_1 = self.conv_layer(self.pool3, "conv4_1") self.conv4_2 = self.conv_layer(self.conv4_1, "conv4_2") self.conv4_3 = self.conv_layer(self.conv4_2, "conv4_3") if i0 == self.timesteps - 1: return self.conv4_3 # self.pool4 = self.max_pool(self.conv4_3, 'pool4') # self.conv5_1 = self.conv_layer(self.pool4, "conv5_1") # self.conv5_2 = self.conv_layer(self.conv5_1, "conv5_2") # self.conv5_3 = self.conv_layer(self.conv5_2, "conv5_3") # if i0 == self.timesteps - 1: # return self.conv5_3 # Resize and conv with tf.variable_scope('fgru'): fgru_1_td = normalization.apply_normalization( activity=self.conv4_3, name='td_norm1_%s' % i0, normalization_type='instance_norm', data_format=self.data_format, training=self.train, trainable=self.train, reuse=self.reuse) # fgru_1_td_2 = normalization.apply_normalization( # activity=self.conv5_3, # name='td_norm12_%s' % i0, # normalization_type='instance_norm', # data_format=self.data_format, # training=self.train, # trainable=self.train, # reuse=self.reuse) # fgru_1_td_2 = self.image_resize( # fgru_1_td_2, # fgru_1_td.get_shape().as_list()[1:3], # align_corners=True) # fgru_1_td = tf.concat([fgru_1_td, fgru_1_td_2], -1) fgru_1_td = self.conv_layer( fgru_1_td, '3_to_2', learned=True, apply_relu=True, shape=[ 1, 1, fgru_1_td.get_shape().as_list()[-1], self.fgru_1.get_shape().as_list()[-1] // 32 ]) fgru_1_td = normalization.apply_normalization( activity=fgru_1_td, name='td_norm1_1%s' % i0, normalization_type='instance_norm', data_format=self.data_format, training=self.train, trainable=self.train, reuse=self.reuse) if extra_convs: fgru_1_td = self.conv_layer( fgru_1_td, '3_to_2_2', learned=True, shape=[ 1, 1, self.fgru_1.get_shape().as_list()[-1] // 32, self.fgru_1.get_shape().as_list()[-1] ]) fgru_1_td = self.image_resize( fgru_1_td, self.fgru_1.get_shape().as_list()[1:3], align_corners=True) error, fgru_activity, self.inh_2 = self.alt_fgru_ops( # h^(1), h^(2) ff_drive=self.fgru_1, h2=fgru_1_td, layer_id=2, i0=i0) self.x_2 = self.fgru_1 self.fgru_1 += fgru_activity # Resize and conv with tf.variable_scope('fgru'): fgru_0_td = normalization.apply_normalization( activity=self.fgru_1, name='td_norm0_%s' % i0, normalization_type='instance_norm', data_format=self.data_format, training=self.train, trainable=self.train, reuse=self.reuse) fgru_0_td = self.conv_layer( fgru_0_td, '2_to_1', learned=True, apply_relu=True, shape=[ 1, 1, fgru_0_td.get_shape().as_list()[-1], self.fgru_0.get_shape().as_list()[-1] // 4 ]) fgru_0_td = normalization.apply_normalization( activity=fgru_0_td, name='td_norm0_1%s' % i0, normalization_type='instance_norm', data_format=self.data_format, training=self.train, trainable=self.train, reuse=self.reuse) if extra_convs: fgru_0_td = self.conv_layer( fgru_0_td, '2_to_1_2', learned=True, shape=[ 1, 1, self.fgru_0.get_shape().as_list()[-1] // 4, self.fgru_0.get_shape().as_list()[-1] ]) fgru_0_td = self.image_resize( fgru_0_td, self.fgru_0.get_shape().as_list()[1:3], align_corners=True) _, fgru_activity, self.inh_3 = self.alt_fgru_ops( # h^(1), h^(2) ff_drive=self.fgru_0, h2=fgru_0_td, layer_id=3, i0=i0) self.x_3 = self.fgru_0 # self.fgru_0 = fgru_activity self.fgru_0 += fgru_activity
def build_model( data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] # norm_moments_training = training # Force instance norm # normalization_type = 'no_param_batch_norm_original' normalization_type = 'no_param_instance_norm' # output_normalization_type = 'batch_norm_original_renorm' output_normalization_type = 'instance_norm' data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Prepare gammanet structure ( compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts) = v2_big_working() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model with tf.variable_scope('vgg', reuse=reuse): aux = get_aux() vgg = vgg16.Vgg16( vgg16_npy_path='/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy', reuse=reuse, aux=aux, train=training, timesteps=8, fgru_normalization_type=normalization_type, ff_normalization_type=normalization_type) vgg(rgb=data_tensor, constructor=gammanet_constructor) # activity = vgg.fgru_0 with tf.variable_scope('fgru', reuse=reuse): # Get side weights hs_0, hs_1 = [], [] h2_rem = [ vgg.fgru_0, vgg.fgru_1, vgg.fgru_2, vgg.conv5_1, vgg.fgru_3] for idx, h in enumerate(h2_rem): res = normalization.apply_normalization( activity=h, name='output_norm1_%s' % idx, normalization_type=output_normalization_type, data_format=data_format, training=training, trainable=training, reuse=reuse) res = aux['image_resize']( res, data_tensor.get_shape().as_list()[1:3], align_corners=True) cnv_0 = tf.layers.conv2d( inputs=res, filters=output_shape, kernel_size=(1, 1), padding='same', data_format=long_data_format, name='readout_aux_00_%s' % idx, # activation=tf.nn.relu, activation=None, trainable=training, use_bias=True, reuse=reuse) cnv_1 = tf.layers.conv2d( inputs=res, filters=output_shape, kernel_size=(1, 1), padding='same', data_format=long_data_format, name='readout_aux_10_%s' % idx, # activation=tf.nn.relu, activation=None, trainable=training, use_bias=True, reuse=reuse) hs_0 += [cnv_0] hs_1 += [cnv_1] s1, s2, s3, s4, s5 = hs_0 s11, s21, s31, s41, s51 = hs_1 o1 = tf.stop_gradient(tf.identity(s1)) o2 = tf.stop_gradient(tf.identity(s2)) o3 = tf.stop_gradient(tf.identity(s3)) o4 = tf.stop_gradient(tf.identity(s4)) o21 = tf.stop_gradient(tf.identity(s21)) o31 = tf.stop_gradient(tf.identity(s31)) o41 = tf.stop_gradient(tf.identity(s41)) o51 = tf.stop_gradient(tf.identity(s51)) p1_1 = s1 p2_1 = s2 + o1 p3_1 = s3 + o2 + o1 p4_1 = s4 + o3 + o2 + o1 p5_1 = s5 + o4 + o3 + o2 + o1 p1_2 = s11 + o21 + o31 + o41 + o51 p2_2 = s21 + o31 + o41 + o51 p3_2 = s31 + o41 + o51 p4_2 = s41 + o51 p5_2 = s51 hs = [p1_1, p2_1, p3_1, p4_1, p5_1, p1_2, p2_2, p3_2, p4_2, p5_2] activity = tf.layers.conv2d( tf.concat(hs, -1), filters=output_shape, kernel_size=(1, 1), padding='same', data_format=long_data_format, name='out', # activation=tf.nn.relu, activation=None, trainable=training, use_bias=False, reuse=reuse) if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) extra_activities = {idx: v for idx, v in enumerate(hs_0)} if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) # return [activity, h_deep], extra_activities return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] # normalization_type = 'no_param_instance_norm' # output_normalization_type = 'instance_norm' fgru_normalization_type = 'no_param_batch_norm_original' ff_normalization_type = 'no_param_batch_norm_original' output_normalization_type = 'batch_norm_original' ff_kernel_size = (5, 5) ff_nl = tf.nn.relu data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Prepare gammanet structure (compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts) = v2_big_working() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model with tf.variable_scope('gammanet', reuse=reuse): activity = tf.layers.conv2d( inputs=data_tensor, filters=gammanet_constructor[0]['features'], kernel_size=ff_kernel_size, padding='same', # strides=(2, 2), data_format=long_data_format, name='l0_0', activation=ff_nl, use_bias=True, reuse=reuse) activity = normalization.apply_normalization( activity=activity, name='input_norm', normalization_type=output_normalization_type, data_format=data_format, training=training, trainable=training, reuse=reuse) # + pre_activity activity = tf.layers.conv2d( inputs=activity, filters=gammanet_constructor[0]['features'], kernel_size=ff_kernel_size, padding='same', # strides=(2, 2), data_format=long_data_format, name='l0_1', activation=ff_nl, use_bias=True, reuse=reuse) # Shift conv-layers to the front # Run fGRU gn = gammanet.GN( layer_name='fgru', gammanet_constructor=gammanet_constructor, data_format=data_format, reuse=reuse, timesteps=3, fgru_connectivity='', # 'all_to_all', additional_readouts=additional_readouts, fgru_normalization_type=fgru_normalization_type, ff_normalization_type=ff_normalization_type, horizontal_padding='SAME', ff_padding='SAME', ff_nl=ff_nl, stop_loop='fgru_3', recurrent_ff=True, horizontal_kernel_initializer=tf.initializers.orthogonal(), kernel_initializer=tf.initializers.orthogonal(), gate_initializer=tf.initializers.orthogonal(), # horizontal_kernel_initializer=tf.contrib.layers.xavier_initializer(), # gate_initializer=tf.contrib.layers.xavier_initializer(), # gate_initializer=tf.contrib.layers.xavier_initializer(), aux=aux, strides=[1, 1, 1, 1], pool_strides=[2, 2], pool_kernel=[2, 2], up_kernel=[4, 4], train=training) h2, h_deep = gn(X=activity) with tf.variable_scope('cv_readout', reuse=reuse): # Apply one more normalization # activity = tf.layers.max_pooling2d( # inputs=h2, # pool_size=(2, 2), # strides=(2, 2), # padding='same', # data_format=long_data_format, # name='output_pool_3') activity = normalization.apply_normalization( activity=h2, name='output_norm_activity', normalization_type=output_normalization_type, data_format=data_format, training=training, trainable=training, reuse=reuse) # + pre_activity # Then read out activity = tf.layers.conv2d( inputs=activity, filters=2048, kernel_size=ff_kernel_size, padding='same', # strides=(2, 2), data_format=long_data_format, name='readout_fc_0', activation=ff_nl, use_bias=True, reuse=reuse) activity = normalization.apply_normalization( activity=activity, name='readout_norm_activity', normalization_type=output_normalization_type, data_format=data_format, training=training, trainable=training, reuse=reuse) # + pre_activity activity = tf.reduce_mean(activity, reduction_indices=[1, 2]) activity = tf.layers.dense(inputs=activity, units=output_shape, activation=None, use_bias=True, bias_initializer=tf.constant_initializer( 1. / 1000.), name='readout_fc_2', reuse=reuse) if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) extra_activities = { # 'activity_low': h2, # 'activity_high': h_deep, } if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) # return [activity, h_deep], extra_activities return activity, extra_activities
def build(self, i0, extra_convs=False): # Convert RGB to BGR with tf.variable_scope('fgru'): error, fgru_activity = self.fgru_ops( # h^(1), h^(2) ff_drive=self.pool2, h2=self.fgru_0, layer_id=0, i0=i0) self.fgru_0 += fgru_activity # + self.conv2_2 self.conv3_1 = self.conv_layer(self.fgru_0, "conv3_1") self.conv3_2 = self.conv_layer(self.conv3_1, "conv3_2") self.conv3_3 = self.conv_layer(self.conv3_2, "conv3_3") self.pool3 = self.max_pool(self.conv3_3, 'pool3') if i0 == 0: self.fgru_1 = self.pool3 with tf.variable_scope('fgru'): error, fgru_activity = self.fgru_ops( # h^(1), h^(2) ff_drive=self.pool3, h2=self.fgru_1, layer_id=1, i0=i0) self.fgru_1 += fgru_activity # + self.conv3_3 self.conv4_1 = self.conv_layer(self.fgru_1, "conv4_1") self.conv4_2 = self.conv_layer(self.conv4_1, "conv4_2") self.conv4_3 = self.conv_layer(self.conv4_2, "conv4_3") self.pool4 = self.max_pool(self.conv4_3, 'pool4') if i0 == 0: self.fgru_2 = self.pool4 with tf.variable_scope('fgru'): error, fgru_activity = self.fgru_ops( # h^(1), h^(2) ff_drive=self.pool4, h2=self.fgru_2, layer_id=2, i0=i0) self.fgru_2 += fgru_activity # + self.conv5_3 self.conv5_1 = self.conv_layer(self.fgru_2, "conv5_1") self.conv5_2 = self.conv_layer(self.conv5_1, "conv5_2") self.conv5_3 = self.conv_layer(self.conv5_2, "conv5_3") if i0 == 0: self.fgru_3 = self.conv5_3 with tf.variable_scope('fgru'): error, fgru_activity = self.fgru_ops( # h^(1), h^(2) ff_drive=self.conv5_3, h2=self.fgru_3, layer_id=3, i0=i0) self.fgru_3 += fgru_activity # + self.conv5_3 # Resize and conv error, fgru_activity = self.fgru_ops( # h^(1), h^(2) ff_drive=self.fgru_2, h2=self.fgru_3, layer_id=4, i0=i0) self.fgru_2 += fgru_activity # Resize and conv with tf.variable_scope('fgru'): fgru_1_td = normalization.apply_normalization( activity=self.fgru_2, name='td_norm1_%s' % i0, normalization_type=self.fgru_normalization_type, data_format=self.data_format, training=self.fgru_normalization_type, trainable=self.train, reuse=self.reuse) fgru_1_td = self.image_resize( fgru_1_td, self.fgru_1.get_shape().as_list()[1:3], align_corners=True) fgru_1_td = self.conv_layer( fgru_1_td, '4_to_3', learned=True, apply_relu=True, shape=[ 1, 1, fgru_1_td.get_shape().as_list()[-1], self.fgru_1.get_shape().as_list()[-1] ]) if extra_convs: fgru_1_td = self.conv_layer( fgru_1_td, '4_to_3_2', learned=True, shape=[ 1, 1, self.fgru_1.get_shape().as_list()[-1], self.fgru_1.get_shape().as_list()[-1] ]) error, fgru_activity = self.fgru_ops( # h^(1), h^(2) ff_drive=self.fgru_1, h2=fgru_1_td, layer_id=5, i0=i0) self.fgru_1 += fgru_activity # Resize and conv with tf.variable_scope('fgru'): fgru_0_td = normalization.apply_normalization( activity=self.fgru_1, name='td_norm0_%s' % i0, normalization_type=self.fgru_normalization_type, data_format=self.data_format, training=self.fgru_normalization_type, trainable=self.train, reuse=self.reuse) fgru_0_td = self.image_resize( fgru_0_td, self.fgru_0.get_shape().as_list()[1:3], align_corners=True) fgru_0_td = self.conv_layer( fgru_0_td, '3_to_2', learned=True, apply_relu=True, shape=[ 1, 1, fgru_0_td.get_shape().as_list()[-1], self.fgru_0.get_shape().as_list()[-1] ]) if extra_convs: fgru_0_td = self.conv_layer( fgru_0_td, '3_to_2_2', learned=True, shape=[ 1, 1, self.fgru_0.get_shape().as_list()[-1], self.fgru_0.get_shape().as_list()[-1] ]) error, fgru_activity = self.fgru_ops( # h^(1), h^(2) ff_drive=self.fgru_0, h2=fgru_0_td, layer_id=6, i0=i0) # self.fgru_0 = fgru_activity self.fgru_0 += fgru_activity
def build_model(data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] # norm_moments_training = training # Force instance norm # normalization_type = 'no_param_batch_norm_original' # output_normalization_type = 'batch_norm_original_renorm' output_normalization_type = 'instance_norm' data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Build model with tf.variable_scope('vgg', reuse=reuse): vgg = vgg16.Vgg16( vgg16_npy_path= '/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy') vgg(rgb=data_tensor, train=training, ff_reuse=reuse) with tf.variable_scope('fgru', reuse=reuse): # Get side weights h2_rem = [ vgg.conv1_2, vgg.conv2_2, vgg.conv3_3, vgg.conv4_3, vgg.conv5_3 ] res_act = [] for idx, h in enumerate(h2_rem): res = normalization.apply_normalization( activity=h, name='output_norm1_%s' % idx, normalization_type=output_normalization_type, data_format=data_format, training=training, trainable=training, reuse=reuse) res_act += [ tf.image.resize_bilinear( res, data_tensor.get_shape().as_list()[1:3], align_corners=True) ] activity = tf.layers.conv2d(tf.concat(res_act, -1), filters=output_shape, kernel_size=(1, 1), padding='same', data_format=long_data_format, name='out', activation=None, trainable=training, use_bias=True, reuse=reuse) if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) extra_activities = {} # idx: v for idx, v in enumerate(hs_0)} if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape, learned_temp=False, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] # norm_moments_training = training # Force instance norm # normalization_type = 'no_param_batch_norm_original' # output_normalization_type = 'batch_norm_original_renorm' output_normalization_type = 'instance_norm' data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Build model with tf.variable_scope('vgg', reuse=reuse): vgg = vgg16.Vgg16( vgg16_npy_path= '/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy') vgg(rgb=data_tensor, train=training, ff_reuse=reuse) with tf.variable_scope('fgru', reuse=reuse): # Get side weights h2_rem = [ vgg.conv1_2, vgg.conv2_2, vgg.conv3_3, vgg.conv4_3, vgg.conv5_3 ] res_act = [] for idx, h in enumerate(h2_rem): res = normalization.apply_normalization( activity=h, name='output_norm1_%s' % idx, normalization_type=output_normalization_type, data_format=data_format, training=training, trainable=training, reuse=reuse) res_act += [ tf.image.resize_bilinear( res, data_tensor.get_shape().as_list()[1:3], align_corners=True) ] # # 3-step readout # (1) Per-pixel competition activity = tf.layers.conv2d(tf.concat(res_act, -1), filters=1, kernel_size=(3, 3), padding='same', data_format=long_data_format, name='out_0', activation=tf.nn.sigmoid, trainable=training, use_bias=True, reuse=reuse) # (2) Softmax across locations activity = tf.layers.conv2d(tf.concat(activity, -1), filters=1, kernel_size=(3, 3), padding='same', data_format=long_data_format, name='out_1', activation=None, trainable=training, use_bias=True, reuse=reuse) act_shape = activity.get_shape().as_list() activity = tf.reshape(activity, [act_shape[0], -1]) if learned_temp: # Potentially add another non-linearity pre-GAP for VAF:q": temperature = tf.layers.dense(tf.reduce_mean(activity, reduction_indices=[1 ]), 1, name='temperature', activation=None, use_bias=True) sigmoid_attention = tf.nn.sigmoid(temperature) map_activity = tf.nn.softmax(activity / sigmoid_attention) else: map_activity = tf.nn.softmax(activity) map_activity = tf.reshape(map_activity, act_shape) # (3) GAP & readout activity = tf.reduce_mean(map_activity, reduction_indices=[1, 2]) activity = tf.layers.dense(activity, output_shape, name='out_2', reuse=reuse, activation=None, use_bias=True) if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) if learned_temp: extra_activities = { 'temperature': temperature, 'map_activity': map_activity } else: extra_activities = {'map_activity': map_activity} if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] input_training = training # training readout_training = training # False norm_moments_training = training # Force instance norm norm_params_training = training fgru_kernel_training = training fgru_param_training = training ff_gate_training = training fgru_gate_training = training remaining_param_training = training normalization_type = 'no_param_batch_norm_original' output_normalization_type = 'batch_norm_original' # normalization_type = 'no_param_instance_norm' # output_normalization_type = 'instance_norm' ff_kernel_size = (3, 3) ff_nl = tf.nn.elu data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Prepare gammanet structure (compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts) = v2_big_working() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model with tf.variable_scope('gammanet', reuse=reuse): aux = get_aux() vgg = vgg16.Vgg16( vgg16_npy_path= '/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy', reuse=reuse, aux=aux, train=training, timesteps=4, fgru_normalization_type=normalization_type, ff_normalization_type=normalization_type) vgg(rgb=data_tensor, constructor=gammanet_constructor) # h2_rem = [vgg.conv5_3, vgg.conv4_3, vgg.conv3_3, vgg.conv2_2, vgg.conv1_1] h2 = vgg.fgru_0 with tf.variable_scope('contour_readout', reuse=reuse): # Apply one more normalization # h_deep = apply_normalization( # activity=h_deep, # name='output_normh_deep', # normalization_type=normalization_type, # data_format=data_format, # training=training, # reuse=reuse) if 0: hs = [] for h in h2_rem: hs += [ tf.image.resize_nearest_neighbor( h, data_tensor.get_shape().as_list()[1:3]) ] activity = normalization.apply_normalization( activity=tf.concat(hs, axis=-1), name='output_norm', normalization_type=output_normalization_type, data_format=data_format, training=norm_moments_training, trainable=training, reuse=reuse) else: activity = normalization.apply_normalization( activity=h2, name='output_norm', normalization_type=output_normalization_type, data_format=data_format, training=norm_moments_training, trainable=training, reuse=reuse) # Then read out activity = tf.layers.conv2d(inputs=activity, filters=output_shape, kernel_size=(1, 1), padding='same', data_format=long_data_format, name='readout_conv', activation=None, trainable=training, use_bias=True, reuse=reuse) if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) extra_activities = {} if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) # return [activity, h_deep], extra_activities return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) normalization_type = 'instance_norm' ff_nl = tf.nn.relu # Prepare gammanet structure compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts = v1( ) # compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts = v3() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model with tf.variable_scope('gammanet', reuse=reuse): in_emb = tf.layers.conv2d(inputs=data_tensor, filters=16, kernel_size=7, strides=(1, 1), padding='same', data_format=long_data_format, activation=ff_nl, trainable=training, use_bias=True, name='l0') in_emb = tf.layers.conv2d( inputs=in_emb, filters=16, kernel_size=7, strides=(1, 1), padding='same', data_format=long_data_format, activation=None, # ff_nl, trainable=training, use_bias=True, name='l01') in_emb = normalization.apply_normalization( activity=in_emb, name='in_norm', normalization_type=normalization_type, data_format=data_format, training=training, reuse=reuse) in_emb = ff_nl(in_emb) # Run fGRU gn = gammanet.GN( layer_name='fgru', gammanet_constructor=gammanet_constructor, data_format=data_format, reuse=reuse, timesteps=8, fgru_connectivity='', # 'all_to_all', additional_readouts=additional_readouts, fgru_normalization_type=normalization_type, ff_normalization_type=normalization_type, horizontal_padding='SAME', ff_padding='SAME', ff_nl=ff_nl, recurrent_ff=False, horizontal_kernel_initializer=tf.initializers.orthogonal(), # horizontal_kernel_initializer=tf_fun.Identity(), kernel_initializer=tf.initializers.orthogonal(), aux=aux, strides=[1, 1, 1, 1], pool_strides=[2, 2], pooling_kernel=[2, 2], up_kernel=[4, 4], train=training) h2, h_deep = gn(X=in_emb) with tf.variable_scope('cv_readout', reuse=reuse): # Apply one more normalization h2 = normalization.apply_normalization( activity=h2, name='output_norm', normalization_type=normalization_type, data_format=data_format, training=training, reuse=reuse) # h_deep = apply_normalization( # activity=h_deep, # name='output_normh_deep', # normalization_type=normalization_type, # data_format=data_format, # training=training, # reuse=reuse) # Then read out activity = tf.layers.conv2d(inputs=h2, filters=output_shape, kernel_size=(1, 1), padding='same', data_format=long_data_format, name='pre_readout_conv', use_bias=True, reuse=reuse) # h_deep = tf.layers.conv2d( # inputs=h_deep, # filters=output_shape, # kernel_size=(1, 1), # padding='same', # data_format=long_data_format, # name='pre_readout_conv1', # use_bias=True, # reuse=reuse) if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) extra_activities = { 'activity_low': h2, 'activity_high': h_deep, } if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) # return [activity, h_deep], extra_activities return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] # norm_moments_training = training # Force instance norm # normalization_type = 'no_param_batch_norm_original' normalization_type = 'no_param_instance_norm' # output_normalization_type = 'batch_norm_original_renorm' output_normalization_type = 'instance_norm' data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Prepare gammanet structure (compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts) = v2_big_working() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model with tf.variable_scope('vgg', reuse=reuse): aux = get_aux() vgg = vgg16.Vgg16( vgg16_npy_path= '/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy', reuse=reuse, aux=aux, train=False, timesteps=8, fgru_normalization_type=normalization_type, ff_normalization_type=normalization_type) vgg(rgb=data_tensor, constructor=gammanet_constructor) activity = vgg.fgru_0 with tf.variable_scope('fgru_gilbert', reuse=reuse): # Then read out # activity = activity ** 2 activity = normalization.apply_normalization( activity=activity, name='output_norm1', normalization_type=output_normalization_type, data_format=data_format, training=training, trainable=training, reuse=reuse) activity = tf.layers.conv2d( inputs=activity, filters=gammanet_constructor[0]['features'], kernel_size=(1, 1), padding='same', data_format=long_data_format, name='readout_l0_1', activation=tf.nn.relu, use_bias=True, trainable=training, reuse=reuse) activity = tf.layers.conv2d(inputs=activity, filters=1, kernel_size=(1, 1), padding='same', data_format=long_data_format, name='readout_l0_2', activation=None, use_bias=True, trainable=training, reuse=reuse) activity = tf.reduce_max(activity, reduction_indices=[1, 2]) if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) extra_activities = {} if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) # return [activity, h_deep], extra_activities return activity, extra_activities