def build_model(data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
normalization_type = 'no_param_instance_norm'
output_normalization_type = 'instance_norm'
ff_kernel_size = (3, 3)
# ff_nl = tf.nn.relu
ff_nl = tf.nn.elu
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor, data_format=data_format)
# Prepare gammanet structure
(compression, ff_kernels, ff_repeats, features, fgru_kernels,
additional_readouts) = spec()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
with tf.variable_scope('gammanet', reuse=reuse):
activity = tf.layers.conv2d(
inputs=data_tensor,
filters=gammanet_constructor[0]['features'],
kernel_size=ff_kernel_size,
padding='same',
data_format=long_data_format,
name='l0_0',
activation=ff_nl,
use_bias=True,
reuse=reuse)
activity = tf.layers.conv2d(
inputs=activity,
filters=gammanet_constructor[0]['features'],
kernel_size=ff_kernel_size,
padding='same',
data_format=long_data_format,
name='l0_1',
activation=ff_nl,
use_bias=True,
reuse=reuse)
activity = normalization.apply_normalization(
activity=activity,
name='input_norm',
normalization_type=output_normalization_type,
data_format=data_format,
training=training,
trainable=training,
reuse=reuse)
# Shift conv-layers to the front
# Run fGRU
gn = gammanet.GN(
layer_name='fgru',
gammanet_constructor=gammanet_constructor,
data_format=data_format,
reuse=reuse,
timesteps=8,
fgru_connectivity='', # 'all_to_all',
additional_readouts=additional_readouts,
fgru_normalization_type=normalization_type,
ff_normalization_type=normalization_type,
horizontal_padding='SAME',
ff_padding='SAME',
ff_nl=ff_nl,
recurrent_ff=True,
horizontal_kernel_initializer=tf.initializers.orthogonal(),
kernel_initializer=tf.initializers.orthogonal(),
gate_initializer=tf.initializers.orthogonal(),
aux=aux,
strides=[1, 1, 1, 1],
pool_strides=[2, 2],
pool_kernel=[2, 2],
up_kernel=[4, 4],
train=training)
h2, h_deep = gn(X=activity)
with tf.variable_scope('cv_readout', reuse=reuse):
# Apply one more normalization
activity = normalization.apply_normalization(
activity=h2,
name='output_norm',
normalization_type=output_normalization_type,
data_format=data_format,
training=training,
trainable=training,
reuse=reuse)
activity = tf.layers.conv2d(
inputs=activity,
filters=output_shape,
kernel_size=(5, 5), # (1, 1),
padding='same',
data_format=long_data_format,
name='readout_conv_2',
activation=None,
use_bias=True,
reuse=reuse)
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
extra_activities = {
'activity_low': h2,
'activity_high': h_deep,
}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
return activity, extra_activities
def build_model(data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
input_training = training
readout_training = training # False
norm_moments_training = training # Force instance norm
norm_params_training = training
fgru_kernel_training = training
fgru_param_training = training
ff_gate_training = training
fgru_gate_training = training
remaining_param_training = training
# normalization_type = 'no_param_batch_norm_original_renorm'
# output_normalization_type = 'batch_norm_original_renorm'
normalization_type = 'no_param_batch_norm_original'
output_normalization_type = 'batch_norm_original'
# normalization_type = 'no_param_instance_norm'
# output_normalization_type = 'instance_norm'
ff_kernel_size = (3, 3)
ff_nl = tf.nn.elu
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor, data_format=data_format)
# Prepare gammanet structure
(compression, ff_kernels, ff_repeats, features, fgru_kernels,
additional_readouts) = v2_big_working()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
with tf.variable_scope('gammanet', reuse=reuse):
activity_1 = tf.layers.conv2d(
inputs=data_tensor,
filters=gammanet_constructor[0]['features'],
kernel_size=ff_kernel_size,
padding='same',
data_format=long_data_format,
name='l0_0',
activation=ff_nl,
use_bias=True,
trainable=input_training,
reuse=reuse)
activity_2 = tf.layers.conv2d(
inputs=activity_1,
filters=gammanet_constructor[0]['features'],
kernel_size=ff_kernel_size,
padding='same',
data_format=long_data_format,
name='l0_1',
activation=ff_nl,
use_bias=True,
trainable=input_training,
reuse=reuse)
# activity = normalization.apply_normalization(
# activity=activity_2,
# name='input_norm_instance',
# normalization_type=normalization_type, # output_normalization_type,
# data_format=data_format,
# training=norm_moments_training,
# trainable=input_training,
# reuse=reuse)
# with tf.variable_scope('contour_readout', reuse=reuse):
# beta = tf.get_variable(
# name='hello_instance_beta',
# shape=[1, 1, 1, gammanet_constructor[0]['features']],
# initializer=tf.initializers.zeros,
# trainable=input_training)
# gamma = tf.get_variable(
# name='helo_instance_gamma',
# shape=[1, 1, 1, gammanet_constructor[0]['features']],
# initializer=tf.initializers.ones,
# trainable=input_training)
# activity = beta + gamma * activity_2
activity = normalization.apply_normalization(
activity=activity_2,
name='input_norm',
normalization_type=output_normalization_type,
data_format=data_format,
training=training,
trainable=training,
reuse=reuse)
with tf.variable_scope('gammanet', reuse=reuse):
# Shift conv-layers to the front
# Run fGRU
gn = gammanet.GN(
layer_name='fgru',
gammanet_constructor=gammanet_constructor,
data_format=data_format,
reuse=reuse,
timesteps=8,
train=remaining_param_training,
train_fgru_gate=fgru_gate_training,
train_ff_gate=ff_gate_training,
train_norm_moments=norm_moments_training,
train_norm_params=norm_params_training,
train_fgru_kernels=fgru_kernel_training,
train_fgru_params=fgru_param_training,
fgru_connectivity='', # 'all_to_all',
additional_readouts=additional_readouts,
fgru_normalization_type=normalization_type,
ff_normalization_type=normalization_type,
horizontal_padding='SAME',
ff_padding='SAME',
ff_nl=ff_nl,
recurrent_ff=True,
horizontal_kernel_initializer=tf.initializers.orthogonal(),
kernel_initializer=tf.initializers.orthogonal(),
gate_initializer=tf.initializers.orthogonal(),
# horizontal_kernel_initializer=tf.contrib.layers.xavier_initializer(),
# kernel_initializer=tf.contrib.layers.xavier_initializer(),
# gate_initializer=tf.contrib.layers.xavier_initializer(),
aux=aux,
strides=[1, 1, 1, 1],
pool_strides=[2, 2],
pool_kernel=[2, 2],
up_kernel=[4, 4])
layers = gn(X=activity)
h2 = layers[0]
layers = layers[1:]
# h2 = h2[0]
# h2 = gn(X=activity)
with tf.variable_scope('contour_readout', reuse=reuse):
activity = normalization.apply_normalization(
activity=h2, # activity,
name='output_norm',
normalization_type=normalization_type,
data_format=data_format,
training=training,
trainable=training,
reuse=reuse)
beta = tf.get_variable(name='hello_instance_beta',
shape=[1, 1, 1,
h2.get_shape().as_list()[-1]],
initializer=tf.initializers.zeros,
trainable=input_training)
gamma = tf.get_variable(name='helo_instance_gamma',
shape=[1, 1, 1,
h2.get_shape().as_list()[-1]],
initializer=tf.initializers.constant(0.1),
trainable=input_training)
activity = beta + gamma * activity
activity = tf.layers.conv2d(
inputs=activity,
filters=512, # gammanet_constructor[0]['features'],
kernel_size=ff_kernel_size,
padding='same',
data_format=long_data_format,
name='readout_conv',
activation=ff_nl,
use_bias=True,
reuse=reuse)
activity = normalization.apply_normalization(
activity=activity, # activity,
name='output_norm_1',
normalization_type=output_normalization_type,
data_format=data_format,
training=training,
trainable=training,
reuse=reuse)
activity = tf.layers.conv2d(
inputs=activity,
filters=128, # gammanet_constructor[0]['features'],
kernel_size=ff_kernel_size,
padding='same',
data_format=long_data_format,
name='readout_conv_1',
activation=ff_nl,
use_bias=True,
reuse=reuse)
# activity = normalization.apply_normalization(
# activity=activity,
# name='output_norm_2',
# normalization_type=output_normalization_type,
# data_format=data_format,
# training=training,
# trainable=training,
# reuse=reuse)
activity = tf.layers.conv2d(inputs=activity,
filters=output_shape,
kernel_size=ff_kernel_size,
padding='same',
data_format=long_data_format,
name='readout_conv_2',
activation=None,
use_bias=True,
reuse=reuse)
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
extra_activities = {}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
# return [activity, h_deep], extra_activities
return activity, extra_activities
def build_model(
data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
output_shape = 2
# norm_moments_training = training # Force instance norm
# normalization_type = 'no_param_batch_norm_original'
normalization_type = 'no_param_instance_norm'
# output_normalization_type = 'batch_norm_original_renorm'
output_normalization_type = 'instance_norm'
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor,
data_format=data_format)
# Prepare gammanet structure
(
compression,
ff_kernels,
ff_repeats,
features,
fgru_kernels,
additional_readouts) = v2_big_working()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
with tf.variable_scope('vgg', reuse=reuse):
vgg = vgg16.Vgg16(
vgg16_npy_path='/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy')
vgg(rgb=data_tensor, train=False, ff_reuse=reuse)
# activity = vgg.fgru_0
with tf.variable_scope('fgru', reuse=reuse):
# Get side weights
h2_rem = [
vgg.fgru_0, vgg.fgru_1, vgg.fgru_2, vgg.fgru_3]
h2s = []
for idx, h in enumerate(h2_rem):
res = aux['image_resize'](
h,
data_tensor.get_shape().as_list()[1:3],
align_corners=True)
h2s += [res]
res = normalization.apply_normalization(
activity=tf.concat(h2s, -1),
name='output_norm1',
normalization_type=output_normalization_type,
data_format=data_format,
training=False,
trainable=False,
reuse=reuse)
activity = tf.layers.conv2d(
inputs=res,
filters=gammanet_constructor[0]['features'],
kernel_size=(1, 1),
padding='same',
data_format=long_data_format,
name='readout_l0_1',
activation=tf.nn.relu,
use_bias=True,
trainable=training,
reuse=reuse)
orientations = tf.layers.conv2d(
inputs=activity,
filters=output_shape,
kernel_size=(1, 1),
padding='same',
data_format=long_data_format,
name='readout_l0_2',
activation=None,
use_bias=True,
trainable=training,
reuse=reuse)
_, h, w, _ = orientations.get_shape().as_list()
activity = orientations[:, h // 2, w // 2, :]
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
extra_activities = {'orientations': orientations}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
# return [activity, h_deep], extra_activities
return activity, extra_activities
def build_model(data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
normalization_type = 'no_param_instance_norm'
output_normalization_type = 'instance_norm'
# normalization_type = 'no_param_batch_norm'
# output_normalization_type = 'batch_norm'
ff_kernel_size = (3, 3)
ff_nl = tf.nn.relu
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor, data_format=data_format)
# Prepare gammanet structure
(compression, ff_kernels, ff_repeats, features, fgru_kernels,
additional_readouts) = v2_big_working()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
fix_training = False
with tf.variable_scope('gammanet', reuse=reuse):
activity = tf.layers.conv2d(
inputs=data_tensor,
filters=gammanet_constructor[0]['features'],
kernel_size=ff_kernel_size,
padding='same',
data_format=long_data_format,
name='l0_0',
activation=ff_nl,
use_bias=True,
# trainable=training,
trainable=fix_training,
reuse=reuse)
activity = tf.layers.conv2d(
inputs=activity,
filters=gammanet_constructor[0]['features'],
kernel_size=ff_kernel_size,
padding='same',
data_format=long_data_format,
name='l0_1',
activation=None,
use_bias=True,
# trainable=training,
trainable=fix_training,
reuse=reuse)
activity = normalization.apply_normalization(
activity=activity,
name='input_norm',
normalization_type=output_normalization_type,
data_format=data_format,
# training=training,
training=fix_training,
reuse=reuse) # + pre_activity
activity = ff_nl(activity)
# # Downsample
# activity = conv.down_block(
# layer_name='l1',
# bottom=activity,
# kernel_size=3 * [[3, 3]],
# num_filters=gammanet_constructor[0]['features'],
# normalization_type=output_normalization_type,
# training=training,
# activation=ff_nl,
# include_pool=False,
# reuse=reuse)
# Shift conv-layers to the front
# Run fGRU
gn = gammanet.GN(
layer_name='fgru',
gammanet_constructor=gammanet_constructor,
data_format=data_format,
reuse=reuse,
timesteps=8,
fgru_connectivity='', # 'all_to_all',
additional_readouts=additional_readouts,
fgru_normalization_type=normalization_type,
ff_normalization_type=normalization_type,
horizontal_padding='SAME',
ff_padding='SAME',
ff_nl=ff_nl,
recurrent_ff=True,
horizontal_kernel_initializer=tf.initializers.orthogonal(),
kernel_initializer=tf.initializers.orthogonal(),
gate_initializer=tf.initializers.orthogonal(),
# horizontal_kernel_initializer=tf.contrib.layers.xavier_initializer(),
# gate_initializer=tf.contrib.layers.xavier_initializer(),
# gate_initializer=tf.contrib.layers.xavier_initializer(),
aux=aux,
strides=[1, 1, 1, 1],
pool_strides=[2, 2],
pool_kernel=[2, 2],
up_kernel=[4, 4],
train=fix_training)
h2, h_deep = gn(X=activity)
with tf.variable_scope('gratings_readout', reuse=reuse):
h2 = normalization.apply_normalization(
activity=h2,
name='output_norm',
normalization_type=output_normalization_type,
data_format=data_format,
training=training,
reuse=reuse) # + pre_activity
# Then read out
activity = tf.layers.conv2d(
inputs=h2,
filters=gammanet_constructor[0]['features'],
kernel_size=(1, 1),
padding='same',
data_format=long_data_format,
name='readout_l0_1',
activation=ff_nl,
use_bias=True,
trainable=training,
reuse=reuse)
activity = tf.layers.conv2d(inputs=activity,
filters=output_shape,
kernel_size=(1, 1),
padding='same',
data_format=long_data_format,
name='readout_l0_2',
activation=None,
use_bias=True,
trainable=training,
reuse=reuse)
_, h, w, _ = activity.get_shape().as_list()
activity = activity[:, h // 2, w // 2, :]
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
extra_activities = {
'activity_low': h2,
'activity_high': h_deep,
}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
# return [activity, h_deep], extra_activities
return activity, extra_activities
def build(self, i0, extra_convs=True):
# Convert RGB to BGR
with tf.variable_scope('fgru'):
error_horizontal_0, fgru_activity = self.fgru_ops( # h^(1), h^(2)
ff_drive=self.conv2_2,
h2=self.fgru_0,
layer_id=0,
i0=i0)
self.fgru_0 = fgru_activity # + self.conv2_2
self.pool2 = self.max_pool(self.fgru_0, 'pool2')
self.conv3_1 = self.conv_layer(self.pool2, "conv3_1")
self.conv3_2 = self.conv_layer(self.conv3_1, "conv3_2")
self.conv3_3 = self.conv_layer(self.conv3_2, "conv3_3")
if i0 == 0:
self.fgru_1 = self.conv3_3
with tf.variable_scope('fgru'):
error, fgru_activity = self.fgru_ops( # h^(1), h^(2)
ff_drive=self.conv3_3,
h2=self.fgru_1,
layer_id=1,
i0=i0)
self.fgru_1 = fgru_activity # + self.conv2_2
self.pool3 = self.max_pool(self.fgru_1, 'pool3')
self.conv4_1 = self.conv_layer(self.pool3, "conv4_1")
self.conv4_2 = self.conv_layer(self.conv4_1, "conv4_2")
self.conv4_3 = self.conv_layer(self.conv4_2, "conv4_3")
if i0 == 0:
self.fgru_2 = self.conv4_3
with tf.variable_scope('fgru'):
error, fgru_activity = self.fgru_ops( # h^(1), h^(2)
ff_drive=self.conv4_3,
h2=self.fgru_2,
layer_id=2,
i0=i0)
self.fgru_2 = fgru_activity # + self.conv3_3
self.pool4 = self.max_pool(self.fgru_2, 'pool4')
self.conv5_1 = self.conv_layer(self.pool4, "conv5_1")
self.conv5_2 = self.conv_layer(self.conv5_1, "conv5_2")
self.conv5_3 = self.conv_layer(self.conv5_2, "conv5_3")
if i0 == 0:
self.fgru_3 = self.conv5_3
with tf.variable_scope('fgru'):
error, fgru_activity = self.fgru_ops( # h^(1), h^(2)
ff_drive=self.conv5_3,
h2=self.fgru_3,
layer_id=3,
i0=i0)
self.fgru_3 = fgru_activity # + self.conv5_3
# Resize and conv
with tf.variable_scope('fgru'):
fgru_2_td = normalization.apply_normalization(
activity=self.fgru_3,
name='td_norm2_%s' % i0,
# normalization_type='instance_norm',
normalization_type=self.fgru_normalization_type,
data_format=self.data_format,
training=self.train,
trainable=self.train,
reuse=self.reuse)
fgru_2_td = self.conv_layer(
fgru_2_td,
'4_to_3',
learned=True,
apply_relu=True,
shape=[
1, 1,
fgru_2_td.get_shape().as_list()[-1],
self.fgru_2.get_shape().as_list()[-1] // 64
])
fgru_2_td = normalization.apply_normalization(
activity=fgru_2_td,
name='td_norm2_1%s' % i0,
# normalization_type='instance_norm',
normalization_type=self.fgru_normalization_type,
data_format=self.data_format,
training=self.train,
trainable=self.train,
reuse=self.reuse)
if extra_convs:
fgru_2_td = self.conv_layer(
fgru_2_td,
'4_to_3_2',
learned=True,
shape=[
1, 1,
self.fgru_2.get_shape().as_list()[-1] // 64,
self.fgru_2.get_shape().as_list()[-1]
])
fgru_2_td = self.image_resize(
fgru_2_td,
self.fgru_2.get_shape().as_list()[1:3],
align_corners=True)
error, fgru_activity = self.fgru_ops( # h^(1), h^(2)
ff_drive=self.fgru_2,
h2=fgru_2_td,
layer_id=4,
i0=i0)
self.fgru_2 += fgru_activity
if i0 == self.last_timestep and self.downsampled:
return self.fgru_2
# Resize and conv
with tf.variable_scope('fgru'):
fgru_1_td = normalization.apply_normalization(
activity=self.fgru_2,
name='td_norm1_%s' % i0,
# normalization_type='instance_norm',
normalization_type=self.fgru_normalization_type,
data_format=self.data_format,
training=self.train,
trainable=self.train,
reuse=self.reuse)
fgru_1_td = self.conv_layer(
fgru_1_td,
'3_to_2',
learned=True,
apply_relu=True,
shape=[
1, 1,
fgru_1_td.get_shape().as_list()[-1],
self.fgru_1.get_shape().as_list()[-1] // 16
])
fgru_1_td = normalization.apply_normalization(
activity=fgru_1_td,
name='td_norm1_1%s' % i0,
# normalization_type='instance_norm',
normalization_type=self.fgru_normalization_type,
data_format=self.data_format,
training=self.train,
trainable=self.train,
reuse=self.reuse)
if extra_convs:
fgru_1_td = self.conv_layer(
fgru_1_td,
'3_to_2_2',
learned=True,
shape=[
1, 1,
self.fgru_1.get_shape().as_list()[-1] // 16,
self.fgru_1.get_shape().as_list()[-1]
])
fgru_1_td = self.image_resize(
fgru_1_td,
self.fgru_1.get_shape().as_list()[1:3],
align_corners=True)
error, fgru_activity = self.fgru_ops( # h^(1), h^(2)
ff_drive=self.fgru_1,
h2=fgru_1_td,
layer_id=5,
i0=i0)
# self.fgru_0 = fgru_activity
self.fgru_1 += fgru_activity
# Resize and conv
with tf.variable_scope('fgru'):
fgru_0_td = normalization.apply_normalization(
activity=self.fgru_1,
name='td_norm0_%s' % i0,
# normalization_type='instance_norm',
normalization_type=self.fgru_normalization_type,
data_format=self.data_format,
training=self.train,
trainable=self.train,
reuse=self.reuse)
fgru_0_td = self.conv_layer(
fgru_0_td,
'2_to_1',
learned=True,
apply_relu=True,
shape=[
1, 1,
fgru_0_td.get_shape().as_list()[-1],
self.fgru_0.get_shape().as_list()[-1] // 4
])
fgru_0_td = normalization.apply_normalization(
activity=fgru_0_td,
name='td_norm0_1%s' % i0,
# normalization_type='instance_norm',
normalization_type=self.fgru_normalization_type,
data_format=self.data_format,
training=self.train,
trainable=self.train,
reuse=self.reuse)
if extra_convs:
fgru_0_td = self.conv_layer(
fgru_0_td,
'2_to_1_2',
learned=True,
shape=[
1, 1,
self.fgru_0.get_shape().as_list()[-1] // 4,
self.fgru_0.get_shape().as_list()[-1]
])
fgru_0_td = self.image_resize(
fgru_0_td,
self.fgru_0.get_shape().as_list()[1:3],
align_corners=True)
error_td_0, fgru_activity = self.fgru_ops( # h^(1), h^(2)
ff_drive=self.fgru_0,
h2=fgru_0_td,
layer_id=6,
i0=i0)
self.fgru_0 = self.fgru_0 + fgru_activity
def build_model(data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
# norm_moments_training = training # Force instance norm
# normalization_type = 'no_param_batch_norm_original'
normalization_type = 'no_param_instance_norm'
# output_normalization_type = 'batch_norm_original_renorm'
output_normalization_type = 'instance_norm'
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor, data_format=data_format)
# Prepare gammanet structure
(compression, ff_kernels, ff_repeats, features, fgru_kernels,
additional_readouts) = v2_big_working()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
with tf.variable_scope('vgg', reuse=reuse):
aux = get_aux()
vgg = vgg16.Vgg16(
vgg16_npy_path=
'/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy',
reuse=reuse,
aux=aux,
train=training,
timesteps=8,
fgru_normalization_type=normalization_type,
ff_normalization_type=normalization_type)
vgg(rgb=data_tensor, constructor=gammanet_constructor)
# activity = vgg.fgru_0
with tf.variable_scope('fgru', reuse=reuse):
# Get side weights
h2_rem = [vgg.fgru_0]
for idx, h in enumerate(h2_rem):
res = normalization.apply_normalization(
activity=h,
name='output_norm1_%s' % idx,
normalization_type=output_normalization_type,
data_format=data_format,
training=training,
trainable=training,
reuse=reuse)
res = aux['image_resize'](res,
data_tensor.get_shape().as_list()[1:3],
align_corners=True)
activity = tf.layers.conv2d(res,
filters=res.get_shape().as_list()[-1],
kernel_size=(3, 3),
padding='same',
data_format=long_data_format,
name='out',
activation=tf.nn.relu,
trainable=training,
use_bias=True,
reuse=reuse)
activity = tf.layers.conv2d(
activity,
filters=output_shape,
kernel_size=(1, 1),
padding='same',
data_format=long_data_format,
name='out2',
# activation=tf.nn.relu,
activation=None,
trainable=training,
use_bias=True,
reuse=reuse)
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
extra_activities = {} # idx: v for idx, v in enumerate(hs_0)}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
# return [activity, h_deep], extra_activities
return activity, extra_activities
def build(self, i0, extra_convs=True):
# Convert RGB to BGR
with tf.variable_scope('fgru'):
_, fgru_activity, self.inh_0 = self.alt_fgru_ops( # h^(1), h^(2)
ff_drive=self.conv2_2,
h2=self.fgru_0,
layer_id=0,
i0=i0)
self.x_0 = self.conv2_2
self.fgru_0 = fgru_activity # + self.conv2_2
self.pool2 = self.max_pool(self.fgru_0, 'pool2')
self.conv3_1 = self.conv_layer(self.pool2, "conv3_1")
self.conv3_2 = self.conv_layer(self.conv3_1, "conv3_2")
self.conv3_3 = self.conv_layer(self.conv3_2, "conv3_3")
if i0 == 0:
self.fgru_1 = self.conv3_3
with tf.variable_scope('fgru'):
_, fgru_activity, self.inh_1 = self.alt_fgru_ops( # h^(1), h^(2)
ff_drive=self.conv3_3,
h2=self.fgru_1,
layer_id=1,
i0=i0)
self.x_1 = self.conv3_3
self.fgru_1 = fgru_activity # + self.conv2_2
self.pool3 = self.max_pool(self.fgru_1, 'pool3')
self.conv4_1 = self.conv_layer(self.pool3, "conv4_1")
self.conv4_2 = self.conv_layer(self.conv4_1, "conv4_2")
self.conv4_3 = self.conv_layer(self.conv4_2, "conv4_3")
if i0 == self.timesteps - 1:
return self.conv4_3
# self.pool4 = self.max_pool(self.conv4_3, 'pool4')
# self.conv5_1 = self.conv_layer(self.pool4, "conv5_1")
# self.conv5_2 = self.conv_layer(self.conv5_1, "conv5_2")
# self.conv5_3 = self.conv_layer(self.conv5_2, "conv5_3")
# if i0 == self.timesteps - 1:
# return self.conv5_3
# Resize and conv
with tf.variable_scope('fgru'):
fgru_1_td = normalization.apply_normalization(
activity=self.conv4_3,
name='td_norm1_%s' % i0,
normalization_type='instance_norm',
data_format=self.data_format,
training=self.train,
trainable=self.train,
reuse=self.reuse)
# fgru_1_td_2 = normalization.apply_normalization(
# activity=self.conv5_3,
# name='td_norm12_%s' % i0,
# normalization_type='instance_norm',
# data_format=self.data_format,
# training=self.train,
# trainable=self.train,
# reuse=self.reuse)
# fgru_1_td_2 = self.image_resize(
# fgru_1_td_2,
# fgru_1_td.get_shape().as_list()[1:3],
# align_corners=True)
# fgru_1_td = tf.concat([fgru_1_td, fgru_1_td_2], -1)
fgru_1_td = self.conv_layer(
fgru_1_td,
'3_to_2',
learned=True,
apply_relu=True,
shape=[
1, 1,
fgru_1_td.get_shape().as_list()[-1],
self.fgru_1.get_shape().as_list()[-1] // 32
])
fgru_1_td = normalization.apply_normalization(
activity=fgru_1_td,
name='td_norm1_1%s' % i0,
normalization_type='instance_norm',
data_format=self.data_format,
training=self.train,
trainable=self.train,
reuse=self.reuse)
if extra_convs:
fgru_1_td = self.conv_layer(
fgru_1_td,
'3_to_2_2',
learned=True,
shape=[
1, 1,
self.fgru_1.get_shape().as_list()[-1] // 32,
self.fgru_1.get_shape().as_list()[-1]
])
fgru_1_td = self.image_resize(
fgru_1_td,
self.fgru_1.get_shape().as_list()[1:3],
align_corners=True)
error, fgru_activity, self.inh_2 = self.alt_fgru_ops( # h^(1), h^(2)
ff_drive=self.fgru_1,
h2=fgru_1_td,
layer_id=2,
i0=i0)
self.x_2 = self.fgru_1
self.fgru_1 += fgru_activity
# Resize and conv
with tf.variable_scope('fgru'):
fgru_0_td = normalization.apply_normalization(
activity=self.fgru_1,
name='td_norm0_%s' % i0,
normalization_type='instance_norm',
data_format=self.data_format,
training=self.train,
trainable=self.train,
reuse=self.reuse)
fgru_0_td = self.conv_layer(
fgru_0_td,
'2_to_1',
learned=True,
apply_relu=True,
shape=[
1, 1,
fgru_0_td.get_shape().as_list()[-1],
self.fgru_0.get_shape().as_list()[-1] // 4
])
fgru_0_td = normalization.apply_normalization(
activity=fgru_0_td,
name='td_norm0_1%s' % i0,
normalization_type='instance_norm',
data_format=self.data_format,
training=self.train,
trainable=self.train,
reuse=self.reuse)
if extra_convs:
fgru_0_td = self.conv_layer(
fgru_0_td,
'2_to_1_2',
learned=True,
shape=[
1, 1,
self.fgru_0.get_shape().as_list()[-1] // 4,
self.fgru_0.get_shape().as_list()[-1]
])
fgru_0_td = self.image_resize(
fgru_0_td,
self.fgru_0.get_shape().as_list()[1:3],
align_corners=True)
_, fgru_activity, self.inh_3 = self.alt_fgru_ops( # h^(1), h^(2)
ff_drive=self.fgru_0,
h2=fgru_0_td,
layer_id=3,
i0=i0)
self.x_3 = self.fgru_0
# self.fgru_0 = fgru_activity
self.fgru_0 += fgru_activity
def build_model(
data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
# norm_moments_training = training # Force instance norm
# normalization_type = 'no_param_batch_norm_original'
normalization_type = 'no_param_instance_norm'
# output_normalization_type = 'batch_norm_original_renorm'
output_normalization_type = 'instance_norm'
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor,
data_format=data_format)
# Prepare gammanet structure
(
compression,
ff_kernels,
ff_repeats,
features,
fgru_kernels,
additional_readouts) = v2_big_working()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
with tf.variable_scope('vgg', reuse=reuse):
aux = get_aux()
vgg = vgg16.Vgg16(
vgg16_npy_path='/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy',
reuse=reuse,
aux=aux,
train=training,
timesteps=8,
fgru_normalization_type=normalization_type,
ff_normalization_type=normalization_type)
vgg(rgb=data_tensor, constructor=gammanet_constructor)
# activity = vgg.fgru_0
with tf.variable_scope('fgru', reuse=reuse):
# Get side weights
hs_0, hs_1 = [], []
h2_rem = [
vgg.fgru_0,
vgg.fgru_1,
vgg.fgru_2,
vgg.conv5_1,
vgg.fgru_3]
for idx, h in enumerate(h2_rem):
res = normalization.apply_normalization(
activity=h,
name='output_norm1_%s' % idx,
normalization_type=output_normalization_type,
data_format=data_format,
training=training,
trainable=training,
reuse=reuse)
res = aux['image_resize'](
res,
data_tensor.get_shape().as_list()[1:3],
align_corners=True)
cnv_0 = tf.layers.conv2d(
inputs=res,
filters=output_shape,
kernel_size=(1, 1),
padding='same',
data_format=long_data_format,
name='readout_aux_00_%s' % idx,
# activation=tf.nn.relu,
activation=None,
trainable=training,
use_bias=True,
reuse=reuse)
cnv_1 = tf.layers.conv2d(
inputs=res,
filters=output_shape,
kernel_size=(1, 1),
padding='same',
data_format=long_data_format,
name='readout_aux_10_%s' % idx,
# activation=tf.nn.relu,
activation=None,
trainable=training,
use_bias=True,
reuse=reuse)
hs_0 += [cnv_0]
hs_1 += [cnv_1]
s1, s2, s3, s4, s5 = hs_0
s11, s21, s31, s41, s51 = hs_1
o1 = tf.stop_gradient(tf.identity(s1))
o2 = tf.stop_gradient(tf.identity(s2))
o3 = tf.stop_gradient(tf.identity(s3))
o4 = tf.stop_gradient(tf.identity(s4))
o21 = tf.stop_gradient(tf.identity(s21))
o31 = tf.stop_gradient(tf.identity(s31))
o41 = tf.stop_gradient(tf.identity(s41))
o51 = tf.stop_gradient(tf.identity(s51))
p1_1 = s1
p2_1 = s2 + o1
p3_1 = s3 + o2 + o1
p4_1 = s4 + o3 + o2 + o1
p5_1 = s5 + o4 + o3 + o2 + o1
p1_2 = s11 + o21 + o31 + o41 + o51
p2_2 = s21 + o31 + o41 + o51
p3_2 = s31 + o41 + o51
p4_2 = s41 + o51
p5_2 = s51
hs = [p1_1, p2_1, p3_1, p4_1, p5_1, p1_2, p2_2, p3_2, p4_2, p5_2]
activity = tf.layers.conv2d(
tf.concat(hs, -1),
filters=output_shape,
kernel_size=(1, 1),
padding='same',
data_format=long_data_format,
name='out',
# activation=tf.nn.relu,
activation=None,
trainable=training,
use_bias=False,
reuse=reuse)
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
extra_activities = {idx: v for idx, v in enumerate(hs_0)}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
# return [activity, h_deep], extra_activities
return activity, extra_activities
def build_model(data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
# normalization_type = 'no_param_instance_norm'
# output_normalization_type = 'instance_norm'
fgru_normalization_type = 'no_param_batch_norm_original'
ff_normalization_type = 'no_param_batch_norm_original'
output_normalization_type = 'batch_norm_original'
ff_kernel_size = (5, 5)
ff_nl = tf.nn.relu
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor, data_format=data_format)
# Prepare gammanet structure
(compression, ff_kernels, ff_repeats, features, fgru_kernels,
additional_readouts) = v2_big_working()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
with tf.variable_scope('gammanet', reuse=reuse):
activity = tf.layers.conv2d(
inputs=data_tensor,
filters=gammanet_constructor[0]['features'],
kernel_size=ff_kernel_size,
padding='same',
# strides=(2, 2),
data_format=long_data_format,
name='l0_0',
activation=ff_nl,
use_bias=True,
reuse=reuse)
activity = normalization.apply_normalization(
activity=activity,
name='input_norm',
normalization_type=output_normalization_type,
data_format=data_format,
training=training,
trainable=training,
reuse=reuse) # + pre_activity
activity = tf.layers.conv2d(
inputs=activity,
filters=gammanet_constructor[0]['features'],
kernel_size=ff_kernel_size,
padding='same',
# strides=(2, 2),
data_format=long_data_format,
name='l0_1',
activation=ff_nl,
use_bias=True,
reuse=reuse)
# Shift conv-layers to the front
# Run fGRU
gn = gammanet.GN(
layer_name='fgru',
gammanet_constructor=gammanet_constructor,
data_format=data_format,
reuse=reuse,
timesteps=3,
fgru_connectivity='', # 'all_to_all',
additional_readouts=additional_readouts,
fgru_normalization_type=fgru_normalization_type,
ff_normalization_type=ff_normalization_type,
horizontal_padding='SAME',
ff_padding='SAME',
ff_nl=ff_nl,
stop_loop='fgru_3',
recurrent_ff=True,
horizontal_kernel_initializer=tf.initializers.orthogonal(),
kernel_initializer=tf.initializers.orthogonal(),
gate_initializer=tf.initializers.orthogonal(),
# horizontal_kernel_initializer=tf.contrib.layers.xavier_initializer(),
# gate_initializer=tf.contrib.layers.xavier_initializer(),
# gate_initializer=tf.contrib.layers.xavier_initializer(),
aux=aux,
strides=[1, 1, 1, 1],
pool_strides=[2, 2],
pool_kernel=[2, 2],
up_kernel=[4, 4],
train=training)
h2, h_deep = gn(X=activity)
with tf.variable_scope('cv_readout', reuse=reuse):
# Apply one more normalization
# activity = tf.layers.max_pooling2d(
# inputs=h2,
# pool_size=(2, 2),
# strides=(2, 2),
# padding='same',
# data_format=long_data_format,
# name='output_pool_3')
activity = normalization.apply_normalization(
activity=h2,
name='output_norm_activity',
normalization_type=output_normalization_type,
data_format=data_format,
training=training,
trainable=training,
reuse=reuse) # + pre_activity
# Then read out
activity = tf.layers.conv2d(
inputs=activity,
filters=2048,
kernel_size=ff_kernel_size,
padding='same',
# strides=(2, 2),
data_format=long_data_format,
name='readout_fc_0',
activation=ff_nl,
use_bias=True,
reuse=reuse)
activity = normalization.apply_normalization(
activity=activity,
name='readout_norm_activity',
normalization_type=output_normalization_type,
data_format=data_format,
training=training,
trainable=training,
reuse=reuse) # + pre_activity
activity = tf.reduce_mean(activity, reduction_indices=[1, 2])
activity = tf.layers.dense(inputs=activity,
units=output_shape,
activation=None,
use_bias=True,
bias_initializer=tf.constant_initializer(
1. / 1000.),
name='readout_fc_2',
reuse=reuse)
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
extra_activities = {
# 'activity_low': h2,
# 'activity_high': h_deep,
}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
# return [activity, h_deep], extra_activities
return activity, extra_activities
def build(self, i0, extra_convs=False):
# Convert RGB to BGR
with tf.variable_scope('fgru'):
error, fgru_activity = self.fgru_ops( # h^(1), h^(2)
ff_drive=self.pool2,
h2=self.fgru_0,
layer_id=0,
i0=i0)
self.fgru_0 += fgru_activity # + self.conv2_2
self.conv3_1 = self.conv_layer(self.fgru_0, "conv3_1")
self.conv3_2 = self.conv_layer(self.conv3_1, "conv3_2")
self.conv3_3 = self.conv_layer(self.conv3_2, "conv3_3")
self.pool3 = self.max_pool(self.conv3_3, 'pool3')
if i0 == 0:
self.fgru_1 = self.pool3
with tf.variable_scope('fgru'):
error, fgru_activity = self.fgru_ops( # h^(1), h^(2)
ff_drive=self.pool3,
h2=self.fgru_1,
layer_id=1,
i0=i0)
self.fgru_1 += fgru_activity # + self.conv3_3
self.conv4_1 = self.conv_layer(self.fgru_1, "conv4_1")
self.conv4_2 = self.conv_layer(self.conv4_1, "conv4_2")
self.conv4_3 = self.conv_layer(self.conv4_2, "conv4_3")
self.pool4 = self.max_pool(self.conv4_3, 'pool4')
if i0 == 0:
self.fgru_2 = self.pool4
with tf.variable_scope('fgru'):
error, fgru_activity = self.fgru_ops( # h^(1), h^(2)
ff_drive=self.pool4,
h2=self.fgru_2,
layer_id=2,
i0=i0)
self.fgru_2 += fgru_activity # + self.conv5_3
self.conv5_1 = self.conv_layer(self.fgru_2, "conv5_1")
self.conv5_2 = self.conv_layer(self.conv5_1, "conv5_2")
self.conv5_3 = self.conv_layer(self.conv5_2, "conv5_3")
if i0 == 0:
self.fgru_3 = self.conv5_3
with tf.variable_scope('fgru'):
error, fgru_activity = self.fgru_ops( # h^(1), h^(2)
ff_drive=self.conv5_3,
h2=self.fgru_3,
layer_id=3,
i0=i0)
self.fgru_3 += fgru_activity # + self.conv5_3
# Resize and conv
error, fgru_activity = self.fgru_ops( # h^(1), h^(2)
ff_drive=self.fgru_2,
h2=self.fgru_3,
layer_id=4,
i0=i0)
self.fgru_2 += fgru_activity
# Resize and conv
with tf.variable_scope('fgru'):
fgru_1_td = normalization.apply_normalization(
activity=self.fgru_2,
name='td_norm1_%s' % i0,
normalization_type=self.fgru_normalization_type,
data_format=self.data_format,
training=self.fgru_normalization_type,
trainable=self.train,
reuse=self.reuse)
fgru_1_td = self.image_resize(
fgru_1_td,
self.fgru_1.get_shape().as_list()[1:3],
align_corners=True)
fgru_1_td = self.conv_layer(
fgru_1_td,
'4_to_3',
learned=True,
apply_relu=True,
shape=[
1, 1,
fgru_1_td.get_shape().as_list()[-1],
self.fgru_1.get_shape().as_list()[-1]
])
if extra_convs:
fgru_1_td = self.conv_layer(
fgru_1_td,
'4_to_3_2',
learned=True,
shape=[
1, 1,
self.fgru_1.get_shape().as_list()[-1],
self.fgru_1.get_shape().as_list()[-1]
])
error, fgru_activity = self.fgru_ops( # h^(1), h^(2)
ff_drive=self.fgru_1,
h2=fgru_1_td,
layer_id=5,
i0=i0)
self.fgru_1 += fgru_activity
# Resize and conv
with tf.variable_scope('fgru'):
fgru_0_td = normalization.apply_normalization(
activity=self.fgru_1,
name='td_norm0_%s' % i0,
normalization_type=self.fgru_normalization_type,
data_format=self.data_format,
training=self.fgru_normalization_type,
trainable=self.train,
reuse=self.reuse)
fgru_0_td = self.image_resize(
fgru_0_td,
self.fgru_0.get_shape().as_list()[1:3],
align_corners=True)
fgru_0_td = self.conv_layer(
fgru_0_td,
'3_to_2',
learned=True,
apply_relu=True,
shape=[
1, 1,
fgru_0_td.get_shape().as_list()[-1],
self.fgru_0.get_shape().as_list()[-1]
])
if extra_convs:
fgru_0_td = self.conv_layer(
fgru_0_td,
'3_to_2_2',
learned=True,
shape=[
1, 1,
self.fgru_0.get_shape().as_list()[-1],
self.fgru_0.get_shape().as_list()[-1]
])
error, fgru_activity = self.fgru_ops( # h^(1), h^(2)
ff_drive=self.fgru_0,
h2=fgru_0_td,
layer_id=6,
i0=i0)
# self.fgru_0 = fgru_activity
self.fgru_0 += fgru_activity
def build_model(data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
# norm_moments_training = training # Force instance norm
# normalization_type = 'no_param_batch_norm_original'
# output_normalization_type = 'batch_norm_original_renorm'
output_normalization_type = 'instance_norm'
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor, data_format=data_format)
# Build model
with tf.variable_scope('vgg', reuse=reuse):
vgg = vgg16.Vgg16(
vgg16_npy_path=
'/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy')
vgg(rgb=data_tensor, train=training, ff_reuse=reuse)
with tf.variable_scope('fgru', reuse=reuse):
# Get side weights
h2_rem = [
vgg.conv1_2, vgg.conv2_2, vgg.conv3_3, vgg.conv4_3, vgg.conv5_3
]
res_act = []
for idx, h in enumerate(h2_rem):
res = normalization.apply_normalization(
activity=h,
name='output_norm1_%s' % idx,
normalization_type=output_normalization_type,
data_format=data_format,
training=training,
trainable=training,
reuse=reuse)
res_act += [
tf.image.resize_bilinear(
res,
data_tensor.get_shape().as_list()[1:3],
align_corners=True)
]
activity = tf.layers.conv2d(tf.concat(res_act, -1),
filters=output_shape,
kernel_size=(1, 1),
padding='same',
data_format=long_data_format,
name='out',
activation=None,
trainable=training,
use_bias=True,
reuse=reuse)
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
extra_activities = {} # idx: v for idx, v in enumerate(hs_0)}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
return activity, extra_activities
def build_model(data_tensor,
reuse,
training,
output_shape,
learned_temp=False,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
# norm_moments_training = training # Force instance norm
# normalization_type = 'no_param_batch_norm_original'
# output_normalization_type = 'batch_norm_original_renorm'
output_normalization_type = 'instance_norm'
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor, data_format=data_format)
# Build model
with tf.variable_scope('vgg', reuse=reuse):
vgg = vgg16.Vgg16(
vgg16_npy_path=
'/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy')
vgg(rgb=data_tensor, train=training, ff_reuse=reuse)
with tf.variable_scope('fgru', reuse=reuse):
# Get side weights
h2_rem = [
vgg.conv1_2, vgg.conv2_2, vgg.conv3_3, vgg.conv4_3, vgg.conv5_3
]
res_act = []
for idx, h in enumerate(h2_rem):
res = normalization.apply_normalization(
activity=h,
name='output_norm1_%s' % idx,
normalization_type=output_normalization_type,
data_format=data_format,
training=training,
trainable=training,
reuse=reuse)
res_act += [
tf.image.resize_bilinear(
res,
data_tensor.get_shape().as_list()[1:3],
align_corners=True)
]
# # 3-step readout
# (1) Per-pixel competition
activity = tf.layers.conv2d(tf.concat(res_act, -1),
filters=1,
kernel_size=(3, 3),
padding='same',
data_format=long_data_format,
name='out_0',
activation=tf.nn.sigmoid,
trainable=training,
use_bias=True,
reuse=reuse)
# (2) Softmax across locations
activity = tf.layers.conv2d(tf.concat(activity, -1),
filters=1,
kernel_size=(3, 3),
padding='same',
data_format=long_data_format,
name='out_1',
activation=None,
trainable=training,
use_bias=True,
reuse=reuse)
act_shape = activity.get_shape().as_list()
activity = tf.reshape(activity, [act_shape[0], -1])
if learned_temp:
# Potentially add another non-linearity pre-GAP for VAF:q":
temperature = tf.layers.dense(tf.reduce_mean(activity,
reduction_indices=[1
]),
1,
name='temperature',
activation=None,
use_bias=True)
sigmoid_attention = tf.nn.sigmoid(temperature)
map_activity = tf.nn.softmax(activity / sigmoid_attention)
else:
map_activity = tf.nn.softmax(activity)
map_activity = tf.reshape(map_activity, act_shape)
# (3) GAP & readout
activity = tf.reduce_mean(map_activity, reduction_indices=[1, 2])
activity = tf.layers.dense(activity,
output_shape,
name='out_2',
reuse=reuse,
activation=None,
use_bias=True)
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
if learned_temp:
extra_activities = {
'temperature': temperature,
'map_activity': map_activity
}
else:
extra_activities = {'map_activity': map_activity}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
return activity, extra_activities
def build_model(data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
input_training = training # training
readout_training = training # False
norm_moments_training = training # Force instance norm
norm_params_training = training
fgru_kernel_training = training
fgru_param_training = training
ff_gate_training = training
fgru_gate_training = training
remaining_param_training = training
normalization_type = 'no_param_batch_norm_original'
output_normalization_type = 'batch_norm_original'
# normalization_type = 'no_param_instance_norm'
# output_normalization_type = 'instance_norm'
ff_kernel_size = (3, 3)
ff_nl = tf.nn.elu
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor, data_format=data_format)
# Prepare gammanet structure
(compression, ff_kernels, ff_repeats, features, fgru_kernels,
additional_readouts) = v2_big_working()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
with tf.variable_scope('gammanet', reuse=reuse):
aux = get_aux()
vgg = vgg16.Vgg16(
vgg16_npy_path=
'/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy',
reuse=reuse,
aux=aux,
train=training,
timesteps=4,
fgru_normalization_type=normalization_type,
ff_normalization_type=normalization_type)
vgg(rgb=data_tensor, constructor=gammanet_constructor)
# h2_rem = [vgg.conv5_3, vgg.conv4_3, vgg.conv3_3, vgg.conv2_2, vgg.conv1_1]
h2 = vgg.fgru_0
with tf.variable_scope('contour_readout', reuse=reuse):
# Apply one more normalization
# h_deep = apply_normalization(
# activity=h_deep,
# name='output_normh_deep',
# normalization_type=normalization_type,
# data_format=data_format,
# training=training,
# reuse=reuse)
if 0:
hs = []
for h in h2_rem:
hs += [
tf.image.resize_nearest_neighbor(
h,
data_tensor.get_shape().as_list()[1:3])
]
activity = normalization.apply_normalization(
activity=tf.concat(hs, axis=-1),
name='output_norm',
normalization_type=output_normalization_type,
data_format=data_format,
training=norm_moments_training,
trainable=training,
reuse=reuse)
else:
activity = normalization.apply_normalization(
activity=h2,
name='output_norm',
normalization_type=output_normalization_type,
data_format=data_format,
training=norm_moments_training,
trainable=training,
reuse=reuse)
# Then read out
activity = tf.layers.conv2d(inputs=activity,
filters=output_shape,
kernel_size=(1, 1),
padding='same',
data_format=long_data_format,
name='readout_conv',
activation=None,
trainable=training,
use_bias=True,
reuse=reuse)
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
extra_activities = {}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
# return [activity, h_deep], extra_activities
return activity, extra_activities
def build_model(data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor, data_format=data_format)
normalization_type = 'instance_norm'
ff_nl = tf.nn.relu
# Prepare gammanet structure
compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts = v1(
)
# compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts = v3()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
with tf.variable_scope('gammanet', reuse=reuse):
in_emb = tf.layers.conv2d(inputs=data_tensor,
filters=16,
kernel_size=7,
strides=(1, 1),
padding='same',
data_format=long_data_format,
activation=ff_nl,
trainable=training,
use_bias=True,
name='l0')
in_emb = tf.layers.conv2d(
inputs=in_emb,
filters=16,
kernel_size=7,
strides=(1, 1),
padding='same',
data_format=long_data_format,
activation=None, # ff_nl,
trainable=training,
use_bias=True,
name='l01')
in_emb = normalization.apply_normalization(
activity=in_emb,
name='in_norm',
normalization_type=normalization_type,
data_format=data_format,
training=training,
reuse=reuse)
in_emb = ff_nl(in_emb)
# Run fGRU
gn = gammanet.GN(
layer_name='fgru',
gammanet_constructor=gammanet_constructor,
data_format=data_format,
reuse=reuse,
timesteps=8,
fgru_connectivity='', # 'all_to_all',
additional_readouts=additional_readouts,
fgru_normalization_type=normalization_type,
ff_normalization_type=normalization_type,
horizontal_padding='SAME',
ff_padding='SAME',
ff_nl=ff_nl,
recurrent_ff=False,
horizontal_kernel_initializer=tf.initializers.orthogonal(),
# horizontal_kernel_initializer=tf_fun.Identity(),
kernel_initializer=tf.initializers.orthogonal(),
aux=aux,
strides=[1, 1, 1, 1],
pool_strides=[2, 2],
pooling_kernel=[2, 2],
up_kernel=[4, 4],
train=training)
h2, h_deep = gn(X=in_emb)
with tf.variable_scope('cv_readout', reuse=reuse):
# Apply one more normalization
h2 = normalization.apply_normalization(
activity=h2,
name='output_norm',
normalization_type=normalization_type,
data_format=data_format,
training=training,
reuse=reuse)
# h_deep = apply_normalization(
# activity=h_deep,
# name='output_normh_deep',
# normalization_type=normalization_type,
# data_format=data_format,
# training=training,
# reuse=reuse)
# Then read out
activity = tf.layers.conv2d(inputs=h2,
filters=output_shape,
kernel_size=(1, 1),
padding='same',
data_format=long_data_format,
name='pre_readout_conv',
use_bias=True,
reuse=reuse)
# h_deep = tf.layers.conv2d(
# inputs=h_deep,
# filters=output_shape,
# kernel_size=(1, 1),
# padding='same',
# data_format=long_data_format,
# name='pre_readout_conv1',
# use_bias=True,
# reuse=reuse)
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
extra_activities = {
'activity_low': h2,
'activity_high': h_deep,
}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
# return [activity, h_deep], extra_activities
return activity, extra_activities
def build_model(data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
# norm_moments_training = training # Force instance norm
# normalization_type = 'no_param_batch_norm_original'
normalization_type = 'no_param_instance_norm'
# output_normalization_type = 'batch_norm_original_renorm'
output_normalization_type = 'instance_norm'
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor, data_format=data_format)
# Prepare gammanet structure
(compression, ff_kernels, ff_repeats, features, fgru_kernels,
additional_readouts) = v2_big_working()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
with tf.variable_scope('vgg', reuse=reuse):
aux = get_aux()
vgg = vgg16.Vgg16(
vgg16_npy_path=
'/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy',
reuse=reuse,
aux=aux,
train=False,
timesteps=8,
fgru_normalization_type=normalization_type,
ff_normalization_type=normalization_type)
vgg(rgb=data_tensor, constructor=gammanet_constructor)
activity = vgg.fgru_0
with tf.variable_scope('fgru_gilbert', reuse=reuse):
# Then read out
# activity = activity ** 2
activity = normalization.apply_normalization(
activity=activity,
name='output_norm1',
normalization_type=output_normalization_type,
data_format=data_format,
training=training,
trainable=training,
reuse=reuse)
activity = tf.layers.conv2d(
inputs=activity,
filters=gammanet_constructor[0]['features'],
kernel_size=(1, 1),
padding='same',
data_format=long_data_format,
name='readout_l0_1',
activation=tf.nn.relu,
use_bias=True,
trainable=training,
reuse=reuse)
activity = tf.layers.conv2d(inputs=activity,
filters=1,
kernel_size=(1, 1),
padding='same',
data_format=long_data_format,
name='readout_l0_2',
activation=None,
use_bias=True,
trainable=training,
reuse=reuse)
activity = tf.reduce_max(activity, reduction_indices=[1, 2])
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
extra_activities = {}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
# return [activity, h_deep], extra_activities
return activity, extra_activities
