def build_model(data_tensor, reuse, training):
"""Create the hgru from Learning long-range..."""
with tf.variable_scope('cnn', reuse=reuse):
with tf.variable_scope('input', reuse=reuse):
conv_aux = {
'pretrained': os.path.join('weights',
'gabors_for_contours_7.npy'),
'pretrained_key': 's1',
'nonlinearity': 'square'
}
x = conv.conv_layer(bottom=data_tensor,
name='gabor_input',
stride=[1, 1, 1, 1],
padding='SAME',
trainable=training,
use_bias=True,
aux=conv_aux)
layer_hgru = hgru.hGRU('hgru_1',
x_shape=x.get_shape().as_list(),
timesteps=8,
h_ext=15,
strides=[1, 1, 1, 1],
padding='SAME',
aux={
'lesion_alpha': True,
'lesion_omega': True
},
train=training)
h2 = layer_hgru.build(x)
h2 = normalization.batch(
bottom=h2,
reuse=reuse,
# renorm=True,
name='hgru_bn',
training=training)
with tf.variable_scope('readout_1', reuse=reuse):
activity = conv.conv_layer(bottom=h2,
name='pre_readout_conv',
num_filters=2,
kernel_size=1,
trainable=training,
use_bias=False)
pool_aux = {'pool_type': 'max'}
activity = pooling.global_pool(bottom=activity,
name='pre_readout_pool',
aux=pool_aux)
activity = normalization.batch(
bottom=activity,
reuse=reuse,
# renorm=True,
name='readout_1_bn',
training=training)
with tf.variable_scope('readout_2', reuse=reuse):
activity = tf.layers.flatten(activity, name='flat_readout')
activity = tf.layers.dense(inputs=activity, units=2)
return activity, h2
def build_model(data_tensor, reuse, training):
"""Create the gru from Learning long-range..."""
with tf.variable_scope('cnn', reuse=reuse):
with tf.variable_scope('input', reuse=reuse):
conv_aux = {
'pretrained': os.path.join('weights',
'gabors_for_contours_7.npy'),
'pretrained_key': 's1',
'nonlinearity': 'square'
}
x = conv.conv_layer(bottom=data_tensor,
name='gabor_input',
stride=[1, 1, 1, 1],
padding='SAME',
trainable=training,
use_bias=True,
aux=conv_aux)
activity = conv.conv_layer(bottom=x,
name='c1',
num_filters=9,
kernel_size=20,
trainable=training,
use_bias=False)
activity = normalization.batch(bottom=activity,
name='c1_bn',
training=training)
activity = tf.nn.relu(activity)
with tf.variable_scope('readout_1', reuse=reuse):
activity = conv.conv_layer(bottom=activity,
name='pre_readout_conv',
num_filters=2,
kernel_size=1,
trainable=training,
use_bias=False)
pool_aux = {'pool_type': 'max'}
activity = pooling.global_pool(bottom=activity,
name='pre_readout_pool',
aux=pool_aux)
activity = normalization.batch(bottom=activity,
name='readout_1_bn',
training=training)
with tf.variable_scope('readout_2', reuse=reuse):
activity = tf.layers.flatten(activity, name='flat_readout')
activity = tf.layers.dense(inputs=activity, units=2)
extra_activities = {'activity': activity}
return activity, extra_activities
def full(self, i0, caps_match, caps2_norm, labels, labels_net_change):
""" DEPRECIATED
"""
labels_new = helpers.label_backward(
labels,
labels,
caps_match,
self.labels_filter,
self.decay_const,
self.b,
labels_gate_scale=self.labels_gate_scale,
labels_gate_bias=self.labels_gate_bias,
caps1_norm=caps2_norm,
labels_update_mode=self.labels_update_mode,
sensitivity_const=self.sensitivity_const,
pushpull_scale=self.pushpull_scale,
pushpull_bias=self.pushpull_bias,
cap_labels=self.cap_labels,
fixed_labels_mask=self.fixed_labels_mask)
if self.bn_per_time:
print('horizontal: using batchnorm per ts')
labels_new = normalization.batch(bottom=labels_new,
renorm=True,
name='in_model_bn_' + str(i0),
training=True)
# Iterate loop
labels_net_change += tf.sqrt(tf.square(labels_new - labels))
i0 += 1
return i0, caps_match, caps2_norm, labels_new, labels_net_change
def up_block(layer_name,
bottom,
skip_activity,
reuse,
kernel_size,
num_filters,
training,
stride=[2, 2, 1],
use_bias=False):
"""Forward block for seung model."""
with tf.variable_scope('%s_block' % layer_name, reuse=reuse):
with tf.variable_scope('%s_layer_1' % layer_name, reuse=reuse):
x = conv3d_transpose_layer(bottom=bottom,
name='%s_1' % layer_name,
stride=stride,
padding='SAME',
num_filters=num_filters,
kernel_size=kernel_size,
trainable=training,
use_bias=use_bias)
x = x + skip_activity # Rethink if this is valid
x = normalization.batch(bottom=x,
name='%s_bn_1' % layer_name,
training=training)
x = tf.nn.elu(x)
return x
def up_block(inputs,
skip,
up_filters,
name,
training,
reuse,
up_kernels=4,
up_strides=(2, 2),
up_padding='same',
nl=tf.nn.relu):
"""Do a unet upsample."""
upact = tf.layers.conv2d_transpose(inputs=inputs,
filters=up_filters,
kernel_size=up_kernels,
strides=up_strides,
name='%s_up' % name,
padding=up_padding)
upact = nl(upact)
upact = normalization.batch(bottom=upact,
name='%s_bn' % name,
training=training,
reuse=reuse)
return conv_block(x=upact + skip,
filters=up_filters,
name='%s_skipped' % name,
training=training,
reuse=reuse)
def up_layer(layer_name,
bottom,
reuse,
kernel_size,
num_filters,
training,
trainable=None,
stride=[2, 2],
padding='same',
kernel_initializer=None,
renorm=False,
use_bias=True):
"""Wrapper for transpose convolutions."""
if trainable is None:
trainable = training
with tf.variable_scope('%s_block' % layer_name, reuse=reuse):
with tf.variable_scope('%s_layer' % layer_name, reuse=reuse):
x = tf.layers.conv2d_transpose(
inputs=bottom,
filters=num_filters,
kernel_size=kernel_size,
name=layer_name,
strides=stride,
padding=padding,
kernel_initializer=kernel_initializer,
trainable=training,
use_bias=use_bias)
x = tf.nn.elu(x)
x = normalization.batch(bottom=x,
name='%s_bn' % layer_name,
renorm=renorm,
training=training)
return x
def conv_block(x,
filters,
kernel_size=3,
strides=(1, 1),
padding='same',
kernel_initializer=tf.initializers.variance_scaling,
data_format='channels_last',
activation=tf.nn.relu,
name=None,
training=True,
reuse=False,
batchnorm=True,
pool=True):
"""VGG conv block."""
assert name is not None, 'Give the conv block a name.'
activity = tf.layers.conv2d(inputs=x,
filters=filters,
kernel_size=kernel_size,
strides=strides,
padding=padding,
data_format=data_format,
activation=activation,
name='%s_conv' % name,
reuse=reuse)
if batchnorm:
activity = normalization.batch(bottom=activity,
name='%s_bn' % name,
training=training,
reuse=reuse)
if pool:
return pooling.max_pool(bottom=activity, name='%s_pool' % name)
else:
return activity
def build_model(data_tensor, reuse, training, output_shape):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[0]
with tf.variable_scope('cnn', reuse=reuse):
# Add input
so_filters = np.load(
os.path.join('weights', 'so_filters.npy')).squeeze().reshape(
11, 11, 3, 8 * 4)[..., 0:32:4]
so_filter_tensor = tf.get_variable(
name='so_filters',
initializer=so_filters,
trainable=training)
so_bias = tf.get_variable(
name='so_bias',
initializer=tf.zeros(so_filters.shape[-1]),
trainable=training)
in_emb = tf.nn.conv2d(
input=data_tensor,
filter=so_filter_tensor,
strides=[1, 1, 1, 1],
padding='SAME',
name='so')
in_emb = tf.nn.bias_add(in_emb, so_bias)
in_emb = in_emb ** 2
layer_hgru = hgru.hGRU(
'hgru_1',
x_shape=in_emb.get_shape().as_list(),
timesteps=12,
h_ext=1,
crf=15,
strides=[1, 1, 1, 1],
padding='SAME',
aux={
'reuse': False,
'constrain': False,
'nonnegative': True,
'while_loop': False,
'hidden_states': 'gru',
'horizontal_dilations': [1, 1, 1, 1]
},
train=training)
h2 = layer_hgru.build(in_emb)
h2 = normalization.batch(
bottom=h2,
renorm=False,
name='hgru_bn',
training=training)
activity = conv.readout_layer(
activity=h2,
reuse=reuse,
training=training,
pool_type='max', # 'select',
output_shape=output_shape,
features=so_filters.shape[-1])
extra_activities = {
'activity': h2
}
return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[0]
with tf.variable_scope('cnn', reuse=reuse):
# Add input
in_emb = conv.skinny_input_layer(
X=data_tensor,
reuse=reuse,
training=training,
features=24,
conv_activation=tf.nn.elu,
conv_kernel_size=7,
pool=False,
name='l0')
layer_hgru = feedback_hgru.hGRU(
layer_name='hgru_1',
x_shape=in_emb.get_shape().as_list(),
timesteps=8,
h_ext=[[9, 9], [5, 5], [1, 1]],
strides=[1, 1, 1, 1],
pool_strides=[4, 4],
padding='SAME',
aux={
'symmetric_weights': True,
'dilations': [[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]],
'batch_norm': True,
'while_loop': False,
'nonnegative': True,
'residual': True,
'final_bn': False,
'renorm': False,
'lesion_alpha': True,
'lesion_omega': True,
'pooling_kernel': [4, 4],
'intermediate_ff': [24, 24], # + filters,
'intermediate_ks': [[5, 5], [5, 5]]},
train=training)
h2 = layer_hgru.build(in_emb)
nh2 = normalization.batch(
bottom=h2,
name='hgru_bn',
fused=True,
renorm=True,
training=training)
activity = conv.conv_layer(
bottom=nh2,
name='pre_readout_conv',
num_filters=output_shape['output'],
kernel_size=1,
trainable=training,
use_bias=True)
extra_activities = {
# 'activity': h2
}
return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[0]
with tf.variable_scope('cnn', reuse=reuse):
# Add input
in_emb = conv.skinny_input_layer(
X=data_tensor,
reuse=reuse,
training=training,
features=24,
conv_activation=tf.nn.elu,
conv_kernel_size=7,
pool=False,
name='l0')
layer_hgru = hgru.hGRU(
'fgru',
x_shape=in_emb.get_shape().as_list(),
timesteps=8,
h_ext=[{'h1': [9, 9]}, {'h2': [5, 5]}, {'fb1': [1, 1]}],
strides=[1, 1, 1, 1],
hgru_ids=[{'h1': 24}, {'h2': 24}, {'fb1': 24}],
hgru_idx=[{'h1': 0}, {'h2': 1}, {'fb1': 2}],
padding='SAME',
aux={
'readout': 'fb',
'intermediate_ff': [24, 24],
'intermediate_ks': [[5, 5], [5, 5]],
'intermediate_repeats': [2, 2],
'while_loop': False,
'skip': False,
'symmetric_weights': True,
'use_homunculus': False,
'include_pooling': True
},
pool_strides=[4, 4],
pooling_kernel=[4, 4],
train=training)
h2 = layer_hgru.build(in_emb)
nh2 = normalization.batch(
bottom=h2,
name='hgru_bn',
fused=True,
renorm=True,
training=training)
activity = conv.conv_layer(
bottom=nh2,
name='pre_readout_conv',
num_filters=output_shape['output'],
kernel_size=1,
trainable=training,
use_bias=True)
extra_activities = {
# 'activity': h2
}
return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[0]
with tf.variable_scope('cnn', reuse=reuse):
# Add input
in_emb = conv.input_layer(
X=data_tensor,
reuse=reuse,
training=training,
features=20,
conv_activation=[tf.nn.relu, None],
conv_kernel_size=7,
pool_kernel_size=[1, 2, 2, 1],
pool_kernel_strides=[1, 2, 2, 1],
pool=True,
name='l0')
layer_hgru = hgru.hGRU(
'fgru',
x_shape=in_emb.get_shape().as_list(),
timesteps=1,
h_ext=[{'h1': [15, 15]}, {'h2': [1, 1]}, {'fb1': [1, 1]}],
strides=[1, 1, 1, 1],
hgru_ids=[{'h1': 20}, {'h2': 128}, {'fb1': 20}],
hgru_idx=[{'h1': 0}, {'h2': 1}, {'fb1': 2}],
padding='SAME',
aux={
'readout': 'fb',
'intermediate_ff': [32, 128],
'intermediate_ks': [[3, 3], [3, 3]],
'intermediate_repeats': [3, 3],
'while_loop': False,
'skip': False,
'force_horizontal': True,
'symmetric_weights': True,
'include_pooling': True
},
pool_strides=[2, 2],
pooling_kernel=[2, 2],
train=training)
h2 = layer_hgru.build(in_emb)
h2 = normalization.batch(
bottom=h2,
renorm=False,
name='hgru_bn',
training=training)
activity = conv.readout_layer(
activity=h2,
reuse=reuse,
training=training,
output_shape=output_shape)
extra_activities = {
'activity': h2
}
return activity, extra_activities
def build(self, caps, labels):
self.prepare_tensors()
i0 = tf.constant(0)
labels_change = tf.zeros_like(labels)
# Custom iterations
caps_intermediate1, caps_match, caps1_norm, caps2_norm = \
helpers.get_caps_match(
caps,
caps,
self.caps_filter,
self.tolerance_const,
self.c,
self.b,
self.eps)
if self.use_while_loop:
elms = [i0, caps_match, caps2_norm, labels, labels_change]
returned = tf.while_loop(self.condition,
self.full,
loop_vars=elms,
back_prop=True,
swap_memory=False)
i0, caps_match, caps2_norm, labels, labels_change = returned
else:
labels_old = labels
for t in range(self.timesteps):
labels = helpers.label_backward(
labels_old,
labels_old,
caps_match,
self.labels_filter,
self.decay_const,
self.b,
labels_gate_scale=self.labels_gate_scale,
labels_gate_bias=self.labels_gate_bias,
caps1_norm=caps2_norm,
labels_update_mode=self.labels_update_mode,
sensitivity_const=self.sensitivity_const,
pushpull_scale=self.pushpull_scale,
pushpull_bias=self.pushpull_bias,
cap_labels=self.cap_labels,
fixed_labels_mask=self.fixed_labels_mask)
if self.bn_per_time:
print('horizontal: using batchnorm per ts')
labels = normalization.batch(bottom=labels,
renorm=True,
name='in_model_bn_' + str(i0),
training=True)
# Iterate loop
labels_change += tf.sqrt(tf.square(labels - labels_old))
labels_old = labels
return caps, labels, labels_change
def build_model(data_tensor, reuse, training, output_shape):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[0]
with tf.variable_scope('cnn', reuse=reuse):
# Add input
so_filters = np.load(
'/media/data_cifs/clicktionary/pretrained_weights/alexnet.npy'
).item()
so_filter_tensor = tf.get_variable(name='so_filters',
initializer=so_filters['conv1'][0],
trainable=training)
so_bias = tf.get_variable(name='so_bias',
initializer=so_filters['conv1'][1],
trainable=training)
in_emb = tf.nn.conv2d(input=data_tensor,
filter=so_filter_tensor,
strides=[1, 1, 1, 1],
padding='SAME',
name='so')
in_emb = tf.nn.bias_add(in_emb, so_bias)
# in_emb = in_emb ** 2
layer_hgru = hgru.hGRU('hgru_1',
x_shape=in_emb.get_shape().as_list(),
timesteps=8,
h_ext=11,
strides=[1, 1, 1, 1],
padding='SAME',
aux={
'reuse': False,
'constrain': False,
'nonnegative': True,
'while_loop': False,
'horizontal_dilations': [1, 2, 2, 1]
},
train=training)
h2 = layer_hgru.build(in_emb)
h2 = normalization.batch(bottom=h2,
renorm=False,
name='hgru_bn',
training=training)
activity = conv.readout_layer(
activity=h2,
reuse=reuse,
training=training,
pool_type='max', # 'select',
output_shape=output_shape,
features=output_shape)
extra_activities = {'activity': h2}
return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[0]
alpha = 5.
beta = 1.
bs = data_tensor.get_shape().as_list()[0]
dist = tf.distributions.Beta(alpha, beta)
rand = dist.sample([])
sel = tf.cast(tf.round(tf.random_uniform([], minval=1, maxval=bs - 1)),
tf.int32)
with tf.variable_scope('cnn', reuse=reuse):
with tf.variable_scope('input', reuse=reuse):
o_x = tf.layers.conv2d(inputs=data_tensor,
filters=24,
kernel_size=11,
name='l0',
strides=(1, 1),
padding='same',
activation=tf.nn.relu,
trainable=training,
use_bias=True)
x = rand * o_x + ((1 - rand) * tf.roll(o_x, sel, 0))
layer_hgru = hgru.hGRU('hgru_1',
x_shape=x.get_shape().as_list(),
timesteps=8,
h_ext=15,
strides=[1, 1, 1, 1],
padding='SAME',
aux={
'reuse': False,
'constrain': False
},
train=training)
h2, inh = layer_hgru.build(x)
with tf.variable_scope('ro', reuse=reuse):
h2 = normalization.batch(bottom=h2,
renorm=False,
name='hgru_bn',
training=training)
activity = conv.readout_layer(activity=h2,
reuse=reuse,
training=training,
output_shape=output_shape)
extra_activities = {'activity': h2}
return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[0]
reduction_filters = 16
with tf.variable_scope('cnn', reuse=reuse):
# Add input
so_filters = np.load(os.path.join('weights',
'so_filters.npy')).squeeze().reshape(
11, 11, 3, 8 * 4)
in_emb = tf.nn.conv2d(input=data_tensor,
filter=so_filters,
strides=[1, 1, 1, 1],
padding='SAME',
name='so')
in_emb = tf.layers.conv2d(inputs=in_emb,
filters=reduction_filters,
kernel_size=(1, 1))
in_emb = in_emb**2
layer_hgru = hgru.hGRU('hgru_1',
x_shape=in_emb.get_shape().as_list(),
timesteps=8,
h_ext=11,
strides=[1, 1, 1, 1],
padding='SAME',
aux={
'reuse': False,
'constrain': False,
'nonnegative': True,
'while_loop': False
},
train=training)
h2 = layer_hgru.build(in_emb)
h2 = normalization.batch(bottom=h2,
renorm=False,
name='hgru_bn',
training=training)
activity = conv.readout_layer(
activity=h2,
reuse=reuse,
training=training,
pool_type='max', # 'select',
output_shape=output_shape,
features=reduction_filters)
extra_activities = {'activity': h2}
return activity, extra_activities
def up_block(
layer_name,
bottom,
skip_activity,
reuse,
kernel_size,
num_filters,
training,
trainable=None,
stride=[2, 2],
padding='same',
kernel_initializer=None,
normalization_type='batch_norm',
data_format='channels_last',
renorm=False,
use_bias=False):
"""Forward block for seung model."""
if trainable is None:
trainable = training
with tf.variable_scope('%s_block' % layer_name, reuse=reuse):
with tf.variable_scope('%s_layer_1' % layer_name, reuse=reuse):
x = tf.layers.conv2d_transpose(
inputs=bottom,
filters=num_filters,
kernel_size=kernel_size,
name='%s_1' % layer_name,
strides=stride,
padding=padding,
kernel_initializer=kernel_initializer,
trainable=trainable,
use_bias=use_bias)
x = x + skip_activity # Rethink if this is valid
if normalization_type is 'batch_norm':
x = normalization.batch(
bottom=x,
name='%s_bn_1' % layer_name,
data_format=data_format,
renorm=renorm,
trainable=trainable,
training=training)
else:
x = normalization.instance(
bottom=x,
training=training)
x = tf.nn.elu(x)
return x
def build_model(data_tensor, reuse, training, output_shape):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[0]
with tf.variable_scope('cnn', reuse=reuse):
with tf.variable_scope('input', reuse=reuse):
x = tf.layers.conv2d(inputs=data_tensor,
filters=24,
kernel_size=11,
name='l0',
strides=(1, 1),
padding='same',
activation=tf.nn.relu,
trainable=training,
use_bias=True)
layer_hgru = hgru.hGRU('hgru_1',
x_shape=x.get_shape().as_list(),
timesteps=8,
h_ext=15,
strides=[1, 1, 1, 1],
padding='SAME',
aux={
'reuse': False,
'constrain': False
},
train=training)
h2 = layer_hgru.build(x)
h2 = normalization.batch(bottom=h2,
renorm=True,
name='hgru_bn',
training=training)
with tf.variable_scope('readout_1', reuse=reuse):
activity = conv.conv_layer(bottom=h2,
name='readout_conv',
num_filters=1,
kernel_size=1,
trainable=training,
use_bias=True)
extra_activities = {'activity': h2}
return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[0]
with tf.variable_scope('cnn', reuse=reuse):
# Add input
in_emb = conv.input_layer(
X=data_tensor,
reuse=reuse,
training=training,
features=20,
conv_activation=[tf.nn.relu, None],
conv_kernel_size=7,
pool_kernel_size=[1, 2, 2, 1],
pool_kernel_strides=[1, 2, 2, 1],
pool=True,
name='l0')
layer_hgru = hgru.hGRU(
'hgru_1',
x_shape=in_emb.get_shape().as_list(),
timesteps=8,
h_ext=15,
strides=[1, 1, 1, 1],
padding='SAME',
aux={'reuse': False, 'constrain': False},
train=training)
h2 = layer_hgru.build(in_emb)
h2 = normalization.batch(
bottom=h2,
renorm=False,
name='hgru_bn',
training=training)
activity = conv.readout_layer(
activity=h2,
reuse=reuse,
training=training,
output_shape=output_shape)
extra_activities = {
'activity': h2
}
return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[0]
with tf.variable_scope('cnn', reuse=reuse):
# Add input
conv_aux = {
'pretrained': os.path.join('weights',
'gabors_for_contours_11.npy'),
'pretrained_key': 's1',
'nonlinearity': 'square'
}
activity = conv.conv_layer(bottom=data_tensor,
name='gabor_input',
stride=[1, 1, 1, 1],
padding='SAME',
trainable=training,
use_bias=True,
aux=conv_aux)
layer_hgru = hgru.hGRU('hgru_1',
x_shape=activity.get_shape().as_list(),
timesteps=8,
h_ext=15,
strides=[1, 1, 1, 1],
padding='SAME',
aux={
'reuse': False,
'constrain': False
},
train=training)
h2 = layer_hgru.build(activity)
h2 = normalization.batch(bottom=h2,
renorm=False,
name='hgru_bn',
training=training)
activity = conv.readout_layer(activity=h2,
reuse=reuse,
training=training,
output_shape=output_shape)
extra_activities = {'activity': h2}
return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[0]
with tf.variable_scope('cnn', reuse=reuse):
# Add input
in_emb = conv.skinny_input_layer(X=data_tensor,
reuse=reuse,
training=training,
features=24,
conv_activation=tf.nn.elu,
conv_kernel_size=7,
pool=False,
name='l0')
layer_hgru = hgru.hGRU('hgru_1',
x_shape=in_emb.get_shape().as_list(),
timesteps=8,
h_ext=9,
strides=[1, 1, 1, 1],
padding='SAME',
aux={
'reuse': False,
'constrain': False,
'nonnegative': True
},
train=training)
h2 = layer_hgru.build(in_emb)
h2 = normalization.batch(bottom=h2,
renorm=True,
name='hgru_bn',
training=training)
activity = conv.conv_layer(bottom=h2,
name='pre_readout_conv',
num_filters=output_shape['output'],
kernel_size=1,
trainable=training,
use_bias=True)
extra_activities = {}
return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[0]
alpha = 5.
beta = 1.
bs = data_tensor.get_shape().as_list()[0]
dist = tf.distributions.Beta(alpha, beta)
rand = dist.sample([])
sel = tf.cast(tf.round(tf.random_uniform([], minval=1, maxval=bs - 1)),
tf.int32)
with tf.variable_scope('cnn', reuse=reuse):
with tf.variable_scope('input', reuse=reuse):
o_x = tf.layers.conv2d(inputs=data_tensor,
filters=24,
kernel_size=11,
name='l0',
strides=(1, 1),
padding='same',
activation=tf.nn.relu,
trainable=training,
use_bias=True)
# x = rand * o_x + ((1 - rand) * tf.roll(o_x, sel, 0))
layer_hgru = hgru.hGRU('hgru_1',
x_shape=o_x.get_shape().as_list(),
timesteps=8,
h_ext=15,
strides=[1, 1, 1, 1],
padding='SAME',
aux={
'reuse': False,
'constrain': False
},
train=training)
h2, inh = layer_hgru.build(o_x)
with tf.variable_scope('ss_readout', reuse=reuse):
o_x = normalization.batch(bottom=o_x,
renorm=False,
name='hgru_x',
training=training)
inh = normalization.batch(bottom=inh,
renorm=False,
name='hgru_inh',
training=training)
layers = [[o_x, inh]]
ds_list = []
for idx, acts in enumerate(zip(layers)):
(xs, inhs) = acts[0]
xs = tf.reshape(xs, [bs, -1])
inhs = tf.reshape(inhs, [bs, -1])
# Augmentation
rolled_xs = []
for r in range(1, bs - 1):
rolled_xs += [tf.roll(xs, r, 0)]
xs = tf.concat([xs] + rolled_xs, 0)
inhs = tf.tile(inhs, [bs - 1, 1])
# CPC distances
ds = tf.reduce_sum((xs * inhs), axis=-1)
ds = tf.reshape(ds, [bs - 1, -1])
ds_list += [ds]
ds_list = tf.transpose(tf.concat(ds_list, -1))
return ds_list, {}
def build_model(data_tensor, reuse, training):
"""Create the hgru from Learning long-range..."""
with tf.variable_scope('cnn', reuse=reuse):
with tf.variable_scope('input', reuse=reuse):
x = tf.layers.conv2d(inputs=data_tensor,
filters=24,
kernel_size=3,
padding='same',
trainable=training,
name='c1_1',
use_bias=True)
x = tf.layers.conv2d(inputs=x,
filters=24,
kernel_size=3,
padding='same',
trainable=training,
name='c1_2',
use_bias=True)
x = tf.layers.conv2d(inputs=x,
filters=24,
kernel_size=3,
padding='same',
trainable=training,
name='c1_3',
use_bias=True)
with tf.variable_scope('hGRU', reuse=reuse):
layer_hgru = hgru.hGRU('hgru',
x_shape=x.get_shape().as_list(),
timesteps=6,
h_ext=7,
strides=[1, 1, 1, 1],
padding='SAME',
aux={
'readout': 'fb',
'pooling_kernel': [1, 4, 4, 1],
'intermediate_ff': [24, 24, 24],
'intermediate_ks': [[3, 3], [3, 3],
[3, 3]],
},
pool_strides=[1, 4, 4, 1],
train=training)
x = layer_hgru.build(x)
x = normalization.batch(bottom=x,
name='hgru_bn',
fused=True,
training=training)
fb = tf.identity(x)
with tf.variable_scope('readout_1', reuse=reuse):
x = conv.conv_layer(bottom=x,
name='pre_readout_conv',
num_filters=2,
kernel_size=1,
trainable=training,
use_bias=True)
pool_aux = {'pool_type': 'max'}
x = pooling.global_pool(bottom=x,
name='pre_readout_pool',
aux=pool_aux)
x = normalization.batch(bottom=x,
name='hgru_bn',
training=training)
with tf.variable_scope('readout_2', reuse=reuse):
x = tf.layers.flatten(x, name='flat_readout')
x = tf.layers.dense(inputs=x, units=2)
extra_activities = {'activity': fb}
return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[0]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
with tf.variable_scope('cnn', reuse=reuse):
# # Concatenate standard deviation
# _, var = tf.nn.moments(data_tensor, axes=[3])
# std = tf.expand_dims(tf.sqrt(var), axis=-1)
# data_tensor = tf.concat([data_tensor, std], axis=-1)
# Add input
in_emb = conv.input_layer(X=data_tensor,
reuse=reuse,
training=training,
features=[12, 12],
conv_activation=[tf.nn.relu, tf.nn.relu],
conv_kernel_size=7,
pool_kernel_size=[1, 4, 4, 1],
pool_kernel_strides=[1, 2, 2, 1],
pool=True,
name='l0')
# Run fGRU
hgru_kernels = OrderedDict()
hgru_kernels['h1'] = [15, 15] # height/width
hgru_kernels['h2'] = [1, 1]
hgru_kernels['fb1'] = [1, 1]
hgru_features = OrderedDict()
hgru_features['h1'] = [12, 12] # Fan-in/fan-out, I and E (match fb1)
hgru_features['h2'] = [48, 48]
hgru_features['fb1'] = [12, 12] # (match h1)
# hgru_features['fb1'] = [24, 12] # (match h1 unless squeeze_fb)
intermediate_ff = [24, 48] # Last feature must match h2
intermediate_ks = [[3, 3], [3, 3]]
intermediate_repeats = [1, 1] # Repeat each interm this many times
layer_hgru = hgru.hGRU(
'fgru',
x_shape=in_emb.get_shape().as_list(),
timesteps=8,
strides=[1, 1, 1, 1],
hgru_features=hgru_features,
hgru_kernels=hgru_kernels,
intermediate_ff=intermediate_ff,
intermediate_ks=intermediate_ks,
intermediate_repeats=intermediate_repeats,
padding='SAME',
aux={
'readout': 'fb',
'squeeze_fb': False, # Compress Inh-hat with a 1x1 conv
'td_gate': True, # Add top-down activity to the in-gate
'attention': 'gala',
'attention_layers': 2,
'td_cell_state': False,
'excite_se': False, # Add S/E in the excitation stage
'residual': True, # intermediate resid connections
'while_loop': False,
'skip': True,
'time_skips': False,
'symmetric_weights': False,
'timestep_output': False,
'include_pooling': True
},
pool_strides=[2, 2],
pooling_kernel=[2, 2],
train=training)
h2 = layer_hgru.build(in_emb)
if isinstance(h2, list):
all_activities = []
for idx, it_h2 in enumerate(h2):
it_h2 = normalization.batch(bottom=it_h2,
renorm=False,
name='hgru_bn_%s' % idx,
training=training)
activity = conv.readout_layer(activity=it_h2,
reuse=reuse,
training=training,
var_scope='readout_%s' % idx,
output_shape=output_shape)
all_activities += [activity]
h2 = all_activities[-1]
activity = all_activities
else:
h2 = normalization.batch(bottom=h2,
renorm=False,
name='hgru_bn',
training=training)
activity = conv.readout_layer(activity=h2,
reuse=reuse,
training=training,
output_shape=output_shape)
extra_activities = {'activity': h2}
return activity, extra_activities
def build_model(data_tensor, reuse, training):
"""Create the hgru from Learning long-range..."""
down_pool_kernel = [1, 2, 2, 1]
down_pool_strides = [1, 2, 2, 1]
down_pool_padding = 'SAME'
with tf.variable_scope('cnn', reuse=reuse):
# Unclear if we should include l0 in the down/upsample cascade
with tf.variable_scope('g1', reuse=reuse):
# Downsample
act11 = conv_block(x=data_tensor,
name='l1_1',
filters=64,
training=training,
reuse=reuse,
pool=False)
act12 = conv_block(x=act11,
name='l1_2',
filters=64,
training=training,
reuse=reuse,
pool=False)
poolact12, poolact12inds = tf.nn.max_pool_with_argmax(
input=act12,
ksize=down_pool_kernel,
strides=down_pool_strides,
padding=down_pool_padding,
name='l1_2_pool')
with tf.variable_scope('g2', reuse=reuse):
# Downsample
act21 = conv_block(x=act12,
name='l2_1',
filters=128,
training=training,
reuse=reuse,
pool=False)
act22 = conv_block(x=act21,
filters=128,
name='l2_2',
training=training,
reuse=reuse,
pool=False)
poolact22, poolact22inds = tf.nn.max_pool_with_argmax(
input=act22,
ksize=down_pool_kernel,
strides=down_pool_strides,
padding=down_pool_padding,
name='l2_2_pool')
with tf.variable_scope('g3', reuse=reuse):
# Downsample
act31 = conv_block(x=poolact22,
name='l3_1',
filters=256,
training=training,
reuse=reuse,
pool=False)
act32 = conv_block(x=act31,
filters=256,
name='l3_2',
training=training,
reuse=reuse,
pool=False)
act33 = conv_block(x=act32,
filters=256,
name='l3_3',
training=training,
reuse=reuse,
pool=False)
poolact33, poolact33inds = tf.nn.max_pool_with_argmax(
input=act33,
ksize=down_pool_kernel,
strides=down_pool_strides,
padding=down_pool_padding,
name='l3_3_pool')
with tf.variable_scope('g4', reuse=reuse):
# Downsample
act41 = conv_block(x=poolact33,
name='l4_1',
filters=512,
training=training,
reuse=reuse,
pool=False)
act42 = conv_block(x=act41,
filters=512,
name='l4_2',
training=training,
reuse=reuse,
pool=False)
act43 = conv_block(x=act42,
filters=512,
name='l4_3',
training=training,
reuse=reuse,
pool=False)
poolact43, poolact43inds = tf.nn.max_pool_with_argmax(
input=act43,
ksize=down_pool_kernel,
strides=down_pool_strides,
padding=down_pool_padding,
name='l4_3_pool')
with tf.variable_scope('g5', reuse=reuse):
# Downsample
act51 = conv_block(x=poolact43,
name='l5_1',
filters=512,
training=training,
reuse=reuse,
pool=False)
act52 = conv_block(x=act51,
filters=512,
name='l5_2',
training=training,
reuse=reuse,
pool=False)
act53 = conv_block(x=act52,
filters=512,
name='l5_3',
training=training,
reuse=reuse,
pool=False)
poolact53, poolact53inds = tf.nn.max_pool_with_argmax(
input=act53,
ksize=down_pool_kernel,
strides=down_pool_strides,
padding=down_pool_padding,
name='l5_3_pool')
with tf.variable_scope('g5_up', reuse=reuse):
upact5 = pooling.unpool_with_argmax_layer(bottom=poolact53,
ind=poolact53inds,
filter_size=[3, 3],
name='l5_unpool')
uact53 = conv_block(x=upact5,
name='ul5_3',
filters=512,
training=training,
reuse=reuse,
pool=False)
uact52 = conv_block(x=uact53,
filters=512,
name='ul5_2',
training=training,
reuse=reuse,
pool=False)
uact51 = conv_block(x=uact52,
filters=512,
name='ul5_1',
training=training,
reuse=reuse,
pool=False)
with tf.variable_scope('g4_up', reuse=reuse):
upact4 = pooling.unpool_with_argmax_layer(bottom=uact51,
ind=poolact43inds,
filter_size=[3, 3],
name='l4_unpool')
uact43 = conv_block(x=upact4,
name='ul4_3',
filters=512,
training=training,
reuse=reuse,
pool=False)
uact42 = conv_block(x=uact43,
filters=512,
name='ul4_2',
training=training,
reuse=reuse,
pool=False)
uact41 = conv_block(x=uact42,
filters=256,
name='ul4_1',
training=training,
reuse=reuse,
pool=False)
with tf.variable_scope('g3_up', reuse=reuse):
upact3 = pooling.unpool_with_argmax_layer(bottom=uact41,
ind=poolact33inds,
filter_size=[3, 3],
name='l3_unpool')
uact33 = conv_block(x=upact3,
name='ul3_3',
filters=256,
training=training,
reuse=reuse,
pool=False)
uact32 = conv_block(x=uact33,
filters=256,
name='ul3_2',
training=training,
reuse=reuse,
pool=False)
uact31 = conv_block(x=uact32,
filters=128,
name='ul3_1',
training=training,
reuse=reuse,
pool=False)
with tf.variable_scope('g2_up', reuse=reuse):
upact2 = pooling.unpool_with_argmax_layer(bottom=uact31,
ind=poolact22inds,
filter_size=[3, 3],
name='l2_unpool')
uact22 = conv_block(x=upact2,
name='ul2_2',
filters=128,
training=training,
reuse=reuse,
pool=False)
uact21 = conv_block(x=uact22,
name='ul2_1',
filters=64,
training=training,
reuse=reuse,
pool=False)
with tf.variable_scope('g1_up', reuse=reuse):
upact1 = pooling.unpool_with_argmax_layer(bottom=uact21,
ind=poolact12inds,
filter_size=[3, 3],
name='l1_unpool')
uact12 = conv_block(x=upact1,
name='ul1_2',
filters=64,
training=training,
reuse=reuse,
pool=False)
uact11 = conv_block(x=uact12,
name='ul1_1',
filters=64,
training=training,
reuse=reuse,
pool=False)
with tf.variable_scope('readout_1', reuse=reuse):
activity = conv.conv_layer(bottom=uact11,
name='pre_readout_conv',
num_filters=2,
kernel_size=1,
trainable=training,
use_bias=False)
pool_aux = {'pool_type': 'max'}
activity = pooling.global_pool(bottom=activity,
name='pre_readout_pool',
aux=pool_aux)
activity = normalization.batch(bottom=activity,
renorm=True,
name='readout_1_bn',
training=training)
with tf.variable_scope('readout_2', reuse=reuse):
activity = tf.layers.flatten(activity, name='flat_readout')
activity = tf.layers.dense(inputs=activity, units=2)
extra_activities = {'activity': activity}
return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[0]
data_format = 'channels_last'
conv_kernel = [
[3, 3],
[3, 3],
[3, 3],
]
up_kernel = [2, 2]
filters = [28, 36, 48, 64, 80]
with tf.variable_scope('cnn', reuse=reuse):
# Unclear if we should include l0 in the down/upsample cascade
with tf.variable_scope('in_embedding', reuse=reuse):
in_emb = tf.layers.conv2d(inputs=data_tensor,
filters=filters[0],
kernel_size=5,
name='l0',
strides=(1, 1),
padding='same',
activation=tf.nn.elu,
data_format=data_format,
trainable=training,
use_bias=True)
# Downsample
l1 = conv.down_block(layer_name='l1',
bottom=in_emb,
kernel_size=conv_kernel,
num_filters=filters[1],
training=training,
reuse=reuse)
l2 = conv.down_block(layer_name='l2',
bottom=l1,
kernel_size=conv_kernel,
num_filters=filters[2],
training=training,
reuse=reuse)
l3 = conv.down_block(layer_name='l3',
bottom=l2,
kernel_size=conv_kernel,
num_filters=filters[3],
training=training,
reuse=reuse)
l4 = conv.down_block(layer_name='l4',
bottom=l3,
kernel_size=conv_kernel,
num_filters=filters[4],
training=training,
reuse=reuse)
# Upsample
ul3 = conv.up_block(layer_name='ul3',
bottom=l4,
skip_activity=l3,
kernel_size=up_kernel,
num_filters=filters[3],
training=training,
reuse=reuse)
ul3 = conv.down_block(layer_name='ul3_d',
bottom=ul3,
kernel_size=conv_kernel,
num_filters=filters[3],
training=training,
reuse=reuse,
include_pool=False)
ul2 = conv.up_block(layer_name='ul2',
bottom=ul3,
skip_activity=l2,
kernel_size=up_kernel,
num_filters=filters[2],
training=training,
reuse=reuse)
ul2 = conv.down_block(layer_name='ul2_d',
bottom=ul2,
kernel_size=conv_kernel,
num_filters=filters[2],
training=training,
reuse=reuse,
include_pool=False)
ul1 = conv.up_block(layer_name='ul1',
bottom=ul2,
skip_activity=l1,
kernel_size=up_kernel,
num_filters=filters[1],
training=training,
reuse=reuse)
ul1 = conv.down_block(layer_name='ul1_d',
bottom=ul1,
kernel_size=conv_kernel,
num_filters=filters[1],
training=training,
reuse=reuse,
include_pool=False)
ul0 = conv.up_block(layer_name='ul0',
bottom=ul1,
skip_activity=in_emb,
kernel_size=up_kernel,
num_filters=filters[0],
training=training,
reuse=reuse)
with tf.variable_scope('readout_1', reuse=reuse):
activity = conv.conv_layer(bottom=ul0,
name='pre_readout_conv',
num_filters=2,
kernel_size=1,
trainable=training,
use_bias=False)
pool_aux = {'pool_type': 'max'}
activity = pooling.global_pool(bottom=activity,
name='pre_readout_pool',
aux=pool_aux)
activity = normalization.batch(bottom=activity,
renorm=True,
name='readout_1_bn',
training=training)
with tf.variable_scope('readout_2', reuse=reuse):
activity = tf.layers.flatten(activity, name='flat_readout')
activity = tf.layers.dense(inputs=activity, units=output_shape)
extra_activities = {'l4': l4}
return activity, extra_activities
def down_block(layer_name,
bottom,
reuse,
kernel_size,
num_filters,
training,
use_bias=False,
include_pool=True):
"""Forward block for seung model."""
with tf.variable_scope('%s_block' % layer_name, reuse=reuse):
with tf.variable_scope('%s_layer_1' % layer_name, reuse=reuse):
#with bfloat16.bfloat16_scope():
print(bottom.dtype)
x = conv3d_layer(bottom=bottom,
name='%s_1' % layer_name,
stride=[1, 1, 1],
padding='SAME',
kernel_size=kernel_size[0],
num_filters=num_filters,
trainable=training,
use_bias=use_bias)
print(x)
x = tf.layers.batch_normalization(inputs=x,
name='%s_bn_1' % layer_name,
scale=True,
center=True,
training=training,
fused=True,
axis=len(x.shape) - 1)
print(x.dtype)
# x = normalization.batch(
# bottom=x,
# name='%s_bn_1' % layer_name,
# training=training)
# print(x.dtype)
x = tf.nn.elu(x)
print(x.dtype)
skip = tf.identity(x)
print(skip.dtype)
with tf.variable_scope('%s_layer_2' % layer_name, reuse=reuse):
x = conv3d_layer(bottom=x,
name='%s_2' % layer_name,
stride=[1, 1, 1],
padding='SAME',
kernel_size=kernel_size[1],
num_filters=num_filters,
trainable=training,
use_bias=use_bias)
x = normalization.batch(bottom=x,
name='%s_bn_2' % layer_name,
training=training)
x = tf.nn.elu(x)
with tf.variable_scope('%s_layer_3' % layer_name, reuse=reuse):
x = conv3d_layer(bottom=x,
name='%s_3' % layer_name,
stride=[1, 1, 1],
padding='SAME',
kernel_size=kernel_size[2],
num_filters=num_filters,
trainable=training,
use_bias=use_bias)
x = tf.nn.elu(x)
x = x + skip
x = normalization.batch(bottom=x,
name='%s_bn_3' % layer_name,
training=training)
if include_pool:
with tf.variable_scope('%s_pool' % layer_name, reuse=reuse):
x = pooling.max_pool3d(
bottom=x,
name='%s_pool' % layer_name,
# TODO: swap back
k=[2, 2, 1],
s=[2, 2, 1])
return x
def build_model(
data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
# output_shape = 2
# norm_moments_training = training # Force instance norm
normalization_type = 'no_param_batch_norm_original'
# normalization_type = 'ada_batch_norm'
# normalization_type = 'no_param_instance_norm'
output_normalization_type = 'batch_norm_original_renorm'
# output_normalization_type = 'instance_norm'
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor,
data_format=data_format)
# Prepare gammanet structure
(
compression,
ff_kernels,
ff_repeats,
features,
fgru_kernels,
additional_readouts) = v2_big_working()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
with tf.variable_scope('vgg', reuse=reuse):
aux = get_aux()
vgg = vgg16.Vgg16(
vgg16_npy_path='/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy',
reuse=reuse,
aux=aux,
train=training,
timesteps=8,
fgru_normalization_type=normalization_type,
ff_normalization_type=normalization_type)
vgg(rgb=data_tensor, constructor=gammanet_constructor)
# activity = vgg.fgru_0
with tf.variable_scope('fgru', reuse=reuse):
# Get side weights
activity = normalization.batch(
bottom=vgg.fgru_0, # activity,
renorm=True,
name='readout_0_bn',
training=training)
# activity = normalization.group(
# bottom=vgg.fgru_0) # , # activity,
# # name='readout_0_bn')
prepool_activity = tf.layers.conv2d(
inputs=activity,
filters=output_shape, # int(activity.get_shape()[-1]),
kernel_size=1,
name='pre_readout_conv0',
strides=(1, 1),
padding='same',
activation=tf.nn.elu,
trainable=training,
use_bias=True)
"""
prepool_activity = normalization.batch(
bottom=prepool_activity,
renorm=True,
name='readout_1_bn',
training=False) # training)
"""
"""
prepool_activity = tf.layers.conv2d(
inputs=prepool_activity,
filters=output_shape,
kernel_size=1,
name='pre_readout_conv1',
strides=(1, 1),
padding='same',
activation=None,
trainable=training,
use_bias=True)
"""
out_activity = tf.reduce_max(prepool_activity, reduction_indices=[1, 2])
activity = tf.layers.dense(
inputs=out_activity,
units=output_shape)
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
# extra_activities = {'orientations': orientations, 'model_output_y': vgg.fgru_0, 'model_output_x': vgg.fgru_0_init}
extra_activities = {'x': vgg.fgru_0}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
# return [activity, h_deep], extra_activities
return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[0]
elif isinstance(output_shape, dict):
nhot_shape = output_shape['aux']
output_shape = output_shape['output']
use_aux = True
with tf.variable_scope('cnn', reuse=reuse):
# Unclear if we should include l0 in the down/upsample cascade
with tf.variable_scope('g1', reuse=reuse):
# Downsample
act11 = conv_block(
x=data_tensor,
name='l1_1',
filters=64,
training=training,
reuse=reuse,
pool=False)
act12 = conv_block(
x=act11,
name='l1_2',
filters=64,
training=training,
reuse=reuse,
pool=False)
poolact12 = pooling.max_pool(
bottom=act12,
name='l1_2_pool')
with tf.variable_scope('g2', reuse=reuse):
# Downsample
act21 = conv_block(
x=poolact12,
name='l2_1',
filters=128,
training=training,
reuse=reuse,
pool=False)
act22 = conv_block(
x=act21,
filters=128,
name='l2_2',
training=training,
reuse=reuse,
pool=False)
poolact22 = pooling.max_pool(
bottom=act22,
name='l2_2_pool')
with tf.variable_scope('g3', reuse=reuse):
# Downsample
act31 = conv_block(
x=poolact22,
name='l3_1',
filters=256,
training=training,
reuse=reuse,
pool=False)
act32 = conv_block(
x=act31,
filters=256,
name='l3_2',
training=training,
reuse=reuse,
pool=False)
act33 = conv_block(
x=act32,
filters=256,
name='l3_3',
training=training,
reuse=reuse,
pool=False)
poolact33 = pooling.max_pool(
bottom=act33,
name='l3_3_pool')
with tf.variable_scope('g4', reuse=reuse):
# Downsample
act41 = conv_block(
x=poolact33,
name='l4_1',
filters=512,
training=training,
reuse=reuse,
pool=False)
act42 = conv_block(
x=act41,
filters=512,
name='l4_2',
training=training,
reuse=reuse,
pool=False)
act43 = conv_block(
x=act42,
filters=512,
name='l4_3',
training=training,
reuse=reuse,
pool=False)
poolact43 = pooling.max_pool(
bottom=act43,
name='l4_3_pool')
with tf.variable_scope('g5', reuse=reuse):
# Downsample
act51 = conv_block(
x=poolact43,
name='l5_1',
filters=512,
training=training,
reuse=reuse,
pool=False)
act52 = conv_block(
x=act51,
filters=512,
name='l5_2',
training=training,
reuse=reuse,
pool=False)
act53 = conv_block(
x=act52,
filters=512,
name='l5_3',
training=training,
reuse=reuse,
pool=False)
poolact53 = pooling.max_pool(
bottom=act53,
name='l5_3_pool')
with tf.variable_scope('g5_skip', reuse=reuse):
upact5 = up_block(
inputs=poolact53,
skip=act53,
up_filters=512,
name='ul5',
training=training,
reuse=reuse)
with tf.variable_scope('g4_skip', reuse=reuse):
upact4 = up_block(
inputs=upact5,
skip=act43,
up_filters=512,
name='ul4',
training=training,
reuse=reuse)
with tf.variable_scope('g3_skip', reuse=reuse):
upact3 = up_block(
inputs=upact4,
skip=act33,
up_filters=256,
name='ul3',
training=training,
reuse=reuse)
with tf.variable_scope('g2_skip', reuse=reuse):
upact2 = up_block(
inputs=upact3,
skip=act22,
up_filters=128,
name='ul2',
training=training,
reuse=reuse)
with tf.variable_scope('g1_skip', reuse=reuse):
upact1 = up_block(
inputs=upact2,
skip=act12,
up_filters=64,
name='ul1',
training=training,
reuse=reuse)
with tf.variable_scope('readout_1', reuse=reuse):
activity = conv.conv_layer(
bottom=upact1,
name='pre_readout_conv',
num_filters=2,
kernel_size=1,
trainable=training,
use_bias=False)
pool_aux = {'pool_type': 'max'}
activity = pooling.global_pool(
bottom=activity,
name='pre_readout_pool',
aux=pool_aux)
activity = normalization.batch(
bottom=activity,
renorm=True,
name='readout_1_bn',
training=training)
with tf.variable_scope('readout_2', reuse=reuse):
pre_activity = tf.layers.flatten(
activity,
name='flat_readout')
activity = tf.layers.dense(
inputs=pre_activity,
units=output_shape)
if use_aux:
nhot = tf.layers.dense(inputs=pre_activity, units=nhot_shape)
else:
nhot = tf.constant(0.)
extra_activities = {
'activity': activity,
'nhot': nhot
}
return activity, extra_activities
def build_model(data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
output_shape = 1
# norm_moments_training = training # Force instance norm
# normalization_type = 'no_param_batch_norm_original'
normalization_type = 'no_param_instance_norm'
# output_normalization_type = 'batch_norm_original_renorm'
output_normalization_type = 'instance_norm'
# data_tensor = tf.concat((data_tensor, data_tensor, data_tensor), -1) # Expand to 3D
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor, data_format=data_format)
# Prepare gammanet structure
(compression, ff_kernels, ff_repeats, features, fgru_kernels,
additional_readouts) = v2_big_working()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
with tf.variable_scope('vgg', reuse=reuse):
aux = get_aux()
vgg = vgg16.Vgg16(
vgg16_npy_path=
'/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy',
reuse=reuse,
aux=aux,
train=False, # training,
timesteps=8,
fgru_normalization_type=normalization_type,
ff_normalization_type=normalization_type)
vgg(rgb=data_tensor, constructor=gammanet_constructor)
# activity = vgg.fgru_0
with tf.variable_scope('fgru', reuse=reuse):
# Get side weights
h2_rem = [vgg.fgru_0, vgg.fgru_1, vgg.fgru_2]
activities = []
for idx, h in enumerate(h2_rem):
# res = normalization.apply_normalization(
# activity=h,
# name='output_norm1_%s' % idx,
# normalization_type=output_normalization_type,
# data_format=data_format,
# training=False,
# trainable=False,
# reuse=reuse)
activities.append(aux['image_resize'](
h, vgg.fgru_0.get_shape().as_list()[1:3], align_corners=True))
activity = tf.concat(activities, -1)
# Then read out
# activity = normalization.apply_normalization(
# activity=res,
# name='output_norm1',
# normalization_type=output_normalization_type,
# data_format=data_format,
# training=training,
# trainable=training,
# reuse=reuse)
with tf.variable_scope('readout', reuse=reuse):
activity = normalization.batch(bottom=activity,
renorm=False,
name='hgru_bn',
training=training)
activity = conv.readout_layer(activity=activity,
reuse=reuse,
training=training,
output_shape=output_shape)
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
# extra_activities = {'orientations': orientations, 'model_output_y': vgg.fgru_0, 'model_output_x': vgg.fgru_0_init}
extra_activities = {}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
# return [activity, h_deep], extra_activities
return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[0]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
with tf.variable_scope('cnn', reuse=reuse):
normalization_type = 'ada_batch_norm' # 'instance_norm'
# # Concatenate standard deviation
# _, var = tf.nn.moments(data_tensor, axes=[3])
# std = tf.expand_dims(tf.sqrt(var), axis=-1)
# data_tensor = tf.concat([data_tensor, std], axis=-1)
# Add input
in_emb = conv.skinny_input_layer(X=data_tensor,
reuse=reuse,
training=training,
features=16,
conv_activation=tf.nn.relu,
conv_kernel_size=7,
pool=False,
name='l0')
# Run fGRU
hgru_kernels = OrderedDict()
hgru_kernels['h1'] = [15, 15] # height/width
hgru_kernels['h2'] = [5, 5]
hgru_kernels['fb1'] = [1, 1]
hgru_features = OrderedDict()
hgru_features['h1'] = [16, 16] # Fan-in/fan-out, I and E (match fb1)
hgru_features['h2'] = [48, 48]
hgru_features['fb1'] = [16, 16] # (match h1)
# hgru_features['fb1'] = [24, 12] # (match h1 unless squeeze_fb)
intermediate_ff = [24, 24, 48] # Last feature must match h2
intermediate_ks = [[5, 5], [5, 5], [5, 5]]
intermediate_repeats = [1, 1, 1] # Repeat each interm this many times
layer_hgru = hgru.hGRU(
'fgru',
x_shape=in_emb.get_shape().as_list(),
timesteps=8,
strides=[1, 1, 1, 1],
hgru_features=hgru_features,
hgru_kernels=hgru_kernels,
intermediate_ff=intermediate_ff,
intermediate_ks=intermediate_ks,
intermediate_repeats=intermediate_repeats,
padding='SAME',
aux={
'readout': 'fb',
'squeeze_fb': False, # Compress Inh-hat with a 1x1 conv
'td_gate': True, # Add top-down activity to the in-gate
'attention': 'gala', # 'gala',
'attention_layers': 2,
'upsample_convs': True,
'td_cell_state': True,
'normalization_type': normalization_type,
'excite_se': False, # Add S/E in the excitation stage
'residual': True, # intermediate resid connections
'while_loop': False,
'skip': True,
'time_skips': False,
'force_horizontal': False,
'symmetric_weights': True,
'timestep_output': False,
'bilinear_init': True,
'include_pooling': True
},
pool_strides=[2, 2],
pooling_kernel=[4, 4],
up_kernel=[4, 4],
train=training)
h2 = layer_hgru.build(in_emb)
if normalization_type is 'batch_norm':
h2 = normalization.batch(bottom=h2,
renorm=False,
name='hgru_bn',
training=training)
elif normalization_type is 'instance_norm':
h2 = normalization.instance(bottom=h2, training=training)
elif normalization_type is 'ada_batch_norm':
h2 = normalization.batch(bottom=h2,
renorm=False,
name='hgru_bn',
training=True)
else:
raise NotImplementedError(normalization_type)
fc = tf.layers.conv2d(inputs=h2,
filters=output_shape,
kernel_size=1,
name='fc')
# activity = tf.reduce_mean(fc, reduction_indices=[1, 2])
activity = tf.reduce_max(fc, reduction_indices=[1, 2])
extra_activities = {'activity': h2}
return activity, extra_activities
