def build_model(data_tensor, reuse, training, output_shape):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[0]
with tf.variable_scope('cnn', reuse=reuse):
# Add input
in_emb = conv.skinny_input_layer(
X=data_tensor,
reuse=reuse,
training=training,
features=24,
conv_activation=tf.nn.elu,
conv_kernel_size=7,
pool=False,
name='l0')
layer_hgru = feedback_hgru.hGRU(
layer_name='hgru_1',
x_shape=in_emb.get_shape().as_list(),
timesteps=8,
h_ext=[[9, 9], [5, 5], [1, 1]],
strides=[1, 1, 1, 1],
pool_strides=[4, 4],
padding='SAME',
aux={
'symmetric_weights': True,
'dilations': [[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]],
'batch_norm': True,
'while_loop': False,
'nonnegative': True,
'residual': True,
'final_bn': False,
'renorm': False,
'lesion_alpha': True,
'lesion_omega': True,
'pooling_kernel': [4, 4],
'intermediate_ff': [24, 24], # + filters,
'intermediate_ks': [[5, 5], [5, 5]]},
train=training)
h2 = layer_hgru.build(in_emb)
nh2 = normalization.batch(
bottom=h2,
name='hgru_bn',
fused=True,
renorm=True,
training=training)
activity = conv.conv_layer(
bottom=nh2,
name='pre_readout_conv',
num_filters=output_shape['output'],
kernel_size=1,
trainable=training,
use_bias=True)
extra_activities = {
# 'activity': h2
}
return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[0]
with tf.variable_scope('cnn', reuse=reuse):
# Add input
in_emb = conv.skinny_input_layer(
X=data_tensor,
reuse=reuse,
training=training,
features=24,
conv_activation=tf.nn.elu,
conv_kernel_size=7,
pool=False,
name='l0')
layer_hgru = hgru.hGRU(
'fgru',
x_shape=in_emb.get_shape().as_list(),
timesteps=8,
h_ext=[{'h1': [9, 9]}, {'h2': [5, 5]}, {'fb1': [1, 1]}],
strides=[1, 1, 1, 1],
hgru_ids=[{'h1': 24}, {'h2': 24}, {'fb1': 24}],
hgru_idx=[{'h1': 0}, {'h2': 1}, {'fb1': 2}],
padding='SAME',
aux={
'readout': 'fb',
'intermediate_ff': [24, 24],
'intermediate_ks': [[5, 5], [5, 5]],
'intermediate_repeats': [2, 2],
'while_loop': False,
'skip': False,
'symmetric_weights': True,
'use_homunculus': False,
'include_pooling': True
},
pool_strides=[4, 4],
pooling_kernel=[4, 4],
train=training)
h2 = layer_hgru.build(in_emb)
nh2 = normalization.batch(
bottom=h2,
name='hgru_bn',
fused=True,
renorm=True,
training=training)
activity = conv.conv_layer(
bottom=nh2,
name='pre_readout_conv',
num_filters=output_shape['output'],
kernel_size=1,
trainable=training,
use_bias=True)
extra_activities = {
# 'activity': h2
}
return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[0]
with tf.variable_scope('cnn', reuse=reuse):
# Add input
in_emb = conv.skinny_input_layer(
X=data_tensor,
reuse=reuse,
training=training,
features=20,
conv_activation=tf.nn.relu,
conv_kernel_size=7,
pool_kernel_size=[1, 2, 2, 1],
pool_kernel_strides=[1, 2, 2, 1],
pool=True,
name='l0')
layer_hgru = hgru.hGRU(
'fgru',
x_shape=in_emb.get_shape().as_list(),
timesteps=8,
h_ext=[{'h1': [15, 15]}, {'h2': [1, 1]}, {'fb1': [1, 1]}],
strides=[1, 1, 1, 1],
hgru_ids=[{'h1': 20}, {'h2': 128}, {'fb1': 20}],
hgru_idx=[{'h1': 0}, {'h2': 1}, {'fb1': 2}],
padding='SAME',
aux={
'readout': 'fb',
'intermediate_ff': [32, 128],
'intermediate_ks': [[3, 3], [3, 3]],
'intermediate_repeats': [3, 3],
'while_loop': False,
'skip': False,
'symmetric_weights': True,
'include_pooling': True
},
pool_strides=[2, 2],
pooling_kernel=[2, 2],
train=training)
h2 = layer_hgru.build(in_emb)
h2 = normalization.batch(
bottom=h2,
renorm=False,
name='hgru_bn',
training=training)
activity = conv.readout_layer(
activity=h2,
reuse=reuse,
training=training,
output_shape=output_shape)
extra_activities = {
'activity': h2
}
return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[0]
with tf.variable_scope('cnn', reuse=reuse):
# Add input
in_emb = conv.skinny_input_layer(X=data_tensor,
reuse=reuse,
training=training,
features=24,
conv_activation=tf.nn.elu,
conv_kernel_size=7,
pool=False,
name='l0')
layer_hgru = hgru.hGRU('hgru_1',
x_shape=in_emb.get_shape().as_list(),
timesteps=8,
h_ext=9,
strides=[1, 1, 1, 1],
padding='SAME',
aux={
'reuse': False,
'constrain': False,
'nonnegative': True
},
train=training)
h2 = layer_hgru.build(in_emb)
h2 = normalization.batch(bottom=h2,
renorm=True,
name='hgru_bn',
training=training)
activity = conv.conv_layer(bottom=h2,
name='pre_readout_conv',
num_filters=output_shape['output'],
kernel_size=1,
trainable=training,
use_bias=True)
extra_activities = {}
return activity, extra_activities
def build_model(data_tensor,
reuse,
training,
output_shape,
data_format='NCHW'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
if data_format is 'NCHW':
data_tensor = tf.transpose(data_tensor, (0, 3, 1, 2))
with tf.variable_scope('cnn', reuse=reuse):
normalization_type = 'ada_batch_norm' #
# # Concatenate standard deviation
# _, var = tf.nn.moments(data_tensor, axes=[3])
# std = tf.expand_dims(tf.sqrt(var), axis=-1)
# data_tensor = tf.concat([data_tensor, std], axis=-1)
# Add input
in_emb = conv.skinny_input_layer(X=data_tensor,
reuse=reuse,
training=training,
features=16,
conv_activation=tf.nn.relu,
conv_kernel_size=7,
pool=False,
data_format=data_format,
name='l0')
# Run fGRU
hgru_kernels = OrderedDict()
hgru_kernels['h1'] = [9, 9] # height/width
hgru_kernels['h2'] = [5, 5]
hgru_kernels['fb1'] = [1, 1]
hgru_features = OrderedDict()
hgru_features['h1'] = [16, 16] # Fan-in/fan-out, I and E (match fb1)
hgru_features['h2'] = [32, 32]
hgru_features['fb1'] = [16, 16] # (match h1)
# hgru_features['fb1'] = [24, 12] # (match h1 unless squeeze_fb)
intermediate_ff = [24, 28, 32] # Last feature must match h2
intermediate_ks = [[5, 5], [5, 5], [5, 5]]
intermediate_repeats = [1, 1, 1] # Repeat each interm this many times
gammanet_layer = gammanet.GN(
layer_name='fgru',
x=in_emb,
data_format=data_format,
reuse=reuse,
timesteps=8,
strides=[1, 1, 1, 1],
hgru_features=hgru_features,
hgru_kernels=hgru_kernels,
intermediate_ff=intermediate_ff,
intermediate_ks=intermediate_ks,
intermediate_repeats=intermediate_repeats,
padding='SAME',
aux={
'readout': 'fb',
'squeeze_fb': False, # Compress Inh-hat with a 1x1 conv
'attention': 'gala', # 'gala', # 'gala', se
'attention_layers': 2,
'saliency_filter': 5,
'use_homunculus': True,
'upsample_convs': True,
'td_cell_state': False,
'td_gate': False, # Add top-down activity to the in-gate
'normalization_type': normalization_type,
'excite_se': False, # Add S/E in the excitation stage
'residual': True, # intermediate resid connections
'while_loop': True,
'skip': True,
'time_skips': False,
'force_horizontal': False,
'symmetric_weights': True,
'timestep_output': False,
'bilinear_init': True,
'include_pooling': True
},
pool_strides=[2, 2],
pooling_kernel=[2, 2],
up_kernel=[4, 4],
train=training)
h2 = gammanet_layer(in_emb)
if normalization_type is 'batch_norm':
h2 = normalization.batch_contrib(bottom=h2,
renorm=False,
name='hgru_bn',
dtype=h2.dtype,
data_format=data_format,
training=training)
elif normalization_type is 'instance_norm':
h2 = normalization.instance(bottom=h2,
data_format=data_format,
training=training)
elif normalization_type is 'ada_batch_norm':
h2 = normalization.batch_contrib(bottom=h2,
renorm=False,
name='hgru_bn',
dtype=h2.dtype,
data_format=data_format,
training=training)
else:
raise NotImplementedError(normalization_type)
with tf.variable_scope('cv_readout', reuse=reuse):
if data_format == 'NCHW':
data_format = 'channels_first'
elif data_format == 'NHWC':
data_format = 'channels_last'
activity = tf.layers.conv2d(inputs=h2,
filters=output_shape,
kernel_size=(1, 1),
padding='same',
data_format=data_format,
name='pre_readout_conv',
use_bias=True,
reuse=reuse)
if data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
extra_activities = {'activity': h2}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[0]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
with tf.variable_scope('cnn', reuse=reuse):
normalization_type = 'ada_batch_norm' # 'instance_norm'
# # Concatenate standard deviation
# _, var = tf.nn.moments(data_tensor, axes=[3])
# std = tf.expand_dims(tf.sqrt(var), axis=-1)
# data_tensor = tf.concat([data_tensor, std], axis=-1)
# Add input
in_emb = conv.skinny_input_layer(X=data_tensor,
reuse=reuse,
training=training,
features=16,
conv_activation=tf.nn.relu,
conv_kernel_size=7,
pool=False,
name='l0')
# Run fGRU
hgru_kernels = OrderedDict()
hgru_kernels['h1'] = [15, 15] # height/width
hgru_kernels['h2'] = [5, 5]
hgru_kernels['fb1'] = [1, 1]
hgru_features = OrderedDict()
hgru_features['h1'] = [16, 16] # Fan-in/fan-out, I and E (match fb1)
hgru_features['h2'] = [48, 48]
hgru_features['fb1'] = [16, 16] # (match h1)
# hgru_features['fb1'] = [24, 12] # (match h1 unless squeeze_fb)
intermediate_ff = [24, 24, 48] # Last feature must match h2
intermediate_ks = [[5, 5], [5, 5], [5, 5]]
intermediate_repeats = [1, 1, 1] # Repeat each interm this many times
layer_hgru = hgru.hGRU(
'fgru',
x_shape=in_emb.get_shape().as_list(),
timesteps=8,
strides=[1, 1, 1, 1],
hgru_features=hgru_features,
hgru_kernels=hgru_kernels,
intermediate_ff=intermediate_ff,
intermediate_ks=intermediate_ks,
intermediate_repeats=intermediate_repeats,
padding='SAME',
aux={
'readout': 'fb',
'squeeze_fb': False, # Compress Inh-hat with a 1x1 conv
'td_gate': True, # Add top-down activity to the in-gate
'attention': 'gala', # 'gala',
'attention_layers': 2,
'upsample_convs': True,
'td_cell_state': True,
'normalization_type': normalization_type,
'excite_se': False, # Add S/E in the excitation stage
'residual': True, # intermediate resid connections
'while_loop': False,
'skip': True,
'time_skips': False,
'force_horizontal': False,
'symmetric_weights': True,
'timestep_output': False,
'bilinear_init': True,
'include_pooling': True
},
pool_strides=[2, 2],
pooling_kernel=[4, 4],
up_kernel=[4, 4],
train=training)
h2 = layer_hgru.build(in_emb)
if normalization_type is 'batch_norm':
h2 = normalization.batch(bottom=h2,
renorm=False,
name='hgru_bn',
training=training)
elif normalization_type is 'instance_norm':
h2 = normalization.instance(bottom=h2, training=training)
elif normalization_type is 'ada_batch_norm':
h2 = normalization.batch(bottom=h2,
renorm=False,
name='hgru_bn',
training=True)
else:
raise NotImplementedError(normalization_type)
fc = tf.layers.conv2d(inputs=h2,
filters=output_shape,
kernel_size=1,
name='fc')
# activity = tf.reduce_mean(fc, reduction_indices=[1, 2])
activity = tf.reduce_max(fc, reduction_indices=[1, 2])
extra_activities = {'activity': h2}
return activity, extra_activities