def build_model(data_tensor, reuse, training, output_shape):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[0]
with tf.variable_scope('cnn', reuse=reuse):
# Add input
in_emb = conv.input_layer(
X=data_tensor,
reuse=reuse,
training=training,
features=20,
conv_activation=[tf.nn.relu, None],
conv_kernel_size=7,
pool_kernel_size=[1, 2, 2, 1],
pool_kernel_strides=[1, 2, 2, 1],
pool=True,
name='l0')
layer_hgru = hgru.hGRU(
'fgru',
x_shape=in_emb.get_shape().as_list(),
timesteps=1,
h_ext=[{'h1': [15, 15]}, {'h2': [1, 1]}, {'fb1': [1, 1]}],
strides=[1, 1, 1, 1],
hgru_ids=[{'h1': 20}, {'h2': 128}, {'fb1': 20}],
hgru_idx=[{'h1': 0}, {'h2': 1}, {'fb1': 2}],
padding='SAME',
aux={
'readout': 'fb',
'intermediate_ff': [32, 128],
'intermediate_ks': [[3, 3], [3, 3]],
'intermediate_repeats': [3, 3],
'while_loop': False,
'skip': False,
'force_horizontal': True,
'symmetric_weights': True,
'include_pooling': True
},
pool_strides=[2, 2],
pooling_kernel=[2, 2],
train=training)
h2 = layer_hgru.build(in_emb)
h2 = normalization.batch(
bottom=h2,
renorm=False,
name='hgru_bn',
training=training)
activity = conv.readout_layer(
activity=h2,
reuse=reuse,
training=training,
output_shape=output_shape)
extra_activities = {
'activity': h2
}
return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[0]
with tf.variable_scope('cnn', reuse=reuse):
# Add input
in_emb = conv.input_layer(
X=data_tensor,
reuse=reuse,
training=training,
features=20,
conv_activation=[tf.nn.relu, None],
conv_kernel_size=7,
pool_kernel_size=[1, 2, 2, 1],
pool_kernel_strides=[1, 2, 2, 1],
pool=True,
name='l0')
layer_hgru = hgru.hGRU(
'hgru_1',
x_shape=in_emb.get_shape().as_list(),
timesteps=8,
h_ext=15,
strides=[1, 1, 1, 1],
padding='SAME',
aux={'reuse': False, 'constrain': False},
train=training)
h2 = layer_hgru.build(in_emb)
h2 = normalization.batch(
bottom=h2,
renorm=False,
name='hgru_bn',
training=training)
activity = conv.readout_layer(
activity=h2,
reuse=reuse,
training=training,
output_shape=output_shape)
extra_activities = {
'activity': h2
}
return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[0]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
with tf.variable_scope('cnn', reuse=reuse):
# # Concatenate standard deviation
# _, var = tf.nn.moments(data_tensor, axes=[3])
# std = tf.expand_dims(tf.sqrt(var), axis=-1)
# data_tensor = tf.concat([data_tensor, std], axis=-1)
# Add input
in_emb = conv.input_layer(X=data_tensor,
reuse=reuse,
training=training,
features=[12, 12],
conv_activation=[tf.nn.relu, tf.nn.relu],
conv_kernel_size=7,
pool_kernel_size=[1, 4, 4, 1],
pool_kernel_strides=[1, 2, 2, 1],
pool=True,
name='l0')
# Run fGRU
hgru_kernels = OrderedDict()
hgru_kernels['h1'] = [15, 15] # height/width
hgru_kernels['h2'] = [1, 1]
hgru_kernels['fb1'] = [1, 1]
hgru_features = OrderedDict()
hgru_features['h1'] = [12, 12] # Fan-in/fan-out, I and E (match fb1)
hgru_features['h2'] = [48, 48]
hgru_features['fb1'] = [12, 12] # (match h1)
# hgru_features['fb1'] = [24, 12] # (match h1 unless squeeze_fb)
intermediate_ff = [24, 48] # Last feature must match h2
intermediate_ks = [[3, 3], [3, 3]]
intermediate_repeats = [1, 1] # Repeat each interm this many times
layer_hgru = hgru.hGRU(
'fgru',
x_shape=in_emb.get_shape().as_list(),
timesteps=8,
strides=[1, 1, 1, 1],
hgru_features=hgru_features,
hgru_kernels=hgru_kernels,
intermediate_ff=intermediate_ff,
intermediate_ks=intermediate_ks,
intermediate_repeats=intermediate_repeats,
padding='SAME',
aux={
'readout': 'fb',
'squeeze_fb': False, # Compress Inh-hat with a 1x1 conv
'td_gate': True, # Add top-down activity to the in-gate
'attention': 'gala',
'attention_layers': 2,
'td_cell_state': False,
'excite_se': False, # Add S/E in the excitation stage
'residual': True, # intermediate resid connections
'while_loop': False,
'skip': True,
'time_skips': False,
'symmetric_weights': False,
'timestep_output': False,
'include_pooling': True
},
pool_strides=[2, 2],
pooling_kernel=[2, 2],
train=training)
h2 = layer_hgru.build(in_emb)
if isinstance(h2, list):
all_activities = []
for idx, it_h2 in enumerate(h2):
it_h2 = normalization.batch(bottom=it_h2,
renorm=False,
name='hgru_bn_%s' % idx,
training=training)
activity = conv.readout_layer(activity=it_h2,
reuse=reuse,
training=training,
var_scope='readout_%s' % idx,
output_shape=output_shape)
all_activities += [activity]
h2 = all_activities[-1]
activity = all_activities
else:
h2 = normalization.batch(bottom=h2,
renorm=False,
name='hgru_bn',
training=training)
activity = conv.readout_layer(activity=h2,
reuse=reuse,
training=training,
output_shape=output_shape)
extra_activities = {'activity': h2}
return activity, extra_activities
