def build(self, x, ffn_seed=None):
"""Run the backprop version of the Circuit."""
self.prepare_tensors()
# Calculate hidden state size
x_shape = x.get_shape().as_list()
l1_h2_shape = x_shape
# l2
int_shape = list(x_shape[1:-1])
str = self.ds_stride_list[0]
for dim in range(len(int_shape)):
int_shape[dim] = self.compute_shape(int_shape[dim], str[dim])
l2_h2_shape = [x_shape[0]] + int_shape + [self.ds_k_list[0]]
# l3
int_shape = list(l2_h2_shape[1:-1])
str = self.ds_stride_list[1]
for dim in range(len(int_shape)):
int_shape[dim] = self.compute_shape(int_shape[dim], str[dim])
l3_h2_shape = [x_shape[0]] + int_shape + [self.ds_k_list[1]]
# Initialize hidden layer activities
if ffn_seed is not None:
l1_h2 = tf.ones(l1_h2_shape, dtype=self.dtype) * ffn_seed * 2 - 1
if self.use_trainable_states:
if ffn_seed is None:
l1_h2 = tf.get_variable(
name='l1_h2',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[1] + l1_h2_shape[1:],
dtype=self.dtype,
uniform=True),
trainable=self.train)
l1_h2 = tf.tile(l1_h2, [x_shape[0]] + [1]*(4 if self.use_3d else 3))
l2_h2 = tf.get_variable(
name='l2_h2',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[1] + l2_h2_shape[1:],
dtype=self.dtype,
uniform=True),
trainable=self.train)
l2_h2 = tf.tile(l2_h2, [x_shape[0]] + [1]*(4 if self.use_3d else 3))
l3_h2 = tf.get_variable(
name='l3_h2',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[1] + l3_h2_shape[1:],
dtype=self.dtype,
uniform=True),
trainable=self.train)
l3_h2 = tf.tile(l3_h2, [x_shape[0]] + [1]*(4 if self.use_3d else 3))
else:
if ffn_seed is None:
l1_h2 = tf.zeros(l1_h2_shape, dtype=self.dtype)
l2_h2 = tf.zeros(l2_h2_shape, dtype=self.dtype)
l3_h2 = tf.zeros(l3_h2_shape, dtype=self.dtype)
for i in range(self.timesteps):
l1_h2_out, l2_h2_out, l3_h2_out = self.full(x, l1_h2, l2_h2, l3_h2, i)
l1_h2 = l1_h2_out
l2_h2 = l2_h2_out
l3_h2 = l3_h2_out
return l1_h2
def prepare_tensors(self):
""" Prepare recurrent/forward weight matrices.
(np.prod([h, w, k]) / 2) - k params in the surround filter
"""
# FEEDFORWARD KERNELS
lower_feats = self.in_k
for idx, (higher_feats,
ff_dhw) in enumerate(zip(self.ff_conv_k, self.ff_conv_dhw)):
with tf.variable_scope('ff_%s' % idx):
setattr(
self, 'ff_%s_weights' % idx,
tf.get_variable(
name='weights',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=ff_dhw + [lower_feats * 3, higher_feats],
dtype=self.dtype,
uniform=self.normal_initializer),
trainable=True))
setattr(
self, 'ff_%s_bias' % idx,
tf.get_variable(name='bias',
dtype=self.dtype,
initializer=tf.ones([higher_feats],
dtype=self.dtype),
trainable=True))
lower_feats = higher_feats
# FEEDBACK KERNELS
lower_feats = self.in_k
for idx, (higher_feats,
fb_dhw) in enumerate(zip(self.fb_k, self.fb_dhw)):
with tf.variable_scope('fb_%s' % idx):
setattr(
self, 'fb_%s_weights' % idx,
tf.get_variable(
name='weights',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=fb_dhw + [lower_feats, higher_feats],
dtype=self.dtype,
uniform=self.normal_initializer),
trainable=True))
setattr(
self, 'fb_%s_bias' % idx,
tf.get_variable(
name='bias',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[lower_feats],
dtype=self.dtype,
uniform=self.normal_initializer),
trainable=True))
lower_feats = higher_feats
# HGRU KERNELS
for idx in range(len(self.hgru_dhw)):
with tf.variable_scope('hgru_%s' % idx):
setattr(
self, 'hgru_%s_W' % idx,
tf.get_variable(
name='W',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.hgru_dhw[idx] +
[self.hgru_k[idx], self.hgru_k[idx]],
dtype=self.dtype,
uniform=self.normal_initializer),
trainable=True))
g_shape = [1, 1, 1] + [self.hgru_k[idx], self.hgru_k[idx]]
setattr(
self, 'hgru_%s_gain_weights' % idx,
tf.get_variable(
name='gain_weights',
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=g_shape,
dtype=self.dtype,
uniform=self.normal_initializer,
mask=None)))
m_shape = [1, 1, 1] + [self.hgru_k[idx], self.hgru_k[idx]]
setattr(
self, 'hgru_%s_mix_weights' % idx,
tf.get_variable(
name='mix_weights',
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=m_shape,
dtype=self.dtype,
uniform=self.normal_initializer,
mask=None)))
# Gate bias
bias_shape = [1, 1, 1, 1, self.hgru_k[idx]]
if self.gate_bias_init == 'chronos':
bias_init = -tf.log(
tf.random_uniform(bias_shape,
minval=1,
maxval=self.timesteps - 1,
dtype=self.dtype))
else:
bias_init = tf.ones(bias_shape, dtype=self.dtype)
setattr(
self, 'hgru_%s_gain_bias' % idx,
tf.get_variable(name='gain_bias',
dtype=self.dtype,
trainable=True,
initializer=bias_init))
if self.gate_bias_init == 'chronos':
bias_init = -bias_init
else:
bias_init = tf.ones(bias_shape, dtype=self.dtype)
setattr(
self, 'hgru_%s_mix_bias' % idx,
tf.get_variable(name='mix_bias',
dtype=self.dtype,
trainable=True,
initializer=bias_init))
# combination params
setattr(
self, 'hgru_%s_alpha' % idx,
tf.get_variable(
name='alpha',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[
1, 1, 1, self.hgru_k[idx] * 3, self.hgru_k[idx]
],
dtype=self.dtype,
uniform=self.normal_initializer,
mask=None)))
setattr(
self, 'hgru_%s_mu' % idx,
tf.get_variable(
name='mu',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[1, 1, 1, 1, self.hgru_k[idx]],
dtype=self.dtype,
uniform=self.normal_initializer,
mask=None)))
setattr(
self, 'hgru_%s_kappa' % idx,
tf.get_variable(
name='kappa',
dtype=self.dtype,
trainable=False,
initializer=initialization.xavier_initializer(
shape=[
1, 1, 1, self.hgru_k[idx] * 3, self.hgru_k[idx]
],
dtype=self.dtype,
uniform=self.normal_initializer,
mask=None)))
setattr(
self, 'hgru_%s_omega' % idx,
tf.get_variable(
name='omega',
dtype=self.dtype,
trainable=False,
initializer=initialization.xavier_initializer(
shape=[1, 1, 1, 1, self.hgru_k[idx]],
dtype=self.dtype,
uniform=self.normal_initializer,
mask=None)))
# adaptation params
if self.adapation:
setattr(
self, 'eta_%s' % idx,
tf.get_variable(name='eta',
dtype=self.dtype,
initializer=tf.ones(
[self.timesteps + 1],
dtype=tf.float32)))
setattr(
self, 'eta2_%s' % idx,
tf.get_variable(name='eta2',
dtype=self.dtype,
initializer=tf.ones(
[self.timesteps + 1],
dtype=tf.float32)))
if self.bn_reuse:
# Make the batchnorm variables
scopes = ['g1_bn', 'g2_bn', 'c1_bn', 'c2_bn']
bn_vars = ['moving_mean', 'moving_variance', 'gamma']
for s in scopes:
with tf.variable_scope(s):
for v in bn_vars:
tf.get_variable(
trainable=self.param_trainable[v],
name=v,
dtype=self.dtype,
shape=[self.hgru_k[idx]],
collections=self.param_collections[v],
initializer=self.param_initializer[v])
self.param_initializer = None
def prepare_tensors(self):
# HGRU KERNELS
self.hgru1.prepare_tensors()
self.hgru2.prepare_tensors()
self.hgru3.prepare_tensors()
self.hgru_td3.prepare_tensors()
self.hgru_td2.prepare_tensors()
self.hgru_td1.prepare_tensors()
# HOMU
# if self.use_homunculus:
# self.homunculus = tf.get_variable(
# name='homunculus',
# dtype=self.dtype,
# initializer=tf.zeros(self.timesteps),
# trainable=self.train)
# DS US KERNELS
in_k = self.in_k
for i, (fsiz, psiz, out_k) in enumerate(zip(self.ds_fsiz_list, self.ds_pool_list, self.ds_k_list)):
for rep in range(self.ds_conv_repeat):
with tf.variable_scope('ds%s_%s' % (i,rep)):
setattr(
self,
'ds%s_%s_w' % (i,rep),
tf.get_variable(
name='w',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=fsiz + [in_k if rep==0 else out_k, out_k],
dtype=self.dtype,
uniform=True),
trainable=self.train))
with tf.variable_scope('us%s_%s' % (i,rep)):
fsiz = [m+n-1 for m,n in zip(fsiz,psiz)] if (rep == self.ds_conv_repeat - 1) else fsiz
self.one_by_one = [1 for s in fsiz]
setattr(
self,
'us%s_%s_w' % (i,rep),
tf.get_variable(
name='w',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=fsiz + [in_k if rep==0 else out_k, out_k],
dtype=self.dtype,
uniform=True),
trainable=self.train))
# if (i < len(self.ds_fsiz_list) - 1) and self.use_dsus_skip:
# with tf.variable_scope('skip%s' % i):
# setattr(
# self,
# 'skip%s_kappa' % i,
# tf.get_variable(
# name='kappa',
# dtype=self.dtype,
# initializer=initialization.xavier_initializer(
# shape=self.one_by_one + [out_k],
# dtype=self.dtype,
# uniform=True),
# trainable=self.train))
# setattr(
# self,
# 'skip%s_gamma' % i,
# tf.get_variable(
# name='gamma',
# dtype=self.dtype,
# initializer=initialization.xavier_initializer(
# shape=self.one_by_one + [out_k],
# dtype=self.dtype,
# uniform=True),
# trainable=self.train))
# setattr(
# self,
# 'skip%s_omega' % i,
# tf.get_variable(
# name='omega',
# dtype=self.dtype,
# initializer=initialization.xavier_initializer(
# shape=self.one_by_one + [out_k],
# dtype=self.dtype,
# uniform=True),
# trainable=self.train))
in_k = out_k
def prepare_tensors(self):
""" Prepare recurrent/forward weight matrices.
(np.prod([h, w, k]) / 2) - k params in the surround filter
"""
# FEEDFORWARD AND FEEDBACK KERNELS
lower_feats = self.in_k
for idx, (higher_feats, ff_dhw, fb_dhw) in enumerate(
zip(self.ff_conv_k, self.ff_conv_dhw, self.fb_dhw)):
setattr(
self, 'fb_kernel_%s' % idx,
tf.get_variable(name='%s_fb_kernel__%s' %
(self.layer_name, idx),
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=fb_dhw + [lower_feats, higher_feats],
dtype=self.dtype,
uniform=self.normal_initializer),
trainable=True))
setattr(
self, 'fb_bias_%s' % idx,
tf.get_variable(name='%s_fb_bias_%s' % (self.layer_name, idx),
dtype=self.dtype,
initializer=tf.ones([lower_feats],
dtype=self.dtype),
trainable=True))
setattr(
self, 'ff_kernel_%s' % idx,
tf.get_variable(name='%s_ff_kernel_%s' %
(self.layer_name, idx),
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=ff_dhw + [lower_feats, higher_feats],
dtype=self.dtype,
uniform=self.normal_initializer),
trainable=True))
setattr(
self, 'ff_bias_%s' % idx,
tf.get_variable(name='%s_ff_bias_%s' % (self.layer_name, idx),
dtype=self.dtype,
initializer=tf.ones([higher_feats],
dtype=self.dtype),
trainable=True))
lower_feats = higher_feats
# HGRU KERNELS
for idx, layer in enumerate(self.hgru_ids):
with tf.variable_scope('%s_hgru_weights_%s' %
(self.layer_name, layer)):
setattr(
self, 'horizontal_kernels_%s' % layer,
tf.get_variable(
name='%s_horizontal' % self.layer_name,
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.hgru_dhw[idx] +
[self.hgru_k[idx], self.hgru_k[idx]],
dtype=self.dtype,
uniform=self.normal_initializer),
trainable=True))
g_shape = self.hgru_gate_dhw[idx] + [
self.hgru_k[idx], self.hgru_k[idx]
]
setattr(
self, 'gain_kernels_%s' % layer,
tf.get_variable(
name='%s_gain' % self.layer_name,
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=g_shape,
dtype=self.dtype,
uniform=self.normal_initializer,
mask=None)))
m_shape = self.hgru_gate_dhw[idx] + [
self.hgru_k[idx], self.hgru_k[idx]
]
setattr(
self, 'mix_kernels_%s' % layer,
tf.get_variable(
name='%s_mix' % self.layer_name,
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=m_shape,
dtype=self.dtype,
uniform=self.normal_initializer,
mask=None)))
# Gain bias
bias_shape = [1, 1, 1, 1, self.hgru_k[idx]]
if self.gate_bias_init == 'chronos':
bias_init = -tf.log(
tf.random_uniform(bias_shape,
minval=1,
maxval=self.timesteps - 1,
dtype=self.dtype))
else:
bias_init = tf.ones(bias_shape, dtype=self.dtype)
setattr(
self, 'gain_bias_%s' % layer,
tf.get_variable(name='%s_gain_bias' % self.layer_name,
dtype=self.dtype,
trainable=True,
initializer=bias_init))
if self.gate_bias_init == 'chronos':
bias_init = -bias_init
else:
bias_init = tf.ones(bias_shape, dtype=self.dtype)
setattr(
self, 'mix_bias_%s' % layer,
tf.get_variable(name='%s_mix_bias' % self.layer_name,
dtype=self.dtype,
trainable=True,
initializer=bias_init))
# Divisive params
if self.alpha and not self.lesion_alpha:
setattr(
self, 'alpha_%s' % layer,
tf.get_variable(
name='%s_alpha' % self.layer_name,
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=bias_shape,
dtype=self.dtype,
uniform=self.normal_initializer,
mask=None)))
elif self.lesion_alpha:
setattr(self, 'alpha_%s' % layer, tf.constant(0.))
else:
setattr(self, 'alpha_%s' % layer, tf.constant(1.))
if self.mu and not self.lesion_mu:
setattr(
self, 'mu_%s' % layer,
tf.get_variable(
name='%s_mu' % self.layer_name,
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=bias_shape,
dtype=self.dtype,
uniform=self.normal_initializer,
mask=None)))
elif self.lesion_mu:
setattr(self, 'mu_%s' % layer, tf.constant(0.))
else:
setattr(self, 'mu_%s' % layer, tf.constant(1.))
if self.gamma:
setattr(
self, 'gamma_%s' % layer,
tf.get_variable(
name='%s_gamma' % self.layer_name,
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=bias_shape,
dtype=self.dtype,
uniform=self.normal_initializer,
mask=None)))
else:
setattr(self, 'gamma_%s' % layer, tf.constant(1.))
if self.multiplicative_excitation:
if self.lesion_kappa:
setattr(self, 'kappa_%s' % layer, tf.constant(0.))
else:
setattr(
self, 'kappa_%s' % layer,
tf.get_variable(
name='%s_kappa' % self.layer_name,
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=bias_shape,
dtype=self.dtype,
uniform=self.normal_initializer,
mask=None)))
if self.lesion_omega:
setattr(self, 'omega_%s' % layer, tf.constant(0.))
else:
setattr(
self, 'omega_%s' % layer,
tf.get_variable(
name='%s_omega' % self.layer_name,
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=bias_shape,
dtype=self.dtype,
uniform=self.normal_initializer,
mask=None)))
else:
setattr(self, 'kappa_%s' % layer, tf.constant(1.))
setattr(self, 'omega_%s' % layer, tf.constant(1.))
if self.adapation:
setattr(
self, 'eta_%s' % layer,
tf.get_variable(
name='%s_eta' % self.layer_name,
dtype=self.dtype,
initializer=tf.random_uniform([self.timesteps],
dtype=tf.float32)))
if self.lesion_omega:
setattr(self, 'omega_%s' % layer, tf.constant(0.))
if self.lesion_kappa:
setattr(self, 'kappa_%s' % layer, tf.constant(0.))
if self.reuse:
# Make the batchnorm variables
scopes = ['g1_bn', 'g2_bn', 'c1_bn', 'c2_bn']
bn_vars = ['moving_mean', 'moving_variance', 'gamma']
for s in scopes:
with tf.variable_scope(s):
for v in bn_vars:
tf.get_variable(
trainable=self.param_trainable[v],
name=v,
dtype=self.dtype,
shape=[self.hgru_k[idx]],
collections=self.param_collections[v],
initializer=self.param_initializer[v])
self.param_initializer = None
def prepare_tensors(self):
local_shape = [1, 1] if not self.use_3d else [1, 1, 1]
self.bn_param_initializer = {
'moving_mean': tf.constant_initializer(0., dtype=self.dtype),
'moving_variance': tf.constant_initializer(1., dtype=self.dtype),
'beta': tf.constant_initializer(0., dtype=self.dtype),
'gamma': tf.constant_initializer(0.1, dtype=self.dtype)
}
with tf.variable_scope(self.var_scope):
tf.get_variable(
name='h1_w',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.fsiz + [self.num_channels, self.num_channels],
dtype=self.dtype,
uniform=True),
trainable=self.train)
tf.get_variable(
name='g1_w',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=local_shape + [self.num_channels, self.num_channels],
dtype=self.dtype,
uniform=True),
trainable=self.train)
tf.get_variable(name='g1_b',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[1] + local_shape + [self.num_channels],
dtype=self.dtype,
uniform=True),
trainable=self.train)
tf.get_variable(name='alpha',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[1] + local_shape + [1],
dtype=self.dtype,
uniform=True),
trainable=self.train)
tf.get_variable(
name='mu',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[1] + local_shape +
[self.num_channels if self.soft_coefficients else 1],
dtype=self.dtype,
uniform=True),
trainable=self.train)
if self.bistream_weights is 'independent':
tf.get_variable(name='h2_w',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.fsiz +
[self.num_channels, self.num_channels],
dtype=self.dtype,
uniform=True),
trainable=self.train)
tf.get_variable(
name='g2_w',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=local_shape + [self.num_channels, self.num_channels],
dtype=self.dtype,
uniform=True),
trainable=self.train)
tf.get_variable(name='g2_b',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[1] + local_shape + [self.num_channels],
dtype=self.dtype,
uniform=True),
trainable=self.train)
tf.get_variable(name='kappa',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[1] + local_shape + [1],
dtype=self.dtype,
uniform=True),
trainable=self.train)
tf.get_variable(
name='gamma',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[1] + local_shape +
[self.num_channels if self.soft_coefficients else 1],
dtype=self.dtype,
uniform=True),
trainable=self.train)
tf.get_variable(
name='omega',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[1] + local_shape +
[self.num_channels if self.soft_coefficients else 1],
dtype=self.dtype,
uniform=True),
trainable=self.train)
if self.bn_reuse:
# Make the batchnorm variables
scopes = ['c1/bn', 'c2/bn', 'g1/bn', 'g1/bn']
shapes = [
self.num_channels, self.num_channels, self.num_channels,
self.num_channels
]
bn_vars = ['moving_mean', 'moving_variance', 'beta', 'gamma']
for (scp, shp) in zip(scopes, shapes):
with tf.variable_scope(scp):
for v in bn_vars:
tf.get_variable(
trainable=self.train,
name=v,
dtype=self.dtype,
shape=[shp],
initializer=self.bn_param_initializer)
def prepare_tensors(self):
""" Prepare recurrent/forward weight matrices.
(np.prod([h, w, k]) / 2) - k params in the surround filter
"""
# Create FF vars
for idx, (ff_filters, ff_kernel) in enumerate(
zip(self.intermediate_ff, self.intermediate_ks)):
setattr(
self, 'intermediate_kernel_%s' % idx,
tf.get_variable(name='%s_ffdrive_kernel_%s' %
(self.layer_name, idx),
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=ff_kernel + [ff_filters, ff_filters],
dtype=self.dtype,
uniform=self.normal_initializer),
trainable=True))
setattr(
self, 'intermediate_bias_%s' % idx,
tf.get_variable(name='%s_ffdrive_bias_%s' %
(self.layer_name, idx),
dtype=self.dtype,
initializer=tf.ones([ff_filters],
dtype=self.dtype),
trainable=True))
# Create recurrent vars
for idx, layer in enumerate(self.hgru_ids):
with tf.variable_scope('%s_hgru_weights_%s' %
(self.layer_name, layer)):
if 'fb' in layer:
# Make the FB resize kernels
setattr(
self, '%s_resize_kernel' % layer,
tf.get_variable(
name='%s_resize_kernel' % layer,
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.pooling_kernel + [self.k, self.k],
dtype=self.dtype,
uniform=self.normal_initializer),
trainable=True))
setattr(
self, '%s_resize_bias' % layer,
tf.get_variable(name='%s_resize_bias' % layer,
dtype=self.dtype,
initializer=tf.ones([self.k],
dtype=self.dtype),
trainable=True))
setattr(
self, 'horizontal_kernels_%s' % layer,
tf.get_variable(
name='%s_horizontal' % self.layer_name,
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.h_ext[idx] + [self.k, self.k],
dtype=self.dtype,
uniform=self.normal_initializer),
trainable=True))
setattr(
self, 'gain_kernels_%s' % layer,
tf.get_variable(
name='%s_gain' % self.layer_name,
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.g_shape,
dtype=self.dtype,
uniform=self.normal_initializer,
mask=None)))
setattr(
self, 'mix_kernels_%s' % layer,
tf.get_variable(
name='%s_mix' % self.layer_name,
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.m_shape,
dtype=self.dtype,
uniform=self.normal_initializer,
mask=None)))
# Gain bias
if self.gate_bias_init == 'chronos':
bias_init = -tf.log(
tf.random_uniform(self.bias_shape,
minval=1,
maxval=self.timesteps - 1,
dtype=self.dtype))
else:
bias_init = tf.ones(self.bias_shape, dtype=self.dtype)
setattr(
self, 'gain_bias_%s' % layer,
tf.get_variable(name='%s_gain_bias' % self.layer_name,
dtype=self.dtype,
trainable=True,
initializer=bias_init))
if self.gate_bias_init == 'chronos':
bias_init = -bias_init
else:
bias_init = tf.ones(self.bias_shape, dtype=self.dtype)
setattr(
self, 'mix_bias_%s' % layer,
tf.get_variable(name='%s_mix_bias' % self.layer_name,
dtype=self.dtype,
trainable=True,
initializer=bias_init))
# Divisive params
if self.alpha and not self.lesion_alpha:
setattr(
self, 'alpha_%s' % layer,
tf.get_variable(
name='%s_alpha' % self.layer_name,
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
dtype=self.dtype,
uniform=self.normal_initializer,
mask=None)))
elif self.lesion_alpha:
setattr(self, 'alpha_%s' % layer, tf.constant(0.))
else:
setattr(self, 'alpha_%s' % layer, tf.constant(1.))
if self.mu and not self.lesion_mu:
setattr(
self, 'mu_%s' % layer,
tf.get_variable(
name='%s_mu' % self.layer_name,
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
dtype=self.dtype,
uniform=self.normal_initializer,
mask=None)))
elif self.lesion_mu:
setattr(self, 'mu_%s' % layer, tf.constant(0.))
else:
setattr(self, 'mu_%s' % layer, tf.constant(1.))
if self.gamma:
setattr(
self, 'gamma_%s' % layer,
tf.get_variable(
name='%s_gamma' % self.layer_name,
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
dtype=self.dtype,
uniform=self.normal_initializer,
mask=None)))
else:
setattr(self, 'gamma_%s' % layer, tf.constant(1.))
if self.multiplicative_excitation:
if self.lesion_kappa:
setattr(self, 'kappa_%s' % layer, tf.constant(0.))
else:
setattr(
self, 'kappa_%s' % layer,
tf.get_variable(
name='%s_kappa' % self.layer_name,
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
dtype=self.dtype,
uniform=self.normal_initializer,
mask=None)))
if self.lesion_omega:
setattr(self, 'omega_%s' % layer, tf.constant(0.))
else:
setattr(
self, 'omega_%s' % layer,
tf.get_variable(
name='%s_omega' % self.layer_name,
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
dtype=self.dtype,
uniform=self.normal_initializer,
mask=None)))
else:
setattr(self, 'kappa_%s' % layer, tf.constant(1.))
setattr(self, 'omega_%s' % layer, tf.constant(1.))
if self.adapation:
setattr(
self, 'eta_%s' % layer,
tf.get_variable(
name='%s_eta' % self.layer_name,
dtype=self.dtype,
initializer=tf.random_uniform([self.timesteps],
dtype=tf.float32)))
if self.lesion_omega:
setattr(self, 'omega_%s' % layer, tf.constant(0.))
if self.lesion_kappa:
setattr(self, 'kappa_%s' % layer, tf.constant(0.))
if self.reuse:
# Make the batchnorm variables
scopes = ['g1_bn', 'g2_bn', 'c1_bn', 'c2_bn']
bn_vars = ['moving_mean', 'moving_variance', 'gamma']
for s in scopes:
with tf.variable_scope(s):
for v in bn_vars:
tf.get_variable(
trainable=self.param_trainable[v],
name=v,
dtype=self.dtype,
shape=[self.k],
collections=self.param_collections[v],
initializer=self.param_initializer[v])
self.param_initializer = None
def prepare_tensors(self):
"""
"""
# HORIZONTAL FILTERS
self.caps_filter = \
tf.get_variable(
name='%s_caps_filter' % (self.name),
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[self.c_h_ext, self.c_h_ext] + [self.f * self.c, self.b * self.f * self.c],
uniform=self.normal_initializer),
trainable=True)
if self.use_independent_labels_filter:
self.labels_filter = \
tf.get_variable(
name='%s_labels_filter' % (self.name),
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[self.l_h_ext, self.l_h_ext] + [self.f, self.b * self.f],
uniform=self.normal_initializer),
trainable=True)
else:
self.labels_filter = \
helpers_mk2.caps2scalar_filter(
self.caps_filter,
self.c,
self.eps)
# CAPS COMPARE FILTER
self.compare_filter = \
tf.get_variable(
name='%s_compare_filter' % (self.name),
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[1, 1] + [self.f * 3, self.f],
uniform=self.normal_initializer),
trainable=True)
# self.compare_filter = None
# LABEL GATES
self.labels_deletegate_filter = \
tf.get_variable(
name='%s_labels_deletegate_filter' % (self.name),
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[self.c_h_ext, self.c_h_ext] + [self.f*2, self.f],
uniform=self.normal_initializer),
trainable=True)
# self.labels_writegate_filter = \
# tf.get_variable(
# name='%s_labels_writegate_filter' % (self.name),
# dtype=self.dtype,
# initializer=initialization.xavier_initializer(
# shape=[self.c_h_ext, self.c_h_ext] + [self.f, self.f],
# uniform=self.normal_initializer),
# trainable=True)
self.labels_writegate_filter = None
# STATIC BIASES
self.bundle_competition_bias = \
tf.get_variable(
name='%s_bundle_competition_bias' % (self.name),
dtype=self.dtype,
shape=[1, 1, 1, self.b],
initializer=tf.constant_initializer([1.0], dtype=self.dtype),
trainable=True)
# self.bundle_competition_bias = \
# tf.constant(
# name='%s_bundle_competition_bias' % (self.name),
# dtype=self.dtype,
# shape=[1, 1, 1, self.b],
# value=1.0)
self.fig_ground_competition_bias = \
tf.get_variable(
name='%s_fig_ground_competition_bias' % (self.name),
dtype=self.dtype,
shape=[1, 1, 1, 2],
initializer=tf.constant_initializer([1.0], dtype=self.dtype),
trainable=True)
# self.fig_ground_competition_bias = \
# tf.constant(
# name='%s_fig_ground_competition_bias' % (self.name),
# dtype=self.dtype,
# shape=[1, 1, 1, 2],
# value=1.0)
self.update_gain = \
tf.get_variable(
name='%s_update_gain' % (self.name),
dtype=self.dtype,
shape=[1, 1, 1, self.f],
initializer=tf.constant_initializer([0.0], dtype=self.dtype),
trainable=True)
import numpy as np
fixed_labels_mask = np.zeros([1, 1, 1, self.f])
if self.fixed_label_ind is not None:
for idx in self.fixed_label_ind:
fixed_labels_mask[0, 0, 0, idx] = 1.
self.fixed_labels_mask = tf.constant(fixed_labels_mask,
dtype=self.dtype)
else:
self.fixed_labels_mask = None
def prepare_tensors(self):
# HGRU KERNELS
self.hgru0.prepare_tensors()
self.hgru1.prepare_tensors()
self.hgru_td1.prepare_tensors()
self.hgru_td0.prepare_tensors()
# FEEDFORWARD KERNELS
lower_feats = self.in_k
for idx, (higher_feats,
ff_dhw) in enumerate(zip(self.ff_conv_k, self.ff_conv_fsiz)):
with tf.variable_scope('ff_%s' % idx):
if idx < 2:
# last conv layer doesn't have spot weights
setattr(
self,
'ff_%s_spot_x' % idx,
tf.get_variable(
name='spot_x',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[1, 1, 1, 1] + [1], #[lower_feats],
dtype=self.dtype,
uniform=True),
trainable=True))
# last conv layer doesn't have spot weights
setattr(
self,
'ff_%s_spot_xy' % idx,
tf.get_variable(
name='spot_xy',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[1, 1, 1, 1] +
[lower_feats], #[lower_feats],
dtype=self.dtype,
uniform=True),
trainable=True))
setattr(
self, 'ff_%s_weights' % idx,
tf.get_variable(
name='weights',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=ff_dhw + [
lower_feats,
higher_feats * self.ff_kpool_multiplier
],
dtype=self.dtype,
uniform=True),
trainable=True))
lower_feats = higher_feats
# FEEDBACK KERNELS
lower_feats = self.in_k
for idx, (higher_feats,
fb_dhw) in enumerate(zip(self.fb_conv_k, self.fb_conv_fsiz)):
with tf.variable_scope('fb_%s' % idx) as scope:
setattr(
self, 'fb_%s_weights' % idx,
tf.get_variable(
name='weights',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=fb_dhw + [lower_feats, higher_feats],
dtype=self.dtype,
uniform=True),
trainable=True))
lower_feats = higher_feats
