def build(self, x):
"""Run the backprop version of the CCircuit."""
self.prepare_tensors()
i0 = tf.constant(0)
if self.hidden_init == 'identity':
h1 = tf.identity(x)
h2 = tf.identity(x)
elif self.hidden_init == 'random':
h1 = initialization.xavier_initializer(
shape=[self.n, self.h, self.w, self.k],
uniform=self.normal_initializer,
mask=None)
h2 = initialization.xavier_initializer(
shape=[self.n, self.h, self.w, self.k],
uniform=self.normal_initializer,
mask=None)
elif self.hidden_init == 'zeros':
h1 = tf.zeros_like(x)
h2 = tf.zeros_like(x)
else:
raise RuntimeError
# While loop
elems = [i0, x, h1, h2]
returned = tf.while_loop(self.condition,
self.full,
loop_vars=elems,
back_prop=True,
swap_memory=True)
# Prepare output
i0, x, h1, h2 = returned
return h2
def build(self, x):
"""Run the backprop version of the CCircuit."""
self.prepare_tensors()
x_shape = x.get_shape().as_list()
i0 = 0 # tf.constant(0)
if self.hidden_init == 'identity':
h2 = tf.identity(x)
elif self.hidden_init == 'random':
h2 = initialization.xavier_initializer(
shape=x_shape, uniform=self.normal_initializer, mask=None)
elif self.hidden_init == 'zeros':
h2 = tf.zeros_like(x)
elif self.hidden_init == 'learned':
h2 = tf.get_variable(name='%s_h2_state' % self.layer_name,
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=x_shape,
uniform=self.normal_initializer),
trainable=self.train)
else:
raise RuntimeError
# Loop
for idx in range(self.timesteps):
i0, x, h2 = self.full(i0=i0, x=x, h2=h2)
return h2
def build(self, x):
"""Run the backprop version of the CCircuit."""
self.prepare_tensors()
x_shape = x.get_shape().as_list()
if self.hidden_init == 'identity':
h1 = tf.identity(x)
h2 = tf.identity(x)
elif self.hidden_init == 'random':
h1 = initialization.xavier_initializer(
shape=x_shape, uniform=self.normal_initializer, mask=None)
h2 = initialization.xavier_initializer(
shape=x_shape, uniform=self.normal_initializer, mask=None)
elif self.hidden_init == 'zeros':
h1 = tf.zeros_like(x)
h2 = tf.zeros_like(x)
else:
raise RuntimeError
if not self.while_loop:
for idx in range(self.timesteps):
_, x, h1, h2 = self.full(i0=idx, x=x, h1=h1, h2=h2)
else:
# While loop
i0 = tf.constant(0)
elems = [i0, x, h1, h2]
returned = tf.while_loop(self.condition,
self.full,
loop_vars=elems,
back_prop=True,
swap_memory=False)
# Prepare output
i0, x, h1, h2 = returned
return h2
def build(self, x):
"""Run the backprop version of the CCircuit."""
self.prepare_tensors()
i0 = tf.constant(0)
if self.hidden_init == 'identity':
h1 = tf.identity(x)
h2 = tf.identity(x)
elif self.hidden_init == 'random':
h1 = initialization.xavier_initializer(
shape=[self.n, self.h, self.w, self.k],
uniform=self.normal_initializer,
mask=None)
h2 = initialization.xavier_initializer(
shape=[self.n, self.h, self.w, self.k],
uniform=self.normal_initializer,
mask=None)
elif self.hidden_init == 'zeros':
h1 = tf.zeros_like(x)
h2 = tf.zeros_like(x)
else:
raise RuntimeError
# While loop
h1_list, h2_list = [], []
for idx in range(self.timesteps):
i0, x, h1, h2 = self.full(i0, x, h1, h2)
h1_list += [h1]
h2_list += [h2]
return h2, h2_list, h1_list
def prepare_tensors(self):
""" Prepare recurrent/forward weight matrices.
(np.prod([h, w, k]) / 2) - k params in the surround filter
"""
with tf.variable_scope('%s_hgru_weights' % self.layer_name):
self.horizontal_kernels = tf.get_variable(
name='%s_horizontal' % self.layer_name,
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.h_shape,
uniform=self.normal_initializer),
trainable=True)
self.h_bias = tf.get_variable(
name='%s_h_bias' % self.layer_name,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
uniform=self.normal_initializer,
mask=None))
self.gain_kernels = tf.get_variable(
name='%s_gain' % self.layer_name,
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.g_shape,
uniform=self.normal_initializer,
mask=None))
self.mix_kernels = tf.get_variable(
name='%s_mix' % self.layer_name,
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.m_shape,
uniform=self.normal_initializer,
mask=None))
# Gain bias
if self.gate_bias_init == 'chronos':
bias_init = -tf.log(
tf.random_uniform(
self.bias_shape, minval=1, maxval=self.timesteps - 1))
else:
bias_init = tf.ones(self.bias_shape)
self.gain_bias = tf.get_variable(
name='%s_gain_bias' % self.layer_name,
dtype=self.dtype,
trainable=True,
initializer=bias_init)
if self.gate_bias_init == 'chronos':
bias_init = -bias_init
else:
bias_init = tf.ones(self.bias_shape)
self.mix_bias = tf.get_variable(
name='%s_mix_bias' % self.layer_name,
dtype=self.dtype,
trainable=True,
initializer=bias_init)
def build(self, reduce_memory=False):
"""Run the backprop version of the CCircuit."""
self.prepare_tensors()
i0 = tf.constant(0)
if self.hidden_init == 'identity':
I = tf.identity(self.X)
O = tf.identity(self.X)
elif self.hidden_init == 'random':
I = initialization.xavier_initializer(
shape=[self.n, self.h, self.w, self.k],
uniform=self.normal_initializer,
mask=None)
O = initialization.xavier_initializer(
shape=[self.n, self.h, self.w, self.k],
uniform=self.normal_initializer,
mask=None)
elif self.hidden_init == 'zeros':
I = tf.zeros_like(self.X)
O = tf.zeros_like(self.X)
else:
raise RuntimeError
if reduce_memory:
print 'Warning: Using FF version of the model.'
for t in range(self.timesteps):
i0, O, I = self.full(i0, O, I)
else:
# While loop
elems = [i0, O, I]
returned = tf.while_loop(self.condition,
self.full,
loop_vars=elems,
back_prop=True,
swap_memory=False)
# Prepare output
i0, O, I = returned # i0, O, I
if self.return_weights:
weights = self.gather_tensors(wak='weight')
tuning = self.gather_tensors(wak='tuning')
new_tuning = {}
for k, v in tuning.iteritems():
key_name = v.name.split('/')[-1].split(':')[0]
new_tuning[key_name] = v
weights = dict(weights, **new_tuning)
activities = self.gather_tensors(wak='activity')
# Attach weights if using association field
if self.association_field:
weights['p_t'] = self.p_r # Make available for regularization
if self.full_far_eCRF:
weights['t_t'] = self.t_r # Make available for regularization
return O, weights, activities
else:
return O
def sparse_pool_layer(self,
bottom,
out_channels,
name,
in_channels=None,
aux=None):
"""Sparse pooling layer."""
def create_gaussian_rf(xy, h, w):
"""Create a gaussian bump for initializing the spatial weights."""
# TODO: implement this.
import ipdb
ipdb.set_trace()
raise NotImplementedError
with tf.variable_scope(name):
bottom_shape = [int(x) for x in bottom.get_shape()]
if in_channels is None:
in_channels = bottom_shape[-1]
# K channel weights
channel_weights = tf.get_variable(
name='%s_channel' % name,
dtype=tf.float32,
initializer=initialization.xavier_initializer(
shape=[in_channels, out_channels], uniform=True, mask=None))
# HxW spatial weights
spatial_weights = tf.get_variable(
name='%s_spatial' % name,
dtype=tf.float32,
initializer=initialization.xavier_initializer(
shape=[1, bottom_shape[1], bottom_shape[2], 1], mask=None))
# If supplied, initialize the spatial weights with RF info
if aux is not None and 'xy' in aux.keys():
gaussian_xy = aux['xy']
if 'h' in aux.keys():
gaussian_h = aux['h']
gaussian_w = aux['w']
k = aux['k']
else:
gaussian_h, gaussian_w, k = None, None, None
spatial_rf = create_gaussian_rf(xy=gaussian_xy,
h=gaussian_h,
w=gaussian_w,
k=k)
spatial_weights += spatial_rf
spatial_sparse = tf.reduce_mean(bottom * spatial_weights,
reduction_indices=[1, 2])
output = tf.matmul(spatial_sparse, channel_weights)
return self, output
def build(self, reduce_memory=False):
"""Run the backprop version of the CCircuit."""
self.prepare_tensors()
i0 = tf.constant(0)
I = initialization.xavier_initializer(
shape=[self.n, self.h, self.w, self.k],
uniform=self.normal_initializer,
mask=None)
O = initialization.xavier_initializer(
shape=[self.n, self.h, self.w, self.k],
uniform=self.normal_initializer,
mask=None)
if reduce_memory:
print 'Warning: Using FF version of the model.'
for t in range(self.timesteps):
i0, O, I = self[self.model_version](i0, O, I)
else:
# While loop
elems = [i0, O, I]
returned = tf.while_loop(self.condition,
self[self.model_version],
loop_vars=elems,
back_prop=True,
swap_memory=False)
# Prepare output
i0, O, I = returned # i0, O, I
if self.return_weights:
weights = self.gather_tensors(wak='weight')
tuning = self.gather_tensors(wak='tuning')
new_tuning = {}
for k, v in tuning.iteritems():
key_name = v.name.split('/')[-1].split(':')[0]
new_tuning[key_name] = v
weights = dict(weights, **new_tuning)
activities = self.gather_tensors(wak='activity')
return O, weights, activities
else:
return O
def build(self, x):
"""Run the backprop version of the CCircuit."""
self.prepare_tensors()
x_shape = x.get_shape().as_list()
i0 = 0 # tf.constant(0)
if self.hidden_init == 'identity':
h1 = tf.identity(x)
h2 = tf.identity(x)
elif self.hidden_init == 'random':
h1 = initialization.xavier_initializer(
shape=x_shape, uniform=self.normal_initializer, mask=None)
h2 = initialization.xavier_initializer(
shape=x_shape, uniform=self.normal_initializer, mask=None)
elif self.hidden_init == 'zeros':
h1 = tf.zeros_like(x)
h2 = tf.zeros_like(x)
else:
raise RuntimeError
# Loop
for idx in range(self.timesteps):
i0, x, h1, h2 = self.full(i0=i0, x=x, h1=h1, h2=h2)
return h2
def prepare_tensors(self):
""" Prepare recurrent/forward weight matrices."""
self.weight_dict = { # Weights lower/activity upper
'U': {
'r': {
'weight': 'u_r',
'activity': 'U_r'
},
'f': {
'weight': 'u_f',
'bias': 'ub_f',
'activity': 'U_f'
}
},
'T': {
'r': {
'weight': 't_r',
'activity': 'T_r'
},
'f': {
'weight': 't_f',
'bias': 'tb_f',
'activity': 'T_f'
}
},
'P': {
'r': {
'weight': 'p_r',
'activity': 'P_r'
},
'f': {
'weight': 'p_f',
'bias': 'pb_f',
'activity': 'P_f'
}
},
'Q': {
'r': {
'weight': 'q_r',
'activity': 'Q_r'
},
'f': {
'weight': 'q_f',
'bias': 'qb_f',
'activity': 'Q_f'
}
},
'I': {
'r': { # Recurrent state
'weight': 'i_r',
'activity': 'I_r'
}
},
'O': {
'r': { # Recurrent state
'weight': 'o_r',
'activity': 'O_r'
}
}
}
# tuned summation: pooling in h, w dimensions
#############################################
setattr(
self, self.weight_dict['Q']['r']['weight'],
tf.get_variable(name=self.weight_dict['Q']['r']['weight'],
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.q_shape,
uniform=self.normal_initializer,
mask=None)))
# untuned suppression: reduction across feature axis
####################################################
setattr(
self, self.weight_dict['U']['r']['weight'],
tf.get_variable(name=self.weight_dict['U']['r']['weight'],
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.u_shape,
uniform=self.normal_initializer,
mask=None)))
# tuned summation: pooling in h, w dimensions
#############################################
p_array = np.zeros(self.p_shape)
for pdx in range(self.k):
p_array[:self.SSN, :self.SSN, pdx, pdx] = 1.0
p_array[self.SSN // 2 - py_utils.ifloor(self.SRF / 2.0):self.SSN // 2 +
py_utils.iceil(self.SRF / 2.0),
self.SSN // 2 - py_utils.ifloor(self.SRF / 2.0):self.SSN // 2 +
py_utils.iceil(self.SRF / 2.0), :, # exclude CRF!
:] = 0.0
setattr(
self, self.weight_dict['P']['r']['weight'],
tf.get_variable(name=self.weight_dict['P']['r']['weight'],
dtype=self.dtype,
initializer=p_array.astype(np.float32),
trainable=False))
# tuned suppression: pooling in h, w dimensions
###############################################
t_array = np.zeros(self.t_shape)
for tdx in range(self.k):
t_array[:self.SSF, :self.SSF, tdx, tdx] = 1.0
t_array[self.SSF // 2 - py_utils.ifloor(self.SSN / 2.0):self.SSF // 2 +
py_utils.iceil(self.SSN / 2.0),
self.SSF // 2 - py_utils.ifloor(self.SSN / 2.0):self.SSF // 2 +
py_utils.iceil(self.SSN / 2.0), :, # exclude near surround!
:] = 0.0
setattr(
self, self.weight_dict['T']['r']['weight'],
tf.get_variable(name=self.weight_dict['T']['r']['weight'],
dtype=self.dtype,
initializer=t_array.astype(np.float32),
trainable=False))
# Scalar weights
self.xi = tf.get_variable(name='xi', initializer=1.)
self.alpha = tf.get_variable(name='alpha', initializer=1.)
self.beta = tf.get_variable(name='beta', initializer=1.)
self.mu = tf.get_variable(name='mu', initializer=1.)
self.nu = tf.get_variable(name='nu', initializer=1.)
self.zeta = tf.get_variable(name='zeta', initializer=1.)
self.gamma = tf.get_variable(name='gamma', initializer=1.)
self.delta = tf.get_variable(name='delta', initializer=1.)
self.eps = tf.get_variable(name='eps', initializer=1.)
self.eta = tf.get_variable(name='eta', initializer=1.)
self.sig = tf.get_variable(name='sig', initializer=1.)
self.tau = tf.get_variable(name='tau', initializer=1.)
def build(self, x):
"""Run the backprop version of the CCircuit."""
self.prepare_tensors()
full_x_shape = x.get_shape().as_list()
i0 = tf.constant(0)
# Remove time dimension
x_shape = [full_x_shape[0]] + full_x_shape[2:]
# Set up hiddens
if self.hidden_init == 'identity':
h1 = tf.identity(x[:, 0])
h2 = tf.identity(x[:, 0])
elif self.hidden_init == 'random':
h1 = initialization.xavier_initializer(
shape=x_shape, uniform=self.normal_initializer, mask=None)
h2 = initialization.xavier_initializer(
shape=x_shape, uniform=self.normal_initializer, mask=None)
elif self.hidden_init == 'zeros':
h1 = tf.zeros(x_shape)
h2 = tf.zeros(x_shape)
else:
raise RuntimeError
# While loop
if self.while_loop:
# Get a dynamic h2 to store all h2s
dh2 = tf.TensorArray(dtype=tf.float32,
size=self.timesteps,
dynamic_size=False,
clear_after_read=True,
name='h2_list')
elems = [i0, x, h1, h2, dh2]
returned = tf.while_loop(self.condition,
self.full,
loop_vars=elems,
back_prop=True,
swap_memory=False)
i0, x, h1, h2, dh2 = returned
dh2 = dh2.stack()
dh2 = tf.transpose(dh2, (1, 0, 2, 3, 4))
dh2.set_shape(full_x_shape)
else:
dh2 = []
timesteps = self.timesteps + self.warmup
count, fake_count = 0, 0
for idx in range(timesteps):
if idx < self.warmup:
# Burn in the 0th frame
it_x = x[:, 0, :, :, :]
i0, _, h1, h2, _ = self.full(i0=i0,
x=it_x,
h1=h1,
h2=h2,
dh2=None,
idx=(1 + idx) * 999)
else:
it_x = x[:, count, :, :, :]
if self.additional_recurrence:
for _ in range(self.additional_recurrence + 1):
i0, _, h1, h2, _ = self.full(i0=i0,
x=it_x,
h1=h1,
h2=h2,
dh2=None,
idx=fake_count)
fake_count += 1
else:
i0, _, h1, h2, _ = self.full(i0=i0,
x=it_x,
h1=h1,
h2=h2,
dh2=None,
idx=count)
dh2 += [h2]
count += 1
fake_count += 1
dh2 = tf.stack(dh2, axis=1)
# Prepare output
return dh2
def prepare_tensors(self):
""" Prepare recurrent/forward weight matrices."""
self.weight_dict = { # Weights lower/activity upper
'U': {
'r': {
'weight': 'u_r',
'activity': 'U_r'
},
'f': {
'weight': 'u_f',
'bias': 'ub_f',
'activity': 'U_f'
}
},
'T': {
'r': {
'weight': 't_r',
'activity': 'T_r'
},
'f': {
'weight': 't_f',
'bias': 'tb_f',
'activity': 'T_f'
}
},
'P': {
'r': {
'weight': 'p_r',
'activity': 'P_r'
},
'f': {
'weight': 'p_f',
'bias': 'pb_f',
'activity': 'P_f'
}
},
'Q': {
'r': {
'weight': 'q_r',
'activity': 'Q_r'
},
'f': {
'weight': 'q_f',
'bias': 'qb_f',
'activity': 'Q_f'
}
},
'I': {
'r': { # Recurrent state
'weight': 'i_r',
'bias': 'ib_r',
'activity': 'I_r'
}
},
'O': {
'r': { # Recurrent state
'weight': 'o_r',
'bias': 'ob_r',
'activity': 'O_r'
}
}
}
# tuned summation: pooling in h, w dimensions
#############################################
setattr(
self,
self.weight_dict['Q']['r']['weight'],
tf.get_variable(
name=self.weight_dict['Q']['r']['weight'],
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.q_shape,
uniform=self.normal_initializer,
mask=None)))
# untuned suppression: reduction across feature axis
####################################################
setattr(
self,
self.weight_dict['U']['r']['weight'],
tf.get_variable(
name=self.weight_dict['U']['r']['weight'],
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.u_shape,
uniform=self.normal_initializer,
mask=None)))
# tuned summation: pooling in h, w dimensions
#############################################
p_array = np.zeros(self.p_shape)
for pdx in range(self.k):
p_array[:self.SSN, :self.SSN, pdx, pdx] = 1.0
p_array[
self.SSN // 2 - py_utils.ifloor(
self.SRF / 2.0):self.SSN // 2 + py_utils.iceil(
self.SRF / 2.0),
self.SSN // 2 - py_utils.ifloor(
self.SRF / 2.0):self.SSN // 2 + py_utils.iceil(
self.SRF / 2.0),
:, # exclude CRF!
:] = 0.0
setattr(
self,
self.weight_dict['P']['r']['weight'],
tf.get_variable(
name=self.weight_dict['P']['r']['weight'],
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.p_shape,
uniform=self.normal_initializer,
mask=p_array)))
# tuned suppression: pooling in h, w dimensions
###############################################
t_array = np.zeros(self.t_shape)
for tdx in range(self.k):
t_array[:self.SSF, :self.SSF, tdx, tdx] = 1.0
t_array[
self.SSF // 2 - py_utils.ifloor(
self.SSN / 2.0):self.SSF // 2 + py_utils.iceil(
self.SSN / 2.0),
self.SSF // 2 - py_utils.ifloor(
self.SSN / 2.0):self.SSF // 2 + py_utils.iceil(
self.SSN / 2.0),
:, # exclude near surround!
:] = 0.0
setattr(
self,
self.weight_dict['T']['r']['weight'],
tf.get_variable(
name=self.weight_dict['T']['r']['weight'],
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.t_shape,
uniform=self.normal_initializer,
mask=t_array)))
if self.model_version != 'no_input_facing':
# Also create input-facing weight matrices
# Q
setattr(
self,
self.weight_dict['Q']['f']['weight'],
tf.get_variable(
name=self.weight_dict['Q']['f']['weight'],
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.q_shape,
uniform=self.normal_initializer)))
setattr(
self,
self.weight_dict['Q']['f']['bias'],
tf.get_variable(
name=self.weight_dict['Q']['f']['bias'],
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[self.q_shape[-1]],
uniform=self.normal_initializer)))
# U
setattr(
self,
self.weight_dict['U']['f']['weight'],
tf.get_variable(
name=self.weight_dict['U']['f']['weight'],
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.u_shape,
uniform=self.normal_initializer)))
setattr(
self,
self.weight_dict['U']['f']['bias'],
tf.get_variable(
name=self.weight_dict['U']['f']['bias'],
dtype=self.dtype,
initializer=initialization.xavier_initializer(
[self.u_shape[-1]],
uniform=self.normal_initializer)))
# P
setattr(
self,
self.weight_dict['P']['f']['weight'],
tf.get_variable(
name=self.weight_dict['P']['f']['weight'],
dtype=self.dtype,
initializer=initialization.xavier_initializer(
self.p_shape,
uniform=self.normal_initializer,
mask=p_array)))
setattr(
self,
self.weight_dict['P']['f']['bias'],
tf.get_variable(
name=self.weight_dict['P']['f']['bias'],
dtype=self.dtype,
initializer=initialization.xavier_initializer(
[self.p_shape[-1]],
uniform=self.normal_initializer,
mask=None)))
# T
setattr(
self,
self.weight_dict['T']['f']['weight'],
tf.get_variable(
name=self.weight_dict['T']['f']['weight'],
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.t_shape,
uniform=self.normal_initializer,
mask=t_array)))
setattr(
self,
self.weight_dict['T']['f']['bias'],
tf.get_variable(
name=self.weight_dict['T']['f']['bias'],
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[self.t_shape[-1]],
uniform=self.normal_initializer,
mask=None)))
if self.model_version == 'full_with_cell_states':
# Input
setattr(
self,
self.weight_dict['I']['r']['weight'],
tf.get_variable(
name=self.weight_dict['I']['r']['weight'],
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.i_shape,
uniform=self.normal_initializer,
mask=None)))
setattr(
self,
self.weight_dict['I']['r']['bias'],
tf.get_variable(
name=self.weight_dict['I']['r']['bias'],
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[self.k],
uniform=self.normal_initializer,
mask=None)))
# Output
setattr(
self,
self.weight_dict['O']['r']['weight'],
tf.get_variable(
name=self.weight_dict['O']['r']['weight'],
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.o_shape,
uniform=self.normal_initializer,
mask=None)))
setattr(
self,
self.weight_dict['O']['r']['bias'],
tf.get_variable(
name=self.weight_dict['O']['r']['bias'],
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[self.k],
uniform=self.normal_initializer,
mask=None)))
# Scalar weights
self.alpha = tf.get_variable(name='alpha', initializer=1.)
self.tau = tf.get_variable(name='tau', initializer=1.)
self.eta = tf.get_variable(name='eta', initializer=1.)
self.omega = tf.get_variable(name='omega', initializer=1.)
self.eps = tf.get_variable(name='eps', initializer=1.)
self.gamma = tf.get_variable(name='gamma', initializer=1.)
def build(self):
"""Run the backprop version of the CCircuit."""
self.prepare_tensors()
i0 = tf.constant(0)
if self.hidden_init == 'identity':
I = tf.identity(self.X)
O = tf.identity(self.X)
elif self.hidden_init == 'random':
I = initialization.xavier_initializer(
shape=[self.n, self.h, self.w, self.k],
uniform=self.normal_initializer,
mask=None)
O = initialization.xavier_initializer(
shape=[self.n, self.h, self.w, self.k],
uniform=self.normal_initializer,
mask=None)
elif self.hidden_init == 'zeros':
I = tf.zeros_like(self.X)
O = tf.zeros_like(self.X)
else:
raise RuntimeError
if self.store_states:
store_I = tf.TensorArray(tf.float32, size=self.timesteps)
store_O = tf.TensorArray(tf.float32, size=self.timesteps)
elems = [i0, O, I, store_I, store_O]
returned = tf.while_loop(self.condition,
self.full,
loop_vars=elems,
back_prop=True,
swap_memory=True)
# Prepare output
i0, O, I, store_I, store_O = returned
setattr(self, 'store_I', store_I.stack('store_I'))
setattr(self, 'store_O', store_O.stack('store_O'))
else:
# While loop
elems = [i0, O, I, tf.constant(0), tf.constant(0)]
returned = tf.while_loop(self.condition,
self.full,
loop_vars=elems,
back_prop=True,
swap_memory=True)
# Prepare output
i0, O, I, _, _ = returned
if self.return_weights:
weights = self.gather_tensors(wak='weight')
tuning = self.gather_tensors(wak='tuning')
new_tuning = {}
for k, v in tuning.iteritems():
key_name = v.name.split('/')[-1].split(':')[0]
new_tuning[key_name] = v
weights = dict(weights, **new_tuning)
activities = self.gather_tensors(wak='activity')
# Attach weights if using association field
if self.association_field:
weights['p_t'] = self.p_r # Make available for regularization
if self.store_states:
weights['store_I'] = store_I
weights['store_O'] = store_O
return O, weights, activities
else:
if self.store_states:
return O # , store_I, store_O
else:
return O
def prepare_tensors(self):
""" Prepare recurrent/forward weight matrices.
(np.prod([h, w, k]) / 2) - k params in the surround filter
"""
# Create FF vars
if self.include_pooling:
# Upsample FF layers then hgru layer
up_filters = self.hgru_ids[0].values() + self.intermediate_ff
for idx in reversed(range(1, len(up_filters))):
label = idx - 1
setattr(
self, 'resize_kernel_%s' % label,
tf.get_variable(
name='%s_resize_kernel_%s' % (self.layer_name, label),
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.up_kernel +
[up_filters[idx - 1], up_filters[idx]],
uniform=self.normal_initializer),
trainable=True))
setattr(
self, 'resize_bias_%s' % label,
tf.get_variable(name='%s_resize_bias_%s' %
(self.layer_name, label),
dtype=self.dtype,
initializer=tf.ones([up_filters[idx - 1]]),
trainable=True))
# Create conv filters that supply top-drive
prev_filters = self.hgru_ids[0].values()[0]
for idx, (ff_filters, ff_kernel, reps) in enumerate(
zip(self.intermediate_ff, self.intermediate_ks,
self.intermediate_repeats)):
for il in range(reps):
setattr(
self, 'intermediate_kernel_%s_%s' % (idx, il),
tf.get_variable(
name='%s_ffdrive_kernel_%s_%s' %
(self.layer_name, idx, il),
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=ff_kernel + [prev_filters, ff_filters],
uniform=self.normal_initializer),
trainable=True))
setattr(
self, 'intermediate_bias_%s_%s' % (idx, il),
tf.get_variable(name='%s_ffdrive_bias_%s_%s' %
(self.layer_name, idx, il),
dtype=self.dtype,
initializer=tf.ones([ff_filters]),
trainable=True))
prev_filters = ff_filters
# Create recurrent vars
self.symm_k_tag = 'sy' if self.symmetric_weights else 'full'
self.symm_g_tag = 'sy' if self.symmetric_gate_weights else 'full'
for idx, hgru_id in enumerate(self.hgru_ids):
layer, rk = hgru_id.items()[0]
self.g_shape = [self.gate_filter, self.gate_filter, rk, rk]
self.m_shape = [self.gate_filter, self.gate_filter, rk, rk]
self.bias_shape = [1, 1, 1, rk]
with tf.variable_scope('%s_hgru_weights_%s' %
(self.layer_name, layer)):
setattr(
self,
'%s_horizontal_kernels_exc_%s' % (self.symm_k_tag, layer),
tf.get_variable(
name='%s_%s_horizontal_exc' %
(self.symm_k_tag, self.layer_name),
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.h_ext[idx].values()[0] + [rk, rk],
uniform=self.normal_initializer),
trainable=True))
setattr(
self,
'%s_horizontal_kernels_inh_%s' % (self.symm_k_tag, layer),
tf.get_variable(
name='%s_%s_horizontal_inh' %
(self.symm_k_tag, self.layer_name),
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.h_ext[idx].values()[0] + [rk, rk],
uniform=self.normal_initializer),
trainable=True))
setattr(
self, '%s_gain_kernels_%s' % (self.symm_g_tag, layer),
tf.get_variable(
name='%s_%s_gain' % (self.symm_g_tag, self.layer_name),
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.g_shape,
uniform=self.normal_initializer,
mask=None)))
setattr(
self, '%s_mix_kernels_%s' % (self.symm_g_tag, layer),
tf.get_variable(
name='%s_%s_mix' % (self.symm_g_tag, self.layer_name),
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.m_shape,
uniform=self.normal_initializer,
mask=None)))
# Gain bias
if self.gate_bias_init == 'chronos':
bias_init = -tf.log(
tf.random_uniform(self.bias_shape,
minval=1,
maxval=self.timesteps - 1))
else:
bias_init = tf.ones(self.bias_shape)
setattr(
self, 'gain_bias_%s' % layer,
tf.get_variable(name='%s_gain_bias' % self.layer_name,
dtype=self.dtype,
trainable=True,
initializer=bias_init))
if self.gate_bias_init == 'chronos':
bias_init = -bias_init
else:
bias_init = tf.ones(self.bias_shape)
setattr(
self, 'mix_bias_%s' % layer,
tf.get_variable(name='%s_mix_bias' % self.layer_name,
dtype=self.dtype,
trainable=True,
initializer=bias_init))
# Divisive params
if self.alpha and not self.lesion_alpha:
setattr(
self, 'alpha_%s' % layer,
tf.get_variable(
name='%s_alpha' % self.layer_name,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
uniform=self.normal_initializer,
mask=None)))
elif self.lesion_alpha:
setattr(self, 'alpha_%s' % layer, tf.constant(0.))
else:
setattr(self, 'alpha_%s' % layer, tf.constant(1.))
if self.mu and not self.lesion_mu:
setattr(
self, 'mu_%s' % layer,
tf.get_variable(
name='%s_mu' % self.layer_name,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
uniform=self.normal_initializer,
mask=None)))
elif self.lesion_mu:
setattr(self, 'mu_%s' % layer, tf.constant(0.))
else:
setattr(self, 'mu_%s' % layer, tf.constant(1.))
if self.gamma:
setattr(
self, 'gamma_%s' % layer,
tf.get_variable(
name='%s_gamma' % self.layer_name,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
uniform=self.normal_initializer,
mask=None)))
else:
setattr(self, 'gamma_%s' % layer, tf.constant(1.))
if self.multiplicative_excitation:
if self.lesion_kappa:
setattr(self, 'kappa_%s' % layer, tf.constant(0.))
else:
setattr(
self, 'kappa_%s' % layer,
tf.get_variable(
name='%s_kappa' % self.layer_name,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
uniform=self.normal_initializer,
mask=None)))
if self.lesion_omega:
setattr(self, 'omega_%s' % layer, tf.constant(0.))
else:
setattr(
self, 'omega_%s' % layer,
tf.get_variable(
name='%s_omega' % self.layer_name,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
uniform=self.normal_initializer,
mask=None)))
else:
setattr(self, 'kappa_%s' % layer, tf.constant(1.))
setattr(self, 'omega_%s' % layer, tf.constant(1.))
if self.adaptation:
setattr(
self, 'eta_%s' % layer,
tf.get_variable(
name='%s_eta' % self.layer_name,
initializer=tf.random_uniform([self.timesteps],
dtype=tf.float32)))
if self.lesion_omega:
setattr(self, 'omega_%s' % layer, tf.constant(0.))
if self.lesion_kappa:
setattr(self, 'kappa_%s' % layer, tf.constant(0.))
if self.reuse:
# Make the batchnorm variables
scopes = ['g1_bn', 'g2_bn', 'c1_bn', 'c2_bn']
bn_vars = ['moving_mean', 'moving_variance', 'gamma']
for s in scopes:
with tf.variable_scope(s):
for v in bn_vars:
tf.get_variable(
trainable=self.param_trainable[v],
name=v,
shape=[rk],
collections=self.param_collections[v],
initializer=self.param_initializer[v])
self.param_initializer = None
def prepare_tensors(self):
""" Prepare recurrent/forward weight matrices.
(np.prod([h, w, k]) / 2) - k params in the surround filter
"""
with tf.variable_scope('%s_hgru_weights' % self.layer_name):
self.horizontal_kernels = tf.get_variable(
name='%s_horizontal' % self.layer_name,
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.h_shape,
uniform=self.normal_initializer),
trainable=self.train)
self.gain_kernels = tf.get_variable(
name='%s_gain' % self.layer_name,
dtype=self.dtype,
trainable=self.train,
initializer=initialization.xavier_initializer(
shape=self.g_shape,
uniform=self.normal_initializer,
mask=None))
self.mix_kernels = tf.get_variable(
name='%s_mix' % self.layer_name,
dtype=self.dtype,
trainable=self.train,
initializer=initialization.xavier_initializer(
shape=self.m_shape,
uniform=self.normal_initializer,
mask=None))
# Gain bias
if self.gate_bias_init == 'chronos':
bias_init = -tf.log(
tf.random_uniform(
self.bias_shape, minval=1, maxval=self.timesteps - 1))
else:
bias_init = -tf.ones(self.bias_shape)
self.gain_bias = tf.get_variable(
name='%s_gain_bias' % self.layer_name,
dtype=self.dtype,
trainable=self.train,
initializer=bias_init)
if self.gate_bias_init == 'chronos':
bias_init = -bias_init
else:
bias_init = tf.ones(self.bias_shape)
self.mix_bias = tf.get_variable(
name='%s_mix_bias' % self.layer_name,
dtype=self.dtype,
trainable=self.train,
initializer=bias_init)
# Divisive params
if self.alpha and not self.lesion_alpha:
self.alpha = tf.get_variable(
name='%s_alpha' % self.layer_name,
trainable=self.train,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
uniform=self.normal_initializer,
mask=None))
elif self.lesion_alpha:
self.alpha = tf.constant(0.)
else:
self.alpha = tf.constant(1.)
if self.mu and not self.lesion_mu:
self.mu = tf.get_variable(
name='%s_mu' % self.layer_name,
trainable=self.train,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
uniform=self.normal_initializer,
mask=None))
elif self.lesion_mu:
self.mu = tf.constant(0.)
else:
self.mu = tf.constant(1.)
if self.gamma:
self.gamma = tf.get_variable(
name='%s_gamma' % self.layer_name,
trainable=self.train,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
uniform=self.normal_initializer,
mask=None))
else:
self.gamma = tf.constant(1.)
if self.multiplicative_excitation:
if self.lesion_kappa:
self.kappa = tf.constant(0.)
else:
self.kappa = tf.get_variable(
name='%s_kappa' % self.layer_name,
trainable=self.train,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
uniform=self.normal_initializer,
mask=None))
if self.lesion_omega:
self.omega = tf.constant(0.)
else:
self.omega = tf.get_variable(
name='%s_omega' % self.layer_name,
trainable=self.train,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
uniform=self.normal_initializer,
mask=None))
else:
self.kappa = tf.constant(1.)
self.omega = tf.constant(1.)
if self.adaptation:
self.eta = tf.get_variable(
trainable=self.train,
name='%s_eta' % self.layer_name,
shape=[self.timesteps],
initializer=tf.random_uniform_initializer)
if self.lesion_omega:
self.omega = tf.constant(0.)
if self.lesion_kappa:
self.kappa = tf.constant(0.)
def prepare_tensors(self):
""" Prepare recurrent/forward weight matrices."""
self.weight_dict = { # Weights lower/activity upper
'U': {
'r': {
'weight': 'u_r',
'activity': 'U_r'
}
},
'T': {
'r': {
'weight': 't_r',
'activity': 'T_r',
'tuning': 't_t'
}
},
'P': {
'r': {
'weight': 'p_r',
'activity': 'P_r',
'tuning': 'p_t'
}
},
'Q': {
'r': {
'weight': 'q_r',
'activity': 'Q_r',
'tuning': 'q_t'
}
},
'I': {
'r': { # Recurrent state
'weight': 'i_r',
'bias': 'i_b',
'activity': 'I_r'
},
'f': { # Recurrent state
'weight': 'i_f',
'activity': 'I_f'
},
},
'O': {
'r': { # Recurrent state
'weight': 'o_r',
'bias': 'o_b',
'activity': 'O_r'
},
'f': { # Recurrent state
'weight': 'o_f',
'activity': 'O_f'
},
},
'xi': {
'r': { # Recurrent state
'weight': 'xi',
}
},
'alpha': {
'r': { # Recurrent state
'weight': 'alpha',
}
},
'beta': {
'r': { # Recurrent state
'weight': 'beta',
}
},
'mu': {
'r': { # Recurrent state
'weight': 'mu',
}
},
'nu': {
'r': { # Recurrent state
'weight': 'nu',
}
},
'zeta': {
'r': { # Recurrent state
'weight': 'zeta',
}
},
'gamma': {
'r': { # Recurrent state
'weight': 'gamma',
}
},
'delta': {
'r': { # Recurrent state
'weight': 'delta',
}
}
}
# tuned summation: pooling in h, w dimensions
#############################################
q_array = np.ones(self.q_shape) / np.prod(self.q_shape)
if 'Q' in self.lesions:
q_array = np.zeros_like(q_array).astype(np.float32)
print 'Lesioning CRF excitation.'
setattr(
self, self.weight_dict['Q']['r']['weight'],
tf.get_variable(name=self.weight_dict['Q']['r']['weight'],
dtype=self.dtype,
initializer=q_array.astype(np.float32),
trainable=False))
# untuned suppression: reduction across feature axis
####################################################
u_array = np.ones(self.u_shape) / np.prod(self.u_shape)
if 'U' in self.lesions:
u_array = np.zeros_like(u_array).astype(np.float32)
print 'Lesioning CRF inhibition.'
setattr(
self, self.weight_dict['U']['r']['weight'],
tf.get_variable(name=self.weight_dict['U']['r']['weight'],
dtype=self.dtype,
initializer=u_array.astype(np.float32),
trainable=False))
# weakly tuned summation: pooling in h, w dimensions
#############################################
p_array = np.ones(self.p_shape)
#Try removing CRF punching
if self.exclude_CRF:
# Punch out the CRF
p_array[self.SSN // 2 -
py_utils.ifloor(self.SRF / 2.0):self.SSF // 2 +
py_utils.iceil(self.SSN / 2.0), self.SSN // 2 -
py_utils.ifloor(self.SRF / 2.0):self.SSF // 2 +
py_utils.iceil(self.SSN / 2.0), :, # exclude CRF!
:] = 0.0
p_array = p_array / p_array.sum()
if 'P' in self.lesions:
print 'Lesioning near eCRF.'
p_array = np.zeros_like(p_array).astype(np.float32)
# Association field is fully learnable
if self.association_field and 'P' not in self.lesions:
setattr(
self, self.weight_dict['P']['r']['weight'],
tf.get_variable(name=self.weight_dict['P']['r']['weight'],
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.p_shape,
uniform=self.normal_initializer),
trainable=True))
else:
setattr(
self, self.weight_dict['P']['r']['weight'],
tf.get_variable(name=self.weight_dict['P']['r']['weight'],
dtype=self.dtype,
initializer=p_array.astype(np.float32),
trainable=False))
# weakly tuned suppression: pooling in h, w dimensions
###############################################
t_array = np.ones(self.t_shape)
#Try without punching CRF
if self.exclude_CRF:
# Punch out the CRF
t_array[self.SSF // 2 -
py_utils.ifloor(self.SSN / 2.0):self.SSF // 2 +
py_utils.iceil(self.SSN / 2.0), self.SSF // 2 -
py_utils.ifloor(self.SSN / 2.0):self.SSF // 2 +
py_utils.iceil(self.SSN /
2.0), :, # exclude near surround!
:] = 0.0
t_array = t_array / t_array.sum()
if 'T' in self.lesions:
print 'Lesioning Far eCRF.'
t_array = np.zeros_like(t_array).astype(np.float32)
#Always set full_far_eCRF to True/initialize with Xavier
if self.full_far_eCRF:
setattr(
self, self.weight_dict['T']['r']['weight'],
tf.get_variable(name=self.weight_dict['T']['r']['weight'],
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.p_shape,
uniform=self.normal_initializer),
trainable=True))
else:
setattr(
self, self.weight_dict['T']['r']['weight'],
tf.get_variable(name=self.weight_dict['T']['r']['weight'],
dtype=self.dtype,
initializer=t_array.astype(np.float32),
trainable=False))
# Connectivity tensors -- Q/P/T
if 'Q' in self.lesions:
print 'Lesioning CRF excitation connectivity.'
setattr(
self, self.weight_dict['Q']['r']['tuning'],
tf.get_variable(name=self.weight_dict['Q']['r']['tuning'],
dtype=self.dtype,
trainable=False,
initializer=np.zeros(self.tuning_shape).astype(
np.float32)))
else:
setattr(
self, self.weight_dict['Q']['r']['tuning'],
tf.get_variable(name=self.weight_dict['Q']['r']['tuning'],
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.tuning_shape,
uniform=self.normal_initializer,
mask=None)))
if not self.association_field:
# Need a tuning tensor for near surround
if 'P' in self.lesions:
print 'Lesioning near eCRF connectivity.'
setattr(
self, self.weight_dict['P']['r']['tuning'],
tf.get_variable(name=self.weight_dict['P']['r']['tuning'],
dtype=self.dtype,
trainable=False,
initializer=np.zeros(
self.tuning_shape).astype(np.float32)))
else:
setattr(
self, self.weight_dict['P']['r']['tuning'],
tf.get_variable(
name=self.weight_dict['P']['r']['tuning'],
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.tuning_shape,
uniform=self.normal_initializer,
mask=None)))
#Again, full_far_eCRF should be set to True for now
import ipdb
ipdb.set_trace()
if not self.full_far_eCRF:
# Need a tuning tensor for near surround
if 'T' in self.lesions:
print 'Lesioning far eCRF connectivity.'
setattr(
self, self.weight_dict['T']['r']['tuning'],
tf.get_variable(name=self.weight_dict['T']['r']['tuning'],
dtype=self.dtype,
trainable=False,
initializer=np.zeros(
self.tuning_shape).astype(np.float32)))
else:
setattr(
self, self.weight_dict['T']['r']['tuning'],
tf.get_variable(name=self.weight_dict['T']['r']['tuning'],
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.tuning_shape,
uniform=self.normal_initializer,
mask=None)))
# Input
setattr(
self, self.weight_dict['I']['r']['weight'],
tf.get_variable(name=self.weight_dict['I']['r']['weight'],
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.i_shape,
uniform=self.normal_initializer,
mask=None)))
setattr(
self, self.weight_dict['I']['f']['weight'],
tf.get_variable(name=self.weight_dict['I']['f']['weight'],
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.i_shape,
uniform=self.normal_initializer,
mask=None)))
setattr(
self, self.weight_dict['I']['r']['bias'],
tf.get_variable(name=self.weight_dict['I']['r']['bias'],
dtype=self.dtype,
trainable=True,
initializer=tf.ones(self.bias_shape)))
# Output
setattr(
self, self.weight_dict['O']['r']['weight'],
tf.get_variable(name=self.weight_dict['O']['r']['weight'],
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.o_shape,
uniform=self.normal_initializer,
mask=None)))
setattr(
self, self.weight_dict['O']['f']['weight'],
tf.get_variable(name=self.weight_dict['O']['f']['weight'],
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.o_shape,
uniform=self.normal_initializer,
mask=None)))
setattr(
self, self.weight_dict['O']['r']['bias'],
tf.get_variable(name=self.weight_dict['O']['r']['bias'],
dtype=self.dtype,
trainable=True,
initializer=tf.ones(self.bias_shape)))
# Vector weights
w_array = np.ones([1, 1, 1, self.k]).astype(np.float32)
b_array = np.zeros([1, 1, 1, self.k]).astype(np.float32)
self.xi = tf.get_variable(name='xi', initializer=w_array)
self.alpha = tf.get_variable(name='alpha', initializer=w_array)
self.beta = tf.get_variable(name='beta', initializer=w_array)
self.mu = tf.get_variable(name='mu', initializer=b_array)
self.nu = tf.get_variable(name='nu', initializer=b_array)
self.zeta = tf.get_variable(name='zeta', initializer=w_array)
self.gamma = tf.get_variable(name='gamma', initializer=w_array)
self.delta = tf.get_variable(name='delta', initializer=w_array)
def prepare_tensors(self):
"""
"""
# recurrent kernels
setattr(
self, 'caps_filter',
tf.get_variable(name='%s_caps_filter' % (self.name),
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[self.h_ext, self.h_ext] +
[self.f * self.c, self.b * self.f * self.c],
uniform=self.normal_initializer),
trainable=True))
if self.use_independent_labels_filter:
setattr(
self, 'labels_filter',
tf.get_variable(name='%s_labels_filter' % (self.name),
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[self.h_ext, self.h_ext] +
[self.f, self.b * self.f],
uniform=self.normal_initializer),
trainable=True))
else:
setattr(self, 'labels_filter',
helpers.caps2scalar_filter(self.caps_filter, self.c))
# label update variables
if self.labels_update_mode == 'pushpull':
setattr(
self, 'pushpull_scale',
tf.get_variable(name='%s_pushpull_scale' % (self.name),
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[1, 1, 1, self.f],
uniform=self.normal_initializer),
trainable=True))
setattr(
self, 'pushpull_bias',
tf.get_variable(name='%s_pushpull_bias' % (self.name),
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[1, 1, 1, self.f],
uniform=self.normal_initializer),
trainable=True))
self.sensitivity_const = None
elif self.labels_update_mode == 'average':
setattr(
self, 'sensitivity_const',
tf.get_variable(name='%s_sensitivity_const' % (self.name),
dtype=self.dtype,
shape=[1, 1, 1, self.f],
initializer=tf.constant_initializer(
[0.5], dtype=self.dtype),
trainable=True))
self.pushpull_scale = None
self.pushpull_bias = None
# label gate variables
if self.use_labels_gate:
setattr(
self, 'labels_gate_scale',
tf.get_variable(name='%s_labels_gate_scale' % (self.name),
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[1, 1, 1, self.f],
uniform=self.normal_initializer),
trainable=True))
setattr(
self, 'labels_gate_bias',
tf.get_variable(name='%s_labels_gate_bias' % (self.name),
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=[1, 1, 1, self.f],
uniform=self.normal_initializer),
trainable=True))
else:
self.labels_gate_scale = None
self.labels_gate_bias = None
# Auxilliary variables
setattr(
self, 'tolerance_const',
tf.get_variable(name='%s_tolerance_const' % (self.name),
dtype=self.dtype,
shape=[1, 1, 1, self.f],
initializer=tf.constant_initializer(
[0.], dtype=self.dtype),
trainable=True))
setattr(
self, 'decay_const',
tf.get_variable(name='%s_decay_const' % (self.name),
dtype=self.dtype,
shape=[1, 1, 1, self.f],
initializer=tf.constant_initializer(
[0.0], dtype=self.dtype),
trainable=True))
import numpy as np
fixed_labels_mask = np.zeros([1, 1, 1, self.f])
if self.fixed_label_ind is not None:
for idx in self.fixed_label_ind:
fixed_labels_mask[0, 0, 0, idx] = 1.
self.fixed_labels_mask = tf.constant(fixed_labels_mask,
dtype=self.dtype)
def prepare_tensors(self):
""" Prepare recurrent/forward weight matrices.
(np.prod([h, w, k]) / 2) - k params in the surround filter
"""
with tf.variable_scope('%s_hgru_weights' % self.layer_name):
self.horizontal_kernels = tf.get_variable(
name='%s_horizontal' % self.layer_name,
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.h_shape, uniform=self.normal_initializer),
trainable=True)
self.h_bias = tf.get_variable(
name='%s_h_bias' % self.layer_name,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
uniform=self.normal_initializer,
mask=None))
self.f_kernels = tf.get_variable(
name='%s_f' % self.layer_name,
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.f_shape,
uniform=self.normal_initializer,
mask=None))
self.i_kernels = tf.get_variable(
name='%s_i' % self.layer_name,
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.i_shape,
uniform=self.normal_initializer,
mask=None))
self.o_kernels = tf.get_variable(
name='%s_o' % self.layer_name,
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.i_shape,
uniform=self.normal_initializer,
mask=None))
# Gain bias
bias_init = tf.ones(self.bias_shape)
self.f_bias = tf.get_variable(name='%s_f_bias' % self.layer_name,
dtype=self.dtype,
trainable=True,
initializer=bias_init)
self.i_bias = tf.get_variable(name='%s_i_bias' % self.layer_name,
dtype=self.dtype,
trainable=True,
initializer=bias_init)
self.o_bias = tf.get_variable(name='%s_o_bias' % self.layer_name,
dtype=self.dtype,
trainable=True,
initializer=bias_init)
# Divisive params
if self.alpha and not self.lesion_alpha:
self.alpha = tf.get_variable(
name='%s_alpha' % self.layer_name,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
uniform=self.normal_initializer,
mask=None))
elif self.lesion_alpha:
self.alpha = tf.constant(0.)
else:
self.alpha = tf.constant(1.)
if self.mu and not self.lesion_mu:
self.mu = tf.get_variable(
name='%s_mu' % self.layer_name,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
uniform=self.normal_initializer,
mask=None))
elif self.lesion_mu:
self.mu = tf.constant(0.)
else:
self.mu = tf.constant(1.)
# if self.gamma:
# self.gamma = tf.get_variable(
# name='%s_gamma' % self.layer_name,
# initializer=initialization.xavier_initializer(
# shape=self.bias_shape,
# uniform=self.normal_initializer,
# mask=None))
# else:
# self.gamma = tf.constant(1.)
if self.multiplicative_excitation:
if self.lesion_kappa:
self.kappa = tf.constant(0.)
else:
self.kappa = tf.get_variable(
name='%s_kappa' % self.layer_name,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
uniform=self.normal_initializer,
mask=None))
if self.lesion_omega:
self.omega = tf.constant(0.)
else:
self.omega = tf.get_variable(
name='%s_omega' % self.layer_name,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
uniform=self.normal_initializer,
mask=None))
else:
self.kappa = tf.constant(1.)
self.omega = tf.constant(1.)
if self.adapation:
self.eta = tf.get_variable(name='%s_eta' % self.layer_name,
initializer=tf.ones(
self.timesteps,
dtype=tf.float32))
if self.lesion_omega:
self.omega = tf.constant(0.)
if self.lesion_kappa:
self.kappa = tf.constant(0.)
def prepare_tensors(self):
# HGRU KERNELS
self.hgru0.prepare_tensors()
self.hgru_td0.prepare_tensors()
# FEEDFORWARD KERNELS
lower_feats = self.hgru_h2_k
for idx, (higher_feats,
ff_dhw) in enumerate(zip(self.ff_conv_k, self.ff_conv_fsiz)):
with tf.variable_scope(self.var_scope + '/ff_%s' % idx):
setattr(
self, 'ff_%s_weights' % idx,
tf.get_variable(
name='weights',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=ff_dhw + [lower_feats, higher_feats],
dtype=self.dtype,
uniform=True),
trainable=True))
lower_feats = higher_feats
# FEEDBACK KERNELS
lower_feats = self.in_k
if not self.share_ff_td_kernels:
for idx, (higher_feats, fb_dhw) in enumerate(
zip(self.fb_conv_k, self.fb_conv_fsiz)):
with tf.variable_scope(self.var_scope + '/fb_%s' % idx):
setattr(
self, 'fb_%s_weights' % idx,
tf.get_variable(
name='weights',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=fb_dhw + [lower_feats, higher_feats],
dtype=self.dtype,
uniform=True),
trainable=True))
lower_feats = higher_feats
else:
higher_feats = self.fb_conv_k[0]
fb_dhw = self.fb_conv_fsiz[0]
idx = 0
with tf.variable_scope(self.var_scope + '/fb_%s' % idx):
setattr(
self, 'fb_%s_weights' % idx,
tf.get_variable(
name='weights',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=fb_dhw + [lower_feats, higher_feats],
dtype=self.dtype,
uniform=True),
trainable=True))
for x in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope=self.var_scope):
fgru = 0
ff_fb = 0
prod = np.prod(x.get_shape().as_list())
if ('fgru' in x.name):
fgru += prod
else:
ff_fb += prod
print('>>>>>>>>>>>>>>>>>>>>>>TRAINABLE VARS: ' + 'fgrus(' +
str(fgru) + ') ff&fb(' + str(ff_fb) + ')')
print('>>>>>>>>>>>>>>>>>>>>>>TRAINABLE VARS: ' + 'total(' +
str(fgru + ff_fb) + ')')
def prepare_tensors(self):
""" Prepare recurrent/forward weight matrices."""
self.weight_dict = { # Weights lower/activity upper
'U': {
'r': {
'weight': 'u_r',
'activity': 'U_r'
}
},
'T': {
'r': {
'weight': 't_r',
'activity': 'T_r',
'tuning': 't_t'
}
},
'P': {
'r': {
'weight': 'p_r',
'activity': 'P_r',
'tuning': 'p_t'
}
},
'Q': {
'r': {
'weight': 'q_r',
'activity': 'Q_r',
'tuning': 'q_t'
}
},
'I': {
'r': { # Recurrent state
'weight': 'i_r',
'activity': 'I_r'
},
'f': { # Recurrent state
'weight': 'i_f',
'activity': 'I_f'
},
},
'O': {
'r': { # Recurrent state
'weight': 'o_r',
'activity': 'O_r'
},
'f': { # Recurrent state
'weight': 'o_f',
'activity': 'O_f'
},
},
'xi': {
'r': { # Recurrent state
'weight': 'xi',
}
},
'alpha': {
'r': { # Recurrent state
'weight': 'alpha',
}
},
'beta': {
'r': { # Recurrent state
'weight': 'beta',
}
},
'mu': {
'r': { # Recurrent state
'weight': 'mu',
}
},
'nu': {
'r': { # Recurrent state
'weight': 'nu',
}
},
'zeta': {
'r': { # Recurrent state
'weight': 'zeta',
}
},
'gamma': {
'r': { # Recurrent state
'weight': 'gamma',
}
},
'delta': {
'r': { # Recurrent state
'weight': 'delta',
}
}
}
# tuned summation: pooling in h, w dimensions
#############################################
q_array = np.ones(self.q_shape) / np.prod(self.q_shape)
setattr(
self, self.weight_dict['Q']['r']['weight'],
tf.get_variable(name=self.weight_dict['Q']['r']['weight'],
dtype=self.dtype,
initializer=q_array.astype(np.float32),
trainable=False))
# untuned suppression: reduction across feature axis
####################################################
u_array = np.ones(self.u_shape) / np.prod(self.u_shape)
setattr(
self, self.weight_dict['U']['r']['weight'],
tf.get_variable(name=self.weight_dict['U']['r']['weight'],
dtype=self.dtype,
initializer=u_array.astype(np.float32),
trainable=False))
# weakly tuned summation: pooling in h, w dimensions
#############################################
p_array = np.ones(self.p_shape)
p_array[self.SSN // 2 - py_utils.ifloor(self.SRF / 2.0):self.SSN // 2 +
py_utils.iceil(self.SRF / 2.0),
self.SSN // 2 - py_utils.ifloor(self.SRF / 2.0):self.SSN // 2 +
py_utils.iceil(self.SRF / 2.0), :, # exclude CRF!
:] = 0.0
p_array = p_array / p_array.sum()
setattr(
self, self.weight_dict['P']['r']['weight'],
tf.get_variable(name=self.weight_dict['P']['r']['weight'],
dtype=self.dtype,
initializer=p_array.astype(np.float32),
trainable=False))
# weakly tuned suppression: pooling in h, w dimensions
###############################################
t_array = np.ones(self.t_shape)
t_array[self.SSF // 2 - py_utils.ifloor(self.SSN / 2.0):self.SSF // 2 +
py_utils.iceil(self.SSN / 2.0),
self.SSF // 2 - py_utils.ifloor(self.SSN / 2.0):self.SSF // 2 +
py_utils.iceil(self.SSN / 2.0), :, # exclude near surround!
:] = 0.0
t_array = t_array / t_array.sum()
setattr(
self, self.weight_dict['T']['r']['weight'],
tf.get_variable(name=self.weight_dict['T']['r']['weight'],
dtype=self.dtype,
initializer=t_array.astype(np.float32),
trainable=False))
# Connectivity tensors -- Q/P/T
setattr(
self, self.weight_dict['Q']['r']['tuning'],
tf.get_variable(name=self.weight_dict['Q']['r']['tuning'],
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.tuning_shape,
uniform=self.normal_initializer,
mask=None)))
setattr(
self, self.weight_dict['P']['r']['tuning'],
tf.get_variable(name=self.weight_dict['P']['r']['tuning'],
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.tuning_shape,
uniform=self.normal_initializer,
mask=None)))
setattr(
self, self.weight_dict['T']['r']['tuning'],
tf.get_variable(name=self.weight_dict['T']['r']['tuning'],
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.tuning_shape,
uniform=self.normal_initializer,
mask=None)))
# Input
setattr(
self, self.weight_dict['I']['r']['weight'],
tf.get_variable(name=self.weight_dict['I']['r']['weight'],
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.i_shape,
uniform=self.normal_initializer,
mask=None)))
setattr(
self, self.weight_dict['I']['f']['weight'],
tf.get_variable(name=self.weight_dict['I']['f']['weight'],
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.i_shape,
uniform=self.normal_initializer,
mask=None)))
# Output
setattr(
self, self.weight_dict['O']['r']['weight'],
tf.get_variable(name=self.weight_dict['O']['r']['weight'],
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.o_shape,
uniform=self.normal_initializer,
mask=None)))
setattr(
self, self.weight_dict['O']['f']['weight'],
tf.get_variable(name=self.weight_dict['O']['f']['weight'],
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.o_shape,
uniform=self.normal_initializer,
mask=None)))
# Vector weights
w_array = np.ones([1, 1, 1, self.k]).astype(np.float32)
b_array = np.zeros([1, 1, 1, self.k]).astype(np.float32)
self.xi = tf.get_variable(name='xi', initializer=w_array)
self.alpha = tf.get_variable(name='alpha', initializer=w_array)
self.beta = tf.get_variable(name='beta', initializer=w_array)
self.mu = tf.get_variable(name='mu', initializer=b_array)
self.nu = tf.get_variable(name='nu', initializer=b_array)
self.zeta = tf.get_variable(name='zeta', initializer=w_array)
self.gamma = tf.get_variable(name='gamma', initializer=w_array)
self.delta = tf.get_variable(name='delta', initializer=w_array)
def prepare_tensors(self):
""" Prepare recurrent/forward weight matrices."""
self.weight_dict = { # Weights lower/activity upper
'U': {
'r': {
'weight': 'u_r',
'activity': 'U_r'
}
},
'T': {
'r': {
'weight': 't_r',
'activity': 'T_r',
'tuning': 't_t'
}
},
'P': {
'r': {
'weight': 'p_r',
'activity': 'P_r',
'tuning': 'p_t'
}
},
'Q': {
'r': {
'weight': 'q_r',
'activity': 'Q_r',
'tuning': 'q_t'
}
},
'I': {
'r': { # Recurrent state
'weight': 'i_r',
'bias': 'i_b',
'activity': 'I_r'
},
'f': { # Recurrent state
'weight': 'i_f',
'activity': 'I_f'
},
},
'O': {
'r': { # Recurrent state
'weight': 'o_r',
'bias': 'o_b',
'activity': 'O_r'
},
'f': { # Recurrent state
'weight': 'o_f',
'activity': 'O_f'
},
},
'xi': {
'r': { # Recurrent state
'weight': 'xi',
}
},
'alpha': {
'r': { # Recurrent state
'weight': 'alpha',
}
},
'beta': {
'r': { # Recurrent state
'weight': 'beta',
}
},
'mu': {
'r': { # Recurrent state
'weight': 'mu',
}
},
'nu': {
'r': { # Recurrent state
'weight': 'nu',
}
},
'zeta': {
'r': { # Recurrent state
'weight': 'zeta',
}
},
'gamma': {
'r': { # Recurrent state
'weight': 'gamma',
}
},
'delta': {
'r': { # Recurrent state
'weight': 'delta',
}
}
}
# tuned summation: pooling in h, w dimensions
#############################################
q_array = np.ones(self.q_shape) / np.prod(self.q_shape)
if 'Q' in self.lesions:
q_array = np.zeros_like(q_array).astype(np.float32)
print 'Lesioning CRF excitation.'
setattr(
self, self.weight_dict['Q']['r']['weight'],
tf.get_variable(name=self.weight_dict['Q']['r']['weight'],
dtype=self.dtype,
initializer=q_array.astype(np.float32),
trainable=False))
# untuned suppression: reduction across feature axis
####################################################
u_array = np.ones(self.u_shape) / np.prod(self.u_shape)
if 'U' in self.lesions:
u_array = np.zeros_like(u_array).astype(np.float32)
print 'Lesioning CRF inhibition.'
setattr(
self, self.weight_dict['U']['r']['weight'],
tf.get_variable(name=self.weight_dict['U']['r']['weight'],
dtype=self.dtype,
initializer=u_array.astype(np.float32),
trainable=False))
# weakly tuned summation: pooling in h, w dimensions
#############################################
p_array = np.ones(self.p_shape)
p_array = p_array / p_array.sum()
if 'P' in self.lesions:
print 'Lesioning near eCRF.'
p_array = np.zeros_like(p_array).astype(np.float32)
# Association field is fully learnable
if self.association_field and 'P' not in self.lesions:
setattr(
self,
self.weight_dict['P']['r']['weight'],
self.symmetric_weights(
tf.get_variable(
#tf.get_variable(
name=self.weight_dict['P']['r']['weight'] +
'_non_symm',
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.p_shape,
uniform=self.normal_initializer),
trainable=True),
self.weight_dict['P']['r']['weight']))
else:
setattr(
self, self.weight_dict['P']['r']['weight'],
tf.get_variable(name=self.weight_dict['P']['r']['weight'],
dtype=self.dtype,
initializer=p_array.astype(np.float32),
trainable=False))
# weakly tuned suppression: pooling in h, w dimensions
###############################################
t_array = np.ones(self.t_shape)
t_array = t_array / t_array.sum()
if 'T' in self.lesions:
print 'Lesioning Far eCRF.'
t_array = np.zeros_like(t_array).astype(np.float32)
setattr(
self, self.weight_dict['T']['r']['weight'],
tf.get_variable(name=self.weight_dict['T']['r']['weight'],
dtype=self.dtype,
initializer=t_array.astype(np.float32),
trainable=False))
# Connectivity tensors -- Q/P/T
if 'Q' in self.lesions:
print 'Lesioning CRF excitation connectivity.'
setattr(
self, self.weight_dict['Q']['r']['tuning'],
tf.get_variable(name=self.weight_dict['Q']['r']['tuning'],
dtype=self.dtype,
trainable=False,
initializer=np.zeros(self.tuning_shape).astype(
np.float32)))
else:
setattr(
self, self.weight_dict['Q']['r']['tuning'],
self.symmetric_weights(
tf.get_variable(
name=self.weight_dict['Q']['r']['tuning'] +
'_non_symm',
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.tuning_shape,
uniform=self.normal_initializer,
mask=None)), self.weight_dict['Q']['r']['tuning']))
if not self.association_field:
# Need a tuning tensor for near surround
if 'P' in self.lesions:
print 'Lesioning near eCRF connectivity.'
setattr(
self, self.weight_dict['P']['r']['tuning'],
tf.get_variable(name=self.weight_dict['P']['r']['tuning'],
dtype=self.dtype,
trainable=False,
initializer=np.zeros(
self.tuning_shape).astype(np.float32)))
else:
setattr(
self, self.weight_dict['P']['r']['tuning'],
self.symmetric_weights(
tf.get_variable(
name=self.weight_dict['P']['r']['tuning'] +
'_non_symm',
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.tuning_shape,
uniform=self.normal_initializer,
mask=None)),
self.weight_dict['P']['r']['tuning']))
if 'T' in self.lesions:
print 'Lesioning far eCRF connectivity.'
setattr(
self, self.weight_dict['T']['r']['tuning'],
tf.get_variable(name=self.weight_dict['T']['r']['tuning'],
dtype=self.dtype,
trainable=False,
initializer=np.zeros(self.tuning_shape).astype(
np.float32)))
else:
setattr(
self, self.weight_dict['T']['r']['tuning'],
self.symmetric_weights(
tf.get_variable(
name=self.weight_dict['T']['r']['tuning'] +
'_non_symm',
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.tuning_shape,
uniform=self.normal_initializer,
mask=None)), self.weight_dict['T']['r']['tuning']))
# Input
setattr(
self, self.weight_dict['I']['r']['weight'],
tf.get_variable(name=self.weight_dict['I']['r']['weight'],
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.i_shape,
uniform=self.normal_initializer,
mask=None)))
setattr(
self, self.weight_dict['I']['f']['weight'],
tf.get_variable(name=self.weight_dict['I']['f']['weight'],
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.i_shape,
uniform=self.normal_initializer,
mask=None)))
setattr(
self, self.weight_dict['I']['r']['bias'],
tf.get_variable(name=self.weight_dict['I']['r']['bias'],
dtype=self.dtype,
trainable=True,
initializer=tf.ones(self.bias_shape)))
# Output
setattr(
self, self.weight_dict['O']['r']['weight'],
tf.get_variable(name=self.weight_dict['O']['r']['weight'],
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.o_shape,
uniform=self.normal_initializer,
mask=None)))
setattr(
self, self.weight_dict['O']['f']['weight'],
tf.get_variable(name=self.weight_dict['O']['f']['weight'],
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.o_shape,
uniform=self.normal_initializer,
mask=None)))
setattr(
self, self.weight_dict['O']['r']['bias'],
tf.get_variable(name=self.weight_dict['O']['r']['bias'],
dtype=self.dtype,
trainable=True,
initializer=tf.ones(self.bias_shape)))
# Vector weights
w_array = np.ones([1, 1, 1, self.k]).astype(np.float32)
b_array = np.zeros([1, 1, 1, self.k]).astype(np.float32)
self.xi = tf.get_variable(name='xi', initializer=w_array)
self.alpha = tf.get_variable(name='alpha', initializer=w_array)
self.beta = tf.get_variable(name='beta', initializer=w_array)
self.mu = tf.get_variable(name='mu', initializer=b_array)
self.nu = tf.get_variable(name='nu', initializer=b_array)
self.zeta = tf.get_variable(name='zeta', initializer=w_array)
self.gamma = tf.get_variable(name='gamma', initializer=w_array)
self.delta = tf.get_variable(name='delta', initializer=w_array)
def prepare_tensors(self):
""" Prepare recurrent/forward weight matrices."""
self.weight_dict = { # Weights lower/activity upper
'U': {
'r': {
'weight': 'u_r',
'activity': 'U_r'
}
},
'P': {
'r': {
'weight': 'p_r',
'activity': 'P_r',
'tuning': 'p_t'
}
},
'Q': {
'r': {
'weight': 'q_r',
'activity': 'Q_r',
'tuning': 'q_t'
}
},
'I': {
'r': { # Recurrent state
'weight': 'i_r',
'bias': 'i_b',
'activity': 'I_r'
},
'f': { # Recurrent state
'weight': 'i_f',
'activity': 'I_f'
},
},
'O': {
'r': { # Recurrent state
'weight': 'o_r',
'bias': 'o_b',
'activity': 'O_r'
},
'f': { # Recurrent state
'weight': 'o_f',
'activity': 'O_f'
},
},
'xi': {
'r': { # Recurrent state
'weight': 'xi',
}
},
'alpha': {
'r': { # Recurrent state
'weight': 'alpha',
}
},
'beta': {
'r': { # Recurrent state
'weight': 'beta',
}
},
'mu': {
'r': { # Recurrent state
'weight': 'mu',
}
},
'nu': {
'r': { # Recurrent state
'weight': 'nu',
}
},
'zeta': {
'r': { # Recurrent state
'weight': 'zeta',
}
},
'gamma': {
'r': { # Recurrent state
'weight': 'gamma',
}
},
'phi': {
'r': { # Recurrent state
'weight': 'phi',
}
},
'kappa': {
'r': { # Recurrent state
'weight': 'kappa',
}
},
'delta': {
'r': { # Recurrent state
'weight': 'delta',
}
}
}
# tuned summation: pooling in h, w dimensions
#############################################
q_array = np.ones(self.q_shape) / np.prod(self.q_shape)
if 'Q' in self.lesions:
q_array = np.zeros_like(q_array).astype(np.float32)
print 'Lesioning CRF excitation.'
setattr(
self, self.weight_dict['Q']['r']['weight'],
tf.get_variable(name=self.weight_dict['Q']['r']['weight'],
dtype=self.dtype,
initializer=q_array.astype(np.float32),
trainable=False))
# untuned suppression: reduction across feature axis
####################################################
u_array = np.ones(self.u_shape) / np.prod(self.u_shape)
if 'U' in self.lesions:
u_array = np.zeros_like(u_array).astype(np.float32)
print 'Lesioning CRF inhibition.'
setattr(
self, self.weight_dict['U']['r']['weight'],
tf.get_variable(name=self.weight_dict['U']['r']['weight'],
dtype=self.dtype,
initializer=u_array.astype(np.float32),
trainable=False))
# weakly tuned summation: pooling in h, w dimensions
#############################################
if isinstance(self.p_shape[0], list) and 'P' not in self.lesions:
# VGG-style filters
for pidx, pext in enumerate(self.p_shape):
if pidx == 0:
it_key = self.weight_dict['P']['r']['weight']
else:
self.weight_dict['P']['r']['weight_%s' %
pidx] = 'p_r_%s' % pidx
it_key = self.weight_dict['P']['r']['weight_%s' % pidx]
setattr(
self, it_key,
tf.get_variable(
name=it_key,
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=pext, uniform=self.normal_initializer),
trainable=True))
else:
p_array = np.ones(self.p_shape)
p_array[self.SSN // 2 -
py_utils.ifloor(self.SRF / 2.0):self.SSF // 2 +
py_utils.iceil(self.SSN / 2.0), self.SSN // 2 -
py_utils.ifloor(self.SRF / 2.0):self.SSF // 2 +
py_utils.iceil(self.SSN / 2.0), :, # exclude CRF!
:] = 0.0
p_array = p_array / p_array.sum()
if 'P' in self.lesions:
print 'Lesioning near eCRF.'
p_array = np.zeros_like(p_array).astype(np.float32)
# Association field is fully learnable
if self.association_field and 'P' not in self.lesions:
setattr(
self, self.weight_dict['P']['r']['weight'],
tf.get_variable(
name=self.weight_dict['P']['r']['weight'],
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.p_shape,
uniform=self.normal_initializer),
trainable=True))
else:
setattr(
self, self.weight_dict['P']['r']['weight'],
tf.get_variable(name=self.weight_dict['P']['r']['weight'],
dtype=self.dtype,
initializer=p_array.astype(np.float32),
trainable=False))
# Connectivity tensors -- Q/P/T
if 'Q' in self.lesions:
print 'Lesioning CRF excitation connectivity.'
setattr(
self, self.weight_dict['Q']['r']['tuning'],
tf.get_variable(name=self.weight_dict['Q']['r']['tuning'],
dtype=self.dtype,
trainable=False,
initializer=np.zeros(self.tuning_shape).astype(
np.float32)))
else:
setattr(
self, self.weight_dict['Q']['r']['tuning'],
tf.get_variable(name=self.weight_dict['Q']['r']['tuning'],
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.tuning_shape,
uniform=self.normal_initializer,
mask=None)))
# Gate weights
setattr(
self, self.weight_dict['I']['r']['weight'],
tf.get_variable(name=self.weight_dict['I']['r']['weight'],
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.i_shape,
uniform=self.normal_initializer,
mask=None)))
setattr(
self, self.weight_dict['I']['f']['weight'],
tf.get_variable(name=self.weight_dict['I']['f']['weight'],
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.i_shape,
uniform=self.normal_initializer,
mask=None)))
setattr(
self, self.weight_dict['I']['r']['bias'],
tf.get_variable(name=self.weight_dict['I']['r']['bias'],
dtype=self.dtype,
trainable=True,
initializer=tf.ones(self.bias_shape)))
# Output
setattr(
self, self.weight_dict['O']['r']['weight'],
tf.get_variable(name=self.weight_dict['O']['r']['weight'],
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.o_shape,
uniform=self.normal_initializer,
mask=None)))
setattr(
self, self.weight_dict['O']['f']['weight'],
tf.get_variable(name=self.weight_dict['O']['f']['weight'],
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.o_shape,
uniform=self.normal_initializer,
mask=None)))
setattr( # TODO: smart initialization of these
self, self.weight_dict['O']['r']['bias'],
tf.get_variable(name=self.weight_dict['O']['r']['bias'],
dtype=self.dtype,
trainable=True,
initializer=tf.ones(self.bias_shape)))
# Degree of freedom weights (vectors)
w_array = np.ones([1, 1, 1, self.k]).astype(np.float32)
b_array = np.zeros([1, 1, 1, self.k]).astype(np.float32)
# Divisive params
self.alpha = tf.get_variable(name='alpha', initializer=w_array)
self.beta = tf.get_variable(name='beta', initializer=w_array)
# Subtractive params
self.mu = tf.get_variable(name='mu', initializer=b_array)
self.nu = tf.get_variable(name='nu', initializer=b_array)
if self.zeta:
self.zeta = tf.get_variable(name='zeta', initializer=w_array)
else:
self.zeta = tf.constant(1.)
if self.gamma:
self.gamma = tf.get_variable(name='gamma', initializer=w_array)
else:
self.gamma = tf.constant(1.)
if self.delta:
self.delta = tf.get_variable(name='delta', initializer=w_array)
else:
self.delta = tf.constant(1.)
if self.xi:
self.xi = tf.get_variable(name='xi', initializer=w_array)
else:
self.xi = tf.constant(1.)
if self.multiplicative_excitation:
self.kappa = tf.get_variable(name='kappa', initializer=w_array)
self.omega = tf.get_variable(name='omega', initializer=w_array)
else:
self.kappa = tf.constant(1.)
self.omega = tf.constant(1.)
def prepare_tensors(self):
""" Prepare recurrent/forward weight matrices.
(np.prod([h, w, k]) / 2) - k params in the surround filter
"""
if self.constrain:
constraint = lambda x: tf.clip_by_value(x, 0, np.infty)
else:
constraint = None
self.var_scope = '%s_hgru_weights' % self.layer_name
with tf.variable_scope(self.var_scope):
if self.symmetric_weights and self.symmetric_inits:
h_init = self.symmetric_init(
initialization.xavier_initializer(
shape=self.h_shape, uniform=self.normal_initializer))
else:
h_init = initialization.xavier_initializer(
shape=self.h_shape, uniform=self.normal_initializer)
self.horizontal_kernels = tf.get_variable(name='%s_horizontal' %
self.layer_name,
dtype=self.dtype,
initializer=h_init,
trainable=self.train)
if self.symmetric_gate_weights and self.symmetric_inits:
g_init = self.symmetric_init(
initialization.xavier_initializer(
shape=self.g_shape, uniform=self.normal_initializer))
m_init = self.symmetric_init(
initialization.xavier_initializer(
shape=self.m_shape, uniform=self.normal_initializer))
else:
g_init = initialization.xavier_initializer(
shape=self.g_shape, uniform=self.normal_initializer)
m_init = initialization.xavier_initializer(
shape=self.m_shape, uniform=self.normal_initializer)
self.gain_kernels = tf.get_variable(name='%s_gain' %
self.layer_name,
dtype=self.dtype,
trainable=self.train,
initializer=g_init)
self.mix_kernels = tf.get_variable(name='%s_mix' % self.layer_name,
dtype=self.dtype,
trainable=self.train,
initializer=m_init)
# Gain bias
if self.gate_bias_init == 'chronos':
bias_init = -tf.log(
tf.random_uniform(
self.bias_shape, minval=1, maxval=self.timesteps - 1))
else:
bias_init = -tf.ones(self.bias_shape)
self.gain_bias = tf.get_variable(name='%s_gain_bias' %
self.layer_name,
dtype=self.dtype,
trainable=self.train,
initializer=bias_init)
if self.gate_bias_init == 'chronos':
bias_init = -bias_init
else:
bias_init = tf.ones(self.bias_shape)
self.mix_bias = tf.get_variable(name='%s_mix_bias' %
self.layer_name,
dtype=self.dtype,
trainable=self.train,
initializer=bias_init)
# Divisive params
if self.alpha and not self.lesion_alpha:
self.alpha = tf.get_variable(
name='%s_alpha' % self.layer_name,
trainable=self.train,
constraint=constraint,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
uniform=self.normal_initializer,
mask=None))
elif self.lesion_alpha:
self.alpha = tf.constant(0.)
else:
self.alpha = tf.constant(1.)
if self.mu and not self.lesion_mu:
self.mu = tf.get_variable(
name='%s_mu' % self.layer_name,
trainable=self.train,
constraint=constraint,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
uniform=self.normal_initializer,
mask=None))
elif self.lesion_mu:
self.mu = tf.constant(0.)
else:
self.mu = tf.constant(1.)
if self.gamma:
self.gamma = tf.get_variable(
name='%s_gamma' % self.layer_name,
trainable=self.train,
constraint=constraint,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
uniform=self.normal_initializer,
mask=None))
else:
self.gamma = tf.constant(1.)
if self.multiplicative_excitation:
if self.lesion_kappa:
self.kappa = tf.constant(0.)
else:
self.kappa = tf.get_variable(
name='%s_kappa' % self.layer_name,
trainable=self.train,
constraint=constraint,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
uniform=self.normal_initializer,
mask=None))
if self.lesion_omega:
self.omega = tf.constant(0.)
else:
self.omega = tf.get_variable(
name='%s_omega' % self.layer_name,
trainable=self.train,
constraint=constraint,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
uniform=self.normal_initializer,
mask=None))
else:
self.kappa = tf.constant(1.)
self.omega = tf.constant(1.)
if self.adaptation:
self.eta = tf.get_variable(
trainable=self.train,
name='%s_eta' % self.layer_name,
shape=[self.timesteps],
initializer=tf.random_uniform_initializer)
if self.lesion_omega:
self.omega = tf.constant(0.)
if self.lesion_kappa:
self.kappa = tf.constant(0.)
if self.reuse:
# Make the batchnorm variables
scopes = ['g1_bn', 'g2_bn', 'c1_bn', 'c2_bn']
bn_vars = ['moving_mean', 'moving_variance', 'gamma']
for s in scopes:
with tf.variable_scope(s) as scope:
for v in bn_vars:
tf.get_variable(
trainable=self.param_trainable[v],
name=v,
shape=[self.k],
collections=self.param_collections[v],
initializer=self.param_initializer[v])
self.param_initializer = None
def prepare_tensors(self):
""" Prepare recurrent/forward weight matrices.
(np.prod([h, w, k]) / 2) - k params in the surround filter
"""
# Create FF vars
setattr(
self, 'resize_kernel',
tf.get_variable(name='%s_resize_kernel' % self.layer_name,
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.pooling_kernel + [self.k, self.k],
uniform=self.normal_initializer),
trainable=True))
setattr(
self, 'resize_bias',
tf.get_variable(name='%s_resize_bias' % self.layer_name,
dtype=self.dtype,
initializer=tf.ones([self.k]),
trainable=True))
for idx, (ff_filters, ff_kernel) in enumerate(
zip(self.intermediate_ff, self.intermediate_ks)):
setattr(
self, 'intermediate_kernel_%s' % idx,
tf.get_variable(name='%s_ffdrive_kernel_%s' %
(self.layer_name, idx),
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=ff_kernel + [ff_filters, ff_filters],
uniform=self.normal_initializer),
trainable=True))
setattr(
self, 'intermediate_bias_%s' % idx,
tf.get_variable(name='%s_ffdrive_bias_%s' %
(self.layer_name, idx),
dtype=self.dtype,
initializer=tf.ones([ff_filters]),
trainable=True))
# Create recurrent vars
for idx, layer in enumerate(self.hgru_ids):
with tf.variable_scope('%s_hgru_weights_%s' %
(self.layer_name, layer)):
setattr(
self, 'horizontal_kernels_%s' % layer,
tf.get_variable(
name='%s_horizontal' % self.layer_name,
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=self.h_ext[idx] + [self.k, self.k],
uniform=self.normal_initializer),
trainable=True))
setattr(
self, 'gain_kernels_%s' % layer,
tf.get_variable(
name='%s_gain' % self.layer_name,
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.g_shape,
uniform=self.normal_initializer,
mask=None)))
setattr(
self, 'mix_kernels_%s' % layer,
tf.get_variable(
name='%s_mix' % self.layer_name,
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.m_shape,
uniform=self.normal_initializer,
mask=None)))
# Gain bias
if self.gate_bias_init == 'chronos':
bias_init = -tf.log(
tf.random_uniform(self.bias_shape,
minval=1,
maxval=self.timesteps - 1))
else:
bias_init = tf.ones(self.bias_shape)
setattr(
self, 'gain_bias_%s' % layer,
tf.get_variable(name='%s_gain_bias' % self.layer_name,
dtype=self.dtype,
trainable=True,
initializer=bias_init))
if self.gate_bias_init == 'chronos':
bias_init = -bias_init
else:
bias_init = tf.ones(self.bias_shape)
setattr(
self, 'mix_bias_%s' % layer,
tf.get_variable(name='%s_mix_bias' % self.layer_name,
dtype=self.dtype,
trainable=True,
initializer=bias_init))
# Divisive params
if self.alpha and not self.lesion_alpha:
setattr(
self, 'alpha_%s' % layer,
tf.get_variable(
name='%s_alpha' % self.layer_name,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
uniform=self.normal_initializer,
mask=None)))
elif self.lesion_alpha:
setattr(self, 'alpha_%s' % layer, tf.constant(0.))
else:
setattr(self, 'alpha_%s' % layer, tf.constant(1.))
if self.mu and not self.lesion_mu:
setattr(
self, 'mu_%s' % layer,
tf.get_variable(
name='%s_mu' % self.layer_name,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
uniform=self.normal_initializer,
mask=None)))
elif self.lesion_mu:
setattr(self, 'mu_%s' % layer, tf.constant(0.))
else:
setattr(self, 'mu_%s' % layer, tf.constant(1.))
if self.gamma:
setattr(
self, 'gamma_%s' % layer,
tf.get_variable(
name='%s_gamma' % self.layer_name,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
uniform=self.normal_initializer,
mask=None)))
else:
setattr(self, 'gamma_%s' % layer, tf.constant(1.))
if self.multiplicative_excitation:
if self.lesion_kappa:
setattr(self, 'kappa_%s' % layer, tf.constant(0.))
else:
setattr(
self, 'kappa_%s' % layer,
tf.get_variable(
name='%s_kappa' % self.layer_name,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
uniform=self.normal_initializer,
mask=None)))
if self.lesion_omega:
setattr(self, 'omega_%s' % layer, tf.constant(0.))
else:
setattr(
self, 'omega_%s' % layer,
tf.get_variable(
name='%s_omega' % self.layer_name,
initializer=initialization.xavier_initializer(
shape=self.bias_shape,
uniform=self.normal_initializer,
mask=None)))
else:
setattr(self, 'kappa_%s' % layer, tf.constant(1.))
setattr(self, 'omega_%s' % layer, tf.constant(1.))
if self.adapation:
setattr(
self, 'eta_%s' % layer,
tf.get_variable(
name='%s_eta' % self.layer_name,
initializer=tf.random_uniform([self.timesteps],
dtype=tf.float32)))
if self.lesion_omega:
setattr(self, 'omega_%s' % layer, tf.constant(0.))
if self.lesion_kappa:
setattr(self, 'kappa_%s' % layer, tf.constant(0.))
if self.reuse:
# Make the batchnorm variables
scopes = ['g1_bn', 'g2_bn', 'c1_bn', 'c2_bn']
bn_vars = ['moving_mean', 'moving_variance', 'gamma']
for s in scopes:
with tf.variable_scope(s):
for v in bn_vars:
tf.get_variable(
trainable=self.param_trainable[v],
name=v,
shape=[self.k],
collections=self.param_collections[v],
initializer=self.param_initializer[v])
self.param_initializer = None
def prepare_tensors(self):
""" Prepare recurrent/forward weight matrices.
9 * k + (2 * k^2) params in the greek letters/gates.
(np.prod([h, w, k]) / 2) - k params in the surround filter
"""
self.weight_dict = { # Weights lower/activity upper
'P': {
'r': {
'weight': 'p_r',
'activity': 'P_r',
'tuning': 'p_t',
# 'bias': 'i_b'
}
},
'I': {
'r': { # Recurrent state
'weight': 'i_r',
'bias': 'i_b',
'activity': 'I_r'
},
# 'f': { # Recurrent state
# 'weight': 'i_f',
# 'activity': 'I_f'
# },
},
'O': {
'r': { # Recurrent state
'weight': 'o_r',
'bias': 'o_b',
'activity': 'O_r'
},
# 'f': { # Recurrent state
# 'weight': 'o_f',
# 'activity': 'O_f'
# },
},
'xi': {
'r': { # Recurrent state
'weight': 'xi',
}
},
# 'alpha': {
# 'r': { # Recurrent state
# 'weight': 'alpha',
# }
# },
'beta': {
'r': { # Recurrent state
'weight': 'beta',
}
},
# 'mu': {
# 'r': { # Recurrent state
# 'weight': 'mu',
# }
# },
'nu': {
'r': { # Recurrent state
'weight': 'nu',
}
},
'zeta': {
'r': { # Recurrent state
'weight': 'zeta',
}
},
'gamma': {
'r': { # Recurrent state
'weight': 'gamma',
}
},
'phi': {
'r': { # Recurrent state
'weight': 'phi',
}
},
'kappa': {
'r': { # Recurrent state
'weight': 'kappa',
}
},
'rho': {
'r': { # Recurrent state
'weight': 'rho',
}
},
}
# weakly tuned summation: pooling in h, w dimensions
#############################################
with tf.variable_scope('contextual_circuit'):
if isinstance(self.p_shape[0], list) and 'P' not in self.lesions:
# VGG-style filters
for pidx, pext in enumerate(self.p_shape):
if pidx == 0:
it_key = self.weight_dict['P']['r']['weight']
else:
self.weight_dict['P']['r']['weight_%s' %
pidx] = 'p_r_%s' % pidx
it_key = self.weight_dict['P']['r']['weight_%s' % pidx]
setattr(
self, it_key,
tf.get_variable(
name=it_key,
dtype=self.dtype,
initializer=initialization.xavier_initializer(
shape=pext, uniform=self.normal_initializer),
trainable=True))
else:
p_array = np.ones(self.p_shape)
p_array[self.SSN // 2 -
py_utils.ifloor(self.SRF / 2.0):self.SSF // 2 +
py_utils.iceil(self.SSN / 2.0), self.SSN // 2 -
py_utils.ifloor(self.SRF / 2.0):self.SSF // 2 +
py_utils.iceil(self.SSN / 2.0), :, # exclude CRF!
:] = 0.0
p_array = p_array / p_array.sum()
if 'P' in self.lesions:
print 'Lesioning near eCRF.'
p_array = np.zeros_like(p_array).astype(np.float32)
# Association field is fully learnable
if self.association_field and 'P' not in self.lesions:
setattr(
self,
self.weight_dict['P']['r']['weight'],
tf.get_variable(
name=self.weight_dict['P']['r']['weight'],
dtype=self.dtype,
# shape=self.p_shape,
initializer=initialization.xavier_initializer(
shape=self.p_shape,
uniform=self.normal_initializer),
trainable=True))
else:
setattr(
self, self.weight_dict['P']['r']['weight'],
tf.get_variable(
name=self.weight_dict['P']['r']['weight'],
dtype=self.dtype,
initializer=p_array.astype(np.float32),
trainable=False))
# Gate weights
setattr(
self, self.weight_dict['I']['r']['weight'],
tf.get_variable(name=self.weight_dict['I']['r']['weight'],
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.i_shape,
uniform=self.normal_initializer,
mask=None)))
# setattr(
# self,
# self.weight_dict['I']['f']['weight'],
# tf.get_variable(
# name=self.weight_dict['I']['f']['weight'],
# dtype=self.dtype,
# trainable=True,
# initializer=initialization.xavier_initializer(
# shape=self.i_shape,
# uniform=self.normal_initializer,
# mask=None)))
if self.gate_bias_init == 'chronos':
bias_init = -tf.log(
tf.random_uniform(
self.bias_shape, minval=1, maxval=self.timesteps - 1))
else:
bias_init = tf.ones(self.bias_shape)
setattr(
self, self.weight_dict['I']['r']['bias'],
tf.get_variable(name=self.weight_dict['I']['r']['bias'],
dtype=self.dtype,
trainable=True,
initializer=bias_init))
# Output
setattr(
self, self.weight_dict['O']['r']['weight'],
tf.get_variable(name=self.weight_dict['O']['r']['weight'],
dtype=self.dtype,
trainable=True,
initializer=initialization.xavier_initializer(
shape=self.o_shape,
uniform=self.normal_initializer,
mask=None)))
# setattr(
# self,
# self.weight_dict['O']['f']['weight'],
# tf.get_variable(
# name=self.weight_dict['O']['f']['weight'],
# dtype=self.dtype,
# trainable=True,
# initializer=initialization.xavier_initializer(
# shape=self.o_shape,
# uniform=self.normal_initializer,
# mask=None)))
if self.gate_bias_init == 'chronos':
# bias_init = -tf.log(
# tf.random_uniform(
# self.bias_shape, minval=1, maxval=self.timesteps - 1))
bias_init = -bias_init
else:
bias_init = tf.ones(self.bias_shape)
setattr( # TODO: smart initialization of these
self, self.weight_dict['O']['r']['bias'],
tf.get_variable(name=self.weight_dict['O']['r']['bias'],
dtype=self.dtype,
trainable=True,
initializer=bias_init))
# Degree of freedom weights (vectors)
w_shape = [1, 1, 1, self.k]
b_shape = [1, 1, 1, self.k]
# w_array = np.ones(w_shape).astype(np.float32)
# b_array = np.zeros(b_shape).astype(np.float32)
# Divisive params
if self.beta and not self.lesion_beta:
self.beta = tf.get_variable(
name='beta',
initializer=initialization.xavier_initializer(
shape=w_shape,
uniform=self.normal_initializer,
mask=None))
# initializer=tf.ones(w_shape, dtype=tf.float32))
elif self.lesion_beta:
self.beta = tf.constant(0.)
else:
self.beta = tf.constant(1.)
if self.nu and not self.lesion_nu:
self.nu = tf.get_variable(
name='nu',
initializer=initialization.xavier_initializer(
shape=b_shape,
uniform=self.normal_initializer,
mask=None))
# initializer=tf.zeros(b_shape, dtype=tf.float32))
elif self.lesion_nu:
self.nu = tf.constant(0.)
else:
self.nu = tf.constant(1.)
if self.zeta:
self.zeta = tf.get_variable(
name='zeta',
initializer=initialization.xavier_initializer(
shape=w_shape,
uniform=self.normal_initializer,
mask=None))
else:
self.zeta = tf.constant(1.)
if self.gamma:
self.gamma = tf.get_variable(
name='gamma',
initializer=initialization.xavier_initializer(
shape=w_shape,
uniform=self.normal_initializer,
mask=None))
else:
self.gamma = tf.constant(1.)
# # TODO
# self.ebias = tf.get_variable(
# name='ebias',
# initializer=initialization.xavier_initializer(
# shape=b_shape,
# uniform=self.normal_initializer,
# mask=None))
if self.xi:
self.xi = tf.get_variable(
name='xi',
initializer=initialization.xavier_initializer(
shape=w_shape,
uniform=self.normal_initializer,
mask=None))
else:
self.xi = tf.constant(1.)
if self.multiplicative_excitation:
if self.lesion_kappa:
self.kappa = tf.constant(0.)
else:
self.kappa = tf.get_variable(
name='kappa',
initializer=initialization.xavier_initializer(
shape=w_shape,
uniform=self.normal_initializer,
mask=None))
# initializer=tf.zeros(w_shape, dtype=tf.float32) + 0.5)
if self.lesion_omega:
self.omega = tf.constant(0.)
else:
self.omega = tf.get_variable(
name='omega',
initializer=initialization.xavier_initializer(
shape=w_shape,
uniform=self.normal_initializer,
mask=None))
# initializer=tf.zeros(w_shape, dtype=tf.float32) + 0.5)
else:
self.kappa = tf.constant(1.)
self.omega = tf.constant(1.)
if self.adapation:
self.rho = tf.get_variable(name='rho',
initializer=tf.ones(
self.timesteps,
dtype=tf.float32))
if self.lesion_omega:
self.omega = tf.constant(0.)
if self.lesion_kappa:
self.kappa = tf.constant(0.)
self.lateral_bias = tf.get_variable(
name='lateral_bias',
initializer=initialization.xavier_initializer(
shape=b_shape, uniform=self.normal_initializer, mask=None))