def build(self, x, ffn_seed=None): """Run the backprop version of the Circuit.""" self.prepare_tensors() # Calculate hidden state size x_shape = x.get_shape().as_list() l1_h2_shape = x_shape # l2 int_shape = list(x_shape[1:-1]) str = self.ds_stride_list[0] for dim in range(len(int_shape)): int_shape[dim] = self.compute_shape(int_shape[dim], str[dim]) l2_h2_shape = [x_shape[0]] + int_shape + [self.ds_k_list[0]] # l3 int_shape = list(l2_h2_shape[1:-1]) str = self.ds_stride_list[1] for dim in range(len(int_shape)): int_shape[dim] = self.compute_shape(int_shape[dim], str[dim]) l3_h2_shape = [x_shape[0]] + int_shape + [self.ds_k_list[1]] # Initialize hidden layer activities if ffn_seed is not None: l1_h2 = tf.ones(l1_h2_shape, dtype=self.dtype) * ffn_seed * 2 - 1 if self.use_trainable_states: if ffn_seed is None: l1_h2 = tf.get_variable( name='l1_h2', dtype=self.dtype, initializer=initialization.xavier_initializer( shape=[1] + l1_h2_shape[1:], dtype=self.dtype, uniform=True), trainable=self.train) l1_h2 = tf.tile(l1_h2, [x_shape[0]] + [1]*(4 if self.use_3d else 3)) l2_h2 = tf.get_variable( name='l2_h2', dtype=self.dtype, initializer=initialization.xavier_initializer( shape=[1] + l2_h2_shape[1:], dtype=self.dtype, uniform=True), trainable=self.train) l2_h2 = tf.tile(l2_h2, [x_shape[0]] + [1]*(4 if self.use_3d else 3)) l3_h2 = tf.get_variable( name='l3_h2', dtype=self.dtype, initializer=initialization.xavier_initializer( shape=[1] + l3_h2_shape[1:], dtype=self.dtype, uniform=True), trainable=self.train) l3_h2 = tf.tile(l3_h2, [x_shape[0]] + [1]*(4 if self.use_3d else 3)) else: if ffn_seed is None: l1_h2 = tf.zeros(l1_h2_shape, dtype=self.dtype) l2_h2 = tf.zeros(l2_h2_shape, dtype=self.dtype) l3_h2 = tf.zeros(l3_h2_shape, dtype=self.dtype) for i in range(self.timesteps): l1_h2_out, l2_h2_out, l3_h2_out = self.full(x, l1_h2, l2_h2, l3_h2, i) l1_h2 = l1_h2_out l2_h2 = l2_h2_out l3_h2 = l3_h2_out return l1_h2
def prepare_tensors(self): """ Prepare recurrent/forward weight matrices. (np.prod([h, w, k]) / 2) - k params in the surround filter """ # FEEDFORWARD KERNELS lower_feats = self.in_k for idx, (higher_feats, ff_dhw) in enumerate(zip(self.ff_conv_k, self.ff_conv_dhw)): with tf.variable_scope('ff_%s' % idx): setattr( self, 'ff_%s_weights' % idx, tf.get_variable( name='weights', dtype=self.dtype, initializer=initialization.xavier_initializer( shape=ff_dhw + [lower_feats * 3, higher_feats], dtype=self.dtype, uniform=self.normal_initializer), trainable=True)) setattr( self, 'ff_%s_bias' % idx, tf.get_variable(name='bias', dtype=self.dtype, initializer=tf.ones([higher_feats], dtype=self.dtype), trainable=True)) lower_feats = higher_feats # FEEDBACK KERNELS lower_feats = self.in_k for idx, (higher_feats, fb_dhw) in enumerate(zip(self.fb_k, self.fb_dhw)): with tf.variable_scope('fb_%s' % idx): setattr( self, 'fb_%s_weights' % idx, tf.get_variable( name='weights', dtype=self.dtype, initializer=initialization.xavier_initializer( shape=fb_dhw + [lower_feats, higher_feats], dtype=self.dtype, uniform=self.normal_initializer), trainable=True)) setattr( self, 'fb_%s_bias' % idx, tf.get_variable( name='bias', dtype=self.dtype, initializer=initialization.xavier_initializer( shape=[lower_feats], dtype=self.dtype, uniform=self.normal_initializer), trainable=True)) lower_feats = higher_feats # HGRU KERNELS for idx in range(len(self.hgru_dhw)): with tf.variable_scope('hgru_%s' % idx): setattr( self, 'hgru_%s_W' % idx, tf.get_variable( name='W', dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.hgru_dhw[idx] + [self.hgru_k[idx], self.hgru_k[idx]], dtype=self.dtype, uniform=self.normal_initializer), trainable=True)) g_shape = [1, 1, 1] + [self.hgru_k[idx], self.hgru_k[idx]] setattr( self, 'hgru_%s_gain_weights' % idx, tf.get_variable( name='gain_weights', dtype=self.dtype, trainable=True, initializer=initialization.xavier_initializer( shape=g_shape, dtype=self.dtype, uniform=self.normal_initializer, mask=None))) m_shape = [1, 1, 1] + [self.hgru_k[idx], self.hgru_k[idx]] setattr( self, 'hgru_%s_mix_weights' % idx, tf.get_variable( name='mix_weights', dtype=self.dtype, trainable=True, initializer=initialization.xavier_initializer( shape=m_shape, dtype=self.dtype, uniform=self.normal_initializer, mask=None))) # Gate bias bias_shape = [1, 1, 1, 1, self.hgru_k[idx]] if self.gate_bias_init == 'chronos': bias_init = -tf.log( tf.random_uniform(bias_shape, minval=1, maxval=self.timesteps - 1, dtype=self.dtype)) else: bias_init = tf.ones(bias_shape, dtype=self.dtype) setattr( self, 'hgru_%s_gain_bias' % idx, tf.get_variable(name='gain_bias', dtype=self.dtype, trainable=True, initializer=bias_init)) if self.gate_bias_init == 'chronos': bias_init = -bias_init else: bias_init = tf.ones(bias_shape, dtype=self.dtype) setattr( self, 'hgru_%s_mix_bias' % idx, tf.get_variable(name='mix_bias', dtype=self.dtype, trainable=True, initializer=bias_init)) # combination params setattr( self, 'hgru_%s_alpha' % idx, tf.get_variable( name='alpha', dtype=self.dtype, initializer=initialization.xavier_initializer( shape=[ 1, 1, 1, self.hgru_k[idx] * 3, self.hgru_k[idx] ], dtype=self.dtype, uniform=self.normal_initializer, mask=None))) setattr( self, 'hgru_%s_mu' % idx, tf.get_variable( name='mu', dtype=self.dtype, initializer=initialization.xavier_initializer( shape=[1, 1, 1, 1, self.hgru_k[idx]], dtype=self.dtype, uniform=self.normal_initializer, mask=None))) setattr( self, 'hgru_%s_kappa' % idx, tf.get_variable( name='kappa', dtype=self.dtype, trainable=False, initializer=initialization.xavier_initializer( shape=[ 1, 1, 1, self.hgru_k[idx] * 3, self.hgru_k[idx] ], dtype=self.dtype, uniform=self.normal_initializer, mask=None))) setattr( self, 'hgru_%s_omega' % idx, tf.get_variable( name='omega', dtype=self.dtype, trainable=False, initializer=initialization.xavier_initializer( shape=[1, 1, 1, 1, self.hgru_k[idx]], dtype=self.dtype, uniform=self.normal_initializer, mask=None))) # adaptation params if self.adapation: setattr( self, 'eta_%s' % idx, tf.get_variable(name='eta', dtype=self.dtype, initializer=tf.ones( [self.timesteps + 1], dtype=tf.float32))) setattr( self, 'eta2_%s' % idx, tf.get_variable(name='eta2', dtype=self.dtype, initializer=tf.ones( [self.timesteps + 1], dtype=tf.float32))) if self.bn_reuse: # Make the batchnorm variables scopes = ['g1_bn', 'g2_bn', 'c1_bn', 'c2_bn'] bn_vars = ['moving_mean', 'moving_variance', 'gamma'] for s in scopes: with tf.variable_scope(s): for v in bn_vars: tf.get_variable( trainable=self.param_trainable[v], name=v, dtype=self.dtype, shape=[self.hgru_k[idx]], collections=self.param_collections[v], initializer=self.param_initializer[v]) self.param_initializer = None
def prepare_tensors(self): # HGRU KERNELS self.hgru1.prepare_tensors() self.hgru2.prepare_tensors() self.hgru3.prepare_tensors() self.hgru_td3.prepare_tensors() self.hgru_td2.prepare_tensors() self.hgru_td1.prepare_tensors() # HOMU # if self.use_homunculus: # self.homunculus = tf.get_variable( # name='homunculus', # dtype=self.dtype, # initializer=tf.zeros(self.timesteps), # trainable=self.train) # DS US KERNELS in_k = self.in_k for i, (fsiz, psiz, out_k) in enumerate(zip(self.ds_fsiz_list, self.ds_pool_list, self.ds_k_list)): for rep in range(self.ds_conv_repeat): with tf.variable_scope('ds%s_%s' % (i,rep)): setattr( self, 'ds%s_%s_w' % (i,rep), tf.get_variable( name='w', dtype=self.dtype, initializer=initialization.xavier_initializer( shape=fsiz + [in_k if rep==0 else out_k, out_k], dtype=self.dtype, uniform=True), trainable=self.train)) with tf.variable_scope('us%s_%s' % (i,rep)): fsiz = [m+n-1 for m,n in zip(fsiz,psiz)] if (rep == self.ds_conv_repeat - 1) else fsiz self.one_by_one = [1 for s in fsiz] setattr( self, 'us%s_%s_w' % (i,rep), tf.get_variable( name='w', dtype=self.dtype, initializer=initialization.xavier_initializer( shape=fsiz + [in_k if rep==0 else out_k, out_k], dtype=self.dtype, uniform=True), trainable=self.train)) # if (i < len(self.ds_fsiz_list) - 1) and self.use_dsus_skip: # with tf.variable_scope('skip%s' % i): # setattr( # self, # 'skip%s_kappa' % i, # tf.get_variable( # name='kappa', # dtype=self.dtype, # initializer=initialization.xavier_initializer( # shape=self.one_by_one + [out_k], # dtype=self.dtype, # uniform=True), # trainable=self.train)) # setattr( # self, # 'skip%s_gamma' % i, # tf.get_variable( # name='gamma', # dtype=self.dtype, # initializer=initialization.xavier_initializer( # shape=self.one_by_one + [out_k], # dtype=self.dtype, # uniform=True), # trainable=self.train)) # setattr( # self, # 'skip%s_omega' % i, # tf.get_variable( # name='omega', # dtype=self.dtype, # initializer=initialization.xavier_initializer( # shape=self.one_by_one + [out_k], # dtype=self.dtype, # uniform=True), # trainable=self.train)) in_k = out_k
def prepare_tensors(self): """ Prepare recurrent/forward weight matrices. (np.prod([h, w, k]) / 2) - k params in the surround filter """ # FEEDFORWARD AND FEEDBACK KERNELS lower_feats = self.in_k for idx, (higher_feats, ff_dhw, fb_dhw) in enumerate( zip(self.ff_conv_k, self.ff_conv_dhw, self.fb_dhw)): setattr( self, 'fb_kernel_%s' % idx, tf.get_variable(name='%s_fb_kernel__%s' % (self.layer_name, idx), dtype=self.dtype, initializer=initialization.xavier_initializer( shape=fb_dhw + [lower_feats, higher_feats], dtype=self.dtype, uniform=self.normal_initializer), trainable=True)) setattr( self, 'fb_bias_%s' % idx, tf.get_variable(name='%s_fb_bias_%s' % (self.layer_name, idx), dtype=self.dtype, initializer=tf.ones([lower_feats], dtype=self.dtype), trainable=True)) setattr( self, 'ff_kernel_%s' % idx, tf.get_variable(name='%s_ff_kernel_%s' % (self.layer_name, idx), dtype=self.dtype, initializer=initialization.xavier_initializer( shape=ff_dhw + [lower_feats, higher_feats], dtype=self.dtype, uniform=self.normal_initializer), trainable=True)) setattr( self, 'ff_bias_%s' % idx, tf.get_variable(name='%s_ff_bias_%s' % (self.layer_name, idx), dtype=self.dtype, initializer=tf.ones([higher_feats], dtype=self.dtype), trainable=True)) lower_feats = higher_feats # HGRU KERNELS for idx, layer in enumerate(self.hgru_ids): with tf.variable_scope('%s_hgru_weights_%s' % (self.layer_name, layer)): setattr( self, 'horizontal_kernels_%s' % layer, tf.get_variable( name='%s_horizontal' % self.layer_name, dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.hgru_dhw[idx] + [self.hgru_k[idx], self.hgru_k[idx]], dtype=self.dtype, uniform=self.normal_initializer), trainable=True)) g_shape = self.hgru_gate_dhw[idx] + [ self.hgru_k[idx], self.hgru_k[idx] ] setattr( self, 'gain_kernels_%s' % layer, tf.get_variable( name='%s_gain' % self.layer_name, dtype=self.dtype, trainable=True, initializer=initialization.xavier_initializer( shape=g_shape, dtype=self.dtype, uniform=self.normal_initializer, mask=None))) m_shape = self.hgru_gate_dhw[idx] + [ self.hgru_k[idx], self.hgru_k[idx] ] setattr( self, 'mix_kernels_%s' % layer, tf.get_variable( name='%s_mix' % self.layer_name, dtype=self.dtype, trainable=True, initializer=initialization.xavier_initializer( shape=m_shape, dtype=self.dtype, uniform=self.normal_initializer, mask=None))) # Gain bias bias_shape = [1, 1, 1, 1, self.hgru_k[idx]] if self.gate_bias_init == 'chronos': bias_init = -tf.log( tf.random_uniform(bias_shape, minval=1, maxval=self.timesteps - 1, dtype=self.dtype)) else: bias_init = tf.ones(bias_shape, dtype=self.dtype) setattr( self, 'gain_bias_%s' % layer, tf.get_variable(name='%s_gain_bias' % self.layer_name, dtype=self.dtype, trainable=True, initializer=bias_init)) if self.gate_bias_init == 'chronos': bias_init = -bias_init else: bias_init = tf.ones(bias_shape, dtype=self.dtype) setattr( self, 'mix_bias_%s' % layer, tf.get_variable(name='%s_mix_bias' % self.layer_name, dtype=self.dtype, trainable=True, initializer=bias_init)) # Divisive params if self.alpha and not self.lesion_alpha: setattr( self, 'alpha_%s' % layer, tf.get_variable( name='%s_alpha' % self.layer_name, dtype=self.dtype, initializer=initialization.xavier_initializer( shape=bias_shape, dtype=self.dtype, uniform=self.normal_initializer, mask=None))) elif self.lesion_alpha: setattr(self, 'alpha_%s' % layer, tf.constant(0.)) else: setattr(self, 'alpha_%s' % layer, tf.constant(1.)) if self.mu and not self.lesion_mu: setattr( self, 'mu_%s' % layer, tf.get_variable( name='%s_mu' % self.layer_name, dtype=self.dtype, initializer=initialization.xavier_initializer( shape=bias_shape, dtype=self.dtype, uniform=self.normal_initializer, mask=None))) elif self.lesion_mu: setattr(self, 'mu_%s' % layer, tf.constant(0.)) else: setattr(self, 'mu_%s' % layer, tf.constant(1.)) if self.gamma: setattr( self, 'gamma_%s' % layer, tf.get_variable( name='%s_gamma' % self.layer_name, dtype=self.dtype, initializer=initialization.xavier_initializer( shape=bias_shape, dtype=self.dtype, uniform=self.normal_initializer, mask=None))) else: setattr(self, 'gamma_%s' % layer, tf.constant(1.)) if self.multiplicative_excitation: if self.lesion_kappa: setattr(self, 'kappa_%s' % layer, tf.constant(0.)) else: setattr( self, 'kappa_%s' % layer, tf.get_variable( name='%s_kappa' % self.layer_name, dtype=self.dtype, initializer=initialization.xavier_initializer( shape=bias_shape, dtype=self.dtype, uniform=self.normal_initializer, mask=None))) if self.lesion_omega: setattr(self, 'omega_%s' % layer, tf.constant(0.)) else: setattr( self, 'omega_%s' % layer, tf.get_variable( name='%s_omega' % self.layer_name, dtype=self.dtype, initializer=initialization.xavier_initializer( shape=bias_shape, dtype=self.dtype, uniform=self.normal_initializer, mask=None))) else: setattr(self, 'kappa_%s' % layer, tf.constant(1.)) setattr(self, 'omega_%s' % layer, tf.constant(1.)) if self.adapation: setattr( self, 'eta_%s' % layer, tf.get_variable( name='%s_eta' % self.layer_name, dtype=self.dtype, initializer=tf.random_uniform([self.timesteps], dtype=tf.float32))) if self.lesion_omega: setattr(self, 'omega_%s' % layer, tf.constant(0.)) if self.lesion_kappa: setattr(self, 'kappa_%s' % layer, tf.constant(0.)) if self.reuse: # Make the batchnorm variables scopes = ['g1_bn', 'g2_bn', 'c1_bn', 'c2_bn'] bn_vars = ['moving_mean', 'moving_variance', 'gamma'] for s in scopes: with tf.variable_scope(s): for v in bn_vars: tf.get_variable( trainable=self.param_trainable[v], name=v, dtype=self.dtype, shape=[self.hgru_k[idx]], collections=self.param_collections[v], initializer=self.param_initializer[v]) self.param_initializer = None
def prepare_tensors(self): local_shape = [1, 1] if not self.use_3d else [1, 1, 1] self.bn_param_initializer = { 'moving_mean': tf.constant_initializer(0., dtype=self.dtype), 'moving_variance': tf.constant_initializer(1., dtype=self.dtype), 'beta': tf.constant_initializer(0., dtype=self.dtype), 'gamma': tf.constant_initializer(0.1, dtype=self.dtype) } with tf.variable_scope(self.var_scope): tf.get_variable( name='h1_w', dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.fsiz + [self.num_channels, self.num_channels], dtype=self.dtype, uniform=True), trainable=self.train) tf.get_variable( name='g1_w', dtype=self.dtype, initializer=initialization.xavier_initializer( shape=local_shape + [self.num_channels, self.num_channels], dtype=self.dtype, uniform=True), trainable=self.train) tf.get_variable(name='g1_b', dtype=self.dtype, initializer=initialization.xavier_initializer( shape=[1] + local_shape + [self.num_channels], dtype=self.dtype, uniform=True), trainable=self.train) tf.get_variable(name='alpha', dtype=self.dtype, initializer=initialization.xavier_initializer( shape=[1] + local_shape + [1], dtype=self.dtype, uniform=True), trainable=self.train) tf.get_variable( name='mu', dtype=self.dtype, initializer=initialization.xavier_initializer( shape=[1] + local_shape + [self.num_channels if self.soft_coefficients else 1], dtype=self.dtype, uniform=True), trainable=self.train) if self.bistream_weights is 'independent': tf.get_variable(name='h2_w', dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.fsiz + [self.num_channels, self.num_channels], dtype=self.dtype, uniform=True), trainable=self.train) tf.get_variable( name='g2_w', dtype=self.dtype, initializer=initialization.xavier_initializer( shape=local_shape + [self.num_channels, self.num_channels], dtype=self.dtype, uniform=True), trainable=self.train) tf.get_variable(name='g2_b', dtype=self.dtype, initializer=initialization.xavier_initializer( shape=[1] + local_shape + [self.num_channels], dtype=self.dtype, uniform=True), trainable=self.train) tf.get_variable(name='kappa', dtype=self.dtype, initializer=initialization.xavier_initializer( shape=[1] + local_shape + [1], dtype=self.dtype, uniform=True), trainable=self.train) tf.get_variable( name='gamma', dtype=self.dtype, initializer=initialization.xavier_initializer( shape=[1] + local_shape + [self.num_channels if self.soft_coefficients else 1], dtype=self.dtype, uniform=True), trainable=self.train) tf.get_variable( name='omega', dtype=self.dtype, initializer=initialization.xavier_initializer( shape=[1] + local_shape + [self.num_channels if self.soft_coefficients else 1], dtype=self.dtype, uniform=True), trainable=self.train) if self.bn_reuse: # Make the batchnorm variables scopes = ['c1/bn', 'c2/bn', 'g1/bn', 'g1/bn'] shapes = [ self.num_channels, self.num_channels, self.num_channels, self.num_channels ] bn_vars = ['moving_mean', 'moving_variance', 'beta', 'gamma'] for (scp, shp) in zip(scopes, shapes): with tf.variable_scope(scp): for v in bn_vars: tf.get_variable( trainable=self.train, name=v, dtype=self.dtype, shape=[shp], initializer=self.bn_param_initializer)
def prepare_tensors(self): """ Prepare recurrent/forward weight matrices. (np.prod([h, w, k]) / 2) - k params in the surround filter """ # Create FF vars for idx, (ff_filters, ff_kernel) in enumerate( zip(self.intermediate_ff, self.intermediate_ks)): setattr( self, 'intermediate_kernel_%s' % idx, tf.get_variable(name='%s_ffdrive_kernel_%s' % (self.layer_name, idx), dtype=self.dtype, initializer=initialization.xavier_initializer( shape=ff_kernel + [ff_filters, ff_filters], dtype=self.dtype, uniform=self.normal_initializer), trainable=True)) setattr( self, 'intermediate_bias_%s' % idx, tf.get_variable(name='%s_ffdrive_bias_%s' % (self.layer_name, idx), dtype=self.dtype, initializer=tf.ones([ff_filters], dtype=self.dtype), trainable=True)) # Create recurrent vars for idx, layer in enumerate(self.hgru_ids): with tf.variable_scope('%s_hgru_weights_%s' % (self.layer_name, layer)): if 'fb' in layer: # Make the FB resize kernels setattr( self, '%s_resize_kernel' % layer, tf.get_variable( name='%s_resize_kernel' % layer, dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.pooling_kernel + [self.k, self.k], dtype=self.dtype, uniform=self.normal_initializer), trainable=True)) setattr( self, '%s_resize_bias' % layer, tf.get_variable(name='%s_resize_bias' % layer, dtype=self.dtype, initializer=tf.ones([self.k], dtype=self.dtype), trainable=True)) setattr( self, 'horizontal_kernels_%s' % layer, tf.get_variable( name='%s_horizontal' % self.layer_name, dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.h_ext[idx] + [self.k, self.k], dtype=self.dtype, uniform=self.normal_initializer), trainable=True)) setattr( self, 'gain_kernels_%s' % layer, tf.get_variable( name='%s_gain' % self.layer_name, dtype=self.dtype, trainable=True, initializer=initialization.xavier_initializer( shape=self.g_shape, dtype=self.dtype, uniform=self.normal_initializer, mask=None))) setattr( self, 'mix_kernels_%s' % layer, tf.get_variable( name='%s_mix' % self.layer_name, dtype=self.dtype, trainable=True, initializer=initialization.xavier_initializer( shape=self.m_shape, dtype=self.dtype, uniform=self.normal_initializer, mask=None))) # Gain bias if self.gate_bias_init == 'chronos': bias_init = -tf.log( tf.random_uniform(self.bias_shape, minval=1, maxval=self.timesteps - 1, dtype=self.dtype)) else: bias_init = tf.ones(self.bias_shape, dtype=self.dtype) setattr( self, 'gain_bias_%s' % layer, tf.get_variable(name='%s_gain_bias' % self.layer_name, dtype=self.dtype, trainable=True, initializer=bias_init)) if self.gate_bias_init == 'chronos': bias_init = -bias_init else: bias_init = tf.ones(self.bias_shape, dtype=self.dtype) setattr( self, 'mix_bias_%s' % layer, tf.get_variable(name='%s_mix_bias' % self.layer_name, dtype=self.dtype, trainable=True, initializer=bias_init)) # Divisive params if self.alpha and not self.lesion_alpha: setattr( self, 'alpha_%s' % layer, tf.get_variable( name='%s_alpha' % self.layer_name, dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.bias_shape, dtype=self.dtype, uniform=self.normal_initializer, mask=None))) elif self.lesion_alpha: setattr(self, 'alpha_%s' % layer, tf.constant(0.)) else: setattr(self, 'alpha_%s' % layer, tf.constant(1.)) if self.mu and not self.lesion_mu: setattr( self, 'mu_%s' % layer, tf.get_variable( name='%s_mu' % self.layer_name, dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.bias_shape, dtype=self.dtype, uniform=self.normal_initializer, mask=None))) elif self.lesion_mu: setattr(self, 'mu_%s' % layer, tf.constant(0.)) else: setattr(self, 'mu_%s' % layer, tf.constant(1.)) if self.gamma: setattr( self, 'gamma_%s' % layer, tf.get_variable( name='%s_gamma' % self.layer_name, dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.bias_shape, dtype=self.dtype, uniform=self.normal_initializer, mask=None))) else: setattr(self, 'gamma_%s' % layer, tf.constant(1.)) if self.multiplicative_excitation: if self.lesion_kappa: setattr(self, 'kappa_%s' % layer, tf.constant(0.)) else: setattr( self, 'kappa_%s' % layer, tf.get_variable( name='%s_kappa' % self.layer_name, dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.bias_shape, dtype=self.dtype, uniform=self.normal_initializer, mask=None))) if self.lesion_omega: setattr(self, 'omega_%s' % layer, tf.constant(0.)) else: setattr( self, 'omega_%s' % layer, tf.get_variable( name='%s_omega' % self.layer_name, dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.bias_shape, dtype=self.dtype, uniform=self.normal_initializer, mask=None))) else: setattr(self, 'kappa_%s' % layer, tf.constant(1.)) setattr(self, 'omega_%s' % layer, tf.constant(1.)) if self.adapation: setattr( self, 'eta_%s' % layer, tf.get_variable( name='%s_eta' % self.layer_name, dtype=self.dtype, initializer=tf.random_uniform([self.timesteps], dtype=tf.float32))) if self.lesion_omega: setattr(self, 'omega_%s' % layer, tf.constant(0.)) if self.lesion_kappa: setattr(self, 'kappa_%s' % layer, tf.constant(0.)) if self.reuse: # Make the batchnorm variables scopes = ['g1_bn', 'g2_bn', 'c1_bn', 'c2_bn'] bn_vars = ['moving_mean', 'moving_variance', 'gamma'] for s in scopes: with tf.variable_scope(s): for v in bn_vars: tf.get_variable( trainable=self.param_trainable[v], name=v, dtype=self.dtype, shape=[self.k], collections=self.param_collections[v], initializer=self.param_initializer[v]) self.param_initializer = None
def prepare_tensors(self): """ """ # HORIZONTAL FILTERS self.caps_filter = \ tf.get_variable( name='%s_caps_filter' % (self.name), dtype=self.dtype, initializer=initialization.xavier_initializer( shape=[self.c_h_ext, self.c_h_ext] + [self.f * self.c, self.b * self.f * self.c], uniform=self.normal_initializer), trainable=True) if self.use_independent_labels_filter: self.labels_filter = \ tf.get_variable( name='%s_labels_filter' % (self.name), dtype=self.dtype, initializer=initialization.xavier_initializer( shape=[self.l_h_ext, self.l_h_ext] + [self.f, self.b * self.f], uniform=self.normal_initializer), trainable=True) else: self.labels_filter = \ helpers_mk2.caps2scalar_filter( self.caps_filter, self.c, self.eps) # CAPS COMPARE FILTER self.compare_filter = \ tf.get_variable( name='%s_compare_filter' % (self.name), dtype=self.dtype, initializer=initialization.xavier_initializer( shape=[1, 1] + [self.f * 3, self.f], uniform=self.normal_initializer), trainable=True) # self.compare_filter = None # LABEL GATES self.labels_deletegate_filter = \ tf.get_variable( name='%s_labels_deletegate_filter' % (self.name), dtype=self.dtype, initializer=initialization.xavier_initializer( shape=[self.c_h_ext, self.c_h_ext] + [self.f*2, self.f], uniform=self.normal_initializer), trainable=True) # self.labels_writegate_filter = \ # tf.get_variable( # name='%s_labels_writegate_filter' % (self.name), # dtype=self.dtype, # initializer=initialization.xavier_initializer( # shape=[self.c_h_ext, self.c_h_ext] + [self.f, self.f], # uniform=self.normal_initializer), # trainable=True) self.labels_writegate_filter = None # STATIC BIASES self.bundle_competition_bias = \ tf.get_variable( name='%s_bundle_competition_bias' % (self.name), dtype=self.dtype, shape=[1, 1, 1, self.b], initializer=tf.constant_initializer([1.0], dtype=self.dtype), trainable=True) # self.bundle_competition_bias = \ # tf.constant( # name='%s_bundle_competition_bias' % (self.name), # dtype=self.dtype, # shape=[1, 1, 1, self.b], # value=1.0) self.fig_ground_competition_bias = \ tf.get_variable( name='%s_fig_ground_competition_bias' % (self.name), dtype=self.dtype, shape=[1, 1, 1, 2], initializer=tf.constant_initializer([1.0], dtype=self.dtype), trainable=True) # self.fig_ground_competition_bias = \ # tf.constant( # name='%s_fig_ground_competition_bias' % (self.name), # dtype=self.dtype, # shape=[1, 1, 1, 2], # value=1.0) self.update_gain = \ tf.get_variable( name='%s_update_gain' % (self.name), dtype=self.dtype, shape=[1, 1, 1, self.f], initializer=tf.constant_initializer([0.0], dtype=self.dtype), trainable=True) import numpy as np fixed_labels_mask = np.zeros([1, 1, 1, self.f]) if self.fixed_label_ind is not None: for idx in self.fixed_label_ind: fixed_labels_mask[0, 0, 0, idx] = 1. self.fixed_labels_mask = tf.constant(fixed_labels_mask, dtype=self.dtype) else: self.fixed_labels_mask = None
def prepare_tensors(self): # HGRU KERNELS self.hgru0.prepare_tensors() self.hgru1.prepare_tensors() self.hgru_td1.prepare_tensors() self.hgru_td0.prepare_tensors() # FEEDFORWARD KERNELS lower_feats = self.in_k for idx, (higher_feats, ff_dhw) in enumerate(zip(self.ff_conv_k, self.ff_conv_fsiz)): with tf.variable_scope('ff_%s' % idx): if idx < 2: # last conv layer doesn't have spot weights setattr( self, 'ff_%s_spot_x' % idx, tf.get_variable( name='spot_x', dtype=self.dtype, initializer=initialization.xavier_initializer( shape=[1, 1, 1, 1] + [1], #[lower_feats], dtype=self.dtype, uniform=True), trainable=True)) # last conv layer doesn't have spot weights setattr( self, 'ff_%s_spot_xy' % idx, tf.get_variable( name='spot_xy', dtype=self.dtype, initializer=initialization.xavier_initializer( shape=[1, 1, 1, 1] + [lower_feats], #[lower_feats], dtype=self.dtype, uniform=True), trainable=True)) setattr( self, 'ff_%s_weights' % idx, tf.get_variable( name='weights', dtype=self.dtype, initializer=initialization.xavier_initializer( shape=ff_dhw + [ lower_feats, higher_feats * self.ff_kpool_multiplier ], dtype=self.dtype, uniform=True), trainable=True)) lower_feats = higher_feats # FEEDBACK KERNELS lower_feats = self.in_k for idx, (higher_feats, fb_dhw) in enumerate(zip(self.fb_conv_k, self.fb_conv_fsiz)): with tf.variable_scope('fb_%s' % idx) as scope: setattr( self, 'fb_%s_weights' % idx, tf.get_variable( name='weights', dtype=self.dtype, initializer=initialization.xavier_initializer( shape=fb_dhw + [lower_feats, higher_feats], dtype=self.dtype, uniform=True), trainable=True)) lower_feats = higher_feats