def test(): """Test the eRF calculator on a vgg16.""" from models.structs.test_vgg16 import test as vgg16 V1_CRF = 0.26 V1_neCRF = 0.54 V1_feCRF = 1.41 eRF = eRF_calculator() calulated_eRFs = eRF.calculate(network=vgg16.layer_structure, image=tf.zeros([1, 224, 224, 3])) layer_idxs = [6, 7, 8, 9] layer_strings = ['conv3_1', 'conv3_2', 'conv3_3', 'pool_3'] for layer_idx, layer_string in zip(layer_idxs, layer_strings): # Derive appropriate filter sizes for a layer layer = { 'padding': 1, 'kernel': vgg16.layer_structure[layer_idx]['filter_size'][0], 'stride': 2 } layer_RF = calulated_eRFs[layer_string] SSN_eRF = py_utils.iceil(layer_RF['r_i'] * (V1_neCRF / V1_CRF)) SSN = eRF.outFromIn(conv=layer, layer=layer_RF, fix_r_out=SSN_eRF) SSF_eRF = py_utils.iceil(layer_RF['r_i'] * (V1_feCRF / V1_CRF)) SSF = eRF.outFromIn(conv=layer, layer=layer_RF, fix_r_out=SSF_eRF) print 'Layers: %s' % layer_string print 'CRF: RF = %s; filter size = %s' % (layer_RF['r_i'], layer['kernel']) print 'near eCRF: RF = %s; filter size = %s' % (SSN_eRF, SSN) print 'far eCRF: RF = %s; filter size = %s' % (SSF_eRF, SSF) print '-' * 20
def set_RFs(self, r_in=None, j_in=None, V1_CRF=0.26, V1_neCRF=0.54, V1_feCRF=1.41, default_stride=1, padding=1): """Set RF sizes for the normalizations. Based on calculation of an effective RF (i.e. not simply kernel size of the current layer, but instead w.r.t. input). Angelucci & Shushruth 2013 V1 RFs: https://www.shadlenlab.columbia.edu/people/Shushruth/Lab_Page/Home_files/GalleyProof.pdf CRF = 0.26 degrees (1x) eCRF near = 0.54 degrees (2x) eCRF far = 1.41 degrees (5.5x) Implementation is to calculate the RF of a computational unit in an activity tensor. Then, near and far eCRFs are derived relative to the CRF size. This means that the *filter size* for the CRF is 1 tensor pixel. And the eRFs are deduced as the appropriate filter size for their calculated RFs. For instance, units in pool_2 of VGG16 have RFs of 16x16 pixels of the input image. Setting the CRF filter size to 1, this means that the near eCRF filter size must capture an RF of ~ 32x32 pixels, and the far eCRF filter size must capture an RF of ~ 80x80 pixels. The eRF calculator can deduce these filter sizes. """ assert r_in is not None, 'You must pass an RF input size.' assert j_in is not None, 'You must pass an input jump.' self.SRF = 1 # See explanation above. self.CRF_excitation = 1 self.CRF_inhibition = 1 SSN_eRF = py_utils.iceil(r_in * (V1_neCRF / V1_CRF)) self.SSN = self.calculate_kernel_from_RF(j_in=j_in, r_in=r_in, fix_r_out=SSN_eRF) SSF_eRF = py_utils.iceil(r_in * (V1_feCRF / V1_CRF)) self.SSF = self.calculate_kernel_from_RF(j_in=j_in, r_in=r_in, fix_r_out=SSF_eRF)
def prepare_tensors(self): """ Prepare recurrent/forward weight matrices.""" self.weight_dict = { # Weights lower/activity upper 'U': { 'r': { 'weight': 'u_r', 'activity': 'U_r' } }, 'P': { 'r': { 'weight': 'p_r', 'activity': 'P_r', 'tuning': 'p_t' } }, 'Q': { 'r': { 'weight': 'q_r', 'activity': 'Q_r', 'tuning': 'q_t' } }, 'I': { 'r': { # Recurrent state 'weight': 'i_r', 'bias': 'i_b', 'activity': 'I_r' }, 'f': { # Recurrent state 'weight': 'i_f', 'activity': 'I_f' }, }, 'O': { 'r': { # Recurrent state 'weight': 'o_r', 'bias': 'o_b', 'activity': 'O_r' }, 'f': { # Recurrent state 'weight': 'o_f', 'activity': 'O_f' }, }, 'xi': { 'r': { # Recurrent state 'weight': 'xi', } }, 'alpha': { 'r': { # Recurrent state 'weight': 'alpha', } }, 'beta': { 'r': { # Recurrent state 'weight': 'beta', } }, 'mu': { 'r': { # Recurrent state 'weight': 'mu', } }, 'nu': { 'r': { # Recurrent state 'weight': 'nu', } }, 'zeta': { 'r': { # Recurrent state 'weight': 'zeta', } }, 'gamma': { 'r': { # Recurrent state 'weight': 'gamma', } }, 'phi': { 'r': { # Recurrent state 'weight': 'phi', } }, 'kappa': { 'r': { # Recurrent state 'weight': 'kappa', } }, 'delta': { 'r': { # Recurrent state 'weight': 'delta', } } } # tuned summation: pooling in h, w dimensions ############################################# q_array = np.ones(self.q_shape) / np.prod(self.q_shape) if 'Q' in self.lesions: q_array = np.zeros_like(q_array).astype(np.float32) print 'Lesioning CRF excitation.' setattr( self, self.weight_dict['Q']['r']['weight'], tf.get_variable(name=self.weight_dict['Q']['r']['weight'], dtype=self.dtype, initializer=q_array.astype(np.float32), trainable=False)) # untuned suppression: reduction across feature axis #################################################### u_array = np.ones(self.u_shape) / np.prod(self.u_shape) if 'U' in self.lesions: u_array = np.zeros_like(u_array).astype(np.float32) print 'Lesioning CRF inhibition.' setattr( self, self.weight_dict['U']['r']['weight'], tf.get_variable(name=self.weight_dict['U']['r']['weight'], dtype=self.dtype, initializer=u_array.astype(np.float32), trainable=False)) # weakly tuned summation: pooling in h, w dimensions ############################################# if isinstance(self.p_shape[0], list) and 'P' not in self.lesions: # VGG-style filters for pidx, pext in enumerate(self.p_shape): if pidx == 0: it_key = self.weight_dict['P']['r']['weight'] else: self.weight_dict['P']['r']['weight_%s' % pidx] = 'p_r_%s' % pidx it_key = self.weight_dict['P']['r']['weight_%s' % pidx] setattr( self, it_key, tf.get_variable( name=it_key, dtype=self.dtype, initializer=initialization.xavier_initializer( shape=pext, uniform=self.normal_initializer), trainable=True)) else: p_array = np.ones(self.p_shape) p_array[self.SSN // 2 - py_utils.ifloor(self.SRF / 2.0):self.SSF // 2 + py_utils.iceil(self.SSN / 2.0), self.SSN // 2 - py_utils.ifloor(self.SRF / 2.0):self.SSF // 2 + py_utils.iceil(self.SSN / 2.0), :, # exclude CRF! :] = 0.0 p_array = p_array / p_array.sum() if 'P' in self.lesions: print 'Lesioning near eCRF.' p_array = np.zeros_like(p_array).astype(np.float32) # Association field is fully learnable if self.association_field and 'P' not in self.lesions: setattr( self, self.weight_dict['P']['r']['weight'], tf.get_variable( name=self.weight_dict['P']['r']['weight'], dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.p_shape, uniform=self.normal_initializer), trainable=True)) else: setattr( self, self.weight_dict['P']['r']['weight'], tf.get_variable(name=self.weight_dict['P']['r']['weight'], dtype=self.dtype, initializer=p_array.astype(np.float32), trainable=False)) # Connectivity tensors -- Q/P/T if 'Q' in self.lesions: print 'Lesioning CRF excitation connectivity.' setattr( self, self.weight_dict['Q']['r']['tuning'], tf.get_variable(name=self.weight_dict['Q']['r']['tuning'], dtype=self.dtype, trainable=False, initializer=np.zeros(self.tuning_shape).astype( np.float32))) else: setattr( self, self.weight_dict['Q']['r']['tuning'], tf.get_variable(name=self.weight_dict['Q']['r']['tuning'], dtype=self.dtype, trainable=True, initializer=initialization.xavier_initializer( shape=self.tuning_shape, uniform=self.normal_initializer, mask=None))) # Gate weights setattr( self, self.weight_dict['I']['r']['weight'], tf.get_variable(name=self.weight_dict['I']['r']['weight'], dtype=self.dtype, trainable=True, initializer=initialization.xavier_initializer( shape=self.i_shape, uniform=self.normal_initializer, mask=None))) setattr( self, self.weight_dict['I']['f']['weight'], tf.get_variable(name=self.weight_dict['I']['f']['weight'], dtype=self.dtype, trainable=True, initializer=initialization.xavier_initializer( shape=self.i_shape, uniform=self.normal_initializer, mask=None))) setattr( self, self.weight_dict['I']['r']['bias'], tf.get_variable(name=self.weight_dict['I']['r']['bias'], dtype=self.dtype, trainable=True, initializer=tf.ones(self.bias_shape))) # Output setattr( self, self.weight_dict['O']['r']['weight'], tf.get_variable(name=self.weight_dict['O']['r']['weight'], dtype=self.dtype, trainable=True, initializer=initialization.xavier_initializer( shape=self.o_shape, uniform=self.normal_initializer, mask=None))) setattr( self, self.weight_dict['O']['f']['weight'], tf.get_variable(name=self.weight_dict['O']['f']['weight'], dtype=self.dtype, trainable=True, initializer=initialization.xavier_initializer( shape=self.o_shape, uniform=self.normal_initializer, mask=None))) setattr( # TODO: smart initialization of these self, self.weight_dict['O']['r']['bias'], tf.get_variable(name=self.weight_dict['O']['r']['bias'], dtype=self.dtype, trainable=True, initializer=tf.ones(self.bias_shape))) # Degree of freedom weights (vectors) w_array = np.ones([1, 1, 1, self.k]).astype(np.float32) b_array = np.zeros([1, 1, 1, self.k]).astype(np.float32) # Divisive params self.alpha = tf.get_variable(name='alpha', initializer=w_array) self.beta = tf.get_variable(name='beta', initializer=w_array) # Subtractive params self.mu = tf.get_variable(name='mu', initializer=b_array) self.nu = tf.get_variable(name='nu', initializer=b_array) if self.zeta: self.zeta = tf.get_variable(name='zeta', initializer=w_array) else: self.zeta = tf.constant(1.) if self.gamma: self.gamma = tf.get_variable(name='gamma', initializer=w_array) else: self.gamma = tf.constant(1.) if self.delta: self.delta = tf.get_variable(name='delta', initializer=w_array) else: self.delta = tf.constant(1.) if self.xi: self.xi = tf.get_variable(name='xi', initializer=w_array) else: self.xi = tf.constant(1.) if self.multiplicative_excitation: self.kappa = tf.get_variable(name='kappa', initializer=w_array) self.omega = tf.get_variable(name='omega', initializer=w_array) else: self.kappa = tf.constant(1.) self.omega = tf.constant(1.)
def prepare_tensors(self): """ Prepare recurrent/forward weight matrices.""" self.weight_dict = { # Weights lower/activity upper 'U': { 'r': { 'weight': 'u_r', 'activity': 'U_r' } }, 'T': { 'r': { 'weight': 't_r', 'activity': 'T_r', 'tuning': 't_t' } }, 'P': { 'r': { 'weight': 'p_r', 'activity': 'P_r', 'tuning': 'p_t' } }, 'Q': { 'r': { 'weight': 'q_r', 'activity': 'Q_r', 'tuning': 'q_t' } }, 'I': { 'r': { # Recurrent state 'weight': 'i_r', 'activity': 'I_r' }, 'f': { # Recurrent state 'weight': 'i_f', 'activity': 'I_f' }, }, 'O': { 'r': { # Recurrent state 'weight': 'o_r', 'activity': 'O_r' }, 'f': { # Recurrent state 'weight': 'o_f', 'activity': 'O_f' }, }, 'xi': { 'r': { # Recurrent state 'weight': 'xi', } }, 'alpha': { 'r': { # Recurrent state 'weight': 'alpha', } }, 'beta': { 'r': { # Recurrent state 'weight': 'beta', } }, 'mu': { 'r': { # Recurrent state 'weight': 'mu', } }, 'nu': { 'r': { # Recurrent state 'weight': 'nu', } }, 'zeta': { 'r': { # Recurrent state 'weight': 'zeta', } }, 'gamma': { 'r': { # Recurrent state 'weight': 'gamma', } }, 'delta': { 'r': { # Recurrent state 'weight': 'delta', } } } # tuned summation: pooling in h, w dimensions ############################################# q_array = np.ones(self.q_shape) / np.prod(self.q_shape) setattr( self, self.weight_dict['Q']['r']['weight'], tf.get_variable(name=self.weight_dict['Q']['r']['weight'], dtype=self.dtype, initializer=q_array.astype(np.float32), trainable=False)) # untuned suppression: reduction across feature axis #################################################### u_array = np.ones(self.u_shape) / np.prod(self.u_shape) setattr( self, self.weight_dict['U']['r']['weight'], tf.get_variable(name=self.weight_dict['U']['r']['weight'], dtype=self.dtype, initializer=u_array.astype(np.float32), trainable=False)) # weakly tuned summation: pooling in h, w dimensions ############################################# p_array = np.ones(self.p_shape) p_array[self.SSN // 2 - py_utils.ifloor(self.SRF / 2.0):self.SSN // 2 + py_utils.iceil(self.SRF / 2.0), self.SSN // 2 - py_utils.ifloor(self.SRF / 2.0):self.SSN // 2 + py_utils.iceil(self.SRF / 2.0), :, # exclude CRF! :] = 0.0 p_array = p_array / p_array.sum() setattr( self, self.weight_dict['P']['r']['weight'], tf.get_variable(name=self.weight_dict['P']['r']['weight'], dtype=self.dtype, initializer=p_array.astype(np.float32), trainable=False)) # weakly tuned suppression: pooling in h, w dimensions ############################################### t_array = np.ones(self.t_shape) t_array[self.SSF // 2 - py_utils.ifloor(self.SSN / 2.0):self.SSF // 2 + py_utils.iceil(self.SSN / 2.0), self.SSF // 2 - py_utils.ifloor(self.SSN / 2.0):self.SSF // 2 + py_utils.iceil(self.SSN / 2.0), :, # exclude near surround! :] = 0.0 t_array = t_array / t_array.sum() setattr( self, self.weight_dict['T']['r']['weight'], tf.get_variable(name=self.weight_dict['T']['r']['weight'], dtype=self.dtype, initializer=t_array.astype(np.float32), trainable=False)) # Connectivity tensors -- Q/P/T setattr( self, self.weight_dict['Q']['r']['tuning'], tf.get_variable(name=self.weight_dict['Q']['r']['tuning'], dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.tuning_shape, uniform=self.normal_initializer, mask=None))) setattr( self, self.weight_dict['P']['r']['tuning'], tf.get_variable(name=self.weight_dict['P']['r']['tuning'], dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.tuning_shape, uniform=self.normal_initializer, mask=None))) setattr( self, self.weight_dict['T']['r']['tuning'], tf.get_variable(name=self.weight_dict['T']['r']['tuning'], dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.tuning_shape, uniform=self.normal_initializer, mask=None))) # Input setattr( self, self.weight_dict['I']['r']['weight'], tf.get_variable(name=self.weight_dict['I']['r']['weight'], dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.i_shape, uniform=self.normal_initializer, mask=None))) setattr( self, self.weight_dict['I']['f']['weight'], tf.get_variable(name=self.weight_dict['I']['f']['weight'], dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.i_shape, uniform=self.normal_initializer, mask=None))) # Output setattr( self, self.weight_dict['O']['r']['weight'], tf.get_variable(name=self.weight_dict['O']['r']['weight'], dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.o_shape, uniform=self.normal_initializer, mask=None))) setattr( self, self.weight_dict['O']['f']['weight'], tf.get_variable(name=self.weight_dict['O']['f']['weight'], dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.o_shape, uniform=self.normal_initializer, mask=None))) # Vector weights w_array = np.ones([1, 1, 1, self.k]).astype(np.float32) b_array = np.zeros([1, 1, 1, self.k]).astype(np.float32) self.xi = tf.get_variable(name='xi', initializer=w_array) self.alpha = tf.get_variable(name='alpha', initializer=w_array) self.beta = tf.get_variable(name='beta', initializer=w_array) self.mu = tf.get_variable(name='mu', initializer=b_array) self.nu = tf.get_variable(name='nu', initializer=b_array) self.zeta = tf.get_variable(name='zeta', initializer=w_array) self.gamma = tf.get_variable(name='gamma', initializer=w_array) self.delta = tf.get_variable(name='delta', initializer=w_array)
def set_RFs(self, eRF, layer, eSRF=None, iSRF=None, SSN=None, SSF=None, V1_CRF=0.26, V1_neCRF=0.54, V1_feCRF=1.41, default_stride=1, rf_calculation='pixel_wise', padding=1): """Set RF sizes for the normalizations. Based on calculation of an effective RF (i.e. not simply kernel size of the current layer, but instead w.r.t. input). Angelucci & Shushruth 2013 V1 RFs: https://www.shadlenlab.columbia.edu/people/Shushruth/Lab_Page/Home_files/GalleyProof.pdf CRF = 0.26 degrees (1x) eCRF near = 0.54 degrees (2x) eCRF far = 1.41 degrees (5.5x) Implementation is to calculate the RF of a computational unit in an activity tensor. Then, near and far eCRFs are derived relative to the CRF size. This means that the *filter size* for the CRF is 1 tensor pixel. And the eRFs are deduced as the appropriate filter size for their calculated RFs. For instance, units in pool_2 of VGG16 have RFs of 16x16 pixels of the input image. Setting the CRF filter size to 1, this means that the near eCRF filter size must capture an RF of ~ 32x32 pixels, and the far eCRF filter size must capture an RF of ~ 80x80 pixels. The eRF calculator can deduce these filter sizes. """ if 'hardcoded_erfs' in layer.keys(): # Use specified effective receptive fields self.SRF = layer['hardcoded_erfs']['SRF'] if 'CRF_excitation' in layer['hardcoded_erfs'].keys(): self.CRF_excitation = layer['hardcoded_erfs']['CRF_excitation'] else: self.CRF_excitation = self.SRF if 'CRF_inhibition' in layer['hardcoded_erfs'].keys(): self.CRF_inhibition = layer['hardcoded_erfs']['CRF_inhibition'] else: self.CRF_inhibition = self.SRF self.SSN = layer['hardcoded_erfs']['SSN'] self.SSF = layer['hardcoded_erfs']['SSF'] else: # Calculate effective receptive field for this layer. # Adjust eCRF filters to yield appropriate RF sizes. eRFc = eRF_calculator() conv = { 'padding': padding, 'kernel': layer['filter_size'][0], 'stride': default_stride } if len(layer['filter_size']) > 1: raise RuntimeError( 'API not implemented for layers with > 1 module.') # Each pixel at this layer corresponds to an input image RF # of eRF['r_i']. if rf_calculation == 'pixel_wise': # self.SRF = 1 # self.CRF_excitation = 1 # self.CRF_inhibition = 1 self.SRF = 7 self.CRF_excitation = 7 self.CRF_inhibition = 7 elif rf_calculation == 'fit_rf': # Adjust SRF filter, then adjust eCRFs w.r.t to that. self.SRF = eRFc.outFromIn(conv=conv, layer=eRF, fix_r_out=eRF['r_i']) self.CRF_excitation = eRF['r_i'] self.CRF_inhibition = eRF['r_i'] if eSRF is not None: self.CRF_excitation if iSRF is not None: self.CRF_inhibition if SSN is None: SSN_eRF = py_utils.iceil(eRF['r_i'] * (V1_neCRF / V1_CRF)) self.SSN = eRFc.outFromIn(conv=conv, layer=eRF, fix_r_out=SSN_eRF) else: self.SSN = SSN if SSF is None: SSF_eRF = py_utils.iceil(eRF['r_i'] * (V1_feCRF / V1_CRF)) self.SSF = eRFc.outFromIn(conv=conv, layer=eRF, fix_r_out=SSF_eRF) else: self.SSF = SSF
def prepare_tensors(self): """ Prepare recurrent/forward weight matrices.""" self.weight_dict = { # Weights lower/activity upper 'U': { 'r': { 'weight': 'u_r', 'activity': 'U_r' }, 'f': { 'weight': 'u_f', 'bias': 'ub_f', 'activity': 'U_f' } }, 'T': { 'r': { 'weight': 't_r', 'activity': 'T_r' }, 'f': { 'weight': 't_f', 'bias': 'tb_f', 'activity': 'T_f' } }, 'P': { 'r': { 'weight': 'p_r', 'activity': 'P_r' }, 'f': { 'weight': 'p_f', 'bias': 'pb_f', 'activity': 'P_f' } }, 'Q': { 'r': { 'weight': 'q_r', 'activity': 'Q_r' }, 'f': { 'weight': 'q_f', 'bias': 'qb_f', 'activity': 'Q_f' } }, 'I': { 'r': { # Recurrent state 'weight': 'i_r', 'activity': 'I_r' } }, 'O': { 'r': { # Recurrent state 'weight': 'o_r', 'activity': 'O_r' } } } # tuned summation: pooling in h, w dimensions ############################################# setattr( self, self.weight_dict['Q']['r']['weight'], tf.get_variable(name=self.weight_dict['Q']['r']['weight'], dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.q_shape, uniform=self.normal_initializer, mask=None))) # untuned suppression: reduction across feature axis #################################################### setattr( self, self.weight_dict['U']['r']['weight'], tf.get_variable(name=self.weight_dict['U']['r']['weight'], dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.u_shape, uniform=self.normal_initializer, mask=None))) # tuned summation: pooling in h, w dimensions ############################################# p_array = np.zeros(self.p_shape) for pdx in range(self.k): p_array[:self.SSN, :self.SSN, pdx, pdx] = 1.0 p_array[self.SSN // 2 - py_utils.ifloor(self.SRF / 2.0):self.SSN // 2 + py_utils.iceil(self.SRF / 2.0), self.SSN // 2 - py_utils.ifloor(self.SRF / 2.0):self.SSN // 2 + py_utils.iceil(self.SRF / 2.0), :, # exclude CRF! :] = 0.0 setattr( self, self.weight_dict['P']['r']['weight'], tf.get_variable(name=self.weight_dict['P']['r']['weight'], dtype=self.dtype, initializer=p_array.astype(np.float32), trainable=False)) # tuned suppression: pooling in h, w dimensions ############################################### t_array = np.zeros(self.t_shape) for tdx in range(self.k): t_array[:self.SSF, :self.SSF, tdx, tdx] = 1.0 t_array[self.SSF // 2 - py_utils.ifloor(self.SSN / 2.0):self.SSF // 2 + py_utils.iceil(self.SSN / 2.0), self.SSF // 2 - py_utils.ifloor(self.SSN / 2.0):self.SSF // 2 + py_utils.iceil(self.SSN / 2.0), :, # exclude near surround! :] = 0.0 setattr( self, self.weight_dict['T']['r']['weight'], tf.get_variable(name=self.weight_dict['T']['r']['weight'], dtype=self.dtype, initializer=t_array.astype(np.float32), trainable=False)) # Scalar weights self.xi = tf.get_variable(name='xi', initializer=1.) self.alpha = tf.get_variable(name='alpha', initializer=1.) self.beta = tf.get_variable(name='beta', initializer=1.) self.mu = tf.get_variable(name='mu', initializer=1.) self.nu = tf.get_variable(name='nu', initializer=1.) self.zeta = tf.get_variable(name='zeta', initializer=1.) self.gamma = tf.get_variable(name='gamma', initializer=1.) self.delta = tf.get_variable(name='delta', initializer=1.) self.eps = tf.get_variable(name='eps', initializer=1.) self.eta = tf.get_variable(name='eta', initializer=1.) self.sig = tf.get_variable(name='sig', initializer=1.) self.tau = tf.get_variable(name='tau', initializer=1.)
def contextual_div_norm_2d(x, CRF_sum_window, CRF_sup_window, eCRF_sum_window, eCRF_sup_window, strides, padding, gamma=None, beta=None, eps=1.0, scope="dn", name="dn_out", return_mean=False): """Applies divisive normalization on CNN feature maps. Collect mean and variances on x on a local window across channels. And apply normalization as below: x_ = gamma * (x - mean) / sqrt(var + eps) + beta https://github.com/renmengye/div-norm/blob/master/div_norm.py Args: x: Input tensor, [B, H, W, C]. sum_window: Summation window size, [H_sum, W_sum]. sup_window: Suppression window size, [H_sup, W_sup]. gamma: Scaling parameter. beta: Bias parameter. eps: Denominator bias. return_mean: Whether to also return the computed mean. Returns: normed: Divisive-normalized variable. mean: Mean used for normalization (optional). """ if not isinstance(CRF_sum_window, list): CRF_sum_window = list(np.repeat(CRF_sum_window, 2)) if not isinstance(CRF_sup_window, list): CRF_sup_window = list(np.repeat(CRF_sup_window, 2)) if not isinstance(eCRF_sum_window, list): eCRF_sum_window = list(np.repeat(eCRF_sum_window, 2)) if not isinstance(eCRF_sup_window, list): eCRF_sup_window = list(np.repeat(eCRF_sup_window, 2)) k = int(x.get_shape()[-1]) with tf.variable_scope(scope): # Q q_array = np.ones((CRF_sum_window + [k, k])) q_array /= q_array.sum() w_sum = tf.cast(tf.constant(q_array), tf.float32) # U u_array = np.ones((CRF_sum_window + [k, 1])) u_array /= u_array.sum() w_sup = tf.cast(tf.constant(u_array), tf.float32) CRF_sum_window = CRF_sum_window[0] CRF_sup_window = CRF_sup_window[0] # P p_shape = eCRF_sum_window + [k, k] eCRF_sum_window = eCRF_sum_window[0] p_array = np.zeros(p_shape) for pdx in range(k): p_array[:eCRF_sum_window, :eCRF_sum_window, pdx, pdx] = 1.0 p_array[eCRF_sum_window // 2 - py_utils.ifloor(CRF_sum_window / 2.0):eCRF_sum_window // 2 + py_utils.iceil(CRF_sum_window / 2.0), CRF_sum_window // 2 - py_utils.ifloor(CRF_sum_window / 2.0):eCRF_sum_window // 2 + py_utils.iceil(CRF_sum_window / 2.0), :, # exclude CRF! :] = 0.0 w_esum = tf.cast(tf.constant(p_array) / p_array.sum(), tf.float32) # T t_shape = eCRF_sup_window + [k, k] eCRF_sup_window = eCRF_sup_window[0] t_array = np.zeros(t_shape) for tdx in range(k): t_array[:eCRF_sup_window, :eCRF_sup_window, tdx, tdx] = 1.0 t_array[eCRF_sup_window // 2 - py_utils.ifloor(CRF_sup_window / 2.0):eCRF_sup_window // 2 + py_utils.iceil(CRF_sup_window / 2.0), eCRF_sup_window // 2 - py_utils.ifloor(CRF_sup_window / 2.0):eCRF_sup_window // 2 + py_utils.iceil(CRF_sup_window / 2.0), :, # exclude near surround! :] = 0.0 w_esup = tf.cast(tf.constant(t_array) / t_array.sum(), tf.float32) # SUM x_mean_CRF = tf.nn.conv2d(x, w_sum, strides=strides, padding=padding) x_mean_eCRF = tf.nn.conv2d(x, w_esum, strides=strides, padding=padding) normed = x - x_mean_CRF - x_mean_eCRF x2 = tf.square(normed) # SUP x2_mean_CRF = tf.nn.conv2d(x2, w_sup, strides=strides, padding=padding) x2_mean_eCRF = tf.nn.conv2d(x2, w_esup, strides=strides, padding=padding) denom = tf.sqrt(x2_mean_CRF + x2_mean_eCRF + eps) normed = normed / denom if gamma is not None: normed *= gamma if beta is not None: normed += beta normed = tf.identity(normed, name=name) if return_mean: return normed, x2 else: return normed
def prepare_tensors(self): """ Prepare recurrent/forward weight matrices.""" self.weight_dict = { # Weights lower/activity upper 'U': { 'r': { 'weight': 'u_r', 'activity': 'U_r' } }, 'T': { 'r': { 'weight': 't_r', 'activity': 'T_r', 'tuning': 't_t' } }, 'P': { 'r': { 'weight': 'p_r', 'activity': 'P_r', 'tuning': 'p_t' } }, 'Q': { 'r': { 'weight': 'q_r', 'activity': 'Q_r', 'tuning': 'q_t' } }, 'I': { 'r': { # Recurrent state 'weight': 'i_r', 'bias': 'i_b', 'activity': 'I_r' }, 'f': { # Recurrent state 'weight': 'i_f', 'activity': 'I_f' }, }, 'O': { 'r': { # Recurrent state 'weight': 'o_r', 'bias': 'o_b', 'activity': 'O_r' }, 'f': { # Recurrent state 'weight': 'o_f', 'activity': 'O_f' }, }, 'xi': { 'r': { # Recurrent state 'weight': 'xi', } }, 'alpha': { 'r': { # Recurrent state 'weight': 'alpha', } }, 'beta': { 'r': { # Recurrent state 'weight': 'beta', } }, 'mu': { 'r': { # Recurrent state 'weight': 'mu', } }, 'nu': { 'r': { # Recurrent state 'weight': 'nu', } }, 'zeta': { 'r': { # Recurrent state 'weight': 'zeta', } }, 'gamma': { 'r': { # Recurrent state 'weight': 'gamma', } }, 'delta': { 'r': { # Recurrent state 'weight': 'delta', } } } # tuned summation: pooling in h, w dimensions ############################################# q_array = np.ones(self.q_shape) / np.prod(self.q_shape) if 'Q' in self.lesions: q_array = np.zeros_like(q_array).astype(np.float32) print 'Lesioning CRF excitation.' setattr( self, self.weight_dict['Q']['r']['weight'], tf.get_variable(name=self.weight_dict['Q']['r']['weight'], dtype=self.dtype, initializer=q_array.astype(np.float32), trainable=False)) # untuned suppression: reduction across feature axis #################################################### u_array = np.ones(self.u_shape) / np.prod(self.u_shape) if 'U' in self.lesions: u_array = np.zeros_like(u_array).astype(np.float32) print 'Lesioning CRF inhibition.' setattr( self, self.weight_dict['U']['r']['weight'], tf.get_variable(name=self.weight_dict['U']['r']['weight'], dtype=self.dtype, initializer=u_array.astype(np.float32), trainable=False)) # weakly tuned summation: pooling in h, w dimensions ############################################# p_array = np.ones(self.p_shape) #Try removing CRF punching if self.exclude_CRF: # Punch out the CRF p_array[self.SSN // 2 - py_utils.ifloor(self.SRF / 2.0):self.SSF // 2 + py_utils.iceil(self.SSN / 2.0), self.SSN // 2 - py_utils.ifloor(self.SRF / 2.0):self.SSF // 2 + py_utils.iceil(self.SSN / 2.0), :, # exclude CRF! :] = 0.0 p_array = p_array / p_array.sum() if 'P' in self.lesions: print 'Lesioning near eCRF.' p_array = np.zeros_like(p_array).astype(np.float32) # Association field is fully learnable if self.association_field and 'P' not in self.lesions: setattr( self, self.weight_dict['P']['r']['weight'], tf.get_variable(name=self.weight_dict['P']['r']['weight'], dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.p_shape, uniform=self.normal_initializer), trainable=True)) else: setattr( self, self.weight_dict['P']['r']['weight'], tf.get_variable(name=self.weight_dict['P']['r']['weight'], dtype=self.dtype, initializer=p_array.astype(np.float32), trainable=False)) # weakly tuned suppression: pooling in h, w dimensions ############################################### t_array = np.ones(self.t_shape) #Try without punching CRF if self.exclude_CRF: # Punch out the CRF t_array[self.SSF // 2 - py_utils.ifloor(self.SSN / 2.0):self.SSF // 2 + py_utils.iceil(self.SSN / 2.0), self.SSF // 2 - py_utils.ifloor(self.SSN / 2.0):self.SSF // 2 + py_utils.iceil(self.SSN / 2.0), :, # exclude near surround! :] = 0.0 t_array = t_array / t_array.sum() if 'T' in self.lesions: print 'Lesioning Far eCRF.' t_array = np.zeros_like(t_array).astype(np.float32) #Always set full_far_eCRF to True/initialize with Xavier if self.full_far_eCRF: setattr( self, self.weight_dict['T']['r']['weight'], tf.get_variable(name=self.weight_dict['T']['r']['weight'], dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.p_shape, uniform=self.normal_initializer), trainable=True)) else: setattr( self, self.weight_dict['T']['r']['weight'], tf.get_variable(name=self.weight_dict['T']['r']['weight'], dtype=self.dtype, initializer=t_array.astype(np.float32), trainable=False)) # Connectivity tensors -- Q/P/T if 'Q' in self.lesions: print 'Lesioning CRF excitation connectivity.' setattr( self, self.weight_dict['Q']['r']['tuning'], tf.get_variable(name=self.weight_dict['Q']['r']['tuning'], dtype=self.dtype, trainable=False, initializer=np.zeros(self.tuning_shape).astype( np.float32))) else: setattr( self, self.weight_dict['Q']['r']['tuning'], tf.get_variable(name=self.weight_dict['Q']['r']['tuning'], dtype=self.dtype, trainable=True, initializer=initialization.xavier_initializer( shape=self.tuning_shape, uniform=self.normal_initializer, mask=None))) if not self.association_field: # Need a tuning tensor for near surround if 'P' in self.lesions: print 'Lesioning near eCRF connectivity.' setattr( self, self.weight_dict['P']['r']['tuning'], tf.get_variable(name=self.weight_dict['P']['r']['tuning'], dtype=self.dtype, trainable=False, initializer=np.zeros( self.tuning_shape).astype(np.float32))) else: setattr( self, self.weight_dict['P']['r']['tuning'], tf.get_variable( name=self.weight_dict['P']['r']['tuning'], dtype=self.dtype, trainable=True, initializer=initialization.xavier_initializer( shape=self.tuning_shape, uniform=self.normal_initializer, mask=None))) #Again, full_far_eCRF should be set to True for now import ipdb ipdb.set_trace() if not self.full_far_eCRF: # Need a tuning tensor for near surround if 'T' in self.lesions: print 'Lesioning far eCRF connectivity.' setattr( self, self.weight_dict['T']['r']['tuning'], tf.get_variable(name=self.weight_dict['T']['r']['tuning'], dtype=self.dtype, trainable=False, initializer=np.zeros( self.tuning_shape).astype(np.float32))) else: setattr( self, self.weight_dict['T']['r']['tuning'], tf.get_variable(name=self.weight_dict['T']['r']['tuning'], dtype=self.dtype, trainable=True, initializer=initialization.xavier_initializer( shape=self.tuning_shape, uniform=self.normal_initializer, mask=None))) # Input setattr( self, self.weight_dict['I']['r']['weight'], tf.get_variable(name=self.weight_dict['I']['r']['weight'], dtype=self.dtype, trainable=True, initializer=initialization.xavier_initializer( shape=self.i_shape, uniform=self.normal_initializer, mask=None))) setattr( self, self.weight_dict['I']['f']['weight'], tf.get_variable(name=self.weight_dict['I']['f']['weight'], dtype=self.dtype, trainable=True, initializer=initialization.xavier_initializer( shape=self.i_shape, uniform=self.normal_initializer, mask=None))) setattr( self, self.weight_dict['I']['r']['bias'], tf.get_variable(name=self.weight_dict['I']['r']['bias'], dtype=self.dtype, trainable=True, initializer=tf.ones(self.bias_shape))) # Output setattr( self, self.weight_dict['O']['r']['weight'], tf.get_variable(name=self.weight_dict['O']['r']['weight'], dtype=self.dtype, trainable=True, initializer=initialization.xavier_initializer( shape=self.o_shape, uniform=self.normal_initializer, mask=None))) setattr( self, self.weight_dict['O']['f']['weight'], tf.get_variable(name=self.weight_dict['O']['f']['weight'], dtype=self.dtype, trainable=True, initializer=initialization.xavier_initializer( shape=self.o_shape, uniform=self.normal_initializer, mask=None))) setattr( self, self.weight_dict['O']['r']['bias'], tf.get_variable(name=self.weight_dict['O']['r']['bias'], dtype=self.dtype, trainable=True, initializer=tf.ones(self.bias_shape))) # Vector weights w_array = np.ones([1, 1, 1, self.k]).astype(np.float32) b_array = np.zeros([1, 1, 1, self.k]).astype(np.float32) self.xi = tf.get_variable(name='xi', initializer=w_array) self.alpha = tf.get_variable(name='alpha', initializer=w_array) self.beta = tf.get_variable(name='beta', initializer=w_array) self.mu = tf.get_variable(name='mu', initializer=b_array) self.nu = tf.get_variable(name='nu', initializer=b_array) self.zeta = tf.get_variable(name='zeta', initializer=w_array) self.gamma = tf.get_variable(name='gamma', initializer=w_array) self.delta = tf.get_variable(name='delta', initializer=w_array)
def prepare_tensors(self): """ Prepare recurrent/forward weight matrices.""" self.weight_dict = { # Weights lower/activity upper 'U': { 'r': { 'weight': 'u_r', 'activity': 'U_r' }, 'f': { 'weight': 'u_f', 'bias': 'ub_f', 'activity': 'U_f' } }, 'T': { 'r': { 'weight': 't_r', 'activity': 'T_r' }, 'f': { 'weight': 't_f', 'bias': 'tb_f', 'activity': 'T_f' } }, 'P': { 'r': { 'weight': 'p_r', 'activity': 'P_r' }, 'f': { 'weight': 'p_f', 'bias': 'pb_f', 'activity': 'P_f' } }, 'Q': { 'r': { 'weight': 'q_r', 'activity': 'Q_r' }, 'f': { 'weight': 'q_f', 'bias': 'qb_f', 'activity': 'Q_f' } }, 'I': { 'r': { # Recurrent state 'weight': 'i_r', 'bias': 'ib_r', 'activity': 'I_r' } }, 'O': { 'r': { # Recurrent state 'weight': 'o_r', 'bias': 'ob_r', 'activity': 'O_r' } } } # tuned summation: pooling in h, w dimensions ############################################# setattr( self, self.weight_dict['Q']['r']['weight'], tf.get_variable( name=self.weight_dict['Q']['r']['weight'], dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.q_shape, uniform=self.normal_initializer, mask=None))) # untuned suppression: reduction across feature axis #################################################### setattr( self, self.weight_dict['U']['r']['weight'], tf.get_variable( name=self.weight_dict['U']['r']['weight'], dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.u_shape, uniform=self.normal_initializer, mask=None))) # tuned summation: pooling in h, w dimensions ############################################# p_array = np.zeros(self.p_shape) for pdx in range(self.k): p_array[:self.SSN, :self.SSN, pdx, pdx] = 1.0 p_array[ self.SSN // 2 - py_utils.ifloor( self.SRF / 2.0):self.SSN // 2 + py_utils.iceil( self.SRF / 2.0), self.SSN // 2 - py_utils.ifloor( self.SRF / 2.0):self.SSN // 2 + py_utils.iceil( self.SRF / 2.0), :, # exclude CRF! :] = 0.0 setattr( self, self.weight_dict['P']['r']['weight'], tf.get_variable( name=self.weight_dict['P']['r']['weight'], dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.p_shape, uniform=self.normal_initializer, mask=p_array))) # tuned suppression: pooling in h, w dimensions ############################################### t_array = np.zeros(self.t_shape) for tdx in range(self.k): t_array[:self.SSF, :self.SSF, tdx, tdx] = 1.0 t_array[ self.SSF // 2 - py_utils.ifloor( self.SSN / 2.0):self.SSF // 2 + py_utils.iceil( self.SSN / 2.0), self.SSF // 2 - py_utils.ifloor( self.SSN / 2.0):self.SSF // 2 + py_utils.iceil( self.SSN / 2.0), :, # exclude near surround! :] = 0.0 setattr( self, self.weight_dict['T']['r']['weight'], tf.get_variable( name=self.weight_dict['T']['r']['weight'], dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.t_shape, uniform=self.normal_initializer, mask=t_array))) if self.model_version != 'no_input_facing': # Also create input-facing weight matrices # Q setattr( self, self.weight_dict['Q']['f']['weight'], tf.get_variable( name=self.weight_dict['Q']['f']['weight'], dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.q_shape, uniform=self.normal_initializer))) setattr( self, self.weight_dict['Q']['f']['bias'], tf.get_variable( name=self.weight_dict['Q']['f']['bias'], dtype=self.dtype, initializer=initialization.xavier_initializer( shape=[self.q_shape[-1]], uniform=self.normal_initializer))) # U setattr( self, self.weight_dict['U']['f']['weight'], tf.get_variable( name=self.weight_dict['U']['f']['weight'], dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.u_shape, uniform=self.normal_initializer))) setattr( self, self.weight_dict['U']['f']['bias'], tf.get_variable( name=self.weight_dict['U']['f']['bias'], dtype=self.dtype, initializer=initialization.xavier_initializer( [self.u_shape[-1]], uniform=self.normal_initializer))) # P setattr( self, self.weight_dict['P']['f']['weight'], tf.get_variable( name=self.weight_dict['P']['f']['weight'], dtype=self.dtype, initializer=initialization.xavier_initializer( self.p_shape, uniform=self.normal_initializer, mask=p_array))) setattr( self, self.weight_dict['P']['f']['bias'], tf.get_variable( name=self.weight_dict['P']['f']['bias'], dtype=self.dtype, initializer=initialization.xavier_initializer( [self.p_shape[-1]], uniform=self.normal_initializer, mask=None))) # T setattr( self, self.weight_dict['T']['f']['weight'], tf.get_variable( name=self.weight_dict['T']['f']['weight'], dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.t_shape, uniform=self.normal_initializer, mask=t_array))) setattr( self, self.weight_dict['T']['f']['bias'], tf.get_variable( name=self.weight_dict['T']['f']['bias'], dtype=self.dtype, initializer=initialization.xavier_initializer( shape=[self.t_shape[-1]], uniform=self.normal_initializer, mask=None))) if self.model_version == 'full_with_cell_states': # Input setattr( self, self.weight_dict['I']['r']['weight'], tf.get_variable( name=self.weight_dict['I']['r']['weight'], dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.i_shape, uniform=self.normal_initializer, mask=None))) setattr( self, self.weight_dict['I']['r']['bias'], tf.get_variable( name=self.weight_dict['I']['r']['bias'], dtype=self.dtype, initializer=initialization.xavier_initializer( shape=[self.k], uniform=self.normal_initializer, mask=None))) # Output setattr( self, self.weight_dict['O']['r']['weight'], tf.get_variable( name=self.weight_dict['O']['r']['weight'], dtype=self.dtype, initializer=initialization.xavier_initializer( shape=self.o_shape, uniform=self.normal_initializer, mask=None))) setattr( self, self.weight_dict['O']['r']['bias'], tf.get_variable( name=self.weight_dict['O']['r']['bias'], dtype=self.dtype, initializer=initialization.xavier_initializer( shape=[self.k], uniform=self.normal_initializer, mask=None))) # Scalar weights self.alpha = tf.get_variable(name='alpha', initializer=1.) self.tau = tf.get_variable(name='tau', initializer=1.) self.eta = tf.get_variable(name='eta', initializer=1.) self.omega = tf.get_variable(name='omega', initializer=1.) self.eps = tf.get_variable(name='eps', initializer=1.) self.gamma = tf.get_variable(name='gamma', initializer=1.)
def prepare_tensors(self): """ Prepare recurrent/forward weight matrices. 9 * k + (2 * k^2) params in the greek letters/gates. (np.prod([h, w, k]) / 2) - k params in the surround filter """ self.weight_dict = { # Weights lower/activity upper 'P': { 'r': { 'weight': 'p_r', 'activity': 'P_r', 'tuning': 'p_t', # 'bias': 'i_b' } }, 'I': { 'r': { # Recurrent state 'weight': 'i_r', 'bias': 'i_b', 'activity': 'I_r' }, # 'f': { # Recurrent state # 'weight': 'i_f', # 'activity': 'I_f' # }, }, 'O': { 'r': { # Recurrent state 'weight': 'o_r', 'bias': 'o_b', 'activity': 'O_r' }, # 'f': { # Recurrent state # 'weight': 'o_f', # 'activity': 'O_f' # }, }, 'xi': { 'r': { # Recurrent state 'weight': 'xi', } }, # 'alpha': { # 'r': { # Recurrent state # 'weight': 'alpha', # } # }, 'beta': { 'r': { # Recurrent state 'weight': 'beta', } }, # 'mu': { # 'r': { # Recurrent state # 'weight': 'mu', # } # }, 'nu': { 'r': { # Recurrent state 'weight': 'nu', } }, 'zeta': { 'r': { # Recurrent state 'weight': 'zeta', } }, 'gamma': { 'r': { # Recurrent state 'weight': 'gamma', } }, 'phi': { 'r': { # Recurrent state 'weight': 'phi', } }, 'kappa': { 'r': { # Recurrent state 'weight': 'kappa', } }, 'rho': { 'r': { # Recurrent state 'weight': 'rho', } }, } # weakly tuned summation: pooling in h, w dimensions ############################################# with tf.variable_scope('contextual_circuit'): if isinstance(self.p_shape[0], list) and 'P' not in self.lesions: # VGG-style filters for pidx, pext in enumerate(self.p_shape): if pidx == 0: it_key = self.weight_dict['P']['r']['weight'] else: self.weight_dict['P']['r']['weight_%s' % pidx] = 'p_r_%s' % pidx it_key = self.weight_dict['P']['r']['weight_%s' % pidx] setattr( self, it_key, tf.get_variable( name=it_key, dtype=self.dtype, initializer=initialization.xavier_initializer( shape=pext, uniform=self.normal_initializer), trainable=True)) else: p_array = np.ones(self.p_shape) p_array[self.SSN // 2 - py_utils.ifloor(self.SRF / 2.0):self.SSF // 2 + py_utils.iceil(self.SSN / 2.0), self.SSN // 2 - py_utils.ifloor(self.SRF / 2.0):self.SSF // 2 + py_utils.iceil(self.SSN / 2.0), :, # exclude CRF! :] = 0.0 p_array = p_array / p_array.sum() if 'P' in self.lesions: print 'Lesioning near eCRF.' p_array = np.zeros_like(p_array).astype(np.float32) # Association field is fully learnable if self.association_field and 'P' not in self.lesions: setattr( self, self.weight_dict['P']['r']['weight'], tf.get_variable( name=self.weight_dict['P']['r']['weight'], dtype=self.dtype, # shape=self.p_shape, initializer=initialization.xavier_initializer( shape=self.p_shape, uniform=self.normal_initializer), trainable=True)) else: setattr( self, self.weight_dict['P']['r']['weight'], tf.get_variable( name=self.weight_dict['P']['r']['weight'], dtype=self.dtype, initializer=p_array.astype(np.float32), trainable=False)) # Gate weights setattr( self, self.weight_dict['I']['r']['weight'], tf.get_variable(name=self.weight_dict['I']['r']['weight'], dtype=self.dtype, trainable=True, initializer=initialization.xavier_initializer( shape=self.i_shape, uniform=self.normal_initializer, mask=None))) # setattr( # self, # self.weight_dict['I']['f']['weight'], # tf.get_variable( # name=self.weight_dict['I']['f']['weight'], # dtype=self.dtype, # trainable=True, # initializer=initialization.xavier_initializer( # shape=self.i_shape, # uniform=self.normal_initializer, # mask=None))) if self.gate_bias_init == 'chronos': bias_init = -tf.log( tf.random_uniform( self.bias_shape, minval=1, maxval=self.timesteps - 1)) else: bias_init = tf.ones(self.bias_shape) setattr( self, self.weight_dict['I']['r']['bias'], tf.get_variable(name=self.weight_dict['I']['r']['bias'], dtype=self.dtype, trainable=True, initializer=bias_init)) # Output setattr( self, self.weight_dict['O']['r']['weight'], tf.get_variable(name=self.weight_dict['O']['r']['weight'], dtype=self.dtype, trainable=True, initializer=initialization.xavier_initializer( shape=self.o_shape, uniform=self.normal_initializer, mask=None))) # setattr( # self, # self.weight_dict['O']['f']['weight'], # tf.get_variable( # name=self.weight_dict['O']['f']['weight'], # dtype=self.dtype, # trainable=True, # initializer=initialization.xavier_initializer( # shape=self.o_shape, # uniform=self.normal_initializer, # mask=None))) if self.gate_bias_init == 'chronos': # bias_init = -tf.log( # tf.random_uniform( # self.bias_shape, minval=1, maxval=self.timesteps - 1)) bias_init = -bias_init else: bias_init = tf.ones(self.bias_shape) setattr( # TODO: smart initialization of these self, self.weight_dict['O']['r']['bias'], tf.get_variable(name=self.weight_dict['O']['r']['bias'], dtype=self.dtype, trainable=True, initializer=bias_init)) # Degree of freedom weights (vectors) w_shape = [1, 1, 1, self.k] b_shape = [1, 1, 1, self.k] # w_array = np.ones(w_shape).astype(np.float32) # b_array = np.zeros(b_shape).astype(np.float32) # Divisive params if self.beta and not self.lesion_beta: self.beta = tf.get_variable( name='beta', initializer=initialization.xavier_initializer( shape=w_shape, uniform=self.normal_initializer, mask=None)) # initializer=tf.ones(w_shape, dtype=tf.float32)) elif self.lesion_beta: self.beta = tf.constant(0.) else: self.beta = tf.constant(1.) if self.nu and not self.lesion_nu: self.nu = tf.get_variable( name='nu', initializer=initialization.xavier_initializer( shape=b_shape, uniform=self.normal_initializer, mask=None)) # initializer=tf.zeros(b_shape, dtype=tf.float32)) elif self.lesion_nu: self.nu = tf.constant(0.) else: self.nu = tf.constant(1.) if self.zeta: self.zeta = tf.get_variable( name='zeta', initializer=initialization.xavier_initializer( shape=w_shape, uniform=self.normal_initializer, mask=None)) else: self.zeta = tf.constant(1.) if self.gamma: self.gamma = tf.get_variable( name='gamma', initializer=initialization.xavier_initializer( shape=w_shape, uniform=self.normal_initializer, mask=None)) else: self.gamma = tf.constant(1.) # # TODO # self.ebias = tf.get_variable( # name='ebias', # initializer=initialization.xavier_initializer( # shape=b_shape, # uniform=self.normal_initializer, # mask=None)) if self.xi: self.xi = tf.get_variable( name='xi', initializer=initialization.xavier_initializer( shape=w_shape, uniform=self.normal_initializer, mask=None)) else: self.xi = tf.constant(1.) if self.multiplicative_excitation: if self.lesion_kappa: self.kappa = tf.constant(0.) else: self.kappa = tf.get_variable( name='kappa', initializer=initialization.xavier_initializer( shape=w_shape, uniform=self.normal_initializer, mask=None)) # initializer=tf.zeros(w_shape, dtype=tf.float32) + 0.5) if self.lesion_omega: self.omega = tf.constant(0.) else: self.omega = tf.get_variable( name='omega', initializer=initialization.xavier_initializer( shape=w_shape, uniform=self.normal_initializer, mask=None)) # initializer=tf.zeros(w_shape, dtype=tf.float32) + 0.5) else: self.kappa = tf.constant(1.) self.omega = tf.constant(1.) if self.adapation: self.rho = tf.get_variable(name='rho', initializer=tf.ones( self.timesteps, dtype=tf.float32)) if self.lesion_omega: self.omega = tf.constant(0.) if self.lesion_kappa: self.kappa = tf.constant(0.) self.lateral_bias = tf.get_variable( name='lateral_bias', initializer=initialization.xavier_initializer( shape=b_shape, uniform=self.normal_initializer, mask=None))