def __init__( self, vgg16_npy_path, train, timesteps, reuse, fgru_normalization_type, ff_normalization_type, perturb=None, layer_name='recurrent_vgg16', ff_nl=tf.nn.relu, horizontal_kernel_initializer=tf.initializers.orthogonal(), # horizontal_kernel_initializer=tf_fun.Identity(), kernel_initializer=tf.initializers.orthogonal(), gate_initializer=tf.initializers.orthogonal(), train_ff_gate=None, train_fgru_gate=None, train_norm_moments=None, train_norm_params=None, train_fgru_kernels=None, train_fgru_params=None, downsampled=False, up_kernel=None, stop_loop=False, recurrent_ff=False, perturb_method="hidden_state", # "kernel" strides=[1, 1, 1, 1], pool_strides=[2, 2], # Because fgrus are every other down-layer pool_kernel=[2, 2], data_format='NHWC', horizontal_padding='SAME', ff_padding='SAME', vgg_dtype=tf.bfloat16, aux=None): if vgg16_npy_path is None: path = inspect.getfile(Vgg16) path = os.path.abspath(os.path.join(path, os.pardir)) path = os.path.join(path, "vgg16.npy") vgg16_npy_path = path print path self.data_format = data_format self.pool_strides = pool_strides self.strides = strides self.pool_kernel = pool_kernel self.fgru_normalization_type = fgru_normalization_type self.ff_normalization_type = ff_normalization_type self.horizontal_padding = horizontal_padding self.ff_padding = ff_padding self.train = train self.layer_name = layer_name self.data_format = data_format self.horizontal_kernel_initializer = horizontal_kernel_initializer self.kernel_initializer = kernel_initializer self.gate_initializer = gate_initializer self.fgru_normalization_type = fgru_normalization_type self.ff_normalization_type = ff_normalization_type self.recurrent_ff = recurrent_ff self.stop_loop = stop_loop self.ff_nl = ff_nl self.fgru_connectivity = '' self.reuse = reuse self.timesteps = timesteps self.downsampled = downsampled self.last_timestep = timesteps - 1 self.perturb = perturb self.perturb_method = perturb_method if train_ff_gate is None: self.train_ff_gate = self.train else: self.train_ff_gate = train_ff_gate if train_fgru_gate is None: self.train_fgru_gate = self.train else: self.train_fgru_gate = train_fgru_gate if train_norm_moments is None: self.train_norm_moments = self.train else: self.train_norm_moments = train_norm_moments if train_norm_moments is None: self.train_norm_params = self.train else: self.train_norm_params = train_norm_params if train_fgru_kernels is None: self.train_fgru_kernels = self.train else: self.train_fgru_kernels = train_fgru_kernels if train_fgru_kernels is None: self.train_fgru_params = self.train else: self.train_fgru_params = train_fgru_params default_vars = defaults() if aux is not None and isinstance(aux, dict): for k, v in aux.iteritems(): default_vars[k] = v self.update_params(default_vars) # Store variables in the order they were created. Hack for python 2.x. self.variable_list = OrderedDict() self.hidden_dict = OrderedDict() # Kernel info if data_format is 'NHWC': self.prepared_pool_kernel = [1] + self.pool_kernel + [1] self.prepared_pool_stride = [1] + self.pool_strides + [1] self.up_strides = [1] + self.pool_strides + [1] else: raise NotImplementedError self.prepared_pool_kernel = [1, 1] + self.pool_kernel self.prepared_pool_stride = [1, 1] + self.pool_strides self.up_strides = [1, 1] + self.pool_strides self.sanity_check() if self.symmetric_weights: self.symmetric_weights = self.symmetric_weights.split('_') # Nonlinearities and initializations if isinstance(self.recurrent_nl, basestring): self.recurrent_nl = tf_fun.interpret_nl(self.recurrent_nl) # Set initializers for greek letters if self.force_alpha_divisive: self.alpha_initializer = tf.initializers.variance_scaling else: self.alpha_initializer = tf.constant_initializer(0.1) self.mu_initializer = tf.constant_initializer(0.) self.omega_initializer = tf.constant_initializer(0.1) self.kappa_initializer = tf.constant_initializer(0.) # Handle BN scope reuse self.scope_reuse = reuse # Load weights self.data_dict = np.load(vgg16_npy_path, allow_pickle=True, encoding='latin1').item() print("npy file loaded")
def __init__( self, layer_name, gammanet_constructor, fgru_normalization_type, ff_normalization_type, train, reuse, fgru_connectivity, ff_nl=tf.nn.relu, additional_readouts=None, horizontal_kernel_initializer=tf.initializers.variance_scaling(), kernel_initializer=tf.initializers.variance_scaling(), gate_initializer=tf.contrib.layers.xavier_initializer(), train_ff_gate=None, train_fgru_gate=None, train_norm_moments=None, train_norm_params=None, train_fgru_kernels=None, train_fgru_params=None, up_kernel=None, stop_loop=False, recurrent_ff=False, timesteps=1, strides=[1, 1, 1, 1], pool_strides=[2, 2], pool_kernel=[4, 4], data_format='NHWC', horizontal_padding='SAME', ff_padding='SAME', aux=None): """Global initializations and settings.""" self.timesteps = timesteps self.strides = strides self.pool_strides = pool_strides self.pool_kernel = pool_kernel self.horizontal_padding = horizontal_padding self.ff_padding = ff_padding self.train = train self.layer_name = layer_name self.data_format = data_format self.horizontal_kernel_initializer = horizontal_kernel_initializer self.kernel_initializer = kernel_initializer self.gate_initializer = gate_initializer self.additional_readouts = additional_readouts self.fgru_normalization_type = fgru_normalization_type self.ff_normalization_type = ff_normalization_type self.recurrent_ff = recurrent_ff self.stop_loop = stop_loop self.ff_nl = ff_nl self.fgru_connectivity = fgru_connectivity self.gammanet_constructor = gammanet_constructor if train_ff_gate is None: self.train_ff_gate = self.train else: self.train_ff_gate = train_ff_gate if train_fgru_gate is None: self.train_fgru_gate = self.train else: self.train_fgru_gate = train_fgru_gate if train_norm_moments is None: self.train_norm_moments = self.train else: self.train_norm_moments = train_norm_moments if train_norm_moments is None: self.train_norm_params = self.train else: self.train_norm_params = train_norm_params if train_fgru_kernels is None: self.train_fgru_kernels = self.train else: self.train_fgru_kernels = train_fgru_kernels if train_fgru_kernels is None: self.train_fgru_params = self.train else: self.train_fgru_params = train_fgru_params if up_kernel is None: self.up_kernel = [ h + w for h, w in zip(self.pool_strides, self.pool_kernel) ] print 'No up-kernel provided. Derived: %s' % self.up_kernel else: self.up_kernel = up_kernel # Sort through and assign the auxilliary variables default_vars = defaults() if aux is not None and isinstance(aux, dict): for k, v in aux.iteritems(): default_vars[k] = v self.update_params(default_vars) if self.time_skips: self.conv_dict = OrderedDict() # Store variables in the order they were created. Hack for python 2.x. self.variable_list = OrderedDict() self.hidden_dict = OrderedDict() # Kernel info if data_format is 'NHWC': self.prepared_pool_kernel = [1] + self.pool_kernel + [1] self.prepared_pool_stride = [1] + self.pool_strides + [1] self.up_strides = [1] + self.pool_strides + [1] else: self.prepared_pool_kernel = [1, 1] + self.pool_kernel self.prepared_pool_stride = [1, 1] + self.pool_strides self.up_strides = [1, 1] + self.pool_strides self.sanity_check() if self.symmetric_weights: self.symmetric_weights = self.symmetric_weights.split('_') # Nonlinearities and initializations if isinstance(self.recurrent_nl, basestring): self.recurrent_nl = tf_fun.interpret_nl(self.recurrent_nl) # Set initializers for greek letters if self.force_alpha_divisive: self.alpha_initializer = tf.initializers.variance_scaling else: self.alpha_initializer = tf.ones_initializer self.mu_initializer = tf.zeros_initializer # self.omega_initializer = tf.initializers.variance_scaling self.omega_initializer = tf.ones_initializer self.kappa_initializer = tf.zeros_initializer # Handle BN scope reuse self.scope_reuse = reuse