def _add_hidden_units(self, num_new_hiddens, position=0, initial_weights='AUTO', initial_bias='AUTO', initial_offsets='AUTO'): """ This function adds new hidden units at the given position to the model. \ .. Warning:: If the parameters are changed. the trainer needs to be reinitialized. :param num_new_hiddens: The number of new hidden units to add. :type num_new_hiddens: int :param position: Position where the units should be added. :type position: int :param initial_weights: The initial weight values for the hidden units. :type initial_weights: 'AUTO' or scalar or numpy array [input_dim, num_new_hiddens] :param initial_bias: The initial hidden bias values. :type initial_bias: 'AUTO' or scalar or numpy array [1, num_new_hiddens] :param initial_offsets: The initial hidden mean values. :type initial_offsets: 'AUTO' or scalar or numpy array [1, num_new_hiddens] """ # AUTO -> Small random values out of # +-4*numx.sqrt(6/(self.input_dim+self.output_dim) # Scalar -> Small Gaussian distributed random values with std_dev # initial_weights # Array -> The corresponding values are used if initial_weights is 'AUTO': new_weights = ( (2.0 * numx.random.rand(self.input_dim, num_new_hiddens) - 1.0) * (4.0 * numx.sqrt( 6.0 / (self.input_dim + self.output_dim + num_new_hiddens)))) else: if numx.isscalar(initial_weights): new_weights = numx.random.randn( self.input_dim, num_new_hiddens) * initial_weights else: new_weights = initial_weights self.w = numx.array( numx.insert(self.w, numx.ones(num_new_hiddens) * position, new_weights, axis=1), self.dtype) # AUTO -> Initialized to Hidden range mean # Scalar -> Initialized to given value # Array -> The corresponding values are used if initial_offsets is 'AUTO': new_oh = numx.zeros((1, num_new_hiddens)) + 0.5 else: if numx.isscalar(initial_offsets): new_oh = numx.zeros((1, num_new_hiddens)) + initial_offsets else: new_oh = initial_offsets self.oh = numx.array( numx.insert(self.oh, numx.ones(num_new_hiddens) * position, new_oh, axis=1), self.dtype) # AUTO -> Initialized to randn()*0.01 # Scalar -> Initialized to given value + randn()*0.01 # Array -> The corresponding values are used if initial_bias is 'AUTO': new_bias = numx.zeros((1, num_new_hiddens)) else: if initial_bias is 'INVERSE_SIGMOID': new_bias = Sigmoid.g(numx.clip(new_oh, 0.01, 0.99)).reshape(new_oh.shape) else: if numx.isscalar(initial_bias): new_bias = initial_bias + numx.zeros((1, num_new_hiddens)) else: new_bias = numx.array(initial_bias, dtype=self.dtype) self.bh = numx.array( numx.insert(self.bh, numx.ones(num_new_hiddens) * position, new_bias, axis=1), self.dtype) self.output_dim = self.w.shape[1]
def __init__(self, number_visibles, number_hiddens, data=None, visible_activation_function=Sigmoid, hidden_activation_function=Sigmoid, initial_weights='AUTO', initial_visible_bias='AUTO', initial_hidden_bias='AUTO', initial_visible_offsets='AUTO', initial_hidden_offsets='AUTO', dtype=numx.float64): """ This function initializes all necessary parameters and data structures. It is recommended to pass the \ training data to initialize the network automatically. :param number_visibles: Number of the visible variables. :type number_visibles: int :param number_hiddens: Number of the hidden variables. :type number_hiddens: int :param data: The training data for parameter initialization if 'AUTO' is chosen for the corresponding parameter. :type data: None or numpy array [num samples, input dim] :param visible_activation_function: Activation function for the visible units. :type visible_activation_function: pydeep.base.activationFunction :param hidden_activation_function: Activation function for the hidden units. :type hidden_activation_function: pydeep.base.activationFunction :param initial_weights: Initial weights. 'AUTO' and a scalar are random init. :type initial_weights: 'AUTO', scalar or numpy array [input dim, output_dim] :param initial_visible_bias: Initial visible bias. 'AUTO' is random, 'INVERSE_SIGMOID' is the inverse Sigmoid \ of the visible mean. If a scalar is passed all values are initialized with it. :type initial_visible_bias: 'AUTO','INVERSE_SIGMOID', scalar or numpy array [1, input dim] :param initial_hidden_bias: Initial hidden bias. 'AUTO' is random, 'INVERSE_SIGMOID' is the inverse Sigmoid of \ the hidden mean. If a scalar is passed all values are initialized with it. :type initial_hidden_bias: 'AUTO','INVERSE_SIGMOID', scalar or numpy array [1, output_dim] :param initial_visible_offsets: Initial visible offset values. AUTO=data mean or 0.5 if no data is given. If a \ scalar is passed all values are initialized with it :type initial_visible_offsets: 'AUTO', scalar or numpy array [1, input dim] :param initial_hidden_offsets: Initial hidden offset values. AUTO = 0.5 If a scalar is passed all values are \ initialized with it. :type initial_hidden_offsets: 'AUTO', scalar or numpy array [1, output_dim] :param dtype: Used data type i.e. numpy.float64. :type dtype: numpy.float32 or numpy.float64 or numpy.float128 """ # Set internal datatype self.dtype = dtype # Set input and output dimension self.input_dim = number_visibles self.output_dim = number_hiddens self.visible_activation_function = visible_activation_function self.hidden_activation_function = hidden_activation_function self._data_mean = 0.5 * numx.ones((1, self.input_dim), self.dtype) self._data_std = numx.ones((1, self.input_dim), self.dtype) if data is not None: if isinstance(data, list): data = numx.concatenate(data) if self.input_dim != data.shape[1]: raise ex.ValueError( "Data dimension and model input dimension have to be equal!" ) self._data_mean = data.mean(axis=0).reshape(1, data.shape[1]) self._data_std = data.std(axis=0).reshape(1, data.shape[1]) # AUTO -> Small random values out of # +-4*numx.sqrt(6/(self.input_dim+self.output_dim) # Scalar -> Small Gaussian distributed random values with std_dev # initial_weights # Array -> The corresponding values are used if initial_weights is 'AUTO': self.w = numx.array( (2.0 * numx.random.rand(self.input_dim, self.output_dim) - 1.0) * (4.0 * numx.sqrt(6.0 / (self.input_dim + self.output_dim))), dtype=dtype) else: if numx.isscalar(initial_weights): self.w = numx.array( numx.random.randn(self.input_dim, self.output_dim) * initial_weights, dtype=dtype) else: self.w = numx.array(initial_weights, dtype=dtype) # AUTO -> data != None -> Initialized to the data mean # data == None -> Initialized to Visible range mean # Scalar -> Initialized to given value # Array -> The corresponding values are used self.ov = numx.zeros((1, self.input_dim)) if initial_visible_offsets is 'AUTO': if data is not None: self.ov += self._data_mean else: self.ov += 0.5 else: if numx.isscalar(initial_visible_offsets): self.ov += initial_visible_offsets else: self.ov += initial_visible_offsets.reshape(1, self.input_dim) self.ov = numx.array(self.ov, dtype=dtype) # AUTO -> data != None -> Initialized to the inverse sigmoid of # data mean # data == Initialized to randn()*0.01 # Scalar -> Initialized to given value + randn()*0.01 # Array -> The corresponding values are used if initial_visible_bias is 'AUTO': if data is None: self.bv = numx.zeros((1, self.input_dim)) else: self.bv = numx.array(Sigmoid.g( numx.clip(self._data_mean, 0.001, 0.999)), dtype=dtype).reshape(self.ov.shape) else: if initial_visible_bias is 'INVERSE_SIGMOID': self.bv = numx.array(Sigmoid.g(numx.clip( self.ov, 0.001, 0.999)), dtype=dtype).reshape(1, self.input_dim) else: if numx.isscalar(initial_visible_bias): self.bv = numx.array(initial_visible_bias + numx.zeros( (1, self.input_dim)), dtype=dtype) else: self.bv = numx.array(initial_visible_bias, dtype=dtype) # AUTO -> Initialized to Hidden range mean # Scalar -> Initialized to given value # Array -> The corresponding values are used self.oh = numx.zeros((1, self.output_dim)) if initial_hidden_offsets is 'AUTO': self.oh += 0.5 else: if numx.isscalar(initial_hidden_offsets): self.oh += initial_hidden_offsets else: self.oh += initial_hidden_offsets.reshape(1, self.output_dim) self.oh = numx.array(self.oh, dtype=dtype) # AUTO -> Initialized to randn()*0.01 # Scalar -> Initialized to given value + randn()*0.01 # Array -> The corresponding values are used if initial_hidden_bias is 'AUTO': self.bh = numx.zeros((1, self.output_dim)) else: if initial_hidden_bias is 'INVERSE_SIGMOID': self.bh = numx.array(Sigmoid.g(numx.clip( self.oh, 0.001, 0.999)), dtype=dtype).reshape(self.oh.shape) else: if numx.isscalar(initial_hidden_bias): self.bh = numx.array(initial_hidden_bias + numx.zeros( (1, self.output_dim)), dtype=dtype) else: self.bh = numx.array(initial_hidden_bias, dtype=dtype)
offset_typ = 'DDD' dbm = MODEL.BinaryBinaryDBM(N, M, O, offset_typ, train_set) # Set the same seed value for all algorithms numx.random.seed(42) # Initialize parameters dbm.W1 = numx.random.randn(N, M) * 0.01 dbm.W2 = numx.random.randn(M, O) * 0.01 dbm.o1 = numx.mean(train_set, axis=0).reshape(1, N) dbm.o2 = numx.zeros((1, M)) + 0.5 dbm.o3 = numx.zeros((1, O)) + 0.5 dbm.b1 = Sigmoid.g(numx.clip(dbm.o1, 0.001, 0.999)) dbm.b2 = Sigmoid.g(numx.clip(dbm.o2, 0.001, 0.999)) dbm.b3 = Sigmoid.g(numx.clip(dbm.o3, 0.001, 0.999)) # Initialize negative Markov chain dbm.m1 = dbm.o1 + numx.zeros((batch_size, N)) dbm.m2 = dbm.o2 + numx.zeros((batch_size, M)) dbm.m3 = dbm.o3 + numx.zeros((batch_size, O)) # Choose trainer CD, PCD, PT trainer = TRAINER.PCD(dbm, batch_size) # Set AIS betas / inv. temps for AIS a = numx.linspace(0.0, 0.5, 100 + 1) a = a[0:a.shape[0] - 1] b = numx.linspace(0.5, 0.9, 800 + 1)
def __init__(self, number_visibles, number_hiddens, data = None, visible_activation_function = Sigmoid, hidden_activation_function = Sigmoid, initial_weights = 'AUTO', initial_visible_bias = 'AUTO', initial_hidden_bias = 'AUTO', initial_visible_offsets = 'AUTO', initial_hidden_offsets = 'AUTO', dtype = numx.float64): ''' This function initializes all necessary parameters and data structures. It is recommended to pass the training data to initialize the network automatically. :Parameters: number_visibles: Number of the visible variables. -type: int number_hiddens Number of hidden variables. -type: int data: The training data for parameter initialization if 'AUTO' is chosen for the corresponding parameter. -type: None or numpy array [num samples, input dim] visible_activation_function Activation function for the visible units. -type: ActivationFunction hidden_activation_function Activation function for the hidden units. -type: ActivationFunction initial_weights: Initial weights. 'AUTO' and a scalar are random init. -type: 'AUTO', scalar or numpy array [input dim, output_dim] initial_visible_bias: Initial visible bias. 'AUTO' is random, 'INVERSE_SIGMOID' is the inverse Sigmoid of the visilbe mean. If a scalar is passed all values are initialized with it. -type: 'AUTO','INVERSE_SIGMOID', scalar or numpy array [1, input dim] initial_hidden_bias: Initial hidden bias. 'AUTO' is random, 'INVERSE_SIGMOID' is the inverse Sigmoid of the hidden mean. If a scalar is passed all values are initialized with it. -type: 'AUTO','INVERSE_SIGMOID', scalar or numpy array [1, output_dim] initial_visible_offsets: Initial visible offset values. AUTO=data mean or 0.5 if no data is given. If a scalar is passed all values are initialized with it. -type: 'AUTO', scalar or numpy array [1, input dim] initial_hidden_offsets: Initial hidden offset values. AUTO = 0.5 If a scalar is passed all values are initialized with it. -type: 'AUTO', scalar or numpy array [1, output_dim] dtype: Used data type i.e. numpy.float64 -type: numpy.float32 or numpy.float64 or numpy.float128 ''' # Set internal datatype self.dtype = dtype # Set input and output dimension self.input_dim = number_visibles self.output_dim = number_hiddens self.visible_activation_function = visible_activation_function self.hidden_activation_function = hidden_activation_function self._data_mean = 0.5*numx.ones((1, self.input_dim),self.dtype) self._data_std = numx.ones((1, self.input_dim),self.dtype) if data is not None: if isinstance(data,list): data = numx.concatenate(data) if self.input_dim != data.shape[1]: raise ex.ValueError("Data dimension and model input \ dimension have to be equal!") self._data_mean = data.mean(axis=0).reshape(1,data.shape[1]) self._data_std = data.std(axis=0).reshape(1,data.shape[1]) # AUTO -> Small random values out of # +-4*numx.sqrt(6/(self.input_dim+self.output_dim) # Scalar -> Small Gaussian distributed random values with std_dev # initial_weights # Array -> The corresponding values are used if initial_weights is 'AUTO': self.w = numx.array((2.0 * numx.random.rand(self.input_dim, self.output_dim) - 1.0) * (4.0 * numx.sqrt(6.0 / (self.input_dim + self.output_dim))) ,dtype=dtype) else: if(numx.isscalar(initial_weights)): self.w = numx.array(numx.random.randn(self.input_dim, self.output_dim) * initial_weights, dtype=dtype) else: self.w = numx.array(initial_weights, dtype=dtype) # AUTO -> data != None -> Initialized to the data mean # data == None -> Initialized to Visible range mean # Scalar -> Initialized to given value # Array -> The corresponding values are used self.ov = numx.zeros((1,self.input_dim)) if initial_visible_offsets is 'AUTO': if data is not None: self.ov += self._data_mean else: self.ov += 0.5 else: if(numx.isscalar(initial_visible_offsets)): self.ov += initial_visible_offsets else: self.ov += initial_visible_offsets.reshape(1,self.input_dim) self.ov = numx.array(self.ov, dtype=dtype) # AUTO -> data != None -> Initialized to the inverse sigmoid of # data mean # data == Initialized to randn()*0.01 # Scalar -> Initialized to given value + randn()*0.01 # Array -> The corresponding values are used if initial_visible_bias is 'AUTO': if data is None: self.bv = numx.zeros((1, self.input_dim)) else: self.bv = numx.array(Sigmoid.g(numx.clip(self._data_mean,0.001, 0.999)), dtype=dtype).reshape(self.ov.shape) else: if initial_visible_bias is 'INVERSE_SIGMOID': self.bv = numx.array(Sigmoid.g(numx.clip(self.ov,0.001, 0.999)), dtype=dtype).reshape(1,self.input_dim) else: if(numx.isscalar(initial_visible_bias)): self.bv = numx.array(initial_visible_bias + numx.zeros((1, self.input_dim)) , dtype=dtype) else: self.bv = numx.array(initial_visible_bias, dtype=dtype) # AUTO -> Initialized to Hidden range mean # Scalar -> Initialized to given value # Array -> The corresponding values are used self.oh = numx.zeros((1,self.output_dim)) if initial_hidden_offsets is 'AUTO': self.oh += 0.5 else: if(numx.isscalar(initial_hidden_offsets)): self.oh += initial_hidden_offsets else: self.oh += initial_hidden_offsets.reshape(1,self.output_dim) self.oh = numx.array(self.oh, dtype=dtype) # AUTO -> Initialized to randn()*0.01 # Scalar -> Initialized to given value + randn()*0.01 # Array -> The corresponding values are used if initial_hidden_bias is 'AUTO': self.bh = numx.zeros((1, self.output_dim)) else: if initial_hidden_bias is 'INVERSE_SIGMOID': self.bh = numx.array( Sigmoid.g(numx.clip(self.oh,0.001,0.999)) , dtype=dtype).reshape(self.oh.shape) else: if(numx.isscalar(initial_hidden_bias)): self.bh = numx.array(initial_hidden_bias + numx.zeros((1, self.output_dim)) , dtype=dtype) else: self.bh = numx.array(initial_hidden_bias, dtype=dtype)
def _add_hidden_units(self, num_new_hiddens, position = 0, initial_weights='AUTO', initial_bias='AUTO', initial_offsets = 'AUTO'): ''' This function adds new hidden units at the given position to the model. Warning: If the parameters are changed. the trainer needs to be reinitialized. :Parameters: num_new_hiddens: The number of new hidden units to add. -type: int position: Position where the units should be added. -type: int initial_weights: The initial weight values for the hidden units. -type: 'AUTO' or scalar or numpy array [input_dim, num_new_hiddens] initial_bias: The initial hidden bias values. -type: 'AUTO' or scalar or numpy array [1, num_new_hiddens] initial_offsets: The initial hidden mean values. -type: 'AUTO' or scalar or numpy array [1, num_new_hiddens] ''' # AUTO -> Small random values out of # +-4*numx.sqrt(6/(self.input_dim+self.output_dim) # Scalar -> Small Gaussian distributed random values with std_dev # initial_weights # Array -> The corresponding values are used new_weights = None if initial_weights == 'AUTO': new_weights = ((2.0 * numx.random.rand(self.input_dim, num_new_hiddens) - 1.0) * (4.0 * numx.sqrt(6.0 / (self.input_dim + self.output_dim + num_new_hiddens)))) else: if(numx.isscalar(initial_weights)): new_weights = numx.random.randn(self.input_dim, num_new_hiddens) * initial_weights else: new_weights = initial_weights self.w = numx.array(numx.insert(self.w, numx.ones((num_new_hiddens) ) * position, new_weights, axis=1),self.dtype) # AUTO -> Initialized to Hidden range mean # Scalar -> Initialized to given value # Array -> The corresponding values are used new_oh = None if initial_offsets == 'AUTO': new_oh = numx.zeros((1, num_new_hiddens)) + 0.5 else: if(numx.isscalar(initial_offsets)): new_oh = numx.zeros((1, num_new_hiddens)) + initial_offsets else: new_oh = initial_offsets self.oh = numx.array(numx.insert(self.oh, numx.ones((num_new_hiddens) ) * position, new_oh, axis=1),self.dtype) # AUTO -> Initialized to randn()*0.01 # Scalar -> Initialized to given value + randn()*0.01 # Array -> The corresponding values are used if initial_bias == 'AUTO': new_bias = numx.zeros((1, num_new_hiddens)) else: if initial_bias == 'INVERSE_SIGMOID': new_bias = Sigmoid.g(numx.clip(new_oh,0.01, 0.99)).reshape(new_oh.shape) else: if(numx.isscalar(initial_bias)): new_bias = initial_bias + numx.zeros((1, num_new_hiddens)) else: new_bias = numx.array(initial_bias, dtype=self.dtype) self.bh = numx.array(numx.insert(self.bh, numx.ones((num_new_hiddens) ) * position, new_bias, axis=1),self.dtype) self.output_dim = self.w.shape[1]
def __init__(self, input_weight_layer, output_weight_layer, data=None, initial_bias='AUTO', initial_offsets='AUTO', dtype=numx.float64): ''' This function initializes the weight layer. :Parameters: input_weight_layer: Referenz to the input weights. -type: Weight_layer or None output_weight_layer Referenz to the output weights. -type: Weight_layer or None data: The training data for initializing the visible bias. -type: None or numpy array [num samples, input dim] or List of numpy arrays [num samples, input dim] initial_bias: Initial visible bias. -type: 'AUTO', scalar or numpy array [1,input dim] initial_offsets: Initial visible mean values. -type: 'AUTO', scalar or numpy array [1, input dim] dtype: Used data type i.e. numpy.float64 -type: numpy.float32 or numpy.float64 or numpy.float128 ''' # Set internal datatype self.dtype = dtype # Set input output dimesnions self.input_weight_layer = input_weight_layer self.output_weight_layer = output_weight_layer # Check that input and output layer match, which has not always to be the case e.g. SoftmaxUnitLayer if input_weight_layer != None: self.input_dim = input_weight_layer.output_dim self.output_dim = self.input_dim else: if output_weight_layer != None: self.output_dim = output_weight_layer.input_dim self.input_dim = self.output_dim else: raise NotImplementedError( "Unit layer needs at least one connected weight layer!") if data is not None: if isinstance(data, list): data = numx.concatenate(data) if self.input_dim != data.shape[1]: raise ValueError("Data dimension and model input \ dimension have to be equal!") data_mean = numx.mean(data, axis=0).reshape(1, data.shape[1]) # AUTO -> data is not None -> Initialized to the data mean # data is None -> Initialized to Visible range mean # Scalar -> Initialized to given value # Array -> The corresponding values are used self.offset = numx.zeros((1, self.input_dim)) if initial_offsets is 'AUTO': if data is not None: self.offset += data_mean else: self.offset += 0.5 else: if (numx.isscalar(initial_offsets)): self.offset += initial_offsets else: self.offset += initial_offsets.reshape(1, self.input_dim) self.offset = numx.array(self.offset, dtype=dtype) # AUTO -> data is not None -> Initialized to the inverse sigmoid of # data mean # data is Initialized to randn()*0.01 # Scalar -> Initialized to given value + randn()*0.01 # Array -> The corresponding values are used self.bias = numx.zeros((1, self.input_dim)) if initial_bias is 'AUTO': if data is not None: self.bias = numx.array(Sigmoid.g( numx.clip(data_mean, 0.001, 0.999)), dtype=dtype).reshape(self.offset.shape) else: if initial_bias is 'INVERSE_SIGMOID': self.bias = numx.array(Sigmoid.g( numx.clip(self.offset, 0.001, 0.999)), dtype=dtype).reshape(1, self.input_dim) else: if (numx.isscalar(initial_bias)): self.bias = numx.array(initial_bias + numx.zeros( (1, self.input_dim)), dtype=dtype) else: self.bias = numx.array(initial_bias, dtype=dtype)
def __init__(self, input_dim, hidden1_dim, hidden2_dim, offset_typ, data, dtype=numx.float64): ''' Initializes the network :Parameters: input_dim: Number of input dimensions. -type: int hidden1_dim: Number of hidden dimensions for the first hidden layer. -type: int hidden2_dim: Number of hidden dimensions for the first hidden layer. -type: int offset_typ: Typs of offset values used for specific initialization 'DDD' -> Centering, 'AAA'-> Enhanced gradient,'MMM' -> Model mean centering -type: string (3 chars) ''' # Set used data type self.dtype = dtype # Set dimensions self.input_dim = input_dim self.hidden1_dim = hidden1_dim self.hidden2_dim = hidden2_dim # Initialize weights self.W1 = numx.random.randn(input_dim, hidden1_dim) * 0.01 self.W2 = numx.random.randn(hidden1_dim, hidden2_dim) * 0.01 # Initialize offsets self.o1 = numx.zeros((1, input_dim)) self.o2 = numx.zeros((1, hidden1_dim)) self.o3 = numx.zeros((1, hidden2_dim)) self.b1 = numx.zeros((1, input_dim)) self.b2 = numx.zeros((1, hidden1_dim)) self.b3 = numx.zeros((1, hidden2_dim)) if data is not None: datamean = numx.mean(data, axis=0).reshape(1, input_dim) if offset_typ[0] is '0': self.b1 = Sigmoid.g(numx.clip(datamean, 0.001, 0.999)) if offset_typ[0] is 'D': self.o1 = numx.copy(datamean) self.b1 = Sigmoid.g(numx.clip(self.o1, 0.001, 0.999)) if offset_typ[0] is 'A': self.o1 = (datamean + 0.5) / 2.0 self.b1 = Sigmoid.g(numx.clip(self.o1, 0.001, 0.999)) if offset_typ[0] is 'M': self.o1 += 0.5 else: if offset_typ[0] != '0': self.o1 += 0.5 if offset_typ[1] != '0': self.o2 += 0.5 if offset_typ[2] != '0': self.o3 += 0.5