def plot_activations(): for activation in common.Activation.available: x = np.linspace(-10.0, 10.0, 100) y = common.Activation(activation).function(x, deriv=False) dy = common.Activation(activation).function(x, deriv=True) viz_client.plot_func(x, y, title=activation) viz_client.plot_func(x, dy, title="d_" + activation)
def test_tanh(self): activation = nnc.Activation("tanh") # Compare scalars self.assertEqual(activation.function(1), math.tanh(1)) self.assertEqual(activation.function(0), math.tanh(0)) # Compare vectors vect = np.array([1, 2, 3]) true_values = np.array( [0.7615941559557649, 0.9640275800758169, 0.9950547536867305] ) self.assertTrue(np.array_equal(activation.function(vect), true_values)) # Compare matrices matrix = np.array([[0.2, 0.0000001, 3], [-0.5, 1000, -10]]) true_values = np.array( [[0.19737532, 0.0000001, 0.99505475], [-0.46211716, 1.0, -1.0]] ) self.assertTrue( np.array_equal(np.round(activation.function(matrix), 8), true_values) ) # Test derivative and scalars self.assertEqual(activation._tanh(1, deriv=True), 1 - math.tanh(1) ** 2) self.assertEqual(activation._tanh(0, deriv=True), 1 - math.tanh(0) ** 2) # Test derivative and vectors true_values = np.array( [0.41997434161402614, 0.07065082485316443, 0.009866037165440211] ) self.assertTrue(np.array_equal(activation._tanh(vect, deriv=True), true_values)) # Test derivative and matrices true_values = np.array( [[0.96104298, 1.0, 0.00986604], [0.78644773, 0.0, 0.00000001]] ) self.assertTrue( np.array_equal( np.round(activation._tanh(matrix, deriv=True), 8), true_values ) )
def __init__(self, num_features, batch_size, num_hidden=0, hidden_sizes=None, activation="sigmoid", learning_rate=0.01, learning_decay=None, weight_decay=None, dropout_rate=None, init_weight_spread=1.0, random_seed=None): """ Initialize a blank numpy net object This object will have input/output layers and weights (neurons and synapses) Both will be lists of numpy arrays having varying sizes Synapses are initialized with random weights with mean 0 :param num_features: (int) Shape of the input layer :param batch_size: (int) Size of the batches you will be running through the net while training :param num_hidden: (int) Number of hidden layers :param hidden_sizes: (list[int]) The sizes you want your hidden layers to be :param activation: (str) or (list[str]) The activation function you want to use, if given a list will specify an activation function for each layer explicitly :param learning_rate: :param learning_decay: :param weight_decay: :param dropout_rate: :param init_weight_spread: :param random_seed: (int) Can specify the random seed if you want to reproduce the same runs """ self.default_layer_size = 16 # Initialize arrays used for neurons and synapses self.batch_size = batch_size self.num_layers = 2 + num_hidden # Input, output, and hidden layers self.num_hidden = num_hidden self.layer = [np.empty(0)] * self.num_layers self.weight = [np.empty(0)] * (self.num_layers - 1) # Set all of the activation functions, you need one for each layer of weights, or one # less than the number of layers. This can be a list of different functions or a string for all the same type if isinstance(activation, str): self.activation_function = [common.Activation(activation).function] * (self.num_layers - 1) self.activation_names = [activation] * self.num_layers elif isinstance(activation, list): if len(activation) == self.num_layers - 1: self.activation_function = list() for i in range(self.num_layers - 1): print(i, activation[i]) self.activation_function.append(common.Activation(activation[i]).function) self.activation_names = activation else: msg = ("activation_function must be one less than the number of layers in your network " "(num_layers-1 = " + str(self.num_layers - 1) + ")") log.out.error(msg) raise ValueError(msg) else: msg = "activation_function must be a string or a list of strings" log.out.error(msg) raise ValueError(msg) # Set network hyperparameters self.learning_rate = learning_rate self.learning_decay = learning_decay self.weight_decay = weight_decay self.dropout_rate = dropout_rate # For diagnostics self.loss_history = list() # This will be appended to a lot so a list is a bit better self.predict_space = None # If left undefined will define by training input bounds self.input_shape = [batch_size, num_features] if hidden_sizes is None: self.hidden_sizes = [self.default_layer_size] * self.num_hidden else: self.hidden_sizes = hidden_sizes self.output_shape = [batch_size, 1] # If requested seed random numbers to make calculation (makes repeatable) if random_seed is not None: np.random.seed(random_seed) else: current_seed = int(np.random.random(1) * 4.0E9) # 4 billion is close to limit of 32 bit unsigned int np.random.seed(current_seed) log.out.info("No random seed selected, using: " + str(current_seed)) # Initialize weights with random noise centered around zero, spread set by init_weight_spread self.weight[0] = (init_weight_spread * 2) * np.random.random([self.input_shape[1], self.hidden_sizes[0]]) - init_weight_spread for i in range(self.num_hidden-1): self.weight[i+1] = (init_weight_spread * 2) * np.random.random([self.weight[i].shape[1], self.hidden_sizes[i+1]]) - init_weight_spread self.weight[self.num_hidden] = (init_weight_spread * 2) * np.random.random([self.weight[self.num_hidden-1].shape[1], self.output_shape[1]]) - init_weight_spread # Initialize layers with zeros self.forward(np.zeros(self.input_shape)) # Add this list of all the layer sizes for easy access later self.layer_sizes = list() for layer in self.layer: self.layer_sizes.append(layer.shape[1])