Esempio n. 1
0
    def populate(self, in_size, out_layer, hidden_layers=[]):
        """
        initializes the layers according to size parameters
        @in_size input size for first lstm layer
        @out_size output size for output layer
        @memory_sizes sizes for state vectors of lstm layers
        """
        layers = hidden_layers
        layers.append(out_layer)

        self.training_layers.append(CachingNeuralLayer(
            in_size=in_size,  # input size equals output(=memory) size of preceding layer
            out_size=layers[0]["size"],
            activation_fn=layers[0]["fn"],
            activation_fn_deriv=layers[0]["fn_deriv"]
        ))
        # for each size in memory_sizes create another lstm layer
        for i in range(1, len(layers)):
            self.training_layers.append(CachingNeuralLayer(
                in_size=self.training_layers[-1].size,  # input size equals output(=memory) size of preceding layer
                out_size=layers[i]["size"],
                activation_fn=layers[i]["fn"],
                activation_fn_deriv=layers[i]["fn_deriv"]
            ))
        # mark network as populated
        Logger.log(str(len(self.training_layers)) + " layers created.")
        self.populated = True
Esempio n. 2
0
 def learn(self, deltas, learning_rate=0.001):
     """
     apply learning algorithm by using deltas from next layer
     @deltas deltas from last (actually next) layer
     """
     Logger.debug("learn(" + str(deltas) + ")")
     # learn recursively over all caches (corresponds to time steps), starting with last cache
     self.learn_recursive(self.last_cache.predecessor, deltas)
     # apply pending weight and bias updates
     self.apply_training(learning_rate)
     # calculate and return deltas for this layer from losses
     deltas = [cache.loss_input for cache in self.caches]
     return deltas
Esempio n. 3
0
    def feed(self, input_data, time_steps=1):
        """
        calculate output vector for given input vector
        @time_steps number of feedforward results to be cached for use in backpropagation through time
        """
        assert time_steps > 0, "time_steps must be at least 1 (for recursive input)!"
        Logger.debug("feed(" + str(input_data) + ")")

        # concatenate input_vector with recurrent input (last output) vector
        concat_in = np.concatenate([input_data, self.last_cache.predecessor.output_values])
        # delete first cache if maximum number of time_steps are already cached
        if time_steps <= len(self.caches):
            self.first_cache.successor.remove()
            self.caches.pop(0)

        # create new cache at end of cache list
        self.last_cache.insert_before(LSTMLayerCache())
        cache = self.last_cache.predecessor
        self.caches.append(cache)

        # cache input and concatenated input values
        cache.input_values = input_data
        cache.concatenated_input = concat_in

        # calculate and cache gate/update_values results
        cache.forget_gate_results = self.forget_gate_layer.feed(concat_in)
        cache.input_gate_results = self.input_gate_layer.feed(concat_in)
        cache.update_values_layer_results = self.update_values_layer.feed(concat_in)
        cache.output_gate_results = self.output_gate_layer.feed(concat_in)

        # calculate state update values
        update_values = np.multiply(
                cache.input_gate_results,
                cache.update_values_layer_results)

        # apply forget layer and apply state update values
        cache.state = cache.predecessor.state * cache.forget_gate_results \
                      + update_values
        # calculate output from new state and output gate
        cache.output_values = Layer.activation_tanh(cache.state) * cache.output_gate_results

        # return calculated output vector
        return cache.output_values
Esempio n. 4
0
 def learn_rec(self, target, layer_id):
     Logger.debug("learn recursive: " + str(layer_id))
     if layer_id > len(self.training_layers):
         raise Exception("invalid layer id!")
     elif layer_id == len(self.training_layers):
         return [target - self.training_layers[-1].caches[0].output_values]
     else:
         last_deltas = self.learn_rec(target, layer_id+1)
     delta = self.training_layers[layer_id].get_delta(
         in_data=self.training_layers[layer_id].caches[0].input_values,
         delta=last_deltas[0],
         predecessor_activation_deriv=self.training_layers[layer_id-1].activation_deriv)
     self.training_layers[layer_id].learn(
         result=self.training_layers[layer_id].caches[0].output_values.T,
         delta=last_deltas[0].T,
         learning_rate=self.config['learning_rate']
     )
     deltas = [delta]
     deltas.extend(last_deltas)
     return deltas
Esempio n. 5
0
    def learn(self, target):
        """
        Calculates the weight updates depending on the loss function derived to the individual weights.
        Requires feedforwarding to be finished for backpropagation through time.
        @targets expected outputs to be compared to actual outputs
        """
        if not self.populated:
            raise Exception("MLP Network needs to be populated first! Have a look at MLPNetwork.populate().")

        Logger.debug("mlpnetwork:learn")
        Logger.debug("target: " + str(target))
        Logger.debug("result: " + str(self.training_layers[-1].caches[0].output_values))
        Logger.debug("out_size: " + str(self.training_layers[-1].size))
        Logger.debug("out_weights: " + str(self.training_layers[-1].weights))
        # calculate output error
        deltas = [target - self.training_layers[-1].caches[0].output_values]
        rng = range(len(self.training_layers) - 2, -1, -1)
        #Logger.log("range: " + str(rng))
        #Logger.log("reversed range: " + str(reversed(rng)))
        #raw_input("press Enter")
        for layer_id, last_weights in reversed(zip(
            rng,
            [l.weights for l in self.training_layers[1:]]
        )):
            Logger.debug(str(layer_id) + "/" + str(len(self.training_layers) - 2))
            deltas.append(
                self.training_layers[layer_id].get_delta(
                    self.training_layers[layer_id].caches[0].output_values,
                    deltas[-1],
                    self.training_layers[layer_id + 1].weights
                )
            )

        error = deltas[0][0]**2

        Logger.debug("deltas: " + str(deltas))
        for delta, layer in zip(reversed(deltas), self.training_layers):
            layer.learn(layer.caches[0].input_values, delta, self.config["learning_rate"])

        #self.bias_1 = min(0, -(learning_rate * delta1) + self.bias_1)
        #self.bias_2 = min(0, -(learning_rate * delta2) + self.bias_2)

        # returns the absolute error (distance of target output and actual output)
        return error

        """
from neural_network.util import Logger
from neural_network.NeuralLayer import NeuralLayer as Layer
from multi_layer_perceptron.MLPNetwork import MLPNetwork

Logger.DEBUG = False

data_in = numpy.array([[[0], [0]], [[0], [1]], [[1], [0]], [[1], [1]]])
data_out_xor = numpy.array([[0], [1], [1], [0]])

data_out_and = numpy.array([[0], [0], [0], [1]])

data_out_or = numpy.array([[0], [1], [1], [1]])

data_out = data_out_xor

Logger.log("data_shape: " + str(numpy.shape(data_in)))
Logger.log("data[0]_shape: " + str(numpy.shape(data_in[0])))

mlp = MLPNetwork()

mlp.populate(
        in_size=2,
        out_layer= {
            "size": 1,
            "fn": None,
            "fn_deriv": None
        },
        hidden_layers=[{
            "size": 3,
            "fn": Layer.activation_sigmoid,
            "fn_deriv": Layer.activation_sigmoid_deriv