コード例 #1
0
ファイル: constructor.py プロジェクト: vamsijkrishna/neupy
    def __init__(self, connection, *args, **kwargs):
        self.connection = clean_layers(connection)

        self.layers = list(self.connection)

        self.input_layer = self.layers[0]
        self.hidden_layers = self.layers[1:]
        self.output_layer = self.layers[-1]

        self.init_layers()
        super(ConstructableNetwork, self).__init__(*args, **kwargs)

        self.logs.message("THEANO", "Initializing Theano variables and "
                                    "functions.")
        start_init_time = time.time()

        self.variables = AttributeKeyDict(
            network_input=create_input_variable(
                self.input_layer, variable_name='x'
            ),
            network_output=create_output_variable(
                self.error, variable_name='y'
            ),
        )
        self.methods = AttributeKeyDict()

        self.init_variables()
        self.init_methods()

        finish_init_time = time.time()
        self.logs.message("THEANO", "Initialization finished sucessfully. "
                          "It took {:.2f} seconds"
                          "".format(finish_init_time - start_init_time))
コード例 #2
0
ファイル: base.py プロジェクト: wjianxz/neupy
    def __init__(self, network, options=None, **kwargs):
        options = options or kwargs

        if isinstance(network, (list, tuple)):
            network = layers.join(*network)

        self.network = network

        if len(self.network.output_layers) != 1:
            n_outputs = len(network.output_layers)

            raise InvalidConnection("Connection should have one output "
                                    "layer, got {}".format(n_outputs))

        target = options.get('target')
        if target is not None and isinstance(target, (list, tuple)):
            options['target'] = tf.placeholder(tf.float32, shape=target)

        self.target = self.network.targets
        super(BaseOptimizer, self).__init__(**options)

        start_init_time = time.time()
        self.logs.message("TENSORFLOW",
                          "Initializing Tensorflow variables and functions.")

        self.variables = AttributeKeyDict()
        self.functions = AttributeKeyDict()
        self.network.outputs
        self.init_functions()

        self.logs.message(
            "TENSORFLOW",
            "Initialization finished successfully. It took {:.2f} seconds"
            "".format(time.time() - start_init_time))
コード例 #3
0
ファイル: docs.py プロジェクト: mayblue9/neupy
def parse_variables_from_docs(instances):
    """
    Parse documentation with NumPy style and returns all
    extracted information.

    Parameters
    ----------
    instances : list
        List of objects that has documentations.

    Returns
    -------
    dict
        Variables parsed from the documentations.
    """
    variables = {}
    # Note: We do not include 'Examples' section because it
    # includes class/function name which will be useless when
    # we inerit documentation for the new object.
    doc_sections = ['Warns', 'Returns', 'Yields', 'Raises', 'See Also',
                    'Parameters', 'Attributes', 'Methods']

    if not instances:
        return variables

    for instance in instances:
        parent_docs = instance.__doc__

        if parent_docs is None:
            continue

        parent_variables = AttributeKeyDict()
        parent_variables.update(iter_parameters(parent_docs))
        parent_variables.update(iter_methods(parent_docs))

        for section_name in doc_sections:
            full_section = parse_full_section(section_name, parent_docs)

            if full_section is not None:
                parent_variables[section_name] = full_section

        parent_name = instance.__name__
        variables[parent_name] = parent_variables

    return variables
コード例 #4
0
ファイル: test_utils.py プロジェクト: disc5/neupy
    def test_attribute_key_dict(self):
        attrdict = AttributeKeyDict(val1='hello', val2='world')

        # Get
        self.assertEqual(attrdict.val1, 'hello')
        self.assertEqual(attrdict.val2, 'world')

        with self.assertRaises(KeyError):
            attrdict.unknown_variable

        # Set
        attrdict.new_value = 'test'
        self.assertEqual(attrdict.new_value, 'test')

        # Delete
        del attrdict.val1
        with self.assertRaises(KeyError):
            attrdict.val1
コード例 #5
0
    def __set__(self, instance, value):
        self.validate(value)

        default_value = self.default.copy()
        default_value.update(value)
        value = default_value

        value = AttributeKeyDict(value)
        instance.__dict__[self.name] = value
コード例 #6
0
ファイル: test_utils.py プロジェクト: itdxer/neupy
    def test_attribute_key_dict(self):
        attrdict = AttributeKeyDict(val1='hello', val2='world')

        # Get
        self.assertEqual(attrdict.val1, 'hello')
        self.assertEqual(attrdict.val2, 'world')

        with self.assertRaises(KeyError):
            attrdict.unknown_variable

        # Set
        attrdict.new_value = 'test'
        self.assertEqual(attrdict.new_value, 'test')

        # Delete
        del attrdict.val1
        with self.assertRaises(KeyError):
            attrdict.val1
コード例 #7
0
    def __init__(self, *args, **options):
        self.errors = self.train_errors = ErrorHistoryList()
        self.validation_errors = ErrorHistoryList()
        self.training = AttributeKeyDict()
        self.last_epoch = 0

        super(BaseNetwork, self).__init__(*args, **options)

        if self.verbose:
            show_network_options(self, highlight_options=options)
コード例 #8
0
ファイル: constructor.py プロジェクト: ravisankaradepu/saddle
    def __init__(self, *args, **kwargs):
        super(BaseAlgorithm, self).__init__(*args, **kwargs)

        self.logs.message("THEANO", "Initializing Theano variables and "
                          "functions.")
        start_init_time = time.time()

        self.variables = AttributeKeyDict()
        self.methods = AttributeKeyDict()

        self.init_input_output_variables()
        self.init_variables()
        self.init_methods()

        finish_init_time = time.time()
        self.logs.message(
            "THEANO", "Initialization finished sucessfully. "
            "It took {:.2f} seconds"
            "".format(finish_init_time - start_init_time))
コード例 #9
0
ファイル: docs.py プロジェクト: vamsijkrishna/neupy
    def __new__(cls, clsname, bases, attrs):
        new_class = super(SharedDocsMeta, cls).__new__(cls, clsname,
                                                       bases, attrs)
        if new_class.__doc__ is None:
            return new_class

        class_docs = new_class.__doc__
        n_indents = find_numpy_doc_indent(class_docs)

        if n_indents is None:
            return new_class

        parameters = {}
        parent_classes = new_class.__mro__

        for parent_class in parent_classes:
            parent_docs = parent_class.__doc__

            if parent_docs is None:
                continue

            parent_name = parent_class.__name__
            parent_params = parameters[parent_name] = AttributeKeyDict()

            for name, type_, desc in iter_parameters(parent_docs):
                parent_params[name] = "{} : {}{}".format(name, type_, desc)

            for name, func_params, desc in iter_methods(parent_docs):
                parent_params[name] = ''.join([name, func_params, desc])

            doc_warns = parse_warns(parent_docs)
            if doc_warns is not None:
                parent_params['Warns'] = doc_warns

        try:
            new_class.__doc__ = new_class.__doc__.format(**parameters)
        except Exception as exception:
            exception_classname = exception.__class__.__name__
            raise SharedDocsException(
                "Can't format documentation for class `{}`. "
                "Catched `{}` exception with message: {}".format(
                    new_class.__name__,
                    exception_classname,
                    exception
                )
            )

        return new_class
コード例 #10
0
    def __set__(self, instance, value):
        self.validate(value)

        if isinstance(value, init.Initializer):
            # All keys will have the same initializer
            dict_value = dict.fromkeys(self.default.keys())

            for key in dict_value:
                dict_value[key] = value

            value = dict_value

        default_value = self.default.copy()
        default_value.update(value)
        value = default_value

        value = AttributeKeyDict(value)
        instance.__dict__[self.name] = value
コード例 #11
0
def parse_variables_from_docs(instances):
    """
    Parse documentation with NumPy style and returns all
    extracted information.

    Parameters
    ----------
    instances : list
        List of objects that has documentations.

    Returns
    -------
    dict
        Variables parsed from the documentations.
    """
    variables = {}
    # Note: We do not include 'Examples' section because it
    # includes class/function name which will be useless when
    # we inerit documentation for the new object.
    doc_sections = [
        'Warns', 'Returns', 'Yields', 'Raises', 'See Also', 'Parameters',
        'Attributes', 'Methods', 'Notes'
    ]

    if not instances:
        return variables

    for instance in instances:
        parent_docs = instance.__doc__

        if parent_docs is None:
            continue

        parent_variables = AttributeKeyDict()
        parent_variables.update(iter_parameters(parent_docs))
        parent_variables.update(iter_methods(parent_docs))

        for section_name in doc_sections:
            full_section = parse_full_section(section_name, parent_docs)

            if full_section is not None:
                parent_variables[section_name] = full_section

        parent_name = instance.__name__
        variables[parent_name] = parent_variables

    return variables
コード例 #12
0
    def train(self, input_train, target_train=None, input_test=None,
              target_test=None, epochs=100, epsilon=None,
              summary='table'):
        """
        Method train neural network.

        Parameters
        ----------
        input_train : array-like

        target_train : array-like or None

        input_test : array-like or None

        target_test : array-like or None

        epochs : int
            Defaults to `100`.

        epsilon : float or None
            Defaults to ``None``.
        """
        show_epoch = self.show_epoch
        logs = self.logs
        training = self.training = AttributeKeyDict()

        if epochs <= 0:
            raise ValueError("Number of epochs needs to be greater than 0.")

        if epsilon is not None and epochs <= 2:
            raise ValueError("Network should train at teast 3 epochs before "
                             "check the difference between errors")

        logging_info_about_the_data(self, input_train, input_test)
        logging_info_about_training(self, epochs, epsilon)
        logs.newline()

        if summary == 'table':
            summary = SummaryTable(
                table_builder=table.TableBuilder(
                    table.Column(name="Epoch #"),
                    table.NumberColumn(name="Train err", places=4),
                    table.NumberColumn(name="Valid err", places=4),
                    table.TimeColumn(name="Time", width=10),
                    stdout=logs.write
                ),
                network=self,
                delay_limit=1.,
                delay_history_length=10,
            )

        elif summary == 'inline':
            summary = InlineSummary(network=self)

        else:
            raise ValueError("`{}` is unknown summary type"
                             "".format(summary))

        iterepochs = create_training_epochs_iterator(self, epochs, epsilon)
        show_epoch = parse_show_epoch_property(self, epochs, epsilon)
        training.show_epoch = show_epoch

        # Storring attributes and methods in local variables we prevent
        # useless __getattr__ call a lot of times in each loop.
        # This variables speed up loop in case on huge amount of
        # iterations.
        training_errors = self.errors
        validation_errors = self.validation_errors
        shuffle_data = self.shuffle_data

        train_epoch = self.train_epoch
        epoch_end_signal = self.epoch_end_signal
        train_end_signal = self.train_end_signal
        on_epoch_start_update = self.on_epoch_start_update

        is_first_iteration = True
        can_compute_validation_error = (input_test is not None)
        last_epoch_shown = 0
#############################################        
	symMatrix = tt.dmatrix("symMatrix")
        symEigenvalues, eigenvectors = tt.nlinalg.eig(symMatrix)
        get_Eigen = theano.function([symMatrix], [symEigenvalues, eigenvectors])
#############################################
        with logs.disable_user_input():
            for epoch in iterepochs:
                validation_error = None
                epoch_start_time = time.time()
                on_epoch_start_update(epoch)

                if shuffle_data:
                    data = shuffle(*as_tuple(input_train, target_train))
                    input_train, target_train = data[:-1], data[-1]

                try:
                    train_error = train_epoch(input_train, target_train)
		    print epoch
		    name=str(self)
		    if(name.split('(')[0]=='Hessian'):
		    	H=self.variables.hessian.get_value()
		   	ev,_=get_Eigen(H)
			print "positive EV ",np.sum(ev>0)
			print "Just zero EV", np.sum(ev==0)
			print "Zero EV ", np.sum(ev==0)+np.sum((ev < 0) & (ev > (np.min(ev)/2.0)))
			print "Neg EV ", np.sum(ev<0)
			print "Max EV ",np.max(ev)
			print "Min EV ",np.min(ev)
			s=str(self.itr)+'.npy'
			np.save(s,ev)
                    if can_compute_validation_error:
                        validation_error = self.prediction_error(input_test,
                                                                 target_test)

                    training_errors.append(train_error)
                    validation_errors.append(validation_error)

                    epoch_finish_time = time.time()
                    training.epoch_time = epoch_finish_time - epoch_start_time

                    if epoch % training.show_epoch == 0 or is_first_iteration:
                        summary.show_last()
                        last_epoch_shown = epoch

                    if epoch_end_signal is not None:
                        epoch_end_signal(self)

                    is_first_iteration = False

                except StopTraining as err:
                    # TODO: This notification breaks table view in terminal.
                    # I need to show it in a different way.
                    logs.message("TRAIN", "Epoch #{} stopped. {}"
                                          "".format(epoch, str(err)))
                    break

            if epoch != last_epoch_shown:
                summary.show_last()

            if train_end_signal is not None:
                train_end_signal(self)

            summary.finish()
            logs.newline()
コード例 #13
0
ファイル: constructor.py プロジェクト: vamsijkrishna/neupy
class ConstructableNetwork(SupervisedLearning, BaseNetwork):
    """ Class contains functionality that helps work with network that have
    constructable layers architecture.

    Parameters
    ----------
    connection : list, tuple or object
        Network architecture. That variables could be described in
        different ways. The simples one is a list or tuple that contains
        integers. Each integer describe layer input size. For example,
        ``(2, 4, 1)`` means that network will have 3 layers with 2 input
        units, 4 hidden units and 1 output unit. The one limitation of that
        method is that all layers automaticaly would with sigmoid actiavtion
        function. Other way is just a list of ``layers.BaseLayer``` class
        instances. For example: ``[Input(2), Tanh(4), Relu(1)]``.
        And the most readable one is pipeline
        ``Input(2) > Tanh(4) > Relu(1)``.
    error : {{'mse', 'rmse', 'mae', 'categorical_crossentropy', \
    'binary_crossentropy'}} or function
        Function that calculate prediction error.
        Defaults to ``mse``.

        * ``mae`` - Mean Absolute Error.

        * ``mse`` - Mean Squared Error.

        * ``rmse`` - Root Mean Squared Error.

        * ``msle`` - Mean Squared Logarithmic Error.

        * ``rmsle`` - Root Mean Squared Logarithmic Error.

        * ``categorical_crossentropy`` - Categorical cross entropy.

        * ``binary_crossentropy`` - Binary cross entropy.

        * Custom function that accept two mandatory arguments.
        The first one is expected value and the second one is
        predicted value. Example: ``custom_func(expected, predicted)``
    {BaseNetwork.step}
    {BaseNetwork.show_epoch}
    {BaseNetwork.shuffle_data}
    {BaseNetwork.epoch_end_signal}
    {BaseNetwork.train_end_signal}
    {Verbose.verbose}

    Attributes
    ----------
    {BaseNetwork.errors}
    {BaseNetwork.train_errors}
    {BaseNetwork.validation_errors}
    {BaseNetwork.last_epoch}
    """
    error = ErrorFunctionProperty(default='mse', choices={
        'mae': errors.mae,
        'mse': errors.mse,
        'rmse': errors.rmse,
        'msle': errors.msle,
        'rmsle': errors.rmsle,
        'binary_crossentropy': errors.binary_crossentropy,
        'categorical_crossentropy': errors.categorical_crossentropy,
    })

    def __init__(self, connection, *args, **kwargs):
        self.connection = clean_layers(connection)

        self.layers = list(self.connection)

        self.input_layer = self.layers[0]
        self.hidden_layers = self.layers[1:]
        self.output_layer = self.layers[-1]

        self.init_layers()
        super(ConstructableNetwork, self).__init__(*args, **kwargs)

        self.logs.message("THEANO", "Initializing Theano variables and "
                                    "functions.")
        start_init_time = time.time()

        self.variables = AttributeKeyDict(
            network_input=create_input_variable(
                self.input_layer, variable_name='x'
            ),
            network_output=create_output_variable(
                self.error, variable_name='y'
            ),
        )
        self.methods = AttributeKeyDict()

        self.init_variables()
        self.init_methods()

        finish_init_time = time.time()
        self.logs.message("THEANO", "Initialization finished sucessfully. "
                          "It took {:.2f} seconds"
                          "".format(finish_init_time - start_init_time))

    def init_variables(self):
        """ Initialize Theano variables.
        """
        network_input = self.variables.network_input
        network_output = self.variables.network_output

        train_prediction = self.connection.output(network_input)
        with self.connection.disable_training_state():
            prediction = self.connection.output(network_input)

        self.variables.update(
            step=theano.shared(name='step', value=asfloat(self.step)),
            epoch=theano.shared(name='epoch', value=asfloat(self.last_epoch)),

            prediction_func=prediction,
            train_prediction_func=train_prediction,

            error_func=self.error(network_output, train_prediction),
            validation_error_func=self.error(network_output, prediction),
        )

    def init_methods(self):
        """ Initialize all methods that needed for prediction and
        training procedures.
        """
        network_input = self.variables.network_input
        network_output = self.variables.network_output

        self.methods.predict = theano.function(
            inputs=[self.variables.network_input],
            outputs=self.variables.prediction_func
        )
        self.methods.train_epoch = theano.function(
            inputs=[network_input, network_output],
            outputs=self.variables.error_func,
            updates=self.init_train_updates(),
        )
        self.methods.prediction_error = theano.function(
            inputs=[network_input, network_output],
            outputs=self.variables.validation_error_func
        )

    def init_layers(self):
        """ Initialize layers in the same order as they were list in
        network initialization step.
        """
        self.connection.initialize()

    def init_train_updates(self):
        """ Initialize train function update in Theano format that
        would be trigger after each training epoch.
        """
        updates = []
        for layer in self.layers:
            updates.extend(self.init_layer_updates(layer))
        return updates

    def init_layer_updates(self, layer):
        """ Initialize train function update in Theano format that
        would be trigger after each training epoch for each layer.

        Parameters
        ----------
        layer : object
            Any layer that inherit from layers.BaseLayer class.

        Returns
        -------
        list
            Update that excaptable by ``theano.function``. There should be
            a lits that contains tuples with 2 elements. First one should
            be parameter that would be updated after epoch and the second one
            should update rules for this parameter. For example parameter
            could be a layer's weight and bias.
        """
        updates = []
        for parameter in layer.parameters:
            updates.extend(self.init_param_updates(layer, parameter))
        updates.extend(layer.updates)
        return updates

    def init_param_updates(self, layer, parameter):
        """ Initialize parameter updates.

        Parameters
        ----------
        layer : object
            Any layer that inherit from BaseLayer class.
        parameter : object
            Usualy it is a weight or bias.

        Returns
        -------
        list
            List of updates related to the specified parameter.
        """
        return []

    def format_input_data(self, input_data):
        """ Input data format is depend on the input layer
        structure.

        Parameters
        ----------
        input_data : array-like or None

        Returns
        -------
        array-like or None
            Function returns formatted array.
        """
        if input_data is not None:
            is_feature1d = does_layer_accept_1d_feature(self.input_layer)
            return format_data(input_data, is_feature1d)

    def format_target_data(self, target_data):
        """ Target data format is depend on the output layer
        structure.

        Parameters
        ----------
        target_data : array-like or None

        Returns
        -------
        array-like or None
            Function returns formatted array.
        """
        if target_data is not None:
            is_feature1d = does_layer_accept_1d_feature(self.output_layer)
            return format_data(target_data, is_feature1d)

    def prediction_error(self, input_data, target_data):
        """ Calculate prediction accuracy for input data.

        Parameters
        ----------
        input_data : array-like
        target_data : array-like

        Returns
        -------
        float
            Prediction error.
        """
        return self.methods.prediction_error(
            self.format_input_data(input_data),
            self.format_target_data(target_data)
        )

    def predict(self, input_data):
        """ Return prediction results for the input data.

        Parameters
        ----------
        input_data : array-like

        Returns
        -------
        array-like
        """
        input_data = self.format_input_data(input_data)
        return self.methods.predict(input_data)

    def on_epoch_start_update(self, epoch):
        """ Function would be trigger before run all training procedure
        related to the current epoch.

        Parameters
        ----------
        epoch : int
            Current epoch number.
        """
        super(ConstructableNetwork, self).on_epoch_start_update(epoch)
        self.variables.epoch.set_value(epoch)

    def train(self, input_train, target_train, input_test=None,
              target_test=None, *args, **kwargs):
        """ Trains neural network.
        """
        return super(ConstructableNetwork, self).train(
            self.format_input_data(input_train),
            self.format_target_data(target_train),
            self.format_input_data(input_test),
            self.format_target_data(target_test),
            *args, **kwargs
        )

    def train_epoch(self, input_train, target_train):
        """ Trains neural network over one epoch.

        Parameters
        ----------
        input_data : array-like
        target_data : array-like

        Returns
        -------
        float
            Prediction error.
        """
        return self.methods.train_epoch(input_train, target_train)

    def architecture(self):
        """ Shows network's architecture in the terminal if
        ``verbose`` parameter is equal to ``True``.
        """
        self.logs.title("Network's architecture")

        values = []
        for index, layer in enumerate(self.layers, start=1):
            input_shape = preformat_layer_shape(layer.input_shape)
            output_shape = preformat_layer_shape(layer.output_shape)
            classname = layer.__class__.__name__

            values.append((index, input_shape, classname, output_shape))

        table.TableBuilder.show_full_table(
            columns=[
                table.Column(name="#"),
                table.Column(name="Input shape"),
                table.Column(name="Layer Type"),
                table.Column(name="Output shape"),
            ],
            values=values,
            stdout=self.logs.write,
        )
        self.logs.newline()

    def __repr__(self):
        n_layers = len(self.connection)

        if n_layers > 5:
            connection = '[... {} layers ...]'.format(n_layers)
        else:
            connection = self.connection

        return "{}({}, {})".format(self.class_name(), connection,
                                   self._repr_options())
コード例 #14
0
    def train(self,
              input_train,
              target_train=None,
              input_test=None,
              target_test=None,
              epochs=100,
              epsilon=None,
              summary='table'):
        """
        Method train neural network.

        Parameters
        ----------
        input_train : array-like

        target_train : array-like or None

        input_test : array-like or None

        target_test : array-like or None

        epochs : int
            Defaults to `100`.

        epsilon : float or None
            Defaults to ``None``.
        """
        show_epoch = self.show_epoch
        logs = self.logs
        training = self.training = AttributeKeyDict()

        if epochs <= 0:
            raise ValueError("Number of epochs needs to be greater than 0.")

        if epsilon is not None and epochs <= 2:
            raise ValueError("Network should train at teast 3 epochs before "
                             "check the difference between errors")

        logging_info_about_the_data(self, input_train, input_test)
        logging_info_about_training(self, epochs, epsilon)
        logs.newline()

        if summary == 'table':
            summary = SummaryTable(
                columns=['Epoch', 'Train err', 'Valid err', 'Time'],
                network=self,
            )

        elif summary == 'inline':
            summary = InlineSummary(network=self)

        else:
            raise ValueError("`{}` is unknown summary type" "".format(summary))

        iterepochs = create_training_epochs_iterator(self, epochs, epsilon)
        show_epoch = parse_show_epoch_property(self, epochs, epsilon)
        training.show_epoch = show_epoch
        training.epoch_time = 0

        # Storring attributes and methods in local variables we prevent
        # useless __getattr__ call a lot of times in each loop.
        # This variables speed up loop in case on huge amount of
        # iterations.
        training_errors = self.errors
        validation_errors = self.validation_errors
        shuffle_data = self.shuffle_data

        train_epoch = self.train_epoch
        epoch_end_signal = self.epoch_end_signal
        train_end_signal = self.train_end_signal
        on_epoch_start_update = self.on_epoch_start_update

        is_first_iteration = True
        can_compute_validation_error = (input_test is not None)
        last_epoch_shown = 0

        with logs.disable_user_input():
            for epoch in iterepochs:
                validation_error = None
                epoch_start_time = time.time()
                on_epoch_start_update(epoch)

                if shuffle_data:
                    data = shuffle(*as_tuple(input_train, target_train))
                    input_train, target_train = data[:-1], data[-1]

                    if len(input_train) == 1:
                        input_train = input_train[0]

                try:
                    train_error = train_epoch(input_train, target_train)

                    if can_compute_validation_error:
                        validation_error = self.prediction_error(
                            input_test, target_test)

                    training_errors.append(train_error)
                    validation_errors.append(validation_error)

                    epoch_finish_time = time.time()
                    training.epoch_time = epoch_finish_time - epoch_start_time

                    if epoch % training.show_epoch == 0 or is_first_iteration:
                        summary.show_last()
                        last_epoch_shown = epoch

                    if epoch_end_signal is not None:
                        epoch_end_signal(self)

                    is_first_iteration = False

                except StopTraining as err:
                    summary.finish()
                    logs.message(
                        "TRAIN", "Epoch #{} stopped. {}"
                        "".format(epoch, str(err)))
                    break

            if epoch != last_epoch_shown:
                summary.show_last()

            if train_end_signal is not None:
                train_end_signal(self)

            summary.finish()
            logs.newline()
コード例 #15
0
    def train(self,
              input_train,
              target_train=None,
              input_test=None,
              target_test=None,
              epochs=100,
              epsilon=None):
        """ Method train neural network.

        Parameters
        ----------
        input_train : array-like
        target_train : array-like or Npne
        input_test : array-like or None
        target_test : array-like or None
        epochs : int
            Defaults to `100`.
        epsilon : float or None
        """

        show_epoch = self.show_epoch
        logs = self.logs
        training = self.training = AttributeKeyDict()

        if epochs <= 0:
            raise ValueError("Number of epochs needs to be greater than 0.")

        if epsilon is not None and epochs <= 2:
            raise ValueError("Network should train at teast 3 epochs before "
                             "check the difference between errors")

        logging_info_about_the_data(self, input_train, input_test)
        logging_info_about_training(self, epochs, epsilon)
        logs.write("")

        iterepochs = create_training_epochs_iterator(self, epochs, epsilon)
        show_epoch = parse_show_epoch_property(self, epochs, epsilon)
        training.show_epoch = show_epoch

        epoch_summary = show_epoch_summary(self)
        next(epoch_summary)

        # Storring attributes and methods in local variables we prevent
        # useless __getattr__ call a lot of times in each loop.
        # This variables speed up loop in case on huge amount of
        # iterations.
        errors = self.errors
        validation_errors = self.validation_errors
        shuffle_data = self.shuffle_data

        train_epoch = self.train_epoch
        epoch_end_signal = self.epoch_end_signal
        train_end_signal = self.train_end_signal
        on_epoch_start_update = self.on_epoch_start_update

        is_first_iteration = True
        can_compute_validation_error = (input_test is not None)
        last_epoch_shown = 0

        with logs.disable_user_input():
            for epoch in iterepochs:
                epoch_start_time = time.time()
                on_epoch_start_update(epoch)

                if shuffle_data:
                    input_train, target_train = shuffle(
                        input_train, target_train)
                try:
                    train_error = train_epoch(input_train, target_train)

                    if can_compute_validation_error:
                        validation_error = self.prediction_error(
                            input_test, target_test)
                        validation_errors.append(validation_error)

                    # It's important that we store error result after
                    # we stored validation error.
                    errors.append(train_error)

                    epoch_finish_time = time.time()
                    training.epoch_time = epoch_finish_time - epoch_start_time

                    if epoch % training.show_epoch == 0 or is_first_iteration:
                        next(epoch_summary)
                        last_epoch_shown = epoch

                    if epoch_end_signal is not None:
                        epoch_end_signal(self)

                    is_first_iteration = False

                except StopNetworkTraining as err:
                    # TODO: This notification breaks table view in terminal.
                    # I need to show it in a different way. Maybe I can
                    # send it in generator using ``throw`` method.
                    logs.message(
                        "TRAIN", "Epoch #{} stopped. {}"
                        "".format(epoch, str(err)))

            if epoch != last_epoch_shown:
                next(epoch_summary)

            if train_end_signal is not None:
                train_end_signal(self)

            epoch_summary.close()

        logs.message("TRAIN", "Trainig finished")
コード例 #16
0
    def train(self, input_train, target_train=None, input_test=None,
              target_test=None, epochs=100, epsilon=None,
              summary_type='table'):
        """ Method train neural network.

        Parameters
        ----------
        input_train : array-like
        target_train : array-like or Npne
        input_test : array-like or None
        target_test : array-like or None
        epochs : int
            Defaults to `100`.
        epsilon : float or None
            Defaults to ``None``.
        """

        show_epoch = self.show_epoch
        logs = self.logs
        training = self.training = AttributeKeyDict()

        if epochs <= 0:
            raise ValueError("Number of epochs needs to be greater than 0.")

        if epsilon is not None and epochs <= 2:
            raise ValueError("Network should train at teast 3 epochs before "
                             "check the difference between errors")

        if summary_type == 'table':
            logging_info_about_the_data(self, input_train, input_test)
            logging_info_about_training(self, epochs, epsilon)
            logs.newline()

            summary = SummaryTable(
                table_builder=table.TableBuilder(
                    table.Column(name="Epoch #"),
                    table.NumberColumn(name="Train err"),
                    table.NumberColumn(name="Valid err"),
                    table.TimeColumn(name="Time", width=10),
                    stdout=logs.write
                ),
                network=self,
                delay_limit=1.,
                delay_history_length=10,
            )

        elif summary_type == 'inline':
            summary = InlineSummary(network=self)

        else:
            raise ValueError("`{}` is unknown summary type"
                             "".format(summary_type))

        iterepochs = create_training_epochs_iterator(self, epochs, epsilon)
        show_epoch = parse_show_epoch_property(self, epochs, epsilon)
        training.show_epoch = show_epoch

        # Storring attributes and methods in local variables we prevent
        # useless __getattr__ call a lot of times in each loop.
        # This variables speed up loop in case on huge amount of
        # iterations.
        training_errors = self.errors
        validation_errors = self.validation_errors
        shuffle_data = self.shuffle_data

        train_epoch = self.train_epoch
        epoch_end_signal = self.epoch_end_signal
        train_end_signal = self.train_end_signal
        on_epoch_start_update = self.on_epoch_start_update

        is_first_iteration = True
        can_compute_validation_error = (input_test is not None)
        last_epoch_shown = 0

        with logs.disable_user_input():
            for epoch in iterepochs:
                validation_error = np.nan
                epoch_start_time = time.time()
                on_epoch_start_update(epoch)

                if shuffle_data:
                    input_train, target_train = shuffle(input_train,
                                                        target_train)
                try:
                    train_error = train_epoch(input_train, target_train)

                    if can_compute_validation_error:
                        validation_error = self.prediction_error(input_test,
                                                                 target_test)

                    training_errors.append(train_error)
                    validation_errors.append(validation_error)

                    epoch_finish_time = time.time()
                    training.epoch_time = epoch_finish_time - epoch_start_time

                    if epoch % training.show_epoch == 0 or is_first_iteration:
                        summary.show_last()
                        last_epoch_shown = epoch

                    if epoch_end_signal is not None:
                        epoch_end_signal(self)

                    is_first_iteration = False

                except StopNetworkTraining as err:
                    # TODO: This notification breaks table view in terminal.
                    # I need to show it in a different way.
                    logs.message("TRAIN", "Epoch #{} stopped. {}"
                                          "".format(epoch, str(err)))
                    break

            if epoch != last_epoch_shown:
                summary.show_last()

            if train_end_signal is not None:
                train_end_signal(self)

            summary.finish()
            logs.newline()

        logs.message("TRAIN", "Trainig finished")
コード例 #17
0
ファイル: base.py プロジェクト: wjianxz/neupy
class BaseOptimizer(BaseNetwork):
    """
    Gradient descent algorithm.

    Parameters
    ----------
    network : list, tuple or LayerConnection instance
        Network's architecture. There are a few ways
        to define it.

        - List of layers.
          For instance, ``[Input(2), Tanh(4), Relu(1)]``.

        - Constructed layers.
          For instance, ``Input(2) >> Tanh(4) >> Relu(1)``.

    regularizer : function or None
        Network's regularizer.

    loss : str or function
        Error/loss function. Defaults to ``mse``.

        - ``mae`` - Mean Absolute Error.

        - ``mse`` - Mean Squared Error.

        - ``rmse`` - Root Mean Squared Error.

        - ``msle`` - Mean Squared Logarithmic Error.

        - ``rmsle`` - Root Mean Squared Logarithmic Error.

        - ``categorical_crossentropy`` - Categorical cross entropy.

        - ``binary_crossentropy`` - Binary cross entropy.

        - ``binary_hinge`` - Binary hinge entropy.

        - ``categorical_hinge`` - Categorical hinge entropy.

        - Custom function which accepts two mandatory arguments.
          The first one is expected value and the second one is
          predicted value. Example:

        .. code-block:: python

            def custom_func(expected, predicted):
                return expected - predicted

    step : float, Variable
        Learning rate, defaults to ``0.1``.

    {BaseNetwork.show_epoch}

    {BaseNetwork.shuffle_data}

    {BaseNetwork.signals}

    {BaseNetwork.verbose}

    Attributes
    ----------
    {BaseNetwork.Attributes}

    Methods
    -------
    {BaseSkeleton.predict}

    train(X_train, y_train, X_test=None, y_test=None, epochs=100)
        Train network. You can control network's training procedure
        with ``epochs`` parameter. The ``X_test`` and ``y_test`` should
        be presented both in case network's validation required
        after each training epoch.

    {BaseSkeleton.fit}
    """
    step = ScalarVariableProperty(default=0.1)
    target = Property(default=None, allow_none=True)
    regularizer = Property(default=None, allow_none=True)
    loss = FunctionWithOptionsProperty(default='mse',
                                       choices={
                                           'mae':
                                           objectives.mae,
                                           'mse':
                                           objectives.mse,
                                           'rmse':
                                           objectives.rmse,
                                           'msle':
                                           objectives.msle,
                                           'rmsle':
                                           objectives.rmsle,
                                           'binary_crossentropy':
                                           objectives.binary_crossentropy,
                                           'categorical_crossentropy':
                                           objectives.categorical_crossentropy,
                                           'binary_hinge':
                                           objectives.binary_hinge,
                                           'categorical_hinge':
                                           objectives.categorical_hinge,
                                       })

    def __init__(self, network, options=None, **kwargs):
        options = options or kwargs

        if isinstance(network, (list, tuple)):
            network = layers.join(*network)

        self.network = network

        if len(self.network.output_layers) != 1:
            n_outputs = len(network.output_layers)

            raise InvalidConnection("Connection should have one output "
                                    "layer, got {}".format(n_outputs))

        target = options.get('target')
        if target is not None and isinstance(target, (list, tuple)):
            options['target'] = tf.placeholder(tf.float32, shape=target)

        self.target = self.network.targets
        super(BaseOptimizer, self).__init__(**options)

        start_init_time = time.time()
        self.logs.message("TENSORFLOW",
                          "Initializing Tensorflow variables and functions.")

        self.variables = AttributeKeyDict()
        self.functions = AttributeKeyDict()
        self.network.outputs
        self.init_functions()

        self.logs.message(
            "TENSORFLOW",
            "Initialization finished successfully. It took {:.2f} seconds"
            "".format(time.time() - start_init_time))

    def init_train_updates(self):
        raise NotImplementedError()

    def init_functions(self):
        loss = self.loss(self.target, self.network.outputs)
        val_loss = self.loss(self.target, self.network.training_outputs)

        if self.regularizer is not None:
            loss += self.regularizer(self.network)

        self.variables.update(
            step=self.step,
            loss=loss,
            val_loss=val_loss,
        )

        with tf.name_scope('training-updates'):
            update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)

            with tf.control_dependencies(update_ops):
                training_updates = self.init_train_updates()
                training_updates.extend(update_ops)

        tf_utils.initialize_uninitialized_variables()

        self.functions.update(
            predict=tf_utils.function(inputs=as_tuple(self.network.inputs),
                                      outputs=self.network.outputs,
                                      name='optimizer/predict'),
            one_training_update=tf_utils.function(
                inputs=as_tuple(self.network.inputs, self.target),
                outputs=loss,
                updates=training_updates,
                name='optimizer/one-update-step'),
            score=tf_utils.function(inputs=as_tuple(self.network.inputs,
                                                    self.target),
                                    outputs=val_loss,
                                    name='optimizer/score'))

    def format_input(self, X):
        X = as_tuple(X)
        X_formatted = []

        if len(X) != len(self.network.input_layers):
            raise ValueError("Number of inputs doesn't match number "
                             "of input layers in the network.")

        for input, input_layer in zip(X, self.network.input_layers):
            input_shape = tf.TensorShape(input_layer.input_shape)
            is_feature1d = (input_shape.ndims == 2 and input_shape[1] == 1)
            formatted_input = format_data(input, is_feature1d=is_feature1d)

            if (formatted_input.ndim + 1) == input_shape.ndims:
                # We assume that when one dimension was missed than user
                # wants to propagate single sample through the network
                formatted_input = np.expand_dims(formatted_input, axis=0)

            X_formatted.append(formatted_input)

        return X_formatted

    def format_target(self, y):
        output_shape = tf.TensorShape(self.network.output_shape)
        is_feature1d = (output_shape.ndims == 2 and output_shape[1] == 1)
        formatted_target = format_data(y, is_feature1d=is_feature1d)

        if (formatted_target.ndim + 1) == len(output_shape):
            # We assume that when one dimension was missed than user
            # wants to propagate single sample through the network
            formatted_target = np.expand_dims(formatted_target, axis=0)

        return formatted_target

    def score(self, X, y):
        """
        Calculate prediction accuracy for input data.

        Parameters
        ----------
        X : array-like
        y : array-like

        Returns
        -------
        float
            Prediction error.
        """
        X = self.format_input(X)
        y = self.format_target(y)
        return self.functions.score(*as_tuple(X, y))

    def predict(self, *X, **kwargs):
        """
        Makes a raw prediction.

        Parameters
        ----------
        X : array-like

        Returns
        -------
        array-like
        """
        default_batch_size = getattr(self, 'batch_size', None)
        predict_kwargs = dict(
            batch_size=kwargs.pop('batch_size', default_batch_size),
            verbose=self.verbose,
        )

        # We require do to this check for python 2 compatibility
        if kwargs:
            raise TypeError("Unknown arguments: {}".format(kwargs))

        return self.network.predict(*self.format_input(X), **predict_kwargs)

    def train(self,
              X_train,
              y_train,
              X_test=None,
              y_test=None,
              *args,
              **kwargs):

        is_test_data_partialy_missing = (
            (X_test is None and y_test is not None)
            or (X_test is not None and y_test is None))

        if is_test_data_partialy_missing:
            raise ValueError("Input or target test samples are missed. They "
                             "must be defined together or none of them.")

        X_train = self.format_input(X_train)
        y_train = self.format_target(y_train)

        if X_test is not None:
            X_test = self.format_input(X_test)
            y_test = self.format_target(y_test)

        return super(BaseOptimizer, self).train(X_train=X_train,
                                                y_train=y_train,
                                                X_test=X_test,
                                                y_test=y_test,
                                                *args,
                                                **kwargs)

    def one_training_update(self, X_train, y_train):
        return self.functions.one_training_update(*as_tuple(X_train, y_train))

    def get_params(self, deep=False, with_network=True):
        params = super(BaseOptimizer, self).get_params()
        if with_network:
            params['network'] = self.network
        return params

    def __reduce__(self):
        parameters = self.get_params(with_network=False)

        # We only need to know placeholders shape
        # in order to be able to reconstruct it
        parameters['target'] = tf_utils.shape_to_tuple(
            parameters['target'].shape)

        args = (self.network, parameters)
        return (self.__class__, args)

    def __repr__(self):
        return "{}({}, {})".format(self.__class__.__name__, self.network,
                                   self.repr_options())
コード例 #18
0
ファイル: base.py プロジェクト: ravisankaradepu/saddle
    def train(self, input_train, target_train=None, input_test=None,
              target_test=None, epochs=100, epsilon=None,
              summary_type='table'):
        """
        Method train neural network.

        Parameters
        ----------
        input_train : array-like
        target_train : array-like or None
        input_test : array-like or None
        target_test : array-like or None
        epochs : int
            Defaults to `100`.
        epsilon : float or None
            Defaults to ``None``.
        """
        show_epoch = self.show_epoch
        logs = self.logs
        training = self.training = AttributeKeyDict()

        if epochs <= 0:
            raise ValueError("Number of epochs needs to be greater than 0.")

        if epsilon is not None and epochs <= 2:
            raise ValueError("Network should train at teast 3 epochs before "
                             "check the difference between errors")

        if summary_type == 'table':
            logging_info_about_the_data(self, input_train, input_test)
            logging_info_about_training(self, epochs, epsilon)
            logs.newline()

            summary = SummaryTable(
                table_builder=table.TableBuilder(
                    table.Column(name="Epoch #"),
                    table.NumberColumn(name="Train err"),
                    table.NumberColumn(name="Valid err"),
                    table.TimeColumn(name="Time", width=10),
                    stdout=logs.write
                ),
                network=self,
                delay_limit=1.,
                delay_history_length=10,
            )

        elif summary_type == 'inline':
            summary = InlineSummary(network=self)

        else:
            raise ValueError("`{}` is unknown summary type"
                             "".format(summary_type))

        iterepochs = create_training_epochs_iterator(self, epochs, epsilon)
        show_epoch = parse_show_epoch_property(self, epochs, epsilon)
        training.show_epoch = show_epoch

        # Storring attributes and methods in local variables we prevent
        # useless __getattr__ call a lot of times in each loop.
        # This variables speed up loop in case on huge amount of
        # iterations.
        training_errors = self.errors
        validation_errors = self.validation_errors
        shuffle_data = self.shuffle_data

        train_epoch = self.train_epoch
        epoch_end_signal = self.epoch_end_signal
        train_end_signal = self.train_end_signal
        on_epoch_start_update = self.on_epoch_start_update

        is_first_iteration = True
        can_compute_validation_error = (input_test is not None)
        last_epoch_shown = 0


        symMatrix = tt.dmatrix("symMatrix")
        symEigenvalues, eigenvectors = tt.nlinalg.eig(symMatrix)
        get_Eigen = theano.function([symMatrix], [symEigenvalues, eigenvectors] )

        epsilon = []
        alpha = []
        alpha0 = []
        with logs.disable_user_input():
            for epoch in iterepochs:
                validation_error = None
                epoch_start_time = time.time()
                on_epoch_start_update(epoch)

                if shuffle_data:
                    input_train, target_train = shuffle(input_train,
                                                        target_train)
                try:
                    train_error = train_epoch(input_train, target_train)
                    H = self.variables.hessian.get_value()
                    ev, _ = get_Eigen(H)
                    if can_compute_validation_error:
                        validation_error = self.prediction_error(input_test,
                                                                 target_test)
                    epsilon.append(train_error)
                    alpha.append(numpy.sum(ev < 0))
                    alpha0.append(numpy.sum(ev == 0))
                    
                    training_errors.append(train_error)
                    validation_errors.append(validation_error)

                    epoch_finish_time = time.time()
                    training.epoch_time = epoch_finish_time - epoch_start_time

                    if epoch % training.show_epoch == 0 or is_first_iteration:
                        summary.show_last()
                        last_epoch_shown = epoch

                    if epoch_end_signal is not None:
                        epoch_end_signal(self)

                    is_first_iteration = False

                except StopNetworkTraining as err:
                    # TODO: This notification breaks table view in terminal.
                    # I need to show it in a different way.
                    logs.message("TRAIN", "Epoch #{} stopped. {}"
                                          "".format(epoch, str(err)))
                    break

            if epoch != last_epoch_shown:
                summary.show_last()

            if train_end_signal is not None:
                train_end_signal(self)

            summary.finish()
            logs.newline()
            plt.plot(alpha,epsilon,'r')
            plt.plot(alpha0,epsilon,'b')
            plt.xlabel('alpha')
            plt.ylabel('epsilon')
            
            # want to collect the output of stdout in a variable
            capture = StringIO()
            capture.truncate(0)
            save_stdout = sys.stdout
            sys.stdout = capture
            print self.connection
            sys.stdout=save_stdout
            s =  capture.getvalue()
            s=s.split('\n')[0:][0]
            str = self.class_name()
                        
            str1 = s+'---'+str+'-alpha-epsilon'+'.eps'
            plt.savefig(str1,format='eps',dpi=1000)
            plt.plot(iterepochs,epsilon)
            plt.xlabel('iterepochs')
            plt.ylabel('epsilon')
            str2=s+'---'+str+'-epsilon-iterepochs'+'.eps'
            plt.savefig(str2,format='eps',dpi=1000)