def __init__(self, connection, *args, **kwargs): self.connection = clean_layers(connection) self.layers = list(self.connection) self.input_layer = self.layers[0] self.hidden_layers = self.layers[1:] self.output_layer = self.layers[-1] self.init_layers() super(ConstructableNetwork, self).__init__(*args, **kwargs) self.logs.message("THEANO", "Initializing Theano variables and " "functions.") start_init_time = time.time() self.variables = AttributeKeyDict( network_input=create_input_variable( self.input_layer, variable_name='x' ), network_output=create_output_variable( self.error, variable_name='y' ), ) self.methods = AttributeKeyDict() self.init_variables() self.init_methods() finish_init_time = time.time() self.logs.message("THEANO", "Initialization finished sucessfully. " "It took {:.2f} seconds" "".format(finish_init_time - start_init_time))
def __init__(self, network, options=None, **kwargs): options = options or kwargs if isinstance(network, (list, tuple)): network = layers.join(*network) self.network = network if len(self.network.output_layers) != 1: n_outputs = len(network.output_layers) raise InvalidConnection("Connection should have one output " "layer, got {}".format(n_outputs)) target = options.get('target') if target is not None and isinstance(target, (list, tuple)): options['target'] = tf.placeholder(tf.float32, shape=target) self.target = self.network.targets super(BaseOptimizer, self).__init__(**options) start_init_time = time.time() self.logs.message("TENSORFLOW", "Initializing Tensorflow variables and functions.") self.variables = AttributeKeyDict() self.functions = AttributeKeyDict() self.network.outputs self.init_functions() self.logs.message( "TENSORFLOW", "Initialization finished successfully. It took {:.2f} seconds" "".format(time.time() - start_init_time))
def parse_variables_from_docs(instances): """ Parse documentation with NumPy style and returns all extracted information. Parameters ---------- instances : list List of objects that has documentations. Returns ------- dict Variables parsed from the documentations. """ variables = {} # Note: We do not include 'Examples' section because it # includes class/function name which will be useless when # we inerit documentation for the new object. doc_sections = ['Warns', 'Returns', 'Yields', 'Raises', 'See Also', 'Parameters', 'Attributes', 'Methods'] if not instances: return variables for instance in instances: parent_docs = instance.__doc__ if parent_docs is None: continue parent_variables = AttributeKeyDict() parent_variables.update(iter_parameters(parent_docs)) parent_variables.update(iter_methods(parent_docs)) for section_name in doc_sections: full_section = parse_full_section(section_name, parent_docs) if full_section is not None: parent_variables[section_name] = full_section parent_name = instance.__name__ variables[parent_name] = parent_variables return variables
def test_attribute_key_dict(self): attrdict = AttributeKeyDict(val1='hello', val2='world') # Get self.assertEqual(attrdict.val1, 'hello') self.assertEqual(attrdict.val2, 'world') with self.assertRaises(KeyError): attrdict.unknown_variable # Set attrdict.new_value = 'test' self.assertEqual(attrdict.new_value, 'test') # Delete del attrdict.val1 with self.assertRaises(KeyError): attrdict.val1
def __set__(self, instance, value): self.validate(value) default_value = self.default.copy() default_value.update(value) value = default_value value = AttributeKeyDict(value) instance.__dict__[self.name] = value
def test_attribute_key_dict(self): attrdict = AttributeKeyDict(val1='hello', val2='world') # Get self.assertEqual(attrdict.val1, 'hello') self.assertEqual(attrdict.val2, 'world') with self.assertRaises(KeyError): attrdict.unknown_variable # Set attrdict.new_value = 'test' self.assertEqual(attrdict.new_value, 'test') # Delete del attrdict.val1 with self.assertRaises(KeyError): attrdict.val1
def __init__(self, *args, **options): self.errors = self.train_errors = ErrorHistoryList() self.validation_errors = ErrorHistoryList() self.training = AttributeKeyDict() self.last_epoch = 0 super(BaseNetwork, self).__init__(*args, **options) if self.verbose: show_network_options(self, highlight_options=options)
def __init__(self, *args, **kwargs): super(BaseAlgorithm, self).__init__(*args, **kwargs) self.logs.message("THEANO", "Initializing Theano variables and " "functions.") start_init_time = time.time() self.variables = AttributeKeyDict() self.methods = AttributeKeyDict() self.init_input_output_variables() self.init_variables() self.init_methods() finish_init_time = time.time() self.logs.message( "THEANO", "Initialization finished sucessfully. " "It took {:.2f} seconds" "".format(finish_init_time - start_init_time))
def __new__(cls, clsname, bases, attrs): new_class = super(SharedDocsMeta, cls).__new__(cls, clsname, bases, attrs) if new_class.__doc__ is None: return new_class class_docs = new_class.__doc__ n_indents = find_numpy_doc_indent(class_docs) if n_indents is None: return new_class parameters = {} parent_classes = new_class.__mro__ for parent_class in parent_classes: parent_docs = parent_class.__doc__ if parent_docs is None: continue parent_name = parent_class.__name__ parent_params = parameters[parent_name] = AttributeKeyDict() for name, type_, desc in iter_parameters(parent_docs): parent_params[name] = "{} : {}{}".format(name, type_, desc) for name, func_params, desc in iter_methods(parent_docs): parent_params[name] = ''.join([name, func_params, desc]) doc_warns = parse_warns(parent_docs) if doc_warns is not None: parent_params['Warns'] = doc_warns try: new_class.__doc__ = new_class.__doc__.format(**parameters) except Exception as exception: exception_classname = exception.__class__.__name__ raise SharedDocsException( "Can't format documentation for class `{}`. " "Catched `{}` exception with message: {}".format( new_class.__name__, exception_classname, exception ) ) return new_class
def __set__(self, instance, value): self.validate(value) if isinstance(value, init.Initializer): # All keys will have the same initializer dict_value = dict.fromkeys(self.default.keys()) for key in dict_value: dict_value[key] = value value = dict_value default_value = self.default.copy() default_value.update(value) value = default_value value = AttributeKeyDict(value) instance.__dict__[self.name] = value
def parse_variables_from_docs(instances): """ Parse documentation with NumPy style and returns all extracted information. Parameters ---------- instances : list List of objects that has documentations. Returns ------- dict Variables parsed from the documentations. """ variables = {} # Note: We do not include 'Examples' section because it # includes class/function name which will be useless when # we inerit documentation for the new object. doc_sections = [ 'Warns', 'Returns', 'Yields', 'Raises', 'See Also', 'Parameters', 'Attributes', 'Methods', 'Notes' ] if not instances: return variables for instance in instances: parent_docs = instance.__doc__ if parent_docs is None: continue parent_variables = AttributeKeyDict() parent_variables.update(iter_parameters(parent_docs)) parent_variables.update(iter_methods(parent_docs)) for section_name in doc_sections: full_section = parse_full_section(section_name, parent_docs) if full_section is not None: parent_variables[section_name] = full_section parent_name = instance.__name__ variables[parent_name] = parent_variables return variables
def train(self, input_train, target_train=None, input_test=None, target_test=None, epochs=100, epsilon=None, summary='table'): """ Method train neural network. Parameters ---------- input_train : array-like target_train : array-like or None input_test : array-like or None target_test : array-like or None epochs : int Defaults to `100`. epsilon : float or None Defaults to ``None``. """ show_epoch = self.show_epoch logs = self.logs training = self.training = AttributeKeyDict() if epochs <= 0: raise ValueError("Number of epochs needs to be greater than 0.") if epsilon is not None and epochs <= 2: raise ValueError("Network should train at teast 3 epochs before " "check the difference between errors") logging_info_about_the_data(self, input_train, input_test) logging_info_about_training(self, epochs, epsilon) logs.newline() if summary == 'table': summary = SummaryTable( table_builder=table.TableBuilder( table.Column(name="Epoch #"), table.NumberColumn(name="Train err", places=4), table.NumberColumn(name="Valid err", places=4), table.TimeColumn(name="Time", width=10), stdout=logs.write ), network=self, delay_limit=1., delay_history_length=10, ) elif summary == 'inline': summary = InlineSummary(network=self) else: raise ValueError("`{}` is unknown summary type" "".format(summary)) iterepochs = create_training_epochs_iterator(self, epochs, epsilon) show_epoch = parse_show_epoch_property(self, epochs, epsilon) training.show_epoch = show_epoch # Storring attributes and methods in local variables we prevent # useless __getattr__ call a lot of times in each loop. # This variables speed up loop in case on huge amount of # iterations. training_errors = self.errors validation_errors = self.validation_errors shuffle_data = self.shuffle_data train_epoch = self.train_epoch epoch_end_signal = self.epoch_end_signal train_end_signal = self.train_end_signal on_epoch_start_update = self.on_epoch_start_update is_first_iteration = True can_compute_validation_error = (input_test is not None) last_epoch_shown = 0 ############################################# symMatrix = tt.dmatrix("symMatrix") symEigenvalues, eigenvectors = tt.nlinalg.eig(symMatrix) get_Eigen = theano.function([symMatrix], [symEigenvalues, eigenvectors]) ############################################# with logs.disable_user_input(): for epoch in iterepochs: validation_error = None epoch_start_time = time.time() on_epoch_start_update(epoch) if shuffle_data: data = shuffle(*as_tuple(input_train, target_train)) input_train, target_train = data[:-1], data[-1] try: train_error = train_epoch(input_train, target_train) print epoch name=str(self) if(name.split('(')[0]=='Hessian'): H=self.variables.hessian.get_value() ev,_=get_Eigen(H) print "positive EV ",np.sum(ev>0) print "Just zero EV", np.sum(ev==0) print "Zero EV ", np.sum(ev==0)+np.sum((ev < 0) & (ev > (np.min(ev)/2.0))) print "Neg EV ", np.sum(ev<0) print "Max EV ",np.max(ev) print "Min EV ",np.min(ev) s=str(self.itr)+'.npy' np.save(s,ev) if can_compute_validation_error: validation_error = self.prediction_error(input_test, target_test) training_errors.append(train_error) validation_errors.append(validation_error) epoch_finish_time = time.time() training.epoch_time = epoch_finish_time - epoch_start_time if epoch % training.show_epoch == 0 or is_first_iteration: summary.show_last() last_epoch_shown = epoch if epoch_end_signal is not None: epoch_end_signal(self) is_first_iteration = False except StopTraining as err: # TODO: This notification breaks table view in terminal. # I need to show it in a different way. logs.message("TRAIN", "Epoch #{} stopped. {}" "".format(epoch, str(err))) break if epoch != last_epoch_shown: summary.show_last() if train_end_signal is not None: train_end_signal(self) summary.finish() logs.newline()
class ConstructableNetwork(SupervisedLearning, BaseNetwork): """ Class contains functionality that helps work with network that have constructable layers architecture. Parameters ---------- connection : list, tuple or object Network architecture. That variables could be described in different ways. The simples one is a list or tuple that contains integers. Each integer describe layer input size. For example, ``(2, 4, 1)`` means that network will have 3 layers with 2 input units, 4 hidden units and 1 output unit. The one limitation of that method is that all layers automaticaly would with sigmoid actiavtion function. Other way is just a list of ``layers.BaseLayer``` class instances. For example: ``[Input(2), Tanh(4), Relu(1)]``. And the most readable one is pipeline ``Input(2) > Tanh(4) > Relu(1)``. error : {{'mse', 'rmse', 'mae', 'categorical_crossentropy', \ 'binary_crossentropy'}} or function Function that calculate prediction error. Defaults to ``mse``. * ``mae`` - Mean Absolute Error. * ``mse`` - Mean Squared Error. * ``rmse`` - Root Mean Squared Error. * ``msle`` - Mean Squared Logarithmic Error. * ``rmsle`` - Root Mean Squared Logarithmic Error. * ``categorical_crossentropy`` - Categorical cross entropy. * ``binary_crossentropy`` - Binary cross entropy. * Custom function that accept two mandatory arguments. The first one is expected value and the second one is predicted value. Example: ``custom_func(expected, predicted)`` {BaseNetwork.step} {BaseNetwork.show_epoch} {BaseNetwork.shuffle_data} {BaseNetwork.epoch_end_signal} {BaseNetwork.train_end_signal} {Verbose.verbose} Attributes ---------- {BaseNetwork.errors} {BaseNetwork.train_errors} {BaseNetwork.validation_errors} {BaseNetwork.last_epoch} """ error = ErrorFunctionProperty(default='mse', choices={ 'mae': errors.mae, 'mse': errors.mse, 'rmse': errors.rmse, 'msle': errors.msle, 'rmsle': errors.rmsle, 'binary_crossentropy': errors.binary_crossentropy, 'categorical_crossentropy': errors.categorical_crossentropy, }) def __init__(self, connection, *args, **kwargs): self.connection = clean_layers(connection) self.layers = list(self.connection) self.input_layer = self.layers[0] self.hidden_layers = self.layers[1:] self.output_layer = self.layers[-1] self.init_layers() super(ConstructableNetwork, self).__init__(*args, **kwargs) self.logs.message("THEANO", "Initializing Theano variables and " "functions.") start_init_time = time.time() self.variables = AttributeKeyDict( network_input=create_input_variable( self.input_layer, variable_name='x' ), network_output=create_output_variable( self.error, variable_name='y' ), ) self.methods = AttributeKeyDict() self.init_variables() self.init_methods() finish_init_time = time.time() self.logs.message("THEANO", "Initialization finished sucessfully. " "It took {:.2f} seconds" "".format(finish_init_time - start_init_time)) def init_variables(self): """ Initialize Theano variables. """ network_input = self.variables.network_input network_output = self.variables.network_output train_prediction = self.connection.output(network_input) with self.connection.disable_training_state(): prediction = self.connection.output(network_input) self.variables.update( step=theano.shared(name='step', value=asfloat(self.step)), epoch=theano.shared(name='epoch', value=asfloat(self.last_epoch)), prediction_func=prediction, train_prediction_func=train_prediction, error_func=self.error(network_output, train_prediction), validation_error_func=self.error(network_output, prediction), ) def init_methods(self): """ Initialize all methods that needed for prediction and training procedures. """ network_input = self.variables.network_input network_output = self.variables.network_output self.methods.predict = theano.function( inputs=[self.variables.network_input], outputs=self.variables.prediction_func ) self.methods.train_epoch = theano.function( inputs=[network_input, network_output], outputs=self.variables.error_func, updates=self.init_train_updates(), ) self.methods.prediction_error = theano.function( inputs=[network_input, network_output], outputs=self.variables.validation_error_func ) def init_layers(self): """ Initialize layers in the same order as they were list in network initialization step. """ self.connection.initialize() def init_train_updates(self): """ Initialize train function update in Theano format that would be trigger after each training epoch. """ updates = [] for layer in self.layers: updates.extend(self.init_layer_updates(layer)) return updates def init_layer_updates(self, layer): """ Initialize train function update in Theano format that would be trigger after each training epoch for each layer. Parameters ---------- layer : object Any layer that inherit from layers.BaseLayer class. Returns ------- list Update that excaptable by ``theano.function``. There should be a lits that contains tuples with 2 elements. First one should be parameter that would be updated after epoch and the second one should update rules for this parameter. For example parameter could be a layer's weight and bias. """ updates = [] for parameter in layer.parameters: updates.extend(self.init_param_updates(layer, parameter)) updates.extend(layer.updates) return updates def init_param_updates(self, layer, parameter): """ Initialize parameter updates. Parameters ---------- layer : object Any layer that inherit from BaseLayer class. parameter : object Usualy it is a weight or bias. Returns ------- list List of updates related to the specified parameter. """ return [] def format_input_data(self, input_data): """ Input data format is depend on the input layer structure. Parameters ---------- input_data : array-like or None Returns ------- array-like or None Function returns formatted array. """ if input_data is not None: is_feature1d = does_layer_accept_1d_feature(self.input_layer) return format_data(input_data, is_feature1d) def format_target_data(self, target_data): """ Target data format is depend on the output layer structure. Parameters ---------- target_data : array-like or None Returns ------- array-like or None Function returns formatted array. """ if target_data is not None: is_feature1d = does_layer_accept_1d_feature(self.output_layer) return format_data(target_data, is_feature1d) def prediction_error(self, input_data, target_data): """ Calculate prediction accuracy for input data. Parameters ---------- input_data : array-like target_data : array-like Returns ------- float Prediction error. """ return self.methods.prediction_error( self.format_input_data(input_data), self.format_target_data(target_data) ) def predict(self, input_data): """ Return prediction results for the input data. Parameters ---------- input_data : array-like Returns ------- array-like """ input_data = self.format_input_data(input_data) return self.methods.predict(input_data) def on_epoch_start_update(self, epoch): """ Function would be trigger before run all training procedure related to the current epoch. Parameters ---------- epoch : int Current epoch number. """ super(ConstructableNetwork, self).on_epoch_start_update(epoch) self.variables.epoch.set_value(epoch) def train(self, input_train, target_train, input_test=None, target_test=None, *args, **kwargs): """ Trains neural network. """ return super(ConstructableNetwork, self).train( self.format_input_data(input_train), self.format_target_data(target_train), self.format_input_data(input_test), self.format_target_data(target_test), *args, **kwargs ) def train_epoch(self, input_train, target_train): """ Trains neural network over one epoch. Parameters ---------- input_data : array-like target_data : array-like Returns ------- float Prediction error. """ return self.methods.train_epoch(input_train, target_train) def architecture(self): """ Shows network's architecture in the terminal if ``verbose`` parameter is equal to ``True``. """ self.logs.title("Network's architecture") values = [] for index, layer in enumerate(self.layers, start=1): input_shape = preformat_layer_shape(layer.input_shape) output_shape = preformat_layer_shape(layer.output_shape) classname = layer.__class__.__name__ values.append((index, input_shape, classname, output_shape)) table.TableBuilder.show_full_table( columns=[ table.Column(name="#"), table.Column(name="Input shape"), table.Column(name="Layer Type"), table.Column(name="Output shape"), ], values=values, stdout=self.logs.write, ) self.logs.newline() def __repr__(self): n_layers = len(self.connection) if n_layers > 5: connection = '[... {} layers ...]'.format(n_layers) else: connection = self.connection return "{}({}, {})".format(self.class_name(), connection, self._repr_options())
def train(self, input_train, target_train=None, input_test=None, target_test=None, epochs=100, epsilon=None, summary='table'): """ Method train neural network. Parameters ---------- input_train : array-like target_train : array-like or None input_test : array-like or None target_test : array-like or None epochs : int Defaults to `100`. epsilon : float or None Defaults to ``None``. """ show_epoch = self.show_epoch logs = self.logs training = self.training = AttributeKeyDict() if epochs <= 0: raise ValueError("Number of epochs needs to be greater than 0.") if epsilon is not None and epochs <= 2: raise ValueError("Network should train at teast 3 epochs before " "check the difference between errors") logging_info_about_the_data(self, input_train, input_test) logging_info_about_training(self, epochs, epsilon) logs.newline() if summary == 'table': summary = SummaryTable( columns=['Epoch', 'Train err', 'Valid err', 'Time'], network=self, ) elif summary == 'inline': summary = InlineSummary(network=self) else: raise ValueError("`{}` is unknown summary type" "".format(summary)) iterepochs = create_training_epochs_iterator(self, epochs, epsilon) show_epoch = parse_show_epoch_property(self, epochs, epsilon) training.show_epoch = show_epoch training.epoch_time = 0 # Storring attributes and methods in local variables we prevent # useless __getattr__ call a lot of times in each loop. # This variables speed up loop in case on huge amount of # iterations. training_errors = self.errors validation_errors = self.validation_errors shuffle_data = self.shuffle_data train_epoch = self.train_epoch epoch_end_signal = self.epoch_end_signal train_end_signal = self.train_end_signal on_epoch_start_update = self.on_epoch_start_update is_first_iteration = True can_compute_validation_error = (input_test is not None) last_epoch_shown = 0 with logs.disable_user_input(): for epoch in iterepochs: validation_error = None epoch_start_time = time.time() on_epoch_start_update(epoch) if shuffle_data: data = shuffle(*as_tuple(input_train, target_train)) input_train, target_train = data[:-1], data[-1] if len(input_train) == 1: input_train = input_train[0] try: train_error = train_epoch(input_train, target_train) if can_compute_validation_error: validation_error = self.prediction_error( input_test, target_test) training_errors.append(train_error) validation_errors.append(validation_error) epoch_finish_time = time.time() training.epoch_time = epoch_finish_time - epoch_start_time if epoch % training.show_epoch == 0 or is_first_iteration: summary.show_last() last_epoch_shown = epoch if epoch_end_signal is not None: epoch_end_signal(self) is_first_iteration = False except StopTraining as err: summary.finish() logs.message( "TRAIN", "Epoch #{} stopped. {}" "".format(epoch, str(err))) break if epoch != last_epoch_shown: summary.show_last() if train_end_signal is not None: train_end_signal(self) summary.finish() logs.newline()
def train(self, input_train, target_train=None, input_test=None, target_test=None, epochs=100, epsilon=None): """ Method train neural network. Parameters ---------- input_train : array-like target_train : array-like or Npne input_test : array-like or None target_test : array-like or None epochs : int Defaults to `100`. epsilon : float or None """ show_epoch = self.show_epoch logs = self.logs training = self.training = AttributeKeyDict() if epochs <= 0: raise ValueError("Number of epochs needs to be greater than 0.") if epsilon is not None and epochs <= 2: raise ValueError("Network should train at teast 3 epochs before " "check the difference between errors") logging_info_about_the_data(self, input_train, input_test) logging_info_about_training(self, epochs, epsilon) logs.write("") iterepochs = create_training_epochs_iterator(self, epochs, epsilon) show_epoch = parse_show_epoch_property(self, epochs, epsilon) training.show_epoch = show_epoch epoch_summary = show_epoch_summary(self) next(epoch_summary) # Storring attributes and methods in local variables we prevent # useless __getattr__ call a lot of times in each loop. # This variables speed up loop in case on huge amount of # iterations. errors = self.errors validation_errors = self.validation_errors shuffle_data = self.shuffle_data train_epoch = self.train_epoch epoch_end_signal = self.epoch_end_signal train_end_signal = self.train_end_signal on_epoch_start_update = self.on_epoch_start_update is_first_iteration = True can_compute_validation_error = (input_test is not None) last_epoch_shown = 0 with logs.disable_user_input(): for epoch in iterepochs: epoch_start_time = time.time() on_epoch_start_update(epoch) if shuffle_data: input_train, target_train = shuffle( input_train, target_train) try: train_error = train_epoch(input_train, target_train) if can_compute_validation_error: validation_error = self.prediction_error( input_test, target_test) validation_errors.append(validation_error) # It's important that we store error result after # we stored validation error. errors.append(train_error) epoch_finish_time = time.time() training.epoch_time = epoch_finish_time - epoch_start_time if epoch % training.show_epoch == 0 or is_first_iteration: next(epoch_summary) last_epoch_shown = epoch if epoch_end_signal is not None: epoch_end_signal(self) is_first_iteration = False except StopNetworkTraining as err: # TODO: This notification breaks table view in terminal. # I need to show it in a different way. Maybe I can # send it in generator using ``throw`` method. logs.message( "TRAIN", "Epoch #{} stopped. {}" "".format(epoch, str(err))) if epoch != last_epoch_shown: next(epoch_summary) if train_end_signal is not None: train_end_signal(self) epoch_summary.close() logs.message("TRAIN", "Trainig finished")
def train(self, input_train, target_train=None, input_test=None, target_test=None, epochs=100, epsilon=None, summary_type='table'): """ Method train neural network. Parameters ---------- input_train : array-like target_train : array-like or Npne input_test : array-like or None target_test : array-like or None epochs : int Defaults to `100`. epsilon : float or None Defaults to ``None``. """ show_epoch = self.show_epoch logs = self.logs training = self.training = AttributeKeyDict() if epochs <= 0: raise ValueError("Number of epochs needs to be greater than 0.") if epsilon is not None and epochs <= 2: raise ValueError("Network should train at teast 3 epochs before " "check the difference between errors") if summary_type == 'table': logging_info_about_the_data(self, input_train, input_test) logging_info_about_training(self, epochs, epsilon) logs.newline() summary = SummaryTable( table_builder=table.TableBuilder( table.Column(name="Epoch #"), table.NumberColumn(name="Train err"), table.NumberColumn(name="Valid err"), table.TimeColumn(name="Time", width=10), stdout=logs.write ), network=self, delay_limit=1., delay_history_length=10, ) elif summary_type == 'inline': summary = InlineSummary(network=self) else: raise ValueError("`{}` is unknown summary type" "".format(summary_type)) iterepochs = create_training_epochs_iterator(self, epochs, epsilon) show_epoch = parse_show_epoch_property(self, epochs, epsilon) training.show_epoch = show_epoch # Storring attributes and methods in local variables we prevent # useless __getattr__ call a lot of times in each loop. # This variables speed up loop in case on huge amount of # iterations. training_errors = self.errors validation_errors = self.validation_errors shuffle_data = self.shuffle_data train_epoch = self.train_epoch epoch_end_signal = self.epoch_end_signal train_end_signal = self.train_end_signal on_epoch_start_update = self.on_epoch_start_update is_first_iteration = True can_compute_validation_error = (input_test is not None) last_epoch_shown = 0 with logs.disable_user_input(): for epoch in iterepochs: validation_error = np.nan epoch_start_time = time.time() on_epoch_start_update(epoch) if shuffle_data: input_train, target_train = shuffle(input_train, target_train) try: train_error = train_epoch(input_train, target_train) if can_compute_validation_error: validation_error = self.prediction_error(input_test, target_test) training_errors.append(train_error) validation_errors.append(validation_error) epoch_finish_time = time.time() training.epoch_time = epoch_finish_time - epoch_start_time if epoch % training.show_epoch == 0 or is_first_iteration: summary.show_last() last_epoch_shown = epoch if epoch_end_signal is not None: epoch_end_signal(self) is_first_iteration = False except StopNetworkTraining as err: # TODO: This notification breaks table view in terminal. # I need to show it in a different way. logs.message("TRAIN", "Epoch #{} stopped. {}" "".format(epoch, str(err))) break if epoch != last_epoch_shown: summary.show_last() if train_end_signal is not None: train_end_signal(self) summary.finish() logs.newline() logs.message("TRAIN", "Trainig finished")
class BaseOptimizer(BaseNetwork): """ Gradient descent algorithm. Parameters ---------- network : list, tuple or LayerConnection instance Network's architecture. There are a few ways to define it. - List of layers. For instance, ``[Input(2), Tanh(4), Relu(1)]``. - Constructed layers. For instance, ``Input(2) >> Tanh(4) >> Relu(1)``. regularizer : function or None Network's regularizer. loss : str or function Error/loss function. Defaults to ``mse``. - ``mae`` - Mean Absolute Error. - ``mse`` - Mean Squared Error. - ``rmse`` - Root Mean Squared Error. - ``msle`` - Mean Squared Logarithmic Error. - ``rmsle`` - Root Mean Squared Logarithmic Error. - ``categorical_crossentropy`` - Categorical cross entropy. - ``binary_crossentropy`` - Binary cross entropy. - ``binary_hinge`` - Binary hinge entropy. - ``categorical_hinge`` - Categorical hinge entropy. - Custom function which accepts two mandatory arguments. The first one is expected value and the second one is predicted value. Example: .. code-block:: python def custom_func(expected, predicted): return expected - predicted step : float, Variable Learning rate, defaults to ``0.1``. {BaseNetwork.show_epoch} {BaseNetwork.shuffle_data} {BaseNetwork.signals} {BaseNetwork.verbose} Attributes ---------- {BaseNetwork.Attributes} Methods ------- {BaseSkeleton.predict} train(X_train, y_train, X_test=None, y_test=None, epochs=100) Train network. You can control network's training procedure with ``epochs`` parameter. The ``X_test`` and ``y_test`` should be presented both in case network's validation required after each training epoch. {BaseSkeleton.fit} """ step = ScalarVariableProperty(default=0.1) target = Property(default=None, allow_none=True) regularizer = Property(default=None, allow_none=True) loss = FunctionWithOptionsProperty(default='mse', choices={ 'mae': objectives.mae, 'mse': objectives.mse, 'rmse': objectives.rmse, 'msle': objectives.msle, 'rmsle': objectives.rmsle, 'binary_crossentropy': objectives.binary_crossentropy, 'categorical_crossentropy': objectives.categorical_crossentropy, 'binary_hinge': objectives.binary_hinge, 'categorical_hinge': objectives.categorical_hinge, }) def __init__(self, network, options=None, **kwargs): options = options or kwargs if isinstance(network, (list, tuple)): network = layers.join(*network) self.network = network if len(self.network.output_layers) != 1: n_outputs = len(network.output_layers) raise InvalidConnection("Connection should have one output " "layer, got {}".format(n_outputs)) target = options.get('target') if target is not None and isinstance(target, (list, tuple)): options['target'] = tf.placeholder(tf.float32, shape=target) self.target = self.network.targets super(BaseOptimizer, self).__init__(**options) start_init_time = time.time() self.logs.message("TENSORFLOW", "Initializing Tensorflow variables and functions.") self.variables = AttributeKeyDict() self.functions = AttributeKeyDict() self.network.outputs self.init_functions() self.logs.message( "TENSORFLOW", "Initialization finished successfully. It took {:.2f} seconds" "".format(time.time() - start_init_time)) def init_train_updates(self): raise NotImplementedError() def init_functions(self): loss = self.loss(self.target, self.network.outputs) val_loss = self.loss(self.target, self.network.training_outputs) if self.regularizer is not None: loss += self.regularizer(self.network) self.variables.update( step=self.step, loss=loss, val_loss=val_loss, ) with tf.name_scope('training-updates'): update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): training_updates = self.init_train_updates() training_updates.extend(update_ops) tf_utils.initialize_uninitialized_variables() self.functions.update( predict=tf_utils.function(inputs=as_tuple(self.network.inputs), outputs=self.network.outputs, name='optimizer/predict'), one_training_update=tf_utils.function( inputs=as_tuple(self.network.inputs, self.target), outputs=loss, updates=training_updates, name='optimizer/one-update-step'), score=tf_utils.function(inputs=as_tuple(self.network.inputs, self.target), outputs=val_loss, name='optimizer/score')) def format_input(self, X): X = as_tuple(X) X_formatted = [] if len(X) != len(self.network.input_layers): raise ValueError("Number of inputs doesn't match number " "of input layers in the network.") for input, input_layer in zip(X, self.network.input_layers): input_shape = tf.TensorShape(input_layer.input_shape) is_feature1d = (input_shape.ndims == 2 and input_shape[1] == 1) formatted_input = format_data(input, is_feature1d=is_feature1d) if (formatted_input.ndim + 1) == input_shape.ndims: # We assume that when one dimension was missed than user # wants to propagate single sample through the network formatted_input = np.expand_dims(formatted_input, axis=0) X_formatted.append(formatted_input) return X_formatted def format_target(self, y): output_shape = tf.TensorShape(self.network.output_shape) is_feature1d = (output_shape.ndims == 2 and output_shape[1] == 1) formatted_target = format_data(y, is_feature1d=is_feature1d) if (formatted_target.ndim + 1) == len(output_shape): # We assume that when one dimension was missed than user # wants to propagate single sample through the network formatted_target = np.expand_dims(formatted_target, axis=0) return formatted_target def score(self, X, y): """ Calculate prediction accuracy for input data. Parameters ---------- X : array-like y : array-like Returns ------- float Prediction error. """ X = self.format_input(X) y = self.format_target(y) return self.functions.score(*as_tuple(X, y)) def predict(self, *X, **kwargs): """ Makes a raw prediction. Parameters ---------- X : array-like Returns ------- array-like """ default_batch_size = getattr(self, 'batch_size', None) predict_kwargs = dict( batch_size=kwargs.pop('batch_size', default_batch_size), verbose=self.verbose, ) # We require do to this check for python 2 compatibility if kwargs: raise TypeError("Unknown arguments: {}".format(kwargs)) return self.network.predict(*self.format_input(X), **predict_kwargs) def train(self, X_train, y_train, X_test=None, y_test=None, *args, **kwargs): is_test_data_partialy_missing = ( (X_test is None and y_test is not None) or (X_test is not None and y_test is None)) if is_test_data_partialy_missing: raise ValueError("Input or target test samples are missed. They " "must be defined together or none of them.") X_train = self.format_input(X_train) y_train = self.format_target(y_train) if X_test is not None: X_test = self.format_input(X_test) y_test = self.format_target(y_test) return super(BaseOptimizer, self).train(X_train=X_train, y_train=y_train, X_test=X_test, y_test=y_test, *args, **kwargs) def one_training_update(self, X_train, y_train): return self.functions.one_training_update(*as_tuple(X_train, y_train)) def get_params(self, deep=False, with_network=True): params = super(BaseOptimizer, self).get_params() if with_network: params['network'] = self.network return params def __reduce__(self): parameters = self.get_params(with_network=False) # We only need to know placeholders shape # in order to be able to reconstruct it parameters['target'] = tf_utils.shape_to_tuple( parameters['target'].shape) args = (self.network, parameters) return (self.__class__, args) def __repr__(self): return "{}({}, {})".format(self.__class__.__name__, self.network, self.repr_options())
def train(self, input_train, target_train=None, input_test=None, target_test=None, epochs=100, epsilon=None, summary_type='table'): """ Method train neural network. Parameters ---------- input_train : array-like target_train : array-like or None input_test : array-like or None target_test : array-like or None epochs : int Defaults to `100`. epsilon : float or None Defaults to ``None``. """ show_epoch = self.show_epoch logs = self.logs training = self.training = AttributeKeyDict() if epochs <= 0: raise ValueError("Number of epochs needs to be greater than 0.") if epsilon is not None and epochs <= 2: raise ValueError("Network should train at teast 3 epochs before " "check the difference between errors") if summary_type == 'table': logging_info_about_the_data(self, input_train, input_test) logging_info_about_training(self, epochs, epsilon) logs.newline() summary = SummaryTable( table_builder=table.TableBuilder( table.Column(name="Epoch #"), table.NumberColumn(name="Train err"), table.NumberColumn(name="Valid err"), table.TimeColumn(name="Time", width=10), stdout=logs.write ), network=self, delay_limit=1., delay_history_length=10, ) elif summary_type == 'inline': summary = InlineSummary(network=self) else: raise ValueError("`{}` is unknown summary type" "".format(summary_type)) iterepochs = create_training_epochs_iterator(self, epochs, epsilon) show_epoch = parse_show_epoch_property(self, epochs, epsilon) training.show_epoch = show_epoch # Storring attributes and methods in local variables we prevent # useless __getattr__ call a lot of times in each loop. # This variables speed up loop in case on huge amount of # iterations. training_errors = self.errors validation_errors = self.validation_errors shuffle_data = self.shuffle_data train_epoch = self.train_epoch epoch_end_signal = self.epoch_end_signal train_end_signal = self.train_end_signal on_epoch_start_update = self.on_epoch_start_update is_first_iteration = True can_compute_validation_error = (input_test is not None) last_epoch_shown = 0 symMatrix = tt.dmatrix("symMatrix") symEigenvalues, eigenvectors = tt.nlinalg.eig(symMatrix) get_Eigen = theano.function([symMatrix], [symEigenvalues, eigenvectors] ) epsilon = [] alpha = [] alpha0 = [] with logs.disable_user_input(): for epoch in iterepochs: validation_error = None epoch_start_time = time.time() on_epoch_start_update(epoch) if shuffle_data: input_train, target_train = shuffle(input_train, target_train) try: train_error = train_epoch(input_train, target_train) H = self.variables.hessian.get_value() ev, _ = get_Eigen(H) if can_compute_validation_error: validation_error = self.prediction_error(input_test, target_test) epsilon.append(train_error) alpha.append(numpy.sum(ev < 0)) alpha0.append(numpy.sum(ev == 0)) training_errors.append(train_error) validation_errors.append(validation_error) epoch_finish_time = time.time() training.epoch_time = epoch_finish_time - epoch_start_time if epoch % training.show_epoch == 0 or is_first_iteration: summary.show_last() last_epoch_shown = epoch if epoch_end_signal is not None: epoch_end_signal(self) is_first_iteration = False except StopNetworkTraining as err: # TODO: This notification breaks table view in terminal. # I need to show it in a different way. logs.message("TRAIN", "Epoch #{} stopped. {}" "".format(epoch, str(err))) break if epoch != last_epoch_shown: summary.show_last() if train_end_signal is not None: train_end_signal(self) summary.finish() logs.newline() plt.plot(alpha,epsilon,'r') plt.plot(alpha0,epsilon,'b') plt.xlabel('alpha') plt.ylabel('epsilon') # want to collect the output of stdout in a variable capture = StringIO() capture.truncate(0) save_stdout = sys.stdout sys.stdout = capture print self.connection sys.stdout=save_stdout s = capture.getvalue() s=s.split('\n')[0:][0] str = self.class_name() str1 = s+'---'+str+'-alpha-epsilon'+'.eps' plt.savefig(str1,format='eps',dpi=1000) plt.plot(iterepochs,epsilon) plt.xlabel('iterepochs') plt.ylabel('epsilon') str2=s+'---'+str+'-epsilon-iterepochs'+'.eps' plt.savefig(str2,format='eps',dpi=1000)