def parse_variables_from_docs(instances):
"""
Parse documentation with NumPy style and returns all
extracted information.
Parameters
----------
instances : list
List of objects that has documentations.
Returns
-------
dict
Variables parsed from the documentations.
"""
variables = {}
# Note: We do not include 'Examples' section because it
# includes class/function name which will be useless when
# we inerit documentation for the new object.
doc_sections = [
'Warns', 'Returns', 'Yields', 'Raises', 'See Also', 'Parameters',
'Attributes', 'Methods', 'Notes'
]
if not instances:
return variables
for instance in instances:
parent_docs = instance.__doc__
if parent_docs is None:
continue
parent_variables = AttributeKeyDict()
parent_variables.update(iter_parameters(parent_docs))
parent_variables.update(iter_methods(parent_docs))
for section_name in doc_sections:
full_section = parse_full_section(section_name, parent_docs)
if full_section is not None:
parent_variables[section_name] = full_section
parent_name = instance.__name__
variables[parent_name] = parent_variables
return variables
def parse_variables_from_docs(instances):
"""
Parse documentation with NumPy style and returns all
extracted information.
Parameters
----------
instances : list
List of objects that has documentations.
Returns
-------
dict
Variables parsed from the documentations.
"""
variables = {}
# Note: We do not include 'Examples' section because it
# includes class/function name which will be useless when
# we inerit documentation for the new object.
doc_sections = ['Warns', 'Returns', 'Yields', 'Raises', 'See Also',
'Parameters', 'Attributes', 'Methods']
if not instances:
return variables
for instance in instances:
parent_docs = instance.__doc__
if parent_docs is None:
continue
parent_variables = AttributeKeyDict()
parent_variables.update(iter_parameters(parent_docs))
parent_variables.update(iter_methods(parent_docs))
for section_name in doc_sections:
full_section = parse_full_section(section_name, parent_docs)
if full_section is not None:
parent_variables[section_name] = full_section
parent_name = instance.__name__
variables[parent_name] = parent_variables
return variables
class BaseOptimizer(BaseNetwork):
"""
Gradient descent algorithm.
Parameters
----------
network : list, tuple or LayerConnection instance
Network's architecture. There are a few ways
to define it.
- List of layers.
For instance, ``[Input(2), Tanh(4), Relu(1)]``.
- Constructed layers.
For instance, ``Input(2) >> Tanh(4) >> Relu(1)``.
regularizer : function or None
Network's regularizer.
loss : str or function
Error/loss function. Defaults to ``mse``.
- ``mae`` - Mean Absolute Error.
- ``mse`` - Mean Squared Error.
- ``rmse`` - Root Mean Squared Error.
- ``msle`` - Mean Squared Logarithmic Error.
- ``rmsle`` - Root Mean Squared Logarithmic Error.
- ``categorical_crossentropy`` - Categorical cross entropy.
- ``binary_crossentropy`` - Binary cross entropy.
- ``binary_hinge`` - Binary hinge entropy.
- ``categorical_hinge`` - Categorical hinge entropy.
- Custom function which accepts two mandatory arguments.
The first one is expected value and the second one is
predicted value. Example:
.. code-block:: python
def custom_func(expected, predicted):
return expected - predicted
step : float, Variable
Learning rate, defaults to ``0.1``.
{BaseNetwork.show_epoch}
{BaseNetwork.shuffle_data}
{BaseNetwork.signals}
{BaseNetwork.verbose}
Attributes
----------
{BaseNetwork.Attributes}
Methods
-------
{BaseSkeleton.predict}
train(X_train, y_train, X_test=None, y_test=None, epochs=100)
Train network. You can control network's training procedure
with ``epochs`` parameter. The ``X_test`` and ``y_test`` should
be presented both in case network's validation required
after each training epoch.
{BaseSkeleton.fit}
"""
step = ScalarVariableProperty(default=0.1)
target = Property(default=None, allow_none=True)
regularizer = Property(default=None, allow_none=True)
loss = FunctionWithOptionsProperty(default='mse',
choices={
'mae':
objectives.mae,
'mse':
objectives.mse,
'rmse':
objectives.rmse,
'msle':
objectives.msle,
'rmsle':
objectives.rmsle,
'binary_crossentropy':
objectives.binary_crossentropy,
'categorical_crossentropy':
objectives.categorical_crossentropy,
'binary_hinge':
objectives.binary_hinge,
'categorical_hinge':
objectives.categorical_hinge,
})
def __init__(self, network, options=None, **kwargs):
options = options or kwargs
if isinstance(network, (list, tuple)):
network = layers.join(*network)
self.network = network
if len(self.network.output_layers) != 1:
n_outputs = len(network.output_layers)
raise InvalidConnection("Connection should have one output "
"layer, got {}".format(n_outputs))
target = options.get('target')
if target is not None and isinstance(target, (list, tuple)):
options['target'] = tf.placeholder(tf.float32, shape=target)
self.target = self.network.targets
super(BaseOptimizer, self).__init__(**options)
start_init_time = time.time()
self.logs.message("TENSORFLOW",
"Initializing Tensorflow variables and functions.")
self.variables = AttributeKeyDict()
self.functions = AttributeKeyDict()
self.network.outputs
self.init_functions()
self.logs.message(
"TENSORFLOW",
"Initialization finished successfully. It took {:.2f} seconds"
"".format(time.time() - start_init_time))
def init_train_updates(self):
raise NotImplementedError()
def init_functions(self):
loss = self.loss(self.target, self.network.outputs)
val_loss = self.loss(self.target, self.network.training_outputs)
if self.regularizer is not None:
loss += self.regularizer(self.network)
self.variables.update(
step=self.step,
loss=loss,
val_loss=val_loss,
)
with tf.name_scope('training-updates'):
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
training_updates = self.init_train_updates()
training_updates.extend(update_ops)
tf_utils.initialize_uninitialized_variables()
self.functions.update(
predict=tf_utils.function(inputs=as_tuple(self.network.inputs),
outputs=self.network.outputs,
name='optimizer/predict'),
one_training_update=tf_utils.function(
inputs=as_tuple(self.network.inputs, self.target),
outputs=loss,
updates=training_updates,
name='optimizer/one-update-step'),
score=tf_utils.function(inputs=as_tuple(self.network.inputs,
self.target),
outputs=val_loss,
name='optimizer/score'))
def format_input(self, X):
X = as_tuple(X)
X_formatted = []
if len(X) != len(self.network.input_layers):
raise ValueError("Number of inputs doesn't match number "
"of input layers in the network.")
for input, input_layer in zip(X, self.network.input_layers):
input_shape = tf.TensorShape(input_layer.input_shape)
is_feature1d = (input_shape.ndims == 2 and input_shape[1] == 1)
formatted_input = format_data(input, is_feature1d=is_feature1d)
if (formatted_input.ndim + 1) == input_shape.ndims:
# We assume that when one dimension was missed than user
# wants to propagate single sample through the network
formatted_input = np.expand_dims(formatted_input, axis=0)
X_formatted.append(formatted_input)
return X_formatted
def format_target(self, y):
output_shape = tf.TensorShape(self.network.output_shape)
is_feature1d = (output_shape.ndims == 2 and output_shape[1] == 1)
formatted_target = format_data(y, is_feature1d=is_feature1d)
if (formatted_target.ndim + 1) == len(output_shape):
# We assume that when one dimension was missed than user
# wants to propagate single sample through the network
formatted_target = np.expand_dims(formatted_target, axis=0)
return formatted_target
def score(self, X, y):
"""
Calculate prediction accuracy for input data.
Parameters
----------
X : array-like
y : array-like
Returns
-------
float
Prediction error.
"""
X = self.format_input(X)
y = self.format_target(y)
return self.functions.score(*as_tuple(X, y))
def predict(self, *X, **kwargs):
"""
Makes a raw prediction.
Parameters
----------
X : array-like
Returns
-------
array-like
"""
default_batch_size = getattr(self, 'batch_size', None)
predict_kwargs = dict(
batch_size=kwargs.pop('batch_size', default_batch_size),
verbose=self.verbose,
)
# We require do to this check for python 2 compatibility
if kwargs:
raise TypeError("Unknown arguments: {}".format(kwargs))
return self.network.predict(*self.format_input(X), **predict_kwargs)
def train(self,
X_train,
y_train,
X_test=None,
y_test=None,
*args,
**kwargs):
is_test_data_partialy_missing = (
(X_test is None and y_test is not None)
or (X_test is not None and y_test is None))
if is_test_data_partialy_missing:
raise ValueError("Input or target test samples are missed. They "
"must be defined together or none of them.")
X_train = self.format_input(X_train)
y_train = self.format_target(y_train)
if X_test is not None:
X_test = self.format_input(X_test)
y_test = self.format_target(y_test)
return super(BaseOptimizer, self).train(X_train=X_train,
y_train=y_train,
X_test=X_test,
y_test=y_test,
*args,
**kwargs)
def one_training_update(self, X_train, y_train):
return self.functions.one_training_update(*as_tuple(X_train, y_train))
def get_params(self, deep=False, with_network=True):
params = super(BaseOptimizer, self).get_params()
if with_network:
params['network'] = self.network
return params
def __reduce__(self):
parameters = self.get_params(with_network=False)
# We only need to know placeholders shape
# in order to be able to reconstruct it
parameters['target'] = tf_utils.shape_to_tuple(
parameters['target'].shape)
args = (self.network, parameters)
return (self.__class__, args)
def __repr__(self):
return "{}({}, {})".format(self.__class__.__name__, self.network,
self.repr_options())
class ConstructableNetwork(SupervisedLearning, BaseNetwork):
""" Class contains functionality that helps work with network that have
constructable layers architecture.
Parameters
----------
connection : list, tuple or object
Network architecture. That variables could be described in
different ways. The simples one is a list or tuple that contains
integers. Each integer describe layer input size. For example,
``(2, 4, 1)`` means that network will have 3 layers with 2 input
units, 4 hidden units and 1 output unit. The one limitation of that
method is that all layers automaticaly would with sigmoid actiavtion
function. Other way is just a list of ``layers.BaseLayer``` class
instances. For example: ``[Input(2), Tanh(4), Relu(1)]``.
And the most readable one is pipeline
``Input(2) > Tanh(4) > Relu(1)``.
error : {{'mse', 'rmse', 'mae', 'categorical_crossentropy', \
'binary_crossentropy'}} or function
Function that calculate prediction error.
Defaults to ``mse``.
* ``mae`` - Mean Absolute Error.
* ``mse`` - Mean Squared Error.
* ``rmse`` - Root Mean Squared Error.
* ``msle`` - Mean Squared Logarithmic Error.
* ``rmsle`` - Root Mean Squared Logarithmic Error.
* ``categorical_crossentropy`` - Categorical cross entropy.
* ``binary_crossentropy`` - Binary cross entropy.
* Custom function that accept two mandatory arguments.
The first one is expected value and the second one is
predicted value. Example: ``custom_func(expected, predicted)``
{BaseNetwork.step}
{BaseNetwork.show_epoch}
{BaseNetwork.shuffle_data}
{BaseNetwork.epoch_end_signal}
{BaseNetwork.train_end_signal}
{Verbose.verbose}
Attributes
----------
{BaseNetwork.errors}
{BaseNetwork.train_errors}
{BaseNetwork.validation_errors}
{BaseNetwork.last_epoch}
"""
error = ErrorFunctionProperty(default='mse', choices={
'mae': errors.mae,
'mse': errors.mse,
'rmse': errors.rmse,
'msle': errors.msle,
'rmsle': errors.rmsle,
'binary_crossentropy': errors.binary_crossentropy,
'categorical_crossentropy': errors.categorical_crossentropy,
})
def __init__(self, connection, *args, **kwargs):
self.connection = clean_layers(connection)
self.layers = list(self.connection)
self.input_layer = self.layers[0]
self.hidden_layers = self.layers[1:]
self.output_layer = self.layers[-1]
self.init_layers()
super(ConstructableNetwork, self).__init__(*args, **kwargs)
self.logs.message("THEANO", "Initializing Theano variables and "
"functions.")
start_init_time = time.time()
self.variables = AttributeKeyDict(
network_input=create_input_variable(
self.input_layer, variable_name='x'
),
network_output=create_output_variable(
self.error, variable_name='y'
),
)
self.methods = AttributeKeyDict()
self.init_variables()
self.init_methods()
finish_init_time = time.time()
self.logs.message("THEANO", "Initialization finished sucessfully. "
"It took {:.2f} seconds"
"".format(finish_init_time - start_init_time))
def init_variables(self):
""" Initialize Theano variables.
"""
network_input = self.variables.network_input
network_output = self.variables.network_output
train_prediction = self.connection.output(network_input)
with self.connection.disable_training_state():
prediction = self.connection.output(network_input)
self.variables.update(
step=theano.shared(name='step', value=asfloat(self.step)),
epoch=theano.shared(name='epoch', value=asfloat(self.last_epoch)),
prediction_func=prediction,
train_prediction_func=train_prediction,
error_func=self.error(network_output, train_prediction),
validation_error_func=self.error(network_output, prediction),
)
def init_methods(self):
""" Initialize all methods that needed for prediction and
training procedures.
"""
network_input = self.variables.network_input
network_output = self.variables.network_output
self.methods.predict = theano.function(
inputs=[self.variables.network_input],
outputs=self.variables.prediction_func
)
self.methods.train_epoch = theano.function(
inputs=[network_input, network_output],
outputs=self.variables.error_func,
updates=self.init_train_updates(),
)
self.methods.prediction_error = theano.function(
inputs=[network_input, network_output],
outputs=self.variables.validation_error_func
)
def init_layers(self):
""" Initialize layers in the same order as they were list in
network initialization step.
"""
self.connection.initialize()
def init_train_updates(self):
""" Initialize train function update in Theano format that
would be trigger after each training epoch.
"""
updates = []
for layer in self.layers:
updates.extend(self.init_layer_updates(layer))
return updates
def init_layer_updates(self, layer):
""" Initialize train function update in Theano format that
would be trigger after each training epoch for each layer.
Parameters
----------
layer : object
Any layer that inherit from layers.BaseLayer class.
Returns
-------
list
Update that excaptable by ``theano.function``. There should be
a lits that contains tuples with 2 elements. First one should
be parameter that would be updated after epoch and the second one
should update rules for this parameter. For example parameter
could be a layer's weight and bias.
"""
updates = []
for parameter in layer.parameters:
updates.extend(self.init_param_updates(layer, parameter))
updates.extend(layer.updates)
return updates
def init_param_updates(self, layer, parameter):
""" Initialize parameter updates.
Parameters
----------
layer : object
Any layer that inherit from BaseLayer class.
parameter : object
Usualy it is a weight or bias.
Returns
-------
list
List of updates related to the specified parameter.
"""
return []
def format_input_data(self, input_data):
""" Input data format is depend on the input layer
structure.
Parameters
----------
input_data : array-like or None
Returns
-------
array-like or None
Function returns formatted array.
"""
if input_data is not None:
is_feature1d = does_layer_accept_1d_feature(self.input_layer)
return format_data(input_data, is_feature1d)
def format_target_data(self, target_data):
""" Target data format is depend on the output layer
structure.
Parameters
----------
target_data : array-like or None
Returns
-------
array-like or None
Function returns formatted array.
"""
if target_data is not None:
is_feature1d = does_layer_accept_1d_feature(self.output_layer)
return format_data(target_data, is_feature1d)
def prediction_error(self, input_data, target_data):
""" Calculate prediction accuracy for input data.
Parameters
----------
input_data : array-like
target_data : array-like
Returns
-------
float
Prediction error.
"""
return self.methods.prediction_error(
self.format_input_data(input_data),
self.format_target_data(target_data)
)
def predict(self, input_data):
""" Return prediction results for the input data.
Parameters
----------
input_data : array-like
Returns
-------
array-like
"""
input_data = self.format_input_data(input_data)
return self.methods.predict(input_data)
def on_epoch_start_update(self, epoch):
""" Function would be trigger before run all training procedure
related to the current epoch.
Parameters
----------
epoch : int
Current epoch number.
"""
super(ConstructableNetwork, self).on_epoch_start_update(epoch)
self.variables.epoch.set_value(epoch)
def train(self, input_train, target_train, input_test=None,
target_test=None, *args, **kwargs):
""" Trains neural network.
"""
return super(ConstructableNetwork, self).train(
self.format_input_data(input_train),
self.format_target_data(target_train),
self.format_input_data(input_test),
self.format_target_data(target_test),
*args, **kwargs
)
def train_epoch(self, input_train, target_train):
""" Trains neural network over one epoch.
Parameters
----------
input_data : array-like
target_data : array-like
Returns
-------
float
Prediction error.
"""
return self.methods.train_epoch(input_train, target_train)
def architecture(self):
""" Shows network's architecture in the terminal if
``verbose`` parameter is equal to ``True``.
"""
self.logs.title("Network's architecture")
values = []
for index, layer in enumerate(self.layers, start=1):
input_shape = preformat_layer_shape(layer.input_shape)
output_shape = preformat_layer_shape(layer.output_shape)
classname = layer.__class__.__name__
values.append((index, input_shape, classname, output_shape))
table.TableBuilder.show_full_table(
columns=[
table.Column(name="#"),
table.Column(name="Input shape"),
table.Column(name="Layer Type"),
table.Column(name="Output shape"),
],
values=values,
stdout=self.logs.write,
)
self.logs.newline()
def __repr__(self):
n_layers = len(self.connection)
if n_layers > 5:
connection = '[... {} layers ...]'.format(n_layers)
else:
connection = self.connection
return "{}({}, {})".format(self.class_name(), connection,
self._repr_options())