def load(identifier):
"""Load the Recurrent Neural Network model from disk.
Args:
identifier: Identifier of model to load
Returns:
result: RNN model
"""
# Turn off verbose logging
memory.setup()
# Setup a Load object
Load = namedtuple('Load', 'model, history')
# Initialize key Variables
_files = files.files(identifier)
# Check for the existence of model files and directories
if os.path.isdir(_files.model_parameters) is False:
print('''\
Directory {} not found. Please create it by training your model first.\
'''.format(_files.model_parameters))
sys.exit(0)
if os.path.isfile(_files.model_weights) is False:
print('File {} not found.'.format(_files.model_weights))
sys.exit(0)
# Load yaml and create model
'''
You have to use this custom_object parameter to read loss values from the
customized loss function cannot be save to a keras model. It does not seem
to work if the load function is in a module different from the one in which
the customized loss function is located. Reference:
https://github.com/keras-team/keras/issues/9377#issuecomment-396187881
'''
ai_model = tf.keras.models.load_model(
_files.model_parameters, custom_objects={'model_loss': model_loss})
# Load weights into new model
ai_model.load_weights(_files.model_weights, by_name=True)
# Load yaml and create model
with open(_files.history, 'r') as yaml_file:
history = yaml.safe_load(yaml_file)
result = Load(model=ai_model, history=history)
return result
def __init__(
self, _data, identifier, batch_size=256, epochs=20,
sequence_length=500, dropout=0.1, divider=1, test_size=0.2,
layers=1, patience=5, units=128, multigpu=False):
"""Instantiate the class.
Args:
data: etl.Data object
batch_size: Size of batch
sequence_length: Length of vectors for for each target
display: Show charts of results if True
Returns:
None
"""
# Setup memory
self._processors = memory.setup()
# Initialize key variables
self._identifier = identifier
if multigpu is True:
self._gpus = len(self._processors.gpus)
else:
self._gpus = 1
_batch_size = int(batch_size * self._gpus)
self._test_size = test_size
# Get data
self._data = _data
# Set steps per epoch
normal = self._data.split()
(training_rows, _) = normal.x_train.shape
steps_per_epoch = int((training_rows // _batch_size) / divider)
# Set key file locations
self._files = files.files(identifier)
# Initialize parameters
self._hyperparameters = HyperParameters(
units=abs(units),
dropout=abs(dropout),
layers=int(abs(layers)),
sequence_length=abs(sequence_length),
patience=abs(patience),
batch_size=_batch_size,
epochs=abs(epochs),
steps_per_epoch=steps_per_epoch
)
# Delete any stale checkpoint file
if os.path.exists(self._files.checkpoint) is True:
os.remove(self._files.checkpoint)
def _compile(self, _hyperparameters):
"""Compile the default model.
Args:
None
Returns:
ai_model: RNN model
"""
# Initialize key variables
use_sigmoid = False
'''
Instantiate the base model (or "template" model).
We recommend doing this with under a CPU device scope,
so that the model's weights are hosted on CPU memory.
Otherwise they may end up hosted on a GPU, which would
complicate weight sharing.
'''
# Intialize key variables realted to data
normal = self._data.split()
(_, x_feature_count) = normal.x_train.shape
(_, y_feature_count) = normal.y_train.shape
# Get GPU information
devices = memory.setup()
gpus = devices.gpus[:max(1, len(devices.gpus) - 1)]
cpus = devices.cpus[0]
# Start creating the model
strategy = tf.distribute.MirroredStrategy(
gpus,
cross_device_ops=tf.distribute.ReductionToOneDevice(
reduce_to_device=cpus))
print('Number of devices: {}'.format(strategy.num_replicas_in_sync))
# Define the model
with strategy.scope():
ai_model = tf.keras.Sequential()
'''
We can now add a Gated Recurrent Unit (GRU) to the network. This
will have 'units' outputs for each time-step in the sequence.
Note that because this is the first layer in the model, Keras
needs to know the shape of its input, which is a batch of sequences
of arbitrary length (indicated by None), where each observation
has a number of input-signals (x_feature_count).
'''
ai_model.add(
tf.keras.layers.GRU(_hyperparameters.units,
return_sequences=True,
recurrent_dropout=_hyperparameters.dropout,
input_shape=(None, x_feature_count)))
for _ in range(1, abs(_hyperparameters.layers)):
ai_model.add(
tf.keras.layers.GRU(
_hyperparameters.units,
return_sequences=True,
recurrent_dropout=_hyperparameters.dropout))
'''
The GRU outputs a batch from
keras_contrib.layers.advanced_activations of sequences of 'units'
values. We want to predict 'y_feature_count' output-signals, so we
add a fully-connected (or dense) layer which maps 'units' values
down to only 'y_feature_count' values.
The output-signals in the data-set have been limited to be between
0 and 1 using a scaler-object. So we also limit the output of the
neural network using the Sigmoid activation function, which
squashes the output to be between 0 and 1.
'''
if bool(use_sigmoid) is True:
ai_model.add(
tf.keras.layers.Dense(y_feature_count,
activation='sigmoid'))
'''
A problem with using the Sigmoid activation function, is that we
can now only output values in the same range as the training-data.
For example, if the training-data only has values between -20 and
+30, then the scaler-object will map -20 to 0 and +30 to 1. So if
we limit the output of the neural network to be between 0 and 1
using the Sigmoid function, this can only be mapped back to values
between -20 and +30.
We can use a linear activation function on the output instead. This
allows for the output to take on arbitrary values. It might work
with the standard initialization for a simple network architecture,
but for more complicated network architectures e.g. with more
layers, it might be necessary to initialize the weights with
smaller values to avoid NaN values during training. You may need to
experiment with this to get it working.
'''
if bool(use_sigmoid) is False:
init = tf.keras.initializers.RandomUniform(minval=-0.03,
maxval=0.03)
ai_model.add(
tf.keras.layers.Dense(y_feature_count,
activation='linear',
kernel_initializer=init))
# Compile Model
'''
This is the optimizer and the beginning learning-rate that we will
use. We then compile the Keras model so it is ready for training.
'''
optimizer = tf.keras.optimizers.RMSprop(lr=1e-3)
ai_model.compile(loss=model_loss,
optimizer=optimizer,
metrics=['accuracy'])
# Display layers
'''
This is a very small model with only two layers. The output shape
of (None, None, 'y_feature_count') means that the model will output
a batch with an arbitrary number of sequences, each of which has an
arbitrary number of observations, and each observation has
'y_feature_count' signals. This corresponds to the
'y_feature_count' target signals we want to predict.
'''
print('\n> Summary (Parallel):\n')
print(ai_model.summary())
return ai_model
def __init__(self,
_data,
batch_size=256,
epochs=20,
sequence_length=1500,
dropout=0.1,
layers=1,
patience=5,
units=128,
multigpu=False):
"""Instantiate the class.
Args:
data: etl.Data object
batch_size: Size of batch
sequence_length: Length of vectors for for each target
warmup_steps:
display: Show charts of results if True
Returns:
None
"""
# Setup memory
self._processors = memory.setup()
# Initialize key variables
if multigpu is True:
self._gpus = len(self._processors.gpus)
else:
self._gpus = 1
self._batch_size = batch_size * self._gpus
self._sequence_length = sequence_length
# Set key file locations
path_prefix = '/tmp/hvass-{}'.format(int(time.time()))
Files = namedtuple('Files',
'checkpoint, model_weights, model_parameters')
self._files = Files(
checkpoint='{}.checkpoint.h5'.format(path_prefix),
model_weights='{}.weights.h5'.format(path_prefix),
model_parameters='{}.model.yaml'.format(path_prefix))
# Initialize parameters
HyperParameters = namedtuple(
'HyperParameters',
'''units, dropout, layers, sequence_length, patience, \
batch_size, epochs''')
self._hyperparameters = HyperParameters(
units=abs(units),
dropout=abs(dropout),
layers=int(abs(layers)),
sequence_length=abs(sequence_length),
patience=abs(patience),
batch_size=int(self._batch_size),
epochs=abs(epochs))
# Delete any stale checkpoint file
if os.path.exists(self._files.checkpoint) is True:
os.remove(self._files.checkpoint)
# Get data
self._data = _data
