def rasa_predict(self, model_data: RasaModelData) -> Dict[Text, tf.Tensor]:
"""Custom prediction method that builds tf graph on the first call.
Args:
model_data: The model data to use for prediction.
Return:
Prediction output.
"""
self._training = False
if not self.prepared_for_prediction:
# in case the model is used for prediction without loading, e.g. directly
# after training, we need to prepare the model for prediction once
self.prepare_for_predict()
self.prepared_for_prediction = True
batch_in = RasaBatchDataGenerator.prepare_batch(model_data.data)
if self._run_eagerly:
return tf_utils.to_numpy_or_python_type(self.predict_step(batch_in))
if self._tf_predict_step is None:
self._tf_predict_step = tf.function(
self.predict_step, input_signature=self._dynamic_signature(batch_in)
)
return tf_utils.to_numpy_or_python_type(self._tf_predict_step(batch_in))
def _save_model(self, epoch, logs):
logs = logs or {}
logs = tf_utils.to_numpy_or_python_type(logs)
filepath = self._get_file_path(epoch, logs)
try:
if self.save_best_only:
current = logs.get(self.monitor)
if current is None:
logging.warning(
'Can save best model only with %s available, skipping.',
self.monitor)
else:
if self.monitor_op(current, self.best):
if self.verbose > 0:
print(
'\nEpoch %05d: %s improved from %0.5f to %0.5f,'
' saving model to %s' %
(epoch + 1, self.monitor, self.best, current,
filepath))
self.best = current
if self.save_weights_only:
self.model.save_weights(filepath,
overwrite=True,
options=self._options)
if self.opt:
optimizer_h5.save_optimizer_to_hdf5(
self.model, self._write_filepath_opt) ##
else:
self.model.save(filepath,
overwrite=True,
options=self._options,
save_format=self.save_format)
elif self.verbose > 0:
print('\nEpoch %05d: %s did not improve from %0.5f' %
(epoch + 1, self.monitor, self.best))
else:
if self.verbose > 0:
print('\nEpoch %05d: saving model to %s' %
(epoch + 1, filepath))
if self.save_weights_only:
self.model.save_weights(filepath,
overwrite=True,
options=self._options)
if self.opt:
optimizer_h5.save_optimizer_to_hdf5(
self.model, self._write_filepath_opt) ##
else:
self.model.save(filepath,
overwrite=True,
options=self._options,
save_format=self.save_format)
self._maybe_remove_file()
except IOError as e:
# `e.errno` appears to be `None` so checking the content of `e.args[0]`.
if 'is a directory' in six.ensure_str(e.args[0]).lower():
raise IOError('Please specify a non-directory filepath for '
'ModelCheckpoint. Filepath used is an existing '
'directory: {}'.format(filepath))
def test_on_batch_custom(self,
x,
y=None,
class_weight=None,
sample_weight=None,
reset_metrics=True,
return_dict=False):
self._assert_compile_was_called()
self._check_call_args('test_on_batch')
with self.distribute_strategy.scope():
iterator = data_adapter.single_batch_iterator(
self.distribute_strategy, x, y, sample_weight, class_weight)
test_function = self.make_test_function()
logs = test_function(iterator)
if reset_metrics:
self.reset_metrics()
logs = tf_utils.to_numpy_or_python_type(logs)
if return_dict:
return logs
else:
results = [logs.get(name, None) for name in self.metrics_names]
if len(results) == 1:
return results[0]
return results
def on_batch_end(self, batch, logs=None):
logs = logs or {}
loss = logs.get('loss')
if loss is not None:
loss = tf_utils.to_numpy_or_python_type(loss)
if np.isnan(loss) or np.isinf(loss):
print('Batch %d: Invalid loss, terminating training' % (batch))
sys.exit(1)
def _save_model(self, epoch, logs):
"""Saves the model.
Arguments:
epoch: the epoch this iteration is in.
logs: the `logs` dict passed in to `on_batch_end` or `on_epoch_end`.
"""
logs = logs or {}
if isinstance(self.save_freq,
int) or self.epochs_since_last_save >= self.period:
# Block only when saving interval is reached.
logs = tf_utils.to_numpy_or_python_type(logs)
self.epochs_since_last_save = 0
filepath = self._get_file_path(epoch, logs)
try:
if self.save_best_only:
current = logs.get(self.monitor)
if current is None:
logging.warning(
'Can save best model only with %s available, '
'skipping.', self.monitor)
else:
if self.monitor_op(current, self.best):
if self.verbose > 0:
print(
'\nEpoch %05d: %s improved from %0.5f to %0.5f,'
' saving model to %s' %
(epoch + 1, self.monitor, self.best,
current, filepath))
self.best = current
self.model.save_pretrained(filepath)
else:
if self.verbose > 0:
print(
'\nEpoch %05d: %s did not improve from %0.5f'
% (epoch + 1, self.monitor, self.best))
else:
if self.verbose > 0:
print('\nEpoch %05d: saving model to %s' %
(epoch + 1, filepath))
self.model.save_pretrained(filepath)
self._maybe_remove_file()
except IOError as e:
# `e.errno` appears to be `None` so checking the content of `e.args[0]`.
if 'is a directory' in six.ensure_str(e.args[0]).lower():
raise IOError(
'Please specify a non-directory filepath for '
'ModelCheckpoint. Filepath used is an existing '
'directory: {}'.format(filepath))
# Re-throw the error for any other causes.
raise e
def _save_model(self, epoch, logs):
"""Saves the model.
Parameters
----------
epoch: The epoch this iteration is in.
logs: The `logs` dict passed in to `on_batch_end` or `on_epoch_end`.
"""
# pylint: disable=too-many-nested-blocks,too-many-branches
logs = logs or {}
if isinstance(self.save_freq, int) or self.epochs_since_last_save >= self.period:
# Block only when saving interval is reached.
logs = tf_utils.to_numpy_or_python_type(logs)
self.epochs_since_last_save = 0
filepath = self._get_file_path(epoch, logs)
try:
if not self._saved and self.save_weights_only:
self.model.save(os.path.dirname(filepath), overwrite=True, include_optimizer=False)
self._saved = True
elif self.save_best_only:
current = logs.get(self.monitor)
if current is None:
logger.warning('Can save best model only with %s available, skipping.', self.monitor)
else:
if self.monitor_op(current, self.best):
if self.verbose > 0:
print('\nEpoch %05d: %s improved from %0.5f to %0.5f, \
saving model to %s' % (epoch + 1, self.monitor, self.best, current, filepath))
self.best = current
if self.save_weights_only:
self.model.save_weights(filepath, overwrite=True, options=self._options)
else:
self.model.save(filepath, overwrite=True, options=self._options)
else:
if self.verbose > 0:
print('\nEpoch %05d: %s did not improve from %0.5f' %
(epoch + 1, self.monitor, self.best)) # noqa: E128
else:
if self.verbose > 0:
print('\nEpoch %05d: saving model to %s' % (epoch + 1, filepath))
if self.save_weights_only:
self.model.save_weights(filepath, overwrite=True, options=self._options)
else:
self.model.save(filepath, overwrite=True, options=self._options)
self._maybe_remove_file()
except IOError as e:
# `e.errno` appears to be `None` so checking the content of `e.args[0]`.
if 'is a directory' in six.ensure_str(e.args[0]).lower():
raise IOError(f'Please specify a non-directory filepath for ModelCheckpoint. \
Filepath used is an existing directory: {filepath}')
def __call__(self, supports_tf_logs: bool) -> Dict:
if supports_tf_logs:
return self.tf_logs
if self.np_logs is None:
try:
self.np_logs = tf_utils.to_numpy_or_python_type(
self.tf_logs)
except AttributeError: # New method as of TF 2.5.
self.np_logs = tf_utils.sync_to_numpy_or_python_type(
self.tf_logs)
return self.np_logs
def on_epoch_end(self, batch, logs=None):
if isinstance(self.save_freq,
int) or self.epochs_since_last_save + 1 >= self.period:
logs = logs or {}
logs = tf_utils.to_numpy_or_python_type(logs)
current = logs.get(self.monitor)
if self.monitor_op(current, self.best):
log_message = f"last_best: {self.best}"
for key in logs:
log_message = ", ".join(
[log_message, f"{key}: {logs[key]}"])
self._logger.info(log_message)
super().on_epoch_end(batch, logs)
def __call__(self, supports_tf_logs: bool) -> Dict:
if supports_tf_logs:
return self.tf_logs
if self.np_logs is None:
try:
self.np_logs = tf_utils.to_numpy_or_python_type(
self.tf_logs)
except AttributeError: # New method as of TF 2.5.
self.np_logs = tf_utils.sync_to_numpy_or_python_type(
self.tf_logs)
# `to_numpy_or_python_type` will return a dict if `self.tf_logs` is a dict,
# but current type annotations aren't good enough to describe that.
return self.np_logs # type: ignore
def on_epoch_end(self, epoch, logs=None):
if self._monitor is not None:
logs = tf_utils.to_numpy_or_python_type(logs)
if self._monitor not in logs:
warnings.warn(
f"'{self._monitor}' does not exist, ignoring confusion matrix for this epoch."
)
return
value = logs[self._monitor]
if self._operator(value, self._best):
self._best = value
self.log_confusion_matrix(epoch, logs)
elif epoch % self._period == 0:
self.log_confusion_matrix(epoch, logs)
def _rasa_predict(
self, batch_in: Tuple[np.ndarray]
) -> Dict[Text, Union[np.ndarray, Dict[Text, Any]]]:
"""Custom prediction method that builds tf graph on the first call.
Args:
batch_in: Prepared batch ready for input to `predict_step` method of model.
Return:
Prediction output, including diagnostic data.
"""
self._training = False
if not self.prepared_for_prediction:
# in case the model is used for prediction without loading, e.g. directly
# after training, we need to prepare the model for prediction once
self.prepare_for_predict()
self.prepared_for_prediction = True
if self._run_eagerly:
outputs = tf_utils.to_numpy_or_python_type(self.predict_step(batch_in))
if DIAGNOSTIC_DATA in outputs:
outputs[DIAGNOSTIC_DATA] = self._empty_lists_to_none_in_dict(
outputs[DIAGNOSTIC_DATA]
)
return outputs
if self._tf_predict_step is None:
self._tf_predict_step = tf.function(
self.predict_step, input_signature=self._dynamic_signature(batch_in)
)
outputs = tf_utils.to_numpy_or_python_type(self._tf_predict_step(batch_in))
if DIAGNOSTIC_DATA in outputs:
outputs[DIAGNOSTIC_DATA] = self._empty_lists_to_none_in_dict(
outputs[DIAGNOSTIC_DATA]
)
return outputs
def _batch_update_progbar(self, batch, logs=None):
"""Updates the progbar."""
logs = logs or {}
self._maybe_init_progbar()
if self.use_steps:
self.seen = batch + 1 # One-indexed.
else:
batch_size = logs.get('size', 0)
num_steps = logs.get('num_steps', 1)
self.seen += batch_size * num_steps
self.display_step += 1
if (self.verbose == 1 and self.seen < self.target
and self.display_step % self.display_per_batches == 0):
logs = tf_utils.to_numpy_or_python_type(logs)
self.progbar.update(self.seen, list(logs.items()), finalize=False)
def _save_model(self, epoch, logs):
super()._save_model(epoch, logs)
logs = tf_utils.to_numpy_or_python_type(logs or {})
filepath = self._get_file_path(epoch, logs)
if not self._checkpoint_exists(filepath):
# Did not save a checkpoint for current epoch
return
value = logs.get(self.monitor)
if self._keep_most_recent:
# delay adding to list of current checkpoints until next save
# if we should always keep the most recent checkpoint
if self._most_recent_checkpoint:
self._checkpoints.update(self._most_recent_checkpoint)
self._most_recent_checkpoint = {filepath: value}
else:
self._checkpoints[filepath] = value
if len(self._checkpoints) > self._keep_count:
self._delete_worst_checkpoint()
def mc_sample(self,
x,
batch_size=None,
steps=None,
max_queue_size=10,
workers=1,
use_multiprocessing=False):
outputs = None
with self.distribute_strategy.scope():
data_handler = data_adapter.DataHandler(
x=x,
batch_size=batch_size,
steps_per_epoch=steps,
initial_epoch=0,
epochs=1,
max_queue_size=max_queue_size,
workers=workers,
use_multiprocessing=use_multiprocessing,
model=self)
predict_function = self.make_mc_sample_function()
for _, iterator in data_handler.enumerate_epochs():
with data_handler.catch_stop_iteration():
for step in data_handler.steps():
tmp_batch_outputs = predict_function(iterator)
if not data_handler.inferred_steps:
context.async_wait()
batch_outputs = tmp_batch_outputs
if outputs is None:
outputs = nest.map_structure(
lambda batch_output: [batch_output],
batch_outputs)
else:
nest.map_structure_up_to(
batch_outputs,
lambda output, batch_output: output.append(
batch_output), outputs, batch_outputs)
all_outputs = nest.map_structure_up_to(batch_outputs, concat, outputs)
return tf_utils.to_numpy_or_python_type(all_outputs)
def _save_optimizer_weights(self, epoch, logs=None):
"""Saves the optimizer weights.
Arguments:
epoch: the epoch this iteration is in.
logs: the `logs` dict passed in to `on_batch_end` or `on_epoch_end`.
"""
logs = logs or {}
if isinstance(self.save_freq, int) or self.epochs_since_last_save >= self.period:
# Block only when saving interval is reached.
logs = tf_utils.to_numpy_or_python_type(logs)
self.epochs_since_last_save = 0
filepath = self._get_file_path(epoch, logs)
try:
if self.verbose > 0:
print('\nEpoch %05d: saving model to %s' % (epoch + 1, filepath))
save_optimizer_weights(self.model, filepath, overwrite=True)
except IOError as e:
# `e.errno` appears to be `None` so checking the content of `e.args[0]`.
if 'is a directory' in six.ensure_str(e.args[0]).lower():
raise IOError('Please specify a non-directory filepath for '
'ModelCheckpoint. Filepath used is an existing '
'directory: {}'.format(filepath))
def __call__(self, supports_tf_logs: bool) -> Dict:
if supports_tf_logs:
return self.tf_logs
if self.np_logs is None:
self.np_logs = tf_utils.to_numpy_or_python_type(self.tf_logs)
return self.np_logs
