def rasa_predict(self, model_data: RasaModelData) -> Dict[Text, tf.Tensor]: """Custom prediction method that builds tf graph on the first call. Args: model_data: The model data to use for prediction. Return: Prediction output. """ self._training = False if not self.prepared_for_prediction: # in case the model is used for prediction without loading, e.g. directly # after training, we need to prepare the model for prediction once self.prepare_for_predict() self.prepared_for_prediction = True batch_in = RasaBatchDataGenerator.prepare_batch(model_data.data) if self._run_eagerly: return tf_utils.to_numpy_or_python_type(self.predict_step(batch_in)) if self._tf_predict_step is None: self._tf_predict_step = tf.function( self.predict_step, input_signature=self._dynamic_signature(batch_in) ) return tf_utils.to_numpy_or_python_type(self._tf_predict_step(batch_in))
def _save_model(self, epoch, logs): logs = logs or {} logs = tf_utils.to_numpy_or_python_type(logs) filepath = self._get_file_path(epoch, logs) try: if self.save_best_only: current = logs.get(self.monitor) if current is None: logging.warning( 'Can save best model only with %s available, skipping.', self.monitor) else: if self.monitor_op(current, self.best): if self.verbose > 0: print( '\nEpoch %05d: %s improved from %0.5f to %0.5f,' ' saving model to %s' % (epoch + 1, self.monitor, self.best, current, filepath)) self.best = current if self.save_weights_only: self.model.save_weights(filepath, overwrite=True, options=self._options) if self.opt: optimizer_h5.save_optimizer_to_hdf5( self.model, self._write_filepath_opt) ## else: self.model.save(filepath, overwrite=True, options=self._options, save_format=self.save_format) elif self.verbose > 0: print('\nEpoch %05d: %s did not improve from %0.5f' % (epoch + 1, self.monitor, self.best)) else: if self.verbose > 0: print('\nEpoch %05d: saving model to %s' % (epoch + 1, filepath)) if self.save_weights_only: self.model.save_weights(filepath, overwrite=True, options=self._options) if self.opt: optimizer_h5.save_optimizer_to_hdf5( self.model, self._write_filepath_opt) ## else: self.model.save(filepath, overwrite=True, options=self._options, save_format=self.save_format) self._maybe_remove_file() except IOError as e: # `e.errno` appears to be `None` so checking the content of `e.args[0]`. if 'is a directory' in six.ensure_str(e.args[0]).lower(): raise IOError('Please specify a non-directory filepath for ' 'ModelCheckpoint. Filepath used is an existing ' 'directory: {}'.format(filepath))
def test_on_batch_custom(self, x, y=None, class_weight=None, sample_weight=None, reset_metrics=True, return_dict=False): self._assert_compile_was_called() self._check_call_args('test_on_batch') with self.distribute_strategy.scope(): iterator = data_adapter.single_batch_iterator( self.distribute_strategy, x, y, sample_weight, class_weight) test_function = self.make_test_function() logs = test_function(iterator) if reset_metrics: self.reset_metrics() logs = tf_utils.to_numpy_or_python_type(logs) if return_dict: return logs else: results = [logs.get(name, None) for name in self.metrics_names] if len(results) == 1: return results[0] return results
def on_batch_end(self, batch, logs=None): logs = logs or {} loss = logs.get('loss') if loss is not None: loss = tf_utils.to_numpy_or_python_type(loss) if np.isnan(loss) or np.isinf(loss): print('Batch %d: Invalid loss, terminating training' % (batch)) sys.exit(1)
def _save_model(self, epoch, logs): """Saves the model. Arguments: epoch: the epoch this iteration is in. logs: the `logs` dict passed in to `on_batch_end` or `on_epoch_end`. """ logs = logs or {} if isinstance(self.save_freq, int) or self.epochs_since_last_save >= self.period: # Block only when saving interval is reached. logs = tf_utils.to_numpy_or_python_type(logs) self.epochs_since_last_save = 0 filepath = self._get_file_path(epoch, logs) try: if self.save_best_only: current = logs.get(self.monitor) if current is None: logging.warning( 'Can save best model only with %s available, ' 'skipping.', self.monitor) else: if self.monitor_op(current, self.best): if self.verbose > 0: print( '\nEpoch %05d: %s improved from %0.5f to %0.5f,' ' saving model to %s' % (epoch + 1, self.monitor, self.best, current, filepath)) self.best = current self.model.save_pretrained(filepath) else: if self.verbose > 0: print( '\nEpoch %05d: %s did not improve from %0.5f' % (epoch + 1, self.monitor, self.best)) else: if self.verbose > 0: print('\nEpoch %05d: saving model to %s' % (epoch + 1, filepath)) self.model.save_pretrained(filepath) self._maybe_remove_file() except IOError as e: # `e.errno` appears to be `None` so checking the content of `e.args[0]`. if 'is a directory' in six.ensure_str(e.args[0]).lower(): raise IOError( 'Please specify a non-directory filepath for ' 'ModelCheckpoint. Filepath used is an existing ' 'directory: {}'.format(filepath)) # Re-throw the error for any other causes. raise e
def _save_model(self, epoch, logs): """Saves the model. Parameters ---------- epoch: The epoch this iteration is in. logs: The `logs` dict passed in to `on_batch_end` or `on_epoch_end`. """ # pylint: disable=too-many-nested-blocks,too-many-branches logs = logs or {} if isinstance(self.save_freq, int) or self.epochs_since_last_save >= self.period: # Block only when saving interval is reached. logs = tf_utils.to_numpy_or_python_type(logs) self.epochs_since_last_save = 0 filepath = self._get_file_path(epoch, logs) try: if not self._saved and self.save_weights_only: self.model.save(os.path.dirname(filepath), overwrite=True, include_optimizer=False) self._saved = True elif self.save_best_only: current = logs.get(self.monitor) if current is None: logger.warning('Can save best model only with %s available, skipping.', self.monitor) else: if self.monitor_op(current, self.best): if self.verbose > 0: print('\nEpoch %05d: %s improved from %0.5f to %0.5f, \ saving model to %s' % (epoch + 1, self.monitor, self.best, current, filepath)) self.best = current if self.save_weights_only: self.model.save_weights(filepath, overwrite=True, options=self._options) else: self.model.save(filepath, overwrite=True, options=self._options) else: if self.verbose > 0: print('\nEpoch %05d: %s did not improve from %0.5f' % (epoch + 1, self.monitor, self.best)) # noqa: E128 else: if self.verbose > 0: print('\nEpoch %05d: saving model to %s' % (epoch + 1, filepath)) if self.save_weights_only: self.model.save_weights(filepath, overwrite=True, options=self._options) else: self.model.save(filepath, overwrite=True, options=self._options) self._maybe_remove_file() except IOError as e: # `e.errno` appears to be `None` so checking the content of `e.args[0]`. if 'is a directory' in six.ensure_str(e.args[0]).lower(): raise IOError(f'Please specify a non-directory filepath for ModelCheckpoint. \ Filepath used is an existing directory: {filepath}')
def __call__(self, supports_tf_logs: bool) -> Dict: if supports_tf_logs: return self.tf_logs if self.np_logs is None: try: self.np_logs = tf_utils.to_numpy_or_python_type( self.tf_logs) except AttributeError: # New method as of TF 2.5. self.np_logs = tf_utils.sync_to_numpy_or_python_type( self.tf_logs) return self.np_logs
def on_epoch_end(self, batch, logs=None): if isinstance(self.save_freq, int) or self.epochs_since_last_save + 1 >= self.period: logs = logs or {} logs = tf_utils.to_numpy_or_python_type(logs) current = logs.get(self.monitor) if self.monitor_op(current, self.best): log_message = f"last_best: {self.best}" for key in logs: log_message = ", ".join( [log_message, f"{key}: {logs[key]}"]) self._logger.info(log_message) super().on_epoch_end(batch, logs)
def __call__(self, supports_tf_logs: bool) -> Dict: if supports_tf_logs: return self.tf_logs if self.np_logs is None: try: self.np_logs = tf_utils.to_numpy_or_python_type( self.tf_logs) except AttributeError: # New method as of TF 2.5. self.np_logs = tf_utils.sync_to_numpy_or_python_type( self.tf_logs) # `to_numpy_or_python_type` will return a dict if `self.tf_logs` is a dict, # but current type annotations aren't good enough to describe that. return self.np_logs # type: ignore
def on_epoch_end(self, epoch, logs=None): if self._monitor is not None: logs = tf_utils.to_numpy_or_python_type(logs) if self._monitor not in logs: warnings.warn( f"'{self._monitor}' does not exist, ignoring confusion matrix for this epoch." ) return value = logs[self._monitor] if self._operator(value, self._best): self._best = value self.log_confusion_matrix(epoch, logs) elif epoch % self._period == 0: self.log_confusion_matrix(epoch, logs)
def _rasa_predict( self, batch_in: Tuple[np.ndarray] ) -> Dict[Text, Union[np.ndarray, Dict[Text, Any]]]: """Custom prediction method that builds tf graph on the first call. Args: batch_in: Prepared batch ready for input to `predict_step` method of model. Return: Prediction output, including diagnostic data. """ self._training = False if not self.prepared_for_prediction: # in case the model is used for prediction without loading, e.g. directly # after training, we need to prepare the model for prediction once self.prepare_for_predict() self.prepared_for_prediction = True if self._run_eagerly: outputs = tf_utils.to_numpy_or_python_type(self.predict_step(batch_in)) if DIAGNOSTIC_DATA in outputs: outputs[DIAGNOSTIC_DATA] = self._empty_lists_to_none_in_dict( outputs[DIAGNOSTIC_DATA] ) return outputs if self._tf_predict_step is None: self._tf_predict_step = tf.function( self.predict_step, input_signature=self._dynamic_signature(batch_in) ) outputs = tf_utils.to_numpy_or_python_type(self._tf_predict_step(batch_in)) if DIAGNOSTIC_DATA in outputs: outputs[DIAGNOSTIC_DATA] = self._empty_lists_to_none_in_dict( outputs[DIAGNOSTIC_DATA] ) return outputs
def _batch_update_progbar(self, batch, logs=None): """Updates the progbar.""" logs = logs or {} self._maybe_init_progbar() if self.use_steps: self.seen = batch + 1 # One-indexed. else: batch_size = logs.get('size', 0) num_steps = logs.get('num_steps', 1) self.seen += batch_size * num_steps self.display_step += 1 if (self.verbose == 1 and self.seen < self.target and self.display_step % self.display_per_batches == 0): logs = tf_utils.to_numpy_or_python_type(logs) self.progbar.update(self.seen, list(logs.items()), finalize=False)
def _save_model(self, epoch, logs): super()._save_model(epoch, logs) logs = tf_utils.to_numpy_or_python_type(logs or {}) filepath = self._get_file_path(epoch, logs) if not self._checkpoint_exists(filepath): # Did not save a checkpoint for current epoch return value = logs.get(self.monitor) if self._keep_most_recent: # delay adding to list of current checkpoints until next save # if we should always keep the most recent checkpoint if self._most_recent_checkpoint: self._checkpoints.update(self._most_recent_checkpoint) self._most_recent_checkpoint = {filepath: value} else: self._checkpoints[filepath] = value if len(self._checkpoints) > self._keep_count: self._delete_worst_checkpoint()
def mc_sample(self, x, batch_size=None, steps=None, max_queue_size=10, workers=1, use_multiprocessing=False): outputs = None with self.distribute_strategy.scope(): data_handler = data_adapter.DataHandler( x=x, batch_size=batch_size, steps_per_epoch=steps, initial_epoch=0, epochs=1, max_queue_size=max_queue_size, workers=workers, use_multiprocessing=use_multiprocessing, model=self) predict_function = self.make_mc_sample_function() for _, iterator in data_handler.enumerate_epochs(): with data_handler.catch_stop_iteration(): for step in data_handler.steps(): tmp_batch_outputs = predict_function(iterator) if not data_handler.inferred_steps: context.async_wait() batch_outputs = tmp_batch_outputs if outputs is None: outputs = nest.map_structure( lambda batch_output: [batch_output], batch_outputs) else: nest.map_structure_up_to( batch_outputs, lambda output, batch_output: output.append( batch_output), outputs, batch_outputs) all_outputs = nest.map_structure_up_to(batch_outputs, concat, outputs) return tf_utils.to_numpy_or_python_type(all_outputs)
def _save_optimizer_weights(self, epoch, logs=None): """Saves the optimizer weights. Arguments: epoch: the epoch this iteration is in. logs: the `logs` dict passed in to `on_batch_end` or `on_epoch_end`. """ logs = logs or {} if isinstance(self.save_freq, int) or self.epochs_since_last_save >= self.period: # Block only when saving interval is reached. logs = tf_utils.to_numpy_or_python_type(logs) self.epochs_since_last_save = 0 filepath = self._get_file_path(epoch, logs) try: if self.verbose > 0: print('\nEpoch %05d: saving model to %s' % (epoch + 1, filepath)) save_optimizer_weights(self.model, filepath, overwrite=True) except IOError as e: # `e.errno` appears to be `None` so checking the content of `e.args[0]`. if 'is a directory' in six.ensure_str(e.args[0]).lower(): raise IOError('Please specify a non-directory filepath for ' 'ModelCheckpoint. Filepath used is an existing ' 'directory: {}'.format(filepath))
def __call__(self, supports_tf_logs: bool) -> Dict: if supports_tf_logs: return self.tf_logs if self.np_logs is None: self.np_logs = tf_utils.to_numpy_or_python_type(self.tf_logs) return self.np_logs