def add_evaluation_result(self, evaluation_result: EvaluationResult):
evaluation_result.set_id(self.get_new_evaluation_id())
if evaluation_result.cost < self._state['best_cost']:
self._state['best_evaluation_id'] = evaluation_result.id
self._state['best_cost'] = evaluation_result.cost
evaluation_result.add_info('best_evaluation_id',
self._state['best_evaluation_id'])
evaluation_result.add_info('best_cost', self._state['best_cost'])
self._state['evaluation_results'].append(evaluation_result)
def uniform_reconstruction_error_cost(config, y_true, y_pred,
**kwargs) -> EvaluationResult:
reconstruction_errors = compute_reconstruction_error(y_true, y_pred)
reconstruction_error_cost = np.sqrt(
np.average(np.power(reconstruction_errors, 2)))
reconstruction_error_z_scores = z_score(reconstruction_errors)
reconstruction_error_z_score_cost = np.sqrt(
np.average(np.power(reconstruction_error_z_scores, 2)))
cost = reconstruction_error_cost * reconstruction_error_z_score_cost
return EvaluationResult(cost=cost,
config=config,
info={
'reconstruction_error_cost':
reconstruction_error_cost,
'reconstruction_error_z_score_cost':
reconstruction_error_z_score_cost,
})
def reconstruction_error_vs_regularized_compression_cost(
config, y_true, y_pred, model: Model, history: dict,
**kwargs) -> EvaluationResult:
compression = compute_model_compression(model)
reconstruction_error = reconstruction_error_cost(config, y_true, y_pred,
**kwargs).cost
if 'activity_regularizer_factor' in config:
regularization_factor = config['activity_regularizer_factor']
cost = reconstruction_error / (compression * regularization_factor)
else:
regularization_factor = 0
cost = reconstruction_error / compression
return EvaluationResult(cost=cost,
config=config,
info={
'compression': compression,
'reconstruction_error': reconstruction_error,
'regularization_factor': regularization_factor,
})
def min_health_score_cost(config,
y_true,
y_pred,
rolling_window_size=200,
**kwargs) -> EvaluationResult:
reconstruction_errors = compute_reconstruction_error(y_true, y_pred)
health_scores = compute_health_score(reconstruction_errors,
reconstruction_errors,
mode='median')
rolling_health_scores = np.convolve(health_scores,
np.ones(rolling_window_size) /
rolling_window_size,
mode='valid')
min_health_score = float(np.min(rolling_health_scores))
cost = 1 - min_health_score
return EvaluationResult(cost=cost,
config=config,
info={
'min_health_score': min_health_score,
})
def uniform_reconstruction_error_vs_compression_cost(
config, y_true, y_pred, model: Model, history: dict,
**kwargs) -> EvaluationResult:
compression = compute_model_compression(model)
reconstruction_error = reconstruction_error_cost(config, y_true, y_pred,
**kwargs).cost
reconstruction_error_z_scores = z_score(
compute_reconstruction_error(y_true, y_pred))
reconstruction_error_z_score_cost = np.sqrt(
np.average(np.power(reconstruction_error_z_scores, 2)))
cost = (reconstruction_error *
reconstruction_error_z_score_cost) / compression
return EvaluationResult(cost=cost,
config=config,
info={
'reconstruction_error': reconstruction_error,
'reconstruction_error_z_score':
reconstruction_error_z_score_cost,
'compression': compression,
})