def get_version_assessments(experiment_resource: ExperimentResource):
"""
Get version assessments using experiment resource.
"""
versions = [experiment_resource.spec.versionInfo.baseline]
if experiment_resource.spec.versionInfo.candidates is not None:
versions += experiment_resource.spec.versionInfo.candidates
messages = []
def check_limits(obj: Objective, value: float) -> bool:
if (obj.upper_limit is not None) and (value > float(obj.upper_limit)):
return False
if (obj.lower_limit is not None) and (value < float(obj.lower_limit)):
return False
return True
aggregated_metric_data = experiment_resource.status.analysis.aggregated_metrics.data
version_assessments = VersionAssessmentsAnalysis(data={})
if experiment_resource.spec.criteria is None or \
experiment_resource.spec.criteria.objectives is None:
return version_assessments
# objectives are available
for version in versions:
version_assessments.data[version.name] = [False] * \
len(experiment_resource.spec.criteria.objectives)
for ind, obj in enumerate(experiment_resource.spec.criteria.objectives):
if obj.metric in aggregated_metric_data:
versions_metric_data = aggregated_metric_data[obj.metric].data
for version in versions:
if version.name in versions_metric_data:
if versions_metric_data[version.name].value is not None:
version_assessments.data[version.name][ind] = \
check_limits(obj, float(versions_metric_data[version.name].value))
else:
messages.append(Message(MessageLevel.WARNING, \
f"Value for {obj.metric} metric and {version.name} version is None."))
else:
messages.append(Message(MessageLevel.WARNING, \
f"Value for {obj.metric} metric and {version.name} version is unavailable."))
else:
messages.append(Message(MessageLevel.WARNING, \
f"Aggregated metric object for {obj.metric} metric is unavailable."))
version_assessments.message = Message.join_messages(messages)
logger.debug("version assessments: %s",
pprint.PrettyPrinter().pformat(version_assessments))
return version_assessments
def test_experiment_abn_response_objects():
AggregatedMetricsAnalysis(**abn_am_response)
VersionAssessmentsAnalysis(**abn_va_response)
WinnerAssessmentAnalysis(**abn_wa_response)
WeightsAnalysis(**abn_w_response)