def test_v2_abn_am_without_candidates():
with requests_mock.mock(real_http=True) as mock:
file_path = os.path.join(os.path.dirname(__file__),
'data/prom_responses',
'prometheus_sample_response.json')
mock.get(abn_er_example["status"]["metrics"][0]["metricObj"]["spec"]
["urlTemplate"],
json=json.load(open(file_path)))
example = copy.deepcopy(abn_er_example)
del example['spec']['versionInfo']['candidates']
expr = ExperimentResource(**example)
get_aggregated_metrics(expr.convert_to_float()).convert_to_quantity()
def provide_aggregated_metrics(ere: ExperimentResource = Body(
..., example=er_example)):
"""
POST iter8 2.0 experiment resource and metric resources and obtain aggregated metrics.
\f
:body er: ExperimentResource
"""
return get_aggregated_metrics(ere.convert_to_float()).convert_to_quantity()
def provide_aggregated_metrics(
ermr: ExperimentResourceAndMetricResources = Body(
..., example=ermr_example)):
"""
POST iter8 2.0 experiment resource and metric resources and obtain aggregated metrics.
\f
:body ermr: ExperimentResourceAndMetricResources
"""
aggregated_metrics = get_aggregated_metrics(ermr)
return aggregated_metrics
def test_mock_metrics():
ercopy = copy.deepcopy(er_example)
ercopy["status"]["metrics"] = mocked_mr_example
expr = ExperimentResource(**ercopy)
agm = get_aggregated_metrics(expr.convert_to_float())
logger.info(agm)
assert agm.data['request-count'].data['default'].value > 100.0
assert agm.data['request-count'].data['canary'].value > 100.0
assert agm.data['mean-latency'].data['default'].value > 15.0
assert agm.data['mean-latency'].data['canary'].value > 9.0
def test_v2_aggregated_metrics_endpoint():
with requests_mock.mock(real_http=True) as mock:
file_path = os.path.join(os.path.dirname(__file__),
'data/prom_responses',
'prometheus_sample_response.json')
response_json = json.load(open(file_path))
mock.get(er_example["status"]["metrics"][0]["metricObj"]["spec"]
["urlTemplate"],
json=response_json)
expr = ExperimentResource(**er_example)
agm = get_aggregated_metrics(
expr.convert_to_float()).convert_to_quantity()
assert agm.data['request-count'].data['default'].value == \
response_json['data']['result'][0]['value'][1]
ercopy = copy.deepcopy(er_example)
del ercopy["status"]["metrics"]
expr = ExperimentResource(**ercopy)
agm = get_aggregated_metrics(
expr.convert_to_float()).convert_to_quantity()
def test_mock_metrics_with_negative_elapsed():
ercopy = copy.deepcopy(er_example)
ercopy["status"]["metrics"] = mocked_mr_example
expr = ExperimentResource(**ercopy)
expr.status.startTime = datetime.now(timezone.utc) + timedelta(hours=10)
agm = get_aggregated_metrics(expr.convert_to_float())
logger.info(agm)
assert agm.data['request-count'].data['default'].value > 0
assert agm.data['request-count'].data['canary'].value > 0
assert agm.data['mean-latency'].data['default'].value > 0
assert agm.data['mean-latency'].data['canary'].value > 0
def get_analytics_results(exp_res: ExperimentResource):
"""
Get analysis results using experiment resource and metric resources.
"""
exp_res.status.analysis = Analysis()
exp_res.status.analysis.aggregated_metrics = get_aggregated_metrics(
exp_res)
exp_res.status.analysis.version_assessments = get_version_assessments(
exp_res)
exp_res.status.analysis.winner_assessment = get_winner_assessment(exp_res)
exp_res.status.analysis.weights = get_weights(exp_res)
return exp_res.status.analysis
def test_v2_no_prometheus_response():
with requests_mock.mock(real_http=True) as mock:
file_path = os.path.join(os.path.dirname(__file__),
'data/prom_responses',
'prometheus_sample_no_response.json')
mock.get(er_example["status"]["metrics"][0]["metricObj"]["spec"]
["urlTemplate"],
json=json.load(open(file_path)))
expr = ExperimentResource(**er_example)
resp = get_aggregated_metrics(
expr.convert_to_float()).convert_to_quantity()
expected_response = {
"request-count": {
"max": None,
"min": None,
"data": {
"default": {
"max": None,
"min": None,
"sample_size": None,
"value": None
},
"canary": {
"max": None,
"min": None,
"sample_size": None,
"value": None
}
}
},
"mean-latency": {
"max": None,
"min": None,
"data": {
"default": {
"max": None,
"min": None,
"sample_size": None,
"value": None
},
"canary": {
"max": None,
"min": None,
"sample_size": None,
"value": None
}
}
}
}
assert resp.data == expected_response
def get_analytics_results(expr: ExperimentResource):
"""
Get analysis results using experiment resource and metric resources.
"""
# if experiment contains aggregated builtin metric histograms, retain it
ana = Analysis()
if expr.status.analysis is not None:
ana.aggregated_builtin_hists = expr.status.analysis.aggregated_builtin_hists
expr.status.analysis = ana
expr.status.analysis.aggregated_metrics = get_aggregated_metrics(expr)
expr.status.analysis.version_assessments = get_version_assessments(expr)
expr.status.analysis.winner_assessment = get_winner_assessment(expr)
expr.status.analysis.weights = get_weights(expr)
return expr.status.analysis
def test_v2_abn_using_previous_metric_status_none():
with requests_mock.mock(real_http=True) as mock:
file_path = os.path.join(os.path.dirname(__file__),
'data/prom_responses',
'prometheus_sample_no_response.json')
mock.get(abn_er_example["status"]["metrics"][0]["metricObj"]["spec"]
["urlTemplate"],
json=json.load(open(file_path)))
example = copy.deepcopy(abn_er_example)
example['status']['metrics'] = abn_mr_example[:2]
expr = ExperimentResource(**example)
resp = get_aggregated_metrics(
expr.convert_to_float()).convert_to_quantity()
assert resp.data['mean-latency'].data['default'].value is None
def test_v2_va_with_no_metric_value():
with requests_mock.mock(real_http=True) as mock:
file_path = os.path.join(os.path.dirname(__file__),
'data/prom_responses',
'prometheus_sample_no_response.json')
mock.get(er_example["status"]["metrics"][0]["metricObj"]["spec"]
["urlTemplate"],
json=json.load(open(file_path)))
expr = ExperimentResource(**er_example)
resp = get_aggregated_metrics(
expr.convert_to_float()).convert_to_quantity()
example = copy.deepcopy(er_example_step1)
example['status']['analysis']["aggregatedMetrics"] = resp
expr = ExperimentResource(**example)
resp2 = get_version_assessments(expr.convert_to_float())
assert resp2.data == {'default': [False], 'canary': [False]}
