def test_v2_analytics_assessment_endpoint():
with requests_mock.mock(real_http=True) as mock:
file_path = os.path.join(os.path.dirname(__file__),
'data/prom_responses',
'prometheus_sample_response.json')
mock.get(er_example["status"]["metrics"][0]["metricObj"]["spec"]
["urlTemplate"],
json=json.load(open(file_path)))
expr = ExperimentResource(**er_example)
get_analytics_results(expr.convert_to_float()).convert_to_quantity()
def test_ab_reward_only_analytics_assessment_endpoint():
with requests_mock.mock(real_http=True) as mock:
file_path = os.path.join(os.path.dirname(__file__),
'data/prom_responses',
'prometheus_sample_response.json')
mock.get(ab_er_example["status"]["metrics"][0]["metricObj"]["spec"]
["urlTemplate"],
json=json.load(open(file_path)))
ab_expr = copy.deepcopy(ab_er_example)
del ab_expr["spec"]["criteria"]["objectives"]
expr = ExperimentResource(**ab_expr)
get_analytics_results(expr.convert_to_float()).convert_to_quantity()
def test_v2_abn_analytics_assessment_conformance():
with requests_mock.mock(real_http=True) as mock:
file_path = os.path.join(os.path.dirname(__file__),
'data/prom_responses',
'prometheus_sample_response.json')
mock.get(abn_er_example["status"]["metrics"][0]["metricObj"]["spec"]
["urlTemplate"],
json=json.load(open(file_path)))
example = copy.deepcopy(abn_er_example)
del example['spec']['versionInfo']['candidates']
example['spec']['strategy']['testingPattern'] = 'Conformance'
expr = ExperimentResource(**example)
get_analytics_results(expr.convert_to_float()).convert_to_quantity()
def provide_analytics_results(expr: ExperimentResource = Body(
..., example=er_example)):
"""
POST iter8 2.0 experiment resource and metric resources and get analytics results.
\f
:body expr: ExperimentResource
"""
return get_analytics_results(expr.convert_to_float()).convert_to_quantity()
def provide_analytics_results(
ermr: ExperimentResourceAndMetricResources = Body(
..., example=ermr_example)):
"""
POST iter8 2.0 experiment resource and metric resources and get analytics results.
\f
:body er: ExperimentResourceAndMetricResources
"""
analytics_results = get_analytics_results(ermr)
return analytics_results
def test_v2_ab_without_reward():
with requests_mock.mock(real_http=True) as mock:
file_path = os.path.join(os.path.dirname(__file__),
'data/prom_responses',
'prometheus_sample_response.json')
mock.get(ab_er_example["status"]["metrics"][0]["metricObj"]["spec"]
["urlTemplate"],
json=json.load(open(file_path)))
example = copy.deepcopy(ab_er_example)
del example['spec']['criteria']['rewards']
expr = ExperimentResource(**example)
resp = get_analytics_results(
expr.convert_to_float()).convert_to_quantity()
assert "No reward metric in experiment" in resp.winner_assessment.message
assert resp.winner_assessment.data.winnerFound is False
