def test_v2_abn_general():
example = copy.deepcopy(abn_er_example_step2)
expr = ExperimentResource(**example)
resp = get_winner_assessment(expr.convert_to_float())
assert resp.data.winnerFound is True
assert resp.data.winner == 'canary1'
def test_v2_ab_without_reward_for_feasible_version():
example = copy.deepcopy(ab_er_example_step2)
del example['status']['analysis']['aggregatedMetrics']['data'][
'business-revenue']['data']['canary']
expr = ExperimentResource(**example)
resp = get_winner_assessment(expr.convert_to_float())
assert "reward value for feasible version canary is not available" in \
resp.message
def provide_winner_assessment(experiment_resource: ExperimentResource = Body(
..., example=er_example_step2)):
"""
POST iter8 2.0 experiment resource, whose status includes
aggregated metrics/version_assessments, and obtain winner assessment.
\f
:body er: ExperimentResource
"""
return get_winner_assessment(experiment_resource.convert_to_float())
def test_v2_abn_with_better_reward_but_not_feasible():
example = copy.deepcopy(abn_er_example_step2)
example['status']['analysis']['versionAssessments']['data']['canary1'] = [
False
]
expr = ExperimentResource(**example)
resp = get_winner_assessment(expr.convert_to_float())
assert resp.data.winnerFound is True
assert resp.data.winner == 'canary2'
def test_v2_no_winner():
example = copy.deepcopy(er_example_step2)
example['status']['analysis']['versionAssessments'] = {
"data": {
"default": [False],
"canary": [False]
},
"message": "All ok"
}
expr = ExperimentResource(**example)
resp = get_winner_assessment(expr.convert_to_float())
assert resp.data.winnerFound is False
def test_canary_is_winner():
example = copy.deepcopy(er_example_step2)
example['status']['analysis']['versionAssessments'] = {
"data": {
"default": [True],
"canary": [True]
},
"message": "All ok"
}
expr = ExperimentResource(**example)
resp = get_winner_assessment(expr.convert_to_float())
assert resp.data.winnerFound == wa_response['data']['winnerFound']
assert resp.data.winner == wa_response['data']['winner']
def test_v2_analytics_assessment_conformance_winner():
example = copy.deepcopy(er_example_step2)
example['status']['analysis']['versionAssessments'] = {
"data": {
"default": [True]
},
"message": "All ok"
}
del example['spec']['versionInfo']['candidates']
example['spec']['strategy']['testingPattern'] = 'Conformance'
expr = ExperimentResource(**example)
resp = get_winner_assessment(expr.convert_to_float())
assert resp.data.winnerFound is True
assert resp.data.winner == 'default'
def test_v2_winner_assessment_endpoint():
expr = ExperimentResource(**er_example_step2)
get_winner_assessment(expr.convert_to_float())
def test_v2_abn_wa_without_va():
expr = ExperimentResource(**abn_er_example)
try:
get_winner_assessment(expr.convert_to_float())
except AttributeError:
pass
