def test_v2_abn_general(): example = copy.deepcopy(abn_er_example_step2) expr = ExperimentResource(**example) resp = get_winner_assessment(expr.convert_to_float()) assert resp.data.winnerFound is True assert resp.data.winner == 'canary1'
def test_v2_ab_without_reward_for_feasible_version(): example = copy.deepcopy(ab_er_example_step2) del example['status']['analysis']['aggregatedMetrics']['data'][ 'business-revenue']['data']['canary'] expr = ExperimentResource(**example) resp = get_winner_assessment(expr.convert_to_float()) assert "reward value for feasible version canary is not available" in \ resp.message
def provide_winner_assessment(experiment_resource: ExperimentResource = Body( ..., example=er_example_step2)): """ POST iter8 2.0 experiment resource, whose status includes aggregated metrics/version_assessments, and obtain winner assessment. \f :body er: ExperimentResource """ return get_winner_assessment(experiment_resource.convert_to_float())
def test_v2_abn_with_better_reward_but_not_feasible(): example = copy.deepcopy(abn_er_example_step2) example['status']['analysis']['versionAssessments']['data']['canary1'] = [ False ] expr = ExperimentResource(**example) resp = get_winner_assessment(expr.convert_to_float()) assert resp.data.winnerFound is True assert resp.data.winner == 'canary2'
def test_v2_no_winner(): example = copy.deepcopy(er_example_step2) example['status']['analysis']['versionAssessments'] = { "data": { "default": [False], "canary": [False] }, "message": "All ok" } expr = ExperimentResource(**example) resp = get_winner_assessment(expr.convert_to_float()) assert resp.data.winnerFound is False
def test_canary_is_winner(): example = copy.deepcopy(er_example_step2) example['status']['analysis']['versionAssessments'] = { "data": { "default": [True], "canary": [True] }, "message": "All ok" } expr = ExperimentResource(**example) resp = get_winner_assessment(expr.convert_to_float()) assert resp.data.winnerFound == wa_response['data']['winnerFound'] assert resp.data.winner == wa_response['data']['winner']
def test_v2_analytics_assessment_conformance_winner(): example = copy.deepcopy(er_example_step2) example['status']['analysis']['versionAssessments'] = { "data": { "default": [True] }, "message": "All ok" } del example['spec']['versionInfo']['candidates'] example['spec']['strategy']['testingPattern'] = 'Conformance' expr = ExperimentResource(**example) resp = get_winner_assessment(expr.convert_to_float()) assert resp.data.winnerFound is True assert resp.data.winner == 'default'
def test_v2_winner_assessment_endpoint(): expr = ExperimentResource(**er_example_step2) get_winner_assessment(expr.convert_to_float())
def test_v2_abn_wa_without_va(): expr = ExperimentResource(**abn_er_example) try: get_winner_assessment(expr.convert_to_float()) except AttributeError: pass