def test_v2_analytics_assessment_endpoint(): with requests_mock.mock(real_http=True) as mock: file_path = os.path.join(os.path.dirname(__file__), 'data/prom_responses', 'prometheus_sample_response.json') mock.get(er_example["status"]["metrics"][0]["metricObj"]["spec"] ["urlTemplate"], json=json.load(open(file_path))) expr = ExperimentResource(**er_example) get_analytics_results(expr.convert_to_float()).convert_to_quantity()
def test_ab_reward_only_analytics_assessment_endpoint(): with requests_mock.mock(real_http=True) as mock: file_path = os.path.join(os.path.dirname(__file__), 'data/prom_responses', 'prometheus_sample_response.json') mock.get(ab_er_example["status"]["metrics"][0]["metricObj"]["spec"] ["urlTemplate"], json=json.load(open(file_path))) ab_expr = copy.deepcopy(ab_er_example) del ab_expr["spec"]["criteria"]["objectives"] expr = ExperimentResource(**ab_expr) get_analytics_results(expr.convert_to_float()).convert_to_quantity()
def test_v2_abn_analytics_assessment_conformance(): with requests_mock.mock(real_http=True) as mock: file_path = os.path.join(os.path.dirname(__file__), 'data/prom_responses', 'prometheus_sample_response.json') mock.get(abn_er_example["status"]["metrics"][0]["metricObj"]["spec"] ["urlTemplate"], json=json.load(open(file_path))) example = copy.deepcopy(abn_er_example) del example['spec']['versionInfo']['candidates'] example['spec']['strategy']['testingPattern'] = 'Conformance' expr = ExperimentResource(**example) get_analytics_results(expr.convert_to_float()).convert_to_quantity()
def provide_analytics_results(expr: ExperimentResource = Body( ..., example=er_example)): """ POST iter8 2.0 experiment resource and metric resources and get analytics results. \f :body expr: ExperimentResource """ return get_analytics_results(expr.convert_to_float()).convert_to_quantity()
def provide_analytics_results( ermr: ExperimentResourceAndMetricResources = Body( ..., example=ermr_example)): """ POST iter8 2.0 experiment resource and metric resources and get analytics results. \f :body er: ExperimentResourceAndMetricResources """ analytics_results = get_analytics_results(ermr) return analytics_results
def test_v2_ab_without_reward(): with requests_mock.mock(real_http=True) as mock: file_path = os.path.join(os.path.dirname(__file__), 'data/prom_responses', 'prometheus_sample_response.json') mock.get(ab_er_example["status"]["metrics"][0]["metricObj"]["spec"] ["urlTemplate"], json=json.load(open(file_path))) example = copy.deepcopy(ab_er_example) del example['spec']['criteria']['rewards'] expr = ExperimentResource(**example) resp = get_analytics_results( expr.convert_to_float()).convert_to_quantity() assert "No reward metric in experiment" in resp.winner_assessment.message assert resp.winner_assessment.data.winnerFound is False