def test_v2_canary_failing_lowerlimit_criteria(): example = copy.deepcopy(er_example_step1) example['spec']['criteria']['objectives'][0].pop('upperLimit') example['spec']['criteria']['objectives'][0]['lowerLimit'] = 500 expr = ExperimentResource(**example) resp = get_version_assessments(expr.convert_to_float()) assert resp.data == {'default': [False], 'canary': [False]}
def test_v2_va_without_mean_latency_metric(): example = copy.deepcopy(er_example_step1) example['status']['analysis']['aggregatedMetrics']["data"].pop( 'mean-latency', None) expr = ExperimentResource(**example) resp = get_version_assessments(expr.convert_to_float()) assert resp.message == \ "Error: ; Warning: Aggregated metric object for mean-latency metric is unavailable.; Info: "
def provide_version_assessments(experiment_resource: ExperimentResource = Body( ..., example=er_example_step1)): """ POST iter8 2.0 experiment resource, whose status includes aggregated metrics, and obtain version assessments. \f :body er: ExperimentResource """ return get_version_assessments(experiment_resource.convert_to_float())
def test_v2_va_with_no_metric_value(): with requests_mock.mock(real_http=True) as mock: file_path = os.path.join(os.path.dirname(__file__), 'data/prom_responses', 'prometheus_sample_no_response.json') mock.get(er_example["status"]["metrics"][0]["metricObj"]["spec"] ["urlTemplate"], json=json.load(open(file_path))) expr = ExperimentResource(**er_example) resp = get_aggregated_metrics( expr.convert_to_float()).convert_to_quantity() example = copy.deepcopy(er_example_step1) example['status']['analysis']["aggregatedMetrics"] = resp expr = ExperimentResource(**example) resp2 = get_version_assessments(expr.convert_to_float()) assert resp2.data == {'default': [False], 'canary': [False]}
def test_v2_abn_va_without_am(): expr = ExperimentResource(**abn_er_example) try: get_version_assessments(expr.convert_to_float()) except AttributeError: pass
def test_v2_version_assessment_endpoint(): expr = ExperimentResource(**er_example_step1) get_version_assessments(expr.convert_to_float())
def test_v2_canary_failing_upperlimit_criteria(): expr = ExperimentResource(**er_example_step1) resp = get_version_assessments(expr.convert_to_float()) assert resp.data == {'default': [True], 'canary': [False]}