async def test_error_rate_with_classifyclient_and_telemetry(mock_aioresponses): mock_aioresponses.get( NORMANDY_URL.format(server=NORMANDY_SERVER), payload=[{ "recipe": { **RECIPE, "filter_expression": ('(normandy.country in ["US"]) &&' "(normandy.telemetry.main.sum > 0)"), } }], ) max_error_percentage = { "default": 0.1, "with_classify_client": 20, "with_telemetry": 30, } with patch_async(f"{MODULE}.fetch_bigquery", return_value=FAKE_ROWS): status, data = await run( server=NORMANDY_SERVER, max_error_percentage=max_error_percentage, ) assert status is False assert data["sources"]["recipe/123"]["error_rate"] == 37.5 assert data["sources"]["recipe/123"]["with_telemetry"] assert data["sources"]["recipe/123"]["with_classify_client"]
async def test_filter_on_runner_uptake(mock_aioresponses): mock_aioresponses.get( NORMANDY_URL.format(server=NORMANDY_SERVER), payload=[{ "recipe": RECIPE }], ) with patch_async(f"{MODULE}.fetch_bigquery", return_value=FAKE_ROWS): status, data = await run( sources=["runner"], server=NORMANDY_SERVER, max_error_percentage=0.1, channels=["release"], ) assert status is False assert data == { "sources": { "runner": { "error_rate": 20.0, "statuses": { "success": 2000, "server_error": 500 }, "ignored": {}, "min_timestamp": "2019-09-16T00:30:00", "max_timestamp": "2019-09-16T00:40:00", } }, "min_rate": 0.0, "max_rate": 20.0, "min_timestamp": "2019-09-16T00:30:00", "max_timestamp": "2019-09-16T01:00:00", }
async def test_min_total_events(mock_aioresponses): mock_aioresponses.get( NORMANDY_URL.format(server=NORMANDY_SERVER), payload=[{ "recipe": { "id": 123 } }], ) with patch_async(f"{MODULE}.fetch_redash", return_value=FAKE_ROWS): status, data = await run( api_key="", server=NORMANDY_SERVER, max_error_percentage=0.1, min_total_events=40001, channels=["release"], ) assert status is True assert data == { "sources": {}, "min_rate": None, "max_rate": None, "min_timestamp": "2019-09-16T00:30:00", "max_timestamp": "2019-09-16T01:00:00", }
async def test_filter_on_action_uptake(mock_aioresponses): mock_aioresponses.get( NORMANDY_URL.format(server=NORMANDY_SERVER), payload=[{ "recipe": RECIPE }], ) with patch_async(f"{MODULE}.fetch_redash", return_value=FAKE_ROWS): status, data = await run( api_key="", sources=["action"], server=NORMANDY_SERVER, max_error_percentage=10, channels=["release"], ) assert status is False assert data == { "sources": { "action/AddonStudyAction": { "error_rate": 10.0, "statuses": { "success": 9000, "action_post_execution_error": 1000 }, "ignored": {}, "min_timestamp": "2019-09-16T00:30:00", "max_timestamp": "2019-09-16T00:40:00", } }, "min_rate": 10.0, "max_rate": 10.0, "min_timestamp": "2019-09-16T00:30:00", "max_timestamp": "2019-09-16T01:00:00", }
async def test_error_rate_with_classify(mock_aioresponses): mock_aioresponses.get( NORMANDY_URL.format(server=NORMANDY_SERVER), payload=[{ "recipe": { **RECIPE, "filter_expression": '(normandy.country in ["US"])' } }], ) with patch_async(f"{MODULE}.fetch_bigquery", return_value=FAKE_ROWS): status, data = await run( server=NORMANDY_SERVER, max_error_percentage=0.1, ) assert status is False assert data["sources"]["recipe/123"]["with_classify_client"]
async def test_error_rate_with_telemetry(mock_aioresponses): mock_aioresponses.get( NORMANDY_URL.format(server=NORMANDY_SERVER), payload=[{ "recipe": { **RECIPE, "filter_expression": "(normandy.telemetry.main.sum > 0)", } }], ) with patch_async(f"{MODULE}.fetch_redash", return_value=FAKE_ROWS): status, data = await run( api_key="", server=NORMANDY_SERVER, max_error_percentage=0.1, ) assert status is False assert data["sources"]["recipe/123"]["with_telemetry"]
async def test_positive(mock_aioresponses): mock_aioresponses.get( NORMANDY_URL.format(server=NORMANDY_SERVER), payload=[{ "recipe": RECIPE }], ) with patch_async(f"{MODULE}.fetch_bigquery", return_value=FAKE_ROWS): status, data = await run( server=NORMANDY_SERVER, max_error_percentage=100.0, channels=["release"], ) assert status is True assert data == { "sources": {}, "min_rate": 33.33, "max_rate": 37.5, "min_timestamp": "2019-09-16T00:30:00", "max_timestamp": "2019-09-16T01:00:00", }
async def test_filter_by_channel(mock_aioresponses): mock_aioresponses.get( NORMANDY_URL.format(server=NORMANDY_SERVER), payload=[{ "recipe": { **RECIPE, "id": 531 } }], ) with patch_async(f"{MODULE}.fetch_redash", return_value=FAKE_ROWS): status, data = await run( api_key="", server=NORMANDY_SERVER, max_error_percentage=0.1, channels=["beta"], ) assert status is False assert data == { "sources": { "recipe/531": { "error_rate": 25.0, "name": "un dos tres", "with_telemetry": False, "with_classify_client": False, "ignored": {}, "max_timestamp": "2019-09-16T01:00:00", "min_timestamp": "2019-09-16T00:50:00", "statuses": { "recipe_didnt_match_filter": 3000, "recipe_filter_broken": 1000, }, } }, "min_rate": 25.0, "max_rate": 25.0, "min_timestamp": "2019-09-16T00:30:00", "max_timestamp": "2019-09-16T01:00:00", }
async def test_negative(mock_aioresponses): mock_aioresponses.get( NORMANDY_URL.format(server=NORMANDY_SERVER), payload=[{ "recipe": RECIPE }], ) with patch_async(f"{MODULE}.fetch_redash", return_value=FAKE_ROWS): status, data = await run( api_key="", server=NORMANDY_SERVER, max_error_percentage=0.1, channels=["release"], ) assert status is False assert data == { "sources": { "recipe/123": { "error_rate": 37.5, "name": "un dos tres", "with_telemetry": False, "with_classify_client": False, "statuses": { "success": 20000, "recipe_didnt_match_filter": 5000, "recipe_execution_error": 10000, "recipe_invalid_action": 5000, }, "ignored": {}, "min_timestamp": "2019-09-16T00:30:00", "max_timestamp": "2019-09-16T00:40:00", } }, "min_rate": 33.33, "max_rate": 37.5, "min_timestamp": "2019-09-16T00:30:00", "max_timestamp": "2019-09-16T01:00:00", }
async def test_ignore_status(mock_aioresponses): mock_aioresponses.get( NORMANDY_URL.format(server=NORMANDY_SERVER), payload=[{ "recipe": RECIPE }], ) with patch_async(f"{MODULE}.fetch_redash", return_value=FAKE_ROWS): status, data = await run( api_key="", server=NORMANDY_SERVER, max_error_percentage=0.1, ignore_status=["recipe_execution_error", "recipe_invalid_action"], channels=["release"], ) assert status is True assert data == { "sources": {}, "min_rate": 0.0, "max_rate": 0.0, "min_timestamp": "2019-09-16T00:30:00", "max_timestamp": "2019-09-16T01:00:00", }