def test_negative_submission(self): errors = validate_submission("2020-08-01", "2020-08-04", IP_FILE_FEW_COUNTRIES, NEGATIVE_SUBMISSION) self.assertIsNotNone(errors) self.assertTrue("negative" in errors[0], f"Expected 'negative' in errors, but got {errors}")
def test_string_daily_cases(self): errors = validate_submission("2020-08-01", "2020-08-04", IP_FILE_FEW_COUNTRIES, STRING_DAILY_CASES_SUBMISSION) self.assertIsNotNone(errors) self.assertTrue( "non numerical" in errors[0], f"Expected 'non numerical' in errors, but got {errors}")
def test_missing_country_submission(self): errors = validate_submission("2020-08-01", "2020-08-04", IP_FILE_FEW_COUNTRIES, MISSING_COUNTRY_SUBMISSION) self.assertIsNotNone(errors) self.assertTrue( "Aruba" in errors[0], f"Expected 'Aruba' in errors, because it's missing, but got {errors}" )
def validate(start_date,end_date,ip_file,output_file): # First, delete any potential old file try: os.remove(output_file) except OSError: pass # Then generate the prediction, calling the official API os.system(f"python predict.py -s {start_date} -e {end_date} -ip {ip_file} -o {output_file}") # And validate it errors = validate_submission(start_date,end_date,ip_file,output_file) if errors: for error in errors: print(error) else: print("All good!")
def test_dates(self): errors = validate_submission("2020-08-01", "2020-08-04", IP_FILE_FEW_COUNTRIES, BAD_DATES_SUBMISSION) self.assertIsNotNone(errors) self.assertEqual(10, len(errors), "Not the expected number of errors") expected_errors = [ 'Afghanistan: Expected prediction for date 2020-08-01 but got 2020-07-01', 'Afghanistan: Expected prediction for date 2020-08-02 but got 2020-07-02', 'Afghanistan: Expected prediction for date 2020-08-03 but got 2020-07-03', 'Afghanistan: Expected prediction for date 2020-08-04 but got 2020-07-04', 'Albania: Expected prediction for date 2020-08-03 but got None', 'Albania: Expected prediction for date 2020-08-04 but got None', 'Angola: Expected prediction for date None but got 2020-08-05', 'Aruba: Expected prediction for date 2020-08-02 but got 2020-08-03', 'Aruba: Expected prediction for date 2020-08-03 but got 2020-08-04', 'Aruba: Expected prediction for date 2020-08-04 but got None' ] self.assertEqual(expected_errors, errors)
predictions[test_predictor_name], ma_df.copy()) temp_df.head(12) # In[ ]: actual_df.head(8) # In[ ]: from covid_xprize.validation.predictor_validation import validate_submission ranking_df = pd.DataFrame() for predictor_name, predictions_file in predictions.items(): print(f"Getting {predictor_name}'s predictions from: {predictions_file}") errors = validate_submission(start_date_str, end_date_str, IP_FILE, predictions_file) if not errors: preds_df = get_predictions_from_file(predictor_name, predictions_file, ma_df) merged_df = actual_df.merge( preds_df, on=['CountryName', 'RegionName', 'Date', 'GeoID'], how='left') ranking_df = ranking_df.append(merged_df) else: print( f"Predictor {predictor_name} did not submit valid predictions! Please check its errors:" ) print(errors) # In[ ]:
def test_nan_submission(self): errors = validate_submission("2020-08-01", "2020-08-04", IP_FILE_FEW_COUNTRIES, NAN_SUBMISSION) self.assertIsNotNone(errors) self.assertTrue("NaN" in errors[0], f"Expected 'NaN' in errors, but got {errors}")
def test_valid_with_additional_columns_submission(self): errors = validate_submission("2020-08-01", "2020-08-04", IP_FILE_FEW_COUNTRIES, VALID_WITH_ADD_COLS_SUBMISSION) self.assertTrue(not errors, f"Unexpected errors: {errors}")
def test_valid_submission(self): errors = validate_submission("2020-08-01", "2020-08-04", IP_FILE_ALL_COUNTRIES, VALID_SUBMISSION) self.assertTrue(not errors, f"Unexpected errors: {errors}")
def test_wrong_columns(self): errors = validate_submission("2020-08-01", "2020-08-01", IP_FILE_ALL_COUNTRIES, WRONG_COLUMNS) self.assertIsNotNone(errors) self.assertTrue(PREDICTED_DAILY_NEW_CASES in errors[0])
import sys sys.path.append("../../../..") from covid_xprize.validation.predictor_validation import validate_submission def validate(start_date, end_date, ip_file, output_file): # First, delete any potential old file try: os.remove(output_file) except OSError: pass # Then generate the prediction, calling the official API get_ipython().system('python predict.py -s {start_date} -e {end_date} -ip {ip_file} -o {output_file}') # And validate it errors = validate_submission(start_date, end_date, ip_file, output_file) if errors: for error in errors: print(error) else: print("All good!") # ## 4 days, no gap # - All countries and regions # - Official number of cases is known up to start_date # - Intervention Plans are the official ones # In[27]: