def test_negative_submission(self):
errors = validate_submission("2020-08-01", "2020-08-04",
IP_FILE_FEW_COUNTRIES,
NEGATIVE_SUBMISSION)
self.assertIsNotNone(errors)
self.assertTrue("negative" in errors[0],
f"Expected 'negative' in errors, but got {errors}")
def test_string_daily_cases(self):
errors = validate_submission("2020-08-01", "2020-08-04",
IP_FILE_FEW_COUNTRIES,
STRING_DAILY_CASES_SUBMISSION)
self.assertIsNotNone(errors)
self.assertTrue(
"non numerical" in errors[0],
f"Expected 'non numerical' in errors, but got {errors}")
def test_missing_country_submission(self):
errors = validate_submission("2020-08-01", "2020-08-04",
IP_FILE_FEW_COUNTRIES,
MISSING_COUNTRY_SUBMISSION)
self.assertIsNotNone(errors)
self.assertTrue(
"Aruba" in errors[0],
f"Expected 'Aruba' in errors, because it's missing, but got {errors}"
)
def validate(start_date,end_date,ip_file,output_file):
# First, delete any potential old file
try:
os.remove(output_file)
except OSError:
pass
# Then generate the prediction, calling the official API
os.system(f"python predict.py -s {start_date} -e {end_date} -ip {ip_file} -o {output_file}")
# And validate it
errors = validate_submission(start_date,end_date,ip_file,output_file)
if errors:
for error in errors:
print(error)
else:
print("All good!")
def test_dates(self):
errors = validate_submission("2020-08-01", "2020-08-04",
IP_FILE_FEW_COUNTRIES,
BAD_DATES_SUBMISSION)
self.assertIsNotNone(errors)
self.assertEqual(10, len(errors), "Not the expected number of errors")
expected_errors = [
'Afghanistan: Expected prediction for date 2020-08-01 but got 2020-07-01',
'Afghanistan: Expected prediction for date 2020-08-02 but got 2020-07-02',
'Afghanistan: Expected prediction for date 2020-08-03 but got 2020-07-03',
'Afghanistan: Expected prediction for date 2020-08-04 but got 2020-07-04',
'Albania: Expected prediction for date 2020-08-03 but got None',
'Albania: Expected prediction for date 2020-08-04 but got None',
'Angola: Expected prediction for date None but got 2020-08-05',
'Aruba: Expected prediction for date 2020-08-02 but got 2020-08-03',
'Aruba: Expected prediction for date 2020-08-03 but got 2020-08-04',
'Aruba: Expected prediction for date 2020-08-04 but got None'
]
self.assertEqual(expected_errors, errors)
predictions[test_predictor_name],
ma_df.copy())
temp_df.head(12)
# In[ ]:
actual_df.head(8)
# In[ ]:
from covid_xprize.validation.predictor_validation import validate_submission
ranking_df = pd.DataFrame()
for predictor_name, predictions_file in predictions.items():
print(f"Getting {predictor_name}'s predictions from: {predictions_file}")
errors = validate_submission(start_date_str, end_date_str, IP_FILE,
predictions_file)
if not errors:
preds_df = get_predictions_from_file(predictor_name, predictions_file,
ma_df)
merged_df = actual_df.merge(
preds_df,
on=['CountryName', 'RegionName', 'Date', 'GeoID'],
how='left')
ranking_df = ranking_df.append(merged_df)
else:
print(
f"Predictor {predictor_name} did not submit valid predictions! Please check its errors:"
)
print(errors)
# In[ ]:
def test_nan_submission(self):
errors = validate_submission("2020-08-01", "2020-08-04",
IP_FILE_FEW_COUNTRIES, NAN_SUBMISSION)
self.assertIsNotNone(errors)
self.assertTrue("NaN" in errors[0],
f"Expected 'NaN' in errors, but got {errors}")
def test_valid_with_additional_columns_submission(self):
errors = validate_submission("2020-08-01", "2020-08-04",
IP_FILE_FEW_COUNTRIES,
VALID_WITH_ADD_COLS_SUBMISSION)
self.assertTrue(not errors, f"Unexpected errors: {errors}")
def test_valid_submission(self):
errors = validate_submission("2020-08-01", "2020-08-04",
IP_FILE_ALL_COUNTRIES, VALID_SUBMISSION)
self.assertTrue(not errors, f"Unexpected errors: {errors}")
def test_wrong_columns(self):
errors = validate_submission("2020-08-01", "2020-08-01",
IP_FILE_ALL_COUNTRIES, WRONG_COLUMNS)
self.assertIsNotNone(errors)
self.assertTrue(PREDICTED_DAILY_NEW_CASES in errors[0])
import sys
sys.path.append("../../../..")
from covid_xprize.validation.predictor_validation import validate_submission
def validate(start_date, end_date, ip_file, output_file):
# First, delete any potential old file
try:
os.remove(output_file)
except OSError:
pass
# Then generate the prediction, calling the official API
get_ipython().system('python predict.py -s {start_date} -e {end_date} -ip {ip_file} -o {output_file}')
# And validate it
errors = validate_submission(start_date, end_date, ip_file, output_file)
if errors:
for error in errors:
print(error)
else:
print("All good!")
# ## 4 days, no gap
# - All countries and regions
# - Official number of cases is known up to start_date
# - Intervention Plans are the official ones
# In[27]:
