def test_invalidate_form_if_user_does_not_have_permission_to_the_state(self):
self.data["state"] = "SP"
form = StateSpreadsheetForm(self.data, self.file_data, user=self.user)
assert not form.is_valid()
assert "state" in form.errors
def new_only_total_spreadsheet(self, state, date, confirmed, deaths):
filename = f"/tmp/{state}-{date}.csv"
with open(filename, mode="w") as fobj:
writer = csv.writer(fobj)
writer.writerow(["municipio", "confirmados", "obitos"])
writer.writerow(["TOTAL NO ESTADO", str(confirmed), str(deaths)])
with open(filename, mode="rb") as fobj:
file_data = fobj.read()
form = StateSpreadsheetForm(
{
"date": date,
"state": state,
"boletim_urls": settings.COVID_19_STATE_TOTALS_URL,
"boletim_notes": NOTES,
},
{
"file": SimpleUploadedFile(filename, file_data),
},
user=self.user,
)
form_valid = form.is_valid()
if not form_valid:
self.debug(
f"{state} - ERROR CREATING - Invalid form: {form.errors}")
return
os.unlink(filename)
return form.save()
def test_skip_sum_validations_if_flagged_in_the_form_data(
self, mocked_format):
mocked_format.return_value = (["results",
"list"], ["warnings", "list"])
self.data.update({
"skip_sum_cases": True,
"skip_sum_deaths": True,
})
form = StateSpreadsheetForm(self.data, self.file_data, user=self.user)
assert form.is_valid(), form.errors
expected = {
"table": ["results", "list"],
"errors": [],
"warnings": ["warnings", "list"],
}
spreadsheet = form.save()
spreadsheet.refresh_from_db()
assert expected == spreadsheet.data
assert 1 == mocked_format.call_count
method_call = mocked_format.call_args_list[0]
data, import_date, state = method_call[0]
kwargs = method_call[1]
assert date.today() == import_date
assert state == "PR"
for entry, expected_entry in zip(
data, rows.import_from_csv(self.file_data["file"])):
assert entry._asdict() == expected_entry._asdict()
assert kwargs["skip_sum_cases"] is True
assert kwargs["skip_sum_deaths"] is True
def new_only_total_spreadsheet(self, state, date, confirmed, deaths):
temp_file = NamedTemporaryFile(delete=False, mode="w", suffix=".csv")
writer = csv.writer(temp_file)
writer.writerow(["municipio", "confirmados", "obitos"])
writer.writerow(["TOTAL NO ESTADO", str(confirmed), str(deaths)])
temp_file.close()
with open(temp_file.name, mode="rb") as fobj:
file_data = fobj.read()
form = StateSpreadsheetForm(
{
"date": date,
"state": state,
"boletim_urls": settings.COVID_19_STATE_TOTALS_URL,
"boletim_notes": NOTES,
},
{
"file": SimpleUploadedFile(temp_file.name, file_data),
},
user=self.user,
)
form_valid = form.is_valid()
if not form_valid:
self.debug(
f"{state} - ERROR CREATING - Invalid form: {form.errors}")
return
os.remove(temp_file.name)
return form.save()
def test_invalidate_if_future_date(self):
self.data['date'] = date.today() + timedelta(days=1)
form = StateSpreadsheetForm(self.data, self.file_data, user=self.user)
assert not form.is_valid()
assert 'date' in form.errors
def test_invalidate_form_if_any_invalid_url(self):
self.data['boletim_urls'] = 'xpto'
form = StateSpreadsheetForm(self.data, self.file_data)
assert not form.is_valid()
assert 'boletim_urls' in form.errors
def test_invalidate_form_if_any_invalid_url(self):
self.data["boletim_urls"] = "xpto"
form = StateSpreadsheetForm(self.data, self.file_data)
assert not form.is_valid()
assert "boletim_urls" in form.errors
def test_import_data_from_ods_with_sucess(self):
valid_ods = SAMPLE_SPREADSHEETS_DATA_DIR / "sample-PR.ods"
assert valid_ods.exists()
self.file_data["file"] = self.gen_file("sample.ods", valid_ods.read_bytes())
form = StateSpreadsheetForm(self.data, self.file_data, user=self.user)
assert form.is_valid(), form.errors
def test_required_fields(self):
required_fields = ['date', 'state', 'file', 'boletim_urls']
form = StateSpreadsheetForm({}, user=self.user)
assert not form.is_valid()
assert len(required_fields) == len(form.errors)
for field in required_fields:
assert field in form.errors
def test_import_data_from_xlsx_with_sucess(self):
valid_xlsx = SAMPLE_SPREADSHEETS_DATA_DIR / 'sample-PR.xlsx'
assert valid_xlsx.exists()
self.file_data['file'] = self.gen_file(f'sample.xlsx',
valid_xlsx.read_bytes())
form = StateSpreadsheetForm(self.data, self.file_data, user=self.user)
assert form.is_valid(), form.errors
def test_raise_validation_error_if_any_error_with_rows_import_functions(self):
valid_xls = SAMPLE_SPREADSHEETS_DATA_DIR / "sample-PR.xls"
assert valid_xls.exists()
# wrong file extension
self.file_data["file"] = self.gen_file("sample.csv", valid_xls.read_bytes())
form = StateSpreadsheetForm(self.data, self.file_data, user=self.user)
assert not form.is_valid()
assert "__all__" in form.errors
def test_invalidate_if_wrong_file_format(self):
valid_formats = ["csv", "xls", "xlsx", "ods"]
for format in valid_formats:
self.file_data["file"] = self.gen_file(f"sample.{format}", "col1,col2")
form = StateSpreadsheetForm(self.data, self.file_data)
assert form.is_valid(), form.errors
self.file_data["file"] = self.gen_file("sample.txt", "col1,col2")
form = StateSpreadsheetForm(self.data, self.file_data)
assert "__all__" in form.errors
def test_list_all_errors_fom_the_import_process(self, mocked_format):
exception = SpreadsheetValidationErrors()
exception.new_error('Error 1')
exception.new_error('Error 2')
mocked_format.side_effect = exception
form = StateSpreadsheetForm(self.data, self.file_data, user=self.user)
assert not form.is_valid()
assert 2 == len(form.errors['__all__'])
assert 'Error 1' in form.errors['__all__']
assert 'Error 2' in form.errors['__all__']
def test_invalidate_if_wrong_file_format(self):
valid_formats = ['csv', 'xls', 'xlsx', 'ods']
for format in valid_formats:
self.file_data['file'] = self.gen_file(f'sample.{format}',
'col1,col2')
form = StateSpreadsheetForm(self.data, self.file_data)
assert form.is_valid(), form.errors
self.file_data['file'] = self.gen_file(f'sample.txt', 'col1,col2')
form = StateSpreadsheetForm(self.data, self.file_data)
assert '__all__' in form.errors
def test_populate_object_data_with_valid_sample(self, mocked_format):
mocked_format.return_value = (['results',
'list'], ['warnings', 'list'])
form = StateSpreadsheetForm(self.data, self.file_data, user=self.user)
assert form.is_valid(), form.errors
expected = {
"table": ['results', 'list'],
"errors": [],
"warnings": ['warnings', 'list'],
}
spreadsheet = form.save()
spreadsheet.refresh_from_db()
assert expected == spreadsheet.data
def test_create_new_spreadsheet_with_valid_data(self):
form = StateSpreadsheetForm(self.data, self.file_data, user=self.user)
assert form.is_valid(), form.errors
spreadsheet = form.save()
spreadsheet.refresh_from_db()
assert self.user == spreadsheet.user
assert date.today() == spreadsheet.date
assert "PR" == spreadsheet.state
assert spreadsheet.file
assert ["http://google.com", "http://brasil.io"] == spreadsheet.boletim_urls
assert "notes" == spreadsheet.boletim_notes
assert StateSpreadsheet.UPLOADED == spreadsheet.status
assert spreadsheet.cancelled is False
assert len(spreadsheet.data) > 0
assert len(spreadsheet.table_data) == 8 # total, undefined + 6 cities
def test_validate_notes_max_length(self):
self.data["boletim_notes"] = "a" * 2001
form = StateSpreadsheetForm(self.data, self.file_data, user=self.user)
assert not form.is_valid()
assert "boletim_notes" in form.errors
def handle(self, *args, **kwargs):
force = self.get_state_option(kwargs, "force")
only = self.get_state_option(kwargs, "only")
username = "******"
debug(f"Getting user object for {username}")
user = get_user_model().objects.get(username=username)
debug(f"Downloading spreadsheet from {STATE_TOTALS_URL}")
response = requests.get(STATE_TOTALS_URL)
debug("Importing spreadsheet")
spreadsheet = rows.import_from_csv(
io.BytesIO(response.content),
encoding="utf-8",
force_types={
"confirmed": rows.fields.IntegerField,
"data_dados": rows.fields.DateField,
"deaths": rows.fields.IntegerField,
},
)
for row in spreadsheet:
state = str(row.state or "").upper()
if only and state not in only:
debug(f"Skipping {state} because of: not in --only")
continue
date = row.data_dados
confirmed = row.confirmed
deaths = row.deaths
recent_deploy = StateSpreadsheet.objects.most_recent_deployed(state, date)
if recent_deploy:
data = recent_deploy.get_total_data()
if confirmed == data["confirmed"] and deaths == data["deaths"]:
debug(f"Skipping {state} because it has the same total for deaths and confirmed")
continue
elif confirmed < data["confirmed"] or deaths < data["deaths"]:
if force and state in force:
debug(
f"WARNING: would skip {state} (already deployed for {date} and numbers of deployed are greater than ours: (ours vs deployed) {confirmed} vs {data['confirmed']}, {deaths} vs {data['deaths']}), but forcing because of --force"
)
else:
debug(
f"Skipping {state} (already deployed for {date} and numbers of deployed are greater than ours: (ours vs deployed) {confirmed} vs {data['confirmed']}, {deaths} vs {data['deaths']})"
)
continue
debug(f"Creating spreadsheet for {state} on {date}")
filename = f"/tmp/{state}-{date}.csv"
with open(filename, mode="w") as fobj:
writer = csv.writer(fobj)
writer.writerow(["municipio", "confirmados", "obitos"])
writer.writerow(["TOTAL NO ESTADO", str(confirmed), str(deaths)])
with open(filename, mode="rb") as fobj:
file_data = fobj.read()
form = StateSpreadsheetForm(
{"date": date, "state": state, "boletim_urls": STATE_TOTALS_URL, "boletim_notes": NOTES,},
{"file": SimpleUploadedFile(filename, file_data),},
user=user,
)
form_valid = form.is_valid()
if not form_valid:
debug(f" Form is NOT valid for {state}! errors = {form.errors}")
continue
os.unlink(filename)
obj = form.save()
StateSpreadsheet.objects.cancel_older_versions(obj)
obj.link_to(obj)
obj.import_to_final_dataset()
obj.refresh_from_db()
debug(f" Spreadsheet created for {state}, id = {obj.id}")
