def test_invalidate_form_if_user_does_not_have_permission_to_the_state(self): self.data["state"] = "SP" form = StateSpreadsheetForm(self.data, self.file_data, user=self.user) assert not form.is_valid() assert "state" in form.errors
def new_only_total_spreadsheet(self, state, date, confirmed, deaths): filename = f"/tmp/{state}-{date}.csv" with open(filename, mode="w") as fobj: writer = csv.writer(fobj) writer.writerow(["municipio", "confirmados", "obitos"]) writer.writerow(["TOTAL NO ESTADO", str(confirmed), str(deaths)]) with open(filename, mode="rb") as fobj: file_data = fobj.read() form = StateSpreadsheetForm( { "date": date, "state": state, "boletim_urls": settings.COVID_19_STATE_TOTALS_URL, "boletim_notes": NOTES, }, { "file": SimpleUploadedFile(filename, file_data), }, user=self.user, ) form_valid = form.is_valid() if not form_valid: self.debug( f"{state} - ERROR CREATING - Invalid form: {form.errors}") return os.unlink(filename) return form.save()
def test_skip_sum_validations_if_flagged_in_the_form_data( self, mocked_format): mocked_format.return_value = (["results", "list"], ["warnings", "list"]) self.data.update({ "skip_sum_cases": True, "skip_sum_deaths": True, }) form = StateSpreadsheetForm(self.data, self.file_data, user=self.user) assert form.is_valid(), form.errors expected = { "table": ["results", "list"], "errors": [], "warnings": ["warnings", "list"], } spreadsheet = form.save() spreadsheet.refresh_from_db() assert expected == spreadsheet.data assert 1 == mocked_format.call_count method_call = mocked_format.call_args_list[0] data, import_date, state = method_call[0] kwargs = method_call[1] assert date.today() == import_date assert state == "PR" for entry, expected_entry in zip( data, rows.import_from_csv(self.file_data["file"])): assert entry._asdict() == expected_entry._asdict() assert kwargs["skip_sum_cases"] is True assert kwargs["skip_sum_deaths"] is True
def new_only_total_spreadsheet(self, state, date, confirmed, deaths): temp_file = NamedTemporaryFile(delete=False, mode="w", suffix=".csv") writer = csv.writer(temp_file) writer.writerow(["municipio", "confirmados", "obitos"]) writer.writerow(["TOTAL NO ESTADO", str(confirmed), str(deaths)]) temp_file.close() with open(temp_file.name, mode="rb") as fobj: file_data = fobj.read() form = StateSpreadsheetForm( { "date": date, "state": state, "boletim_urls": settings.COVID_19_STATE_TOTALS_URL, "boletim_notes": NOTES, }, { "file": SimpleUploadedFile(temp_file.name, file_data), }, user=self.user, ) form_valid = form.is_valid() if not form_valid: self.debug( f"{state} - ERROR CREATING - Invalid form: {form.errors}") return os.remove(temp_file.name) return form.save()
def test_invalidate_if_future_date(self): self.data['date'] = date.today() + timedelta(days=1) form = StateSpreadsheetForm(self.data, self.file_data, user=self.user) assert not form.is_valid() assert 'date' in form.errors
def test_invalidate_form_if_any_invalid_url(self): self.data['boletim_urls'] = 'xpto' form = StateSpreadsheetForm(self.data, self.file_data) assert not form.is_valid() assert 'boletim_urls' in form.errors
def test_invalidate_form_if_any_invalid_url(self): self.data["boletim_urls"] = "xpto" form = StateSpreadsheetForm(self.data, self.file_data) assert not form.is_valid() assert "boletim_urls" in form.errors
def test_import_data_from_ods_with_sucess(self): valid_ods = SAMPLE_SPREADSHEETS_DATA_DIR / "sample-PR.ods" assert valid_ods.exists() self.file_data["file"] = self.gen_file("sample.ods", valid_ods.read_bytes()) form = StateSpreadsheetForm(self.data, self.file_data, user=self.user) assert form.is_valid(), form.errors
def test_required_fields(self): required_fields = ['date', 'state', 'file', 'boletim_urls'] form = StateSpreadsheetForm({}, user=self.user) assert not form.is_valid() assert len(required_fields) == len(form.errors) for field in required_fields: assert field in form.errors
def test_import_data_from_xlsx_with_sucess(self): valid_xlsx = SAMPLE_SPREADSHEETS_DATA_DIR / 'sample-PR.xlsx' assert valid_xlsx.exists() self.file_data['file'] = self.gen_file(f'sample.xlsx', valid_xlsx.read_bytes()) form = StateSpreadsheetForm(self.data, self.file_data, user=self.user) assert form.is_valid(), form.errors
def test_raise_validation_error_if_any_error_with_rows_import_functions(self): valid_xls = SAMPLE_SPREADSHEETS_DATA_DIR / "sample-PR.xls" assert valid_xls.exists() # wrong file extension self.file_data["file"] = self.gen_file("sample.csv", valid_xls.read_bytes()) form = StateSpreadsheetForm(self.data, self.file_data, user=self.user) assert not form.is_valid() assert "__all__" in form.errors
def test_invalidate_if_wrong_file_format(self): valid_formats = ["csv", "xls", "xlsx", "ods"] for format in valid_formats: self.file_data["file"] = self.gen_file(f"sample.{format}", "col1,col2") form = StateSpreadsheetForm(self.data, self.file_data) assert form.is_valid(), form.errors self.file_data["file"] = self.gen_file("sample.txt", "col1,col2") form = StateSpreadsheetForm(self.data, self.file_data) assert "__all__" in form.errors
def test_list_all_errors_fom_the_import_process(self, mocked_format): exception = SpreadsheetValidationErrors() exception.new_error('Error 1') exception.new_error('Error 2') mocked_format.side_effect = exception form = StateSpreadsheetForm(self.data, self.file_data, user=self.user) assert not form.is_valid() assert 2 == len(form.errors['__all__']) assert 'Error 1' in form.errors['__all__'] assert 'Error 2' in form.errors['__all__']
def test_invalidate_if_wrong_file_format(self): valid_formats = ['csv', 'xls', 'xlsx', 'ods'] for format in valid_formats: self.file_data['file'] = self.gen_file(f'sample.{format}', 'col1,col2') form = StateSpreadsheetForm(self.data, self.file_data) assert form.is_valid(), form.errors self.file_data['file'] = self.gen_file(f'sample.txt', 'col1,col2') form = StateSpreadsheetForm(self.data, self.file_data) assert '__all__' in form.errors
def test_populate_object_data_with_valid_sample(self, mocked_format): mocked_format.return_value = (['results', 'list'], ['warnings', 'list']) form = StateSpreadsheetForm(self.data, self.file_data, user=self.user) assert form.is_valid(), form.errors expected = { "table": ['results', 'list'], "errors": [], "warnings": ['warnings', 'list'], } spreadsheet = form.save() spreadsheet.refresh_from_db() assert expected == spreadsheet.data
def test_create_new_spreadsheet_with_valid_data(self): form = StateSpreadsheetForm(self.data, self.file_data, user=self.user) assert form.is_valid(), form.errors spreadsheet = form.save() spreadsheet.refresh_from_db() assert self.user == spreadsheet.user assert date.today() == spreadsheet.date assert "PR" == spreadsheet.state assert spreadsheet.file assert ["http://google.com", "http://brasil.io"] == spreadsheet.boletim_urls assert "notes" == spreadsheet.boletim_notes assert StateSpreadsheet.UPLOADED == spreadsheet.status assert spreadsheet.cancelled is False assert len(spreadsheet.data) > 0 assert len(spreadsheet.table_data) == 8 # total, undefined + 6 cities
def test_validate_notes_max_length(self): self.data["boletim_notes"] = "a" * 2001 form = StateSpreadsheetForm(self.data, self.file_data, user=self.user) assert not form.is_valid() assert "boletim_notes" in form.errors
def handle(self, *args, **kwargs): force = self.get_state_option(kwargs, "force") only = self.get_state_option(kwargs, "only") username = "******" debug(f"Getting user object for {username}") user = get_user_model().objects.get(username=username) debug(f"Downloading spreadsheet from {STATE_TOTALS_URL}") response = requests.get(STATE_TOTALS_URL) debug("Importing spreadsheet") spreadsheet = rows.import_from_csv( io.BytesIO(response.content), encoding="utf-8", force_types={ "confirmed": rows.fields.IntegerField, "data_dados": rows.fields.DateField, "deaths": rows.fields.IntegerField, }, ) for row in spreadsheet: state = str(row.state or "").upper() if only and state not in only: debug(f"Skipping {state} because of: not in --only") continue date = row.data_dados confirmed = row.confirmed deaths = row.deaths recent_deploy = StateSpreadsheet.objects.most_recent_deployed(state, date) if recent_deploy: data = recent_deploy.get_total_data() if confirmed == data["confirmed"] and deaths == data["deaths"]: debug(f"Skipping {state} because it has the same total for deaths and confirmed") continue elif confirmed < data["confirmed"] or deaths < data["deaths"]: if force and state in force: debug( f"WARNING: would skip {state} (already deployed for {date} and numbers of deployed are greater than ours: (ours vs deployed) {confirmed} vs {data['confirmed']}, {deaths} vs {data['deaths']}), but forcing because of --force" ) else: debug( f"Skipping {state} (already deployed for {date} and numbers of deployed are greater than ours: (ours vs deployed) {confirmed} vs {data['confirmed']}, {deaths} vs {data['deaths']})" ) continue debug(f"Creating spreadsheet for {state} on {date}") filename = f"/tmp/{state}-{date}.csv" with open(filename, mode="w") as fobj: writer = csv.writer(fobj) writer.writerow(["municipio", "confirmados", "obitos"]) writer.writerow(["TOTAL NO ESTADO", str(confirmed), str(deaths)]) with open(filename, mode="rb") as fobj: file_data = fobj.read() form = StateSpreadsheetForm( {"date": date, "state": state, "boletim_urls": STATE_TOTALS_URL, "boletim_notes": NOTES,}, {"file": SimpleUploadedFile(filename, file_data),}, user=user, ) form_valid = form.is_valid() if not form_valid: debug(f" Form is NOT valid for {state}! errors = {form.errors}") continue os.unlink(filename) obj = form.save() StateSpreadsheet.objects.cancel_older_versions(obj) obj.link_to(obj) obj.import_to_final_dataset() obj.refresh_from_db() debug(f" Spreadsheet created for {state}, id = {obj.id}")