Exemplo n.º 1
0
    def test_invalidate_form_if_user_does_not_have_permission_to_the_state(self):
        self.data["state"] = "SP"

        form = StateSpreadsheetForm(self.data, self.file_data, user=self.user)

        assert not form.is_valid()
        assert "state" in form.errors
Exemplo n.º 2
0
    def new_only_total_spreadsheet(self, state, date, confirmed, deaths):
        filename = f"/tmp/{state}-{date}.csv"
        with open(filename, mode="w") as fobj:
            writer = csv.writer(fobj)
            writer.writerow(["municipio", "confirmados", "obitos"])
            writer.writerow(["TOTAL NO ESTADO", str(confirmed), str(deaths)])

        with open(filename, mode="rb") as fobj:
            file_data = fobj.read()
            form = StateSpreadsheetForm(
                {
                    "date": date,
                    "state": state,
                    "boletim_urls": settings.COVID_19_STATE_TOTALS_URL,
                    "boletim_notes": NOTES,
                },
                {
                    "file": SimpleUploadedFile(filename, file_data),
                },
                user=self.user,
            )
            form_valid = form.is_valid()
            if not form_valid:
                self.debug(
                    f"{state} - ERROR CREATING - Invalid form: {form.errors}")
                return
        os.unlink(filename)

        return form.save()
Exemplo n.º 3
0
    def test_skip_sum_validations_if_flagged_in_the_form_data(
            self, mocked_format):
        mocked_format.return_value = (["results",
                                       "list"], ["warnings", "list"])
        self.data.update({
            "skip_sum_cases": True,
            "skip_sum_deaths": True,
        })
        form = StateSpreadsheetForm(self.data, self.file_data, user=self.user)
        assert form.is_valid(), form.errors
        expected = {
            "table": ["results", "list"],
            "errors": [],
            "warnings": ["warnings", "list"],
        }

        spreadsheet = form.save()
        spreadsheet.refresh_from_db()

        assert expected == spreadsheet.data
        assert 1 == mocked_format.call_count
        method_call = mocked_format.call_args_list[0]
        data, import_date, state = method_call[0]
        kwargs = method_call[1]
        assert date.today() == import_date
        assert state == "PR"
        for entry, expected_entry in zip(
                data, rows.import_from_csv(self.file_data["file"])):
            assert entry._asdict() == expected_entry._asdict()
        assert kwargs["skip_sum_cases"] is True
        assert kwargs["skip_sum_deaths"] is True
Exemplo n.º 4
0
    def new_only_total_spreadsheet(self, state, date, confirmed, deaths):
        temp_file = NamedTemporaryFile(delete=False, mode="w", suffix=".csv")
        writer = csv.writer(temp_file)
        writer.writerow(["municipio", "confirmados", "obitos"])
        writer.writerow(["TOTAL NO ESTADO", str(confirmed), str(deaths)])
        temp_file.close()

        with open(temp_file.name, mode="rb") as fobj:
            file_data = fobj.read()
            form = StateSpreadsheetForm(
                {
                    "date": date,
                    "state": state,
                    "boletim_urls": settings.COVID_19_STATE_TOTALS_URL,
                    "boletim_notes": NOTES,
                },
                {
                    "file": SimpleUploadedFile(temp_file.name, file_data),
                },
                user=self.user,
            )
            form_valid = form.is_valid()
            if not form_valid:
                self.debug(
                    f"{state} - ERROR CREATING - Invalid form: {form.errors}")
                return

        os.remove(temp_file.name)
        return form.save()
Exemplo n.º 5
0
    def test_invalidate_if_future_date(self):
        self.data['date'] = date.today() + timedelta(days=1)

        form = StateSpreadsheetForm(self.data, self.file_data, user=self.user)

        assert not form.is_valid()
        assert 'date' in form.errors
Exemplo n.º 6
0
    def test_invalidate_form_if_any_invalid_url(self):
        self.data['boletim_urls'] = 'xpto'

        form = StateSpreadsheetForm(self.data, self.file_data)

        assert not form.is_valid()
        assert 'boletim_urls' in form.errors
Exemplo n.º 7
0
    def test_invalidate_form_if_any_invalid_url(self):
        self.data["boletim_urls"] = "xpto"

        form = StateSpreadsheetForm(self.data, self.file_data)

        assert not form.is_valid()
        assert "boletim_urls" in form.errors
Exemplo n.º 8
0
    def test_import_data_from_ods_with_sucess(self):
        valid_ods = SAMPLE_SPREADSHEETS_DATA_DIR / "sample-PR.ods"
        assert valid_ods.exists()

        self.file_data["file"] = self.gen_file("sample.ods", valid_ods.read_bytes())
        form = StateSpreadsheetForm(self.data, self.file_data, user=self.user)

        assert form.is_valid(), form.errors
Exemplo n.º 9
0
    def test_required_fields(self):
        required_fields = ['date', 'state', 'file', 'boletim_urls']

        form = StateSpreadsheetForm({}, user=self.user)

        assert not form.is_valid()
        assert len(required_fields) == len(form.errors)
        for field in required_fields:
            assert field in form.errors
Exemplo n.º 10
0
    def test_import_data_from_xlsx_with_sucess(self):
        valid_xlsx = SAMPLE_SPREADSHEETS_DATA_DIR / 'sample-PR.xlsx'
        assert valid_xlsx.exists()

        self.file_data['file'] = self.gen_file(f'sample.xlsx',
                                               valid_xlsx.read_bytes())
        form = StateSpreadsheetForm(self.data, self.file_data, user=self.user)

        assert form.is_valid(), form.errors
Exemplo n.º 11
0
    def test_raise_validation_error_if_any_error_with_rows_import_functions(self):
        valid_xls = SAMPLE_SPREADSHEETS_DATA_DIR / "sample-PR.xls"
        assert valid_xls.exists()

        # wrong file extension
        self.file_data["file"] = self.gen_file("sample.csv", valid_xls.read_bytes())
        form = StateSpreadsheetForm(self.data, self.file_data, user=self.user)

        assert not form.is_valid()
        assert "__all__" in form.errors
Exemplo n.º 12
0
    def test_invalidate_if_wrong_file_format(self):
        valid_formats = ["csv", "xls", "xlsx", "ods"]
        for format in valid_formats:
            self.file_data["file"] = self.gen_file(f"sample.{format}", "col1,col2")

            form = StateSpreadsheetForm(self.data, self.file_data)
            assert form.is_valid(), form.errors

        self.file_data["file"] = self.gen_file("sample.txt", "col1,col2")
        form = StateSpreadsheetForm(self.data, self.file_data)
        assert "__all__" in form.errors
Exemplo n.º 13
0
    def test_list_all_errors_fom_the_import_process(self, mocked_format):
        exception = SpreadsheetValidationErrors()
        exception.new_error('Error 1')
        exception.new_error('Error 2')
        mocked_format.side_effect = exception

        form = StateSpreadsheetForm(self.data, self.file_data, user=self.user)
        assert not form.is_valid()

        assert 2 == len(form.errors['__all__'])
        assert 'Error 1' in form.errors['__all__']
        assert 'Error 2' in form.errors['__all__']
Exemplo n.º 14
0
    def test_invalidate_if_wrong_file_format(self):
        valid_formats = ['csv', 'xls', 'xlsx', 'ods']
        for format in valid_formats:
            self.file_data['file'] = self.gen_file(f'sample.{format}',
                                                   'col1,col2')

            form = StateSpreadsheetForm(self.data, self.file_data)
            assert form.is_valid(), form.errors

        self.file_data['file'] = self.gen_file(f'sample.txt', 'col1,col2')
        form = StateSpreadsheetForm(self.data, self.file_data)
        assert '__all__' in form.errors
Exemplo n.º 15
0
    def test_populate_object_data_with_valid_sample(self, mocked_format):
        mocked_format.return_value = (['results',
                                       'list'], ['warnings', 'list'])
        form = StateSpreadsheetForm(self.data, self.file_data, user=self.user)
        assert form.is_valid(), form.errors
        expected = {
            "table": ['results', 'list'],
            "errors": [],
            "warnings": ['warnings', 'list'],
        }

        spreadsheet = form.save()
        spreadsheet.refresh_from_db()

        assert expected == spreadsheet.data
Exemplo n.º 16
0
    def test_create_new_spreadsheet_with_valid_data(self):
        form = StateSpreadsheetForm(self.data, self.file_data, user=self.user)
        assert form.is_valid(), form.errors

        spreadsheet = form.save()
        spreadsheet.refresh_from_db()

        assert self.user == spreadsheet.user
        assert date.today() == spreadsheet.date
        assert "PR" == spreadsheet.state
        assert spreadsheet.file
        assert ["http://google.com", "http://brasil.io"] == spreadsheet.boletim_urls
        assert "notes" == spreadsheet.boletim_notes
        assert StateSpreadsheet.UPLOADED == spreadsheet.status
        assert spreadsheet.cancelled is False
        assert len(spreadsheet.data) > 0
        assert len(spreadsheet.table_data) == 8  # total, undefined + 6 cities
Exemplo n.º 17
0
 def test_validate_notes_max_length(self):
     self.data["boletim_notes"] = "a" * 2001
     form = StateSpreadsheetForm(self.data, self.file_data, user=self.user)
     assert not form.is_valid()
     assert "boletim_notes" in form.errors
Exemplo n.º 18
0
    def handle(self, *args, **kwargs):
        force = self.get_state_option(kwargs, "force")
        only = self.get_state_option(kwargs, "only")

        username = "******"
        debug(f"Getting user object for {username}")
        user = get_user_model().objects.get(username=username)

        debug(f"Downloading spreadsheet from {STATE_TOTALS_URL}")
        response = requests.get(STATE_TOTALS_URL)

        debug("Importing spreadsheet")
        spreadsheet = rows.import_from_csv(
            io.BytesIO(response.content),
            encoding="utf-8",
            force_types={
                "confirmed": rows.fields.IntegerField,
                "data_dados": rows.fields.DateField,
                "deaths": rows.fields.IntegerField,
            },
        )

        for row in spreadsheet:
            state = str(row.state or "").upper()
            if only and state not in only:
                debug(f"Skipping {state} because of: not in --only")
                continue

            date = row.data_dados
            confirmed = row.confirmed
            deaths = row.deaths

            recent_deploy = StateSpreadsheet.objects.most_recent_deployed(state, date)
            if recent_deploy:
                data = recent_deploy.get_total_data()

                if confirmed == data["confirmed"] and deaths == data["deaths"]:
                    debug(f"Skipping {state} because it has the same total for deaths and confirmed")
                    continue
                elif confirmed < data["confirmed"] or deaths < data["deaths"]:
                    if force and state in force:
                        debug(
                            f"WARNING: would skip {state} (already deployed for {date} and numbers of deployed are greater than ours: (ours vs deployed) {confirmed} vs {data['confirmed']}, {deaths} vs {data['deaths']}), but forcing because of --force"
                        )
                    else:
                        debug(
                            f"Skipping {state} (already deployed for {date} and numbers of deployed are greater than ours: (ours vs deployed) {confirmed} vs {data['confirmed']}, {deaths} vs {data['deaths']})"
                        )
                        continue

            debug(f"Creating spreadsheet for {state} on {date}")

            filename = f"/tmp/{state}-{date}.csv"
            with open(filename, mode="w") as fobj:
                writer = csv.writer(fobj)
                writer.writerow(["municipio", "confirmados", "obitos"])
                writer.writerow(["TOTAL NO ESTADO", str(confirmed), str(deaths)])

            with open(filename, mode="rb") as fobj:
                file_data = fobj.read()
                form = StateSpreadsheetForm(
                    {"date": date, "state": state, "boletim_urls": STATE_TOTALS_URL, "boletim_notes": NOTES,},
                    {"file": SimpleUploadedFile(filename, file_data),},
                    user=user,
                )
                form_valid = form.is_valid()
                if not form_valid:
                    debug(f"  Form is NOT valid for {state}! errors = {form.errors}")
                    continue
            os.unlink(filename)

            obj = form.save()
            StateSpreadsheet.objects.cancel_older_versions(obj)
            obj.link_to(obj)
            obj.import_to_final_dataset()
            obj.refresh_from_db()
            debug(f"  Spreadsheet created for {state}, id = {obj.id}")