def handle(self, *args, **kwargs):
truncate = kwargs.get("truncate", False)
update_functions = [
(Dataset, dataset_update_data),
(Link, link_update_data),
(Version, version_update_data),
(Table, table_update_data),
(Field, field_update_data),
]
if truncate:
print("Deleting metadata to create new objects...")
for Model, _ in update_functions:
Model.objects.all().delete()
else:
print("WARNING: updating data only. If some field was removed "
"this change will not be reflected on your database. "
"Consider using --truncate")
self.datasets, self.tables, self.versions = {}, {}, {}
response_data = http_get(settings.DATA_URL, 5)
if response_data is None:
raise RuntimeError(f"Cannot download {settings.DATA_URL}")
for Model, update_data_function in update_functions:
table = rows.import_from_xlsx(io.BytesIO(response_data),
sheet_name=Model.__name__,
workbook_kwargs={"read_only": False})
self._update_data(Model, table, update_data_function)
def handle(self, *args, **kwargs):
truncate = kwargs.get("truncate", False)
update_functions = [
(Dataset, dataset_update_data),
(Link, link_update_data),
(Version, version_update_data),
(Table, table_update_data),
(Field, field_update_data),
]
data_tables_map = {}
for table in Table.with_hidden.all():
key = (table.dataset.slug, table.name)
data_tables_map[key] = table.data_table
if truncate:
print("Deleting metadata to create new objects...")
for Model, _ in update_functions:
Model.objects.all().delete()
else:
print("WARNING: updating data only. If some field was removed "
"this change will not be reflected on your database. "
"Consider using --truncate")
self.datasets, self.tables, self.versions = {}, {}, {}
response_data = http_get(settings.DATA_URL, 5)
if response_data is None:
raise RuntimeError(f"Cannot download {settings.DATA_URL}")
for Model, update_data_function in update_functions:
table = rows.import_from_xlsx(io.BytesIO(response_data),
sheet_name=Model.__name__,
workbook_kwargs={"read_only": False})
self._update_data(Model, table, update_data_function)
print("Updating DataTable...", end="", flush=True)
total_created, total_updated, total_skipped = 0, 0, 0
for table in Table.with_hidden.select_related("dataset"):
key = (table.dataset.slug, table.name)
data_table = data_tables_map.get(key, None)
if data_table is None: # create DataTable if new Table or if previous was None
data_table = DataTable.new_data_table(table, suffix_size=0)
data_table.activate()
total_created += 1
elif data_table.table != table: # Tables were truncated so previous DataTables get updated
total_updated += 1
data_table.table = table
data_table.save()
else: # Same table as before, so no need to update
total_skipped += 1
if table.filtering_fields: # avoid None
table.fields.filter(name__in=table.filtering_fields).update(
frontend_filter=True)
if table.search_fields:
table.fields.filter(name__in=table.search_fields).update(
searchable=True)
print(" created: {}, updated: {}, skipped: {}.".format(
total_created, total_updated, total_skipped))
def get_state_data_from_google_spreadsheets(state, timeout=5):
state_spreadsheet_url = import_info_by_state(state).planilha_brasilio
state_spreadsheet_download_url = spreadsheet_download_url(
state_spreadsheet_url, "xlsx")
data = http_get(state_spreadsheet_download_url, timeout)
reports = rows.import_from_xlsx(io.BytesIO(data),
sheet_name=BOLETIM_SPREADSHEET,
force_types=FIELDS_BOLETIM)
cases = rows.import_from_xlsx(io.BytesIO(data),
sheet_name=CASOS_SPREADSHEET)
return {
"reports": [dict(row._asdict()) for row in reports if row.date],
"cases": [dict(row._asdict()) for row in cases if row.municipio],
}
def get_general_spreadsheet(timeout=5):
data = http_get(
spreadsheet_download_url(STATE_LINKS_SPREADSHEET_ID, "csv"), timeout)
table = rows.import_from_csv(io.BytesIO(data), encoding="utf-8")
return {row.uf: row._asdict() for row in table}
def get_base_data():
result = http_get("https://data.brasil.io/meta/covid19-base-data.json.gz",
timeout=5)
json_data = gzip.decompress(result)
data = json.loads(json_data)
return data