def main():
print("Importing data into db")
Base.metadata.create_all(engine)
db_instance = SessionLocal()
with open('02-29-2020.csv') as csv_file:
csv_reader = csv.reader(csv_file, delimiter=',')
next(csv_reader, None) # skip the headers
for row in csv_reader:
dr = DailyReport(
province_state=row[0],
country_region=row[1],
last_update=datetime.fromisoformat(row[2]),
confirmed=row[3],
deaths=row[4],
recovered=row[5],
)
db_instance.add(dr)
db_instance.commit()
def import_daily_report(report):
db_instance = SessionLocal()
to_deduplicate = defaultdict(list)
for row in report:
last_update_str = row.get('Last Update') or row['Last_Update']
last_update = datetime.fromisoformat(last_update_str)
# Region identifiers
country_region = row.get('Country_Region') or row['Country/Region']
province_state = row.get('Province_State') or row.get('Province/State')
admin2 = row.get('Admin2')
fips = row.get('fips')
province_state = clean_optional_field(province_state)
fips = clean_optional_field(fips)
admin2 = clean_admin2(admin2)
# Measures
confirmed = int(row['Confirmed'])
deaths = int(row['Deaths'])
recovered = int(row['Recovered'])
try:
dr = get_daily_report_by_region_and_date(
db=db_instance,
province_state=province_state,
country_region=country_region,
admin2=admin2,
fips=fips,
last_update=last_update)
except NoResultFound:
dr = DailyReport(
province_state=province_state,
country_region=country_region,
admin2=admin2,
fips=fips,
last_update=last_update,
confirmed=confirmed,
deaths=deaths,
recovered=recovered,
)
except Exception:
print(f'Invalid row: {row!r}')
raise
else:
if dr.confirmed > confirmed or dr.deaths > deaths or dr.recovered > recovered:
print(f'Warning: NOT lowering estimates for {dr!r}')
print(
f'New data: confirmed={confirmed}, deaths={deaths}, recovered={recovered}'
)
continue
else:
dr.confirmed = confirmed
dr.deaths = deaths
dr.recovered = recovered
db_instance.add(dr)
# Track potentially duplicated admin2s
if dr.admin2 in DUPLICATE_ADMIN2.values():
to_deduplicate[dr.admin2].append(dr)
deduplicate(db_instance, to_deduplicate)
db_instance.flush()
sanity_check(db_instance)
db_instance.commit()