def update_provincial_breakdown_collection():
"""Update provincial breakdown"""
n_docs = 0
response = {"status": "ko", "regions": [], "updated": False, "errors": []}
updated, msg = False, ""
try:
df = load_df(URL_PROVINCIAL)
pattern = "|".join(PROVINCES)
df = df[df[PROVINCE_KEY].str.contains(pattern)]
df = preprocess_provincial_df(df)
breakdowns = build_provincial_breakdowns(df)
for b in breakdowns:
_filter = {REGION_KEY: b[REGION_KEY]}
update = {"$set": b}
res = prov_bdown_coll.update_one(_filter, update, upsert=True)
if res.modified_count:
n_docs += 1
response["regions"].append(b[REGION_KEY])
response["n_docs"] = n_docs
response["updated"] = res.acknowledged
updated = True
msg = f"Updated {n_docs} docs in {prov_bdown_coll.name}"
if not updated:
msg = f"Nothing to update in {prov_bdown_coll.name}"
app.logger.warning(msg)
response["status"], response["msg"] = "ok", msg
app.logger.warning(msg)
except Exception as e:
response["errors"].append(f"{e}")
app.logger.error(f"{e}")
return response
def update_provincial_collection():
"""Update provincial data collection"""
response = {"status": "ko", "updated": False, "errors": [], "msg": ""}
inserted_ids = []
try:
df = load_df(URL_PROVINCIAL)
df = preprocess_provincial_df(df)
latest_dt = df[DATE_KEY].max()
cursor = prov_data_coll.find().sort(DATE_KEY, -1).limit(1)
latest_dt_db = next(cursor)[DATE_KEY]
if latest_dt.date() == latest_dt_db.date():
msg = "DB up-to-date"
app.logger.warning(msg)
else:
msg = f"Latest data missing in {prov_data_coll.name}! Updating..."
app.logger.warning(msg)
df = df[df[DATE_KEY] > latest_dt_db]
new_records = df.to_dict(orient='records')
r = prov_data_coll.insert_many(new_records, ordered=True)
inserted_ids.extend(r.inserted_ids)
msg = f"{len(inserted_ids)} docs updated in {prov_data_coll.name}"
response["updated"] = True
response["status"], response["msg"] = "ok", msg
app.logger.warning(msg)
except Exception as e:
err = f"{e}"
app.logger.error(err)
response["errors"].append(err)
response["n_docs"] = len(inserted_ids)
return response
def create_provincial_trends_collection():
"""Create provincial trends data collection"""
df = pd.read_csv(URL_PROVINCIAL, parse_dates=[DATE_KEY])
df.drop(columns=COLUMNS_TO_DROP, inplace=True)
df_provincial_augmented = preprocess_provincial_df(df)
provincial_trends = build_provincial_trends(df_provincial_augmented)
try:
app.logger.info("Creating provincial trends collection")
prov_trends_coll.drop()
prov_trends_coll.insert_many(provincial_trends)
except Exception as e:
app.logger.error(e)
def create_provincial_collections():
"""Drop and recreate provincial collection"""
df = pd.read_csv(URL_PROVINCIAL, parse_dates=[DATE_KEY])
df.drop(columns=COLUMNS_TO_DROP, inplace=True)
df_provincial_augmented = preprocess_provincial_df(df)
provincial_records = df_provincial_augmented.to_dict(orient='records')
try:
app.logger.info("Creating provincial")
prov_data_coll.drop()
prov_data_coll.insert_many(provincial_records, ordered=True)
except Exception as e:
app.logger.error(e)
def update_provincial_series_or_trends_collection(coll_type):
"""Update provincial series or trends collection"""
n_docs = 0
response = {"status": "ko", "provs": [], "updated": False, "errors": []}
updated, msg = False, ""
df = load_df(URL_PROVINCIAL)
pattern = "|".join(PROVINCES)
df = df[df[PROVINCE_KEY].str.contains(pattern)]
df = preprocess_provincial_df(df)
if coll_type == "trends":
records = build_provincial_trends(df)
coll = prov_trends_coll
elif coll_type == "series":
records = build_provincial_series(df)
coll = prov_series_coll
else:
msg = "Invalid collection type"
app.logger.error(msg)
response["errors"].append(msg)
return response
try:
for r in records:
_filter = {PROVINCE_KEY: r[PROVINCE_KEY]}
update = {"$set": r}
results = coll.update_one(_filter, update, upsert=True)
if results.modified_count:
n_docs += 1
response["provs"].append(r[PROVINCE_KEY])
response["n_docs"] = n_docs
response["updated"] = results.acknowledged
updated = True
msg = f"Updated {n_docs} docs in {prov_bdown_coll.name}"
if not updated:
msg = f"Nothing to update in {prov_bdown_coll.name}"
app.logger.warning(msg)
response["status"], response["msg"] = "ok", msg
except Exception as e:
response["errors"].append(f"{e}")
app.logger.error(f"{e}")
return response