def update_national_trends_collection():
"""Update national trends collection"""
response = {"ids": [], "updated": False, "errors": []}
df = load_df(URL_NATIONAL)
df = preprocess_national_df(df)
n_docs = 0
for col in TREND_CARDS:
response["status"] = "ok"
try:
_filter = {'id': col}
trend = {"$set": build_trend(df, col)}
results = nat_trends_coll.update_one(_filter, trend, upsert=True)
if results.modified_count:
n_docs += 1
response["ids"].append(col)
except Exception as e:
response["status"] = "ko"
response["errors"].append(f"{e}")
app.logger.error(f"{e}")
continue
response["n_docs"] = n_docs
msg = f"{n_docs} docs updated in {nat_trends_coll.name}"
app.logger.warning(msg)
response["msg"] = msg
return response
def update_national_collection():
"""Update national collection"""
response = {"status": "ko", "n_inserted_docs": 0, "errors": []}
try:
df = load_df(URL_NATIONAL)
df = preprocess_national_df(df)
df['_id'] = df[DATE_KEY]
inserted_ids = []
records_in_db = list(nat_data_coll.find())
if records_in_db:
df_mongo = pd.DataFrame(records_in_db)
common = df.merge(df_mongo, on=[DATE_KEY])
df_to_db = df[(~df[DATE_KEY].isin(common[DATE_KEY]))]
if not df_to_db.empty:
new_records = df_to_db.to_dict(orient='records')
r = nat_data_coll.insert_many(new_records, ordered=True)
inserted_ids.extend(r.inserted_ids)
response["n_inserted_docs"] = len(inserted_ids)
response["inserted_ids"] = inserted_ids
else:
msg = f"Filling empty {nat_data_coll.name}"
app.logger.warning(msg)
r = nat_data_coll.insert_many(df.to_dict(orient='records'))
inserted_ids.extend(r.inserted_ids)
response["n_inserted_docs"] = len(inserted_ids)
response["msg"] = msg
msg = f"{len(inserted_ids)} docs updated in {nat_data_coll.name}"
app.logger.warning(msg)
response["status"] = "ok"
except Exception as e:
response["errors"], response["msg"] = f"{e}", f"{e}"
return response
def update_provincial_breakdown_collection():
"""Update provincial breakdown"""
n_docs = 0
response = {"status": "ko", "regions": [], "updated": False, "errors": []}
updated, msg = False, ""
try:
df = load_df(URL_PROVINCIAL)
pattern = "|".join(PROVINCES)
df = df[df[PROVINCE_KEY].str.contains(pattern)]
df = preprocess_provincial_df(df)
breakdowns = build_provincial_breakdowns(df)
for b in breakdowns:
_filter = {REGION_KEY: b[REGION_KEY]}
update = {"$set": b}
res = prov_bdown_coll.update_one(_filter, update, upsert=True)
if res.modified_count:
n_docs += 1
response["regions"].append(b[REGION_KEY])
response["n_docs"] = n_docs
response["updated"] = res.acknowledged
updated = True
msg = f"Updated {n_docs} docs in {prov_bdown_coll.name}"
if not updated:
msg = f"Nothing to update in {prov_bdown_coll.name}"
app.logger.warning(msg)
response["status"], response["msg"] = "ok", msg
app.logger.warning(msg)
except Exception as e:
response["errors"].append(f"{e}")
app.logger.error(f"{e}")
return response
def update_provincial_collection():
"""Update provincial data collection"""
response = {"status": "ko", "updated": False, "errors": [], "msg": ""}
inserted_ids = []
try:
df = load_df(URL_PROVINCIAL)
df = preprocess_provincial_df(df)
latest_dt = df[DATE_KEY].max()
cursor = prov_data_coll.find().sort(DATE_KEY, -1).limit(1)
latest_dt_db = next(cursor)[DATE_KEY]
if latest_dt.date() == latest_dt_db.date():
msg = "DB up-to-date"
app.logger.warning(msg)
else:
msg = f"Latest data missing in {prov_data_coll.name}! Updating..."
app.logger.warning(msg)
df = df[df[DATE_KEY] > latest_dt_db]
new_records = df.to_dict(orient='records')
r = prov_data_coll.insert_many(new_records, ordered=True)
inserted_ids.extend(r.inserted_ids)
msg = f"{len(inserted_ids)} docs updated in {prov_data_coll.name}"
response["updated"] = True
response["status"], response["msg"] = "ok", msg
app.logger.warning(msg)
except Exception as e:
err = f"{e}"
app.logger.error(err)
response["errors"].append(err)
response["n_docs"] = len(inserted_ids)
return response
def update_regional_breakdown_collection():
"""Update regional breakdown"""
response = {"status": "ko", "updated": False, "errors": []}
try:
df = load_df(URL_REGIONAL)
df = preprocess_regional_df(df)
breakdown = build_regional_breakdown(df)
try:
doc = reg_bdown_coll.find().next()
mongo_id = doc["_id"]
_filter = {"_id": mongo_id}
update = {"$set": breakdown}
res = reg_bdown_coll.update_one(_filter, update, upsert=True)
msg = f"Updated regional breakdown in {reg_bdown_coll.name}"
response["updated"], response["msg"] = res.acknowledged, msg
response["status"] = "ok"
app.logger.warning(msg)
except StopIteration as e:
msg = f"{e}"
app.logger.error(msg)
response["msg"] = msg
except Exception as e:
response["errors"].append(f"{e}")
app.logger.error(f"{e}")
return response
def update_regional_trends_collection():
"""Update regional trends collection"""
n_docs = 0
response = {"status": "ko", "regions": [], "updated": False, "errors": []}
try:
df = load_df(URL_REGIONAL)
df = preprocess_regional_df(df)
for r in REGIONS:
_filter = {REGION_KEY: r}
update = {
"$set": {
REGION_KEY: r,
"trends": build_national_trends(df[df[REGION_KEY] == r])
}
}
results = reg_trends_coll.update_one(_filter, update, upsert=True)
if results.modified_count:
n_docs += 1
response["regions"].append(r)
response["n_docs"] = n_docs
response["updated"] = results.acknowledged
msg = f"{n_docs} docs updated in {reg_trends_coll.name}"
response["msg"] = msg
if not response["updated"]:
msg = f"Nothing to update in {reg_trends_coll.name}"
response["msg"] = msg
response["status"] = "ok"
app.logger.warning(msg)
except Exception as e:
response["errors"].append(f"{e}")
app.logger.error(f"{e}")
return response
def update_provincial_series_or_trends_collection(coll_type):
"""Update provincial series or trends collection"""
n_docs = 0
response = {"status": "ko", "provs": [], "updated": False, "errors": []}
updated, msg = False, ""
df = load_df(URL_PROVINCIAL)
pattern = "|".join(PROVINCES)
df = df[df[PROVINCE_KEY].str.contains(pattern)]
df = preprocess_provincial_df(df)
if coll_type == "trends":
records = build_provincial_trends(df)
coll = prov_trends_coll
elif coll_type == "series":
records = build_provincial_series(df)
coll = prov_series_coll
else:
msg = "Invalid collection type"
app.logger.error(msg)
response["errors"].append(msg)
return response
try:
for r in records:
_filter = {PROVINCE_KEY: r[PROVINCE_KEY]}
update = {"$set": r}
results = coll.update_one(_filter, update, upsert=True)
if results.modified_count:
n_docs += 1
response["provs"].append(r[PROVINCE_KEY])
response["n_docs"] = n_docs
response["updated"] = results.acknowledged
updated = True
msg = f"Updated {n_docs} docs in {prov_bdown_coll.name}"
if not updated:
msg = f"Nothing to update in {prov_bdown_coll.name}"
app.logger.warning(msg)
response["status"], response["msg"] = "ok", msg
except Exception as e:
response["errors"].append(f"{e}")
app.logger.error(f"{e}")
return response
def update_national_series_collection():
"""Update national series collection"""
response = {"status": "ko", "updated": False, "errors": []}
df = load_df(URL_NATIONAL)
df = preprocess_national_df(df)
df['_id'] = df[DATE_KEY]
national_series = build_national_series(df)
cursor = nat_series_coll.find({})
try:
doc = cursor.next()
mongo_id = doc['_id']
_filter, update = {'_id': mongo_id}, {"$set": national_series}
r = nat_series_coll.update_one(_filter, update, upsert=True)
msg = f"Updated {nat_series_coll.name}"
app.logger.warning(msg)
response["status"], response["updated"] = "ok", r.acknowledged
except StopIteration:
r = nat_series_coll.insert_one(national_series)
msg = f"Filled empty collection {nat_series_coll.name}"
response["status"], response["updated"] = "ok", r.acknowledged
response["msg"] = msg
return response