def update_national_trends_collection():
"""Update national trends collection"""
response = {"ids": [], "updated": False, "errors": []}
df = load_df(URL_NATIONAL)
df = preprocess_national_df(df)
n_docs = 0
for col in TREND_CARDS:
response["status"] = "ok"
try:
_filter = {'id': col}
trend = {"$set": build_trend(df, col)}
results = nat_trends_coll.update_one(_filter, trend, upsert=True)
if results.modified_count:
n_docs += 1
response["ids"].append(col)
except Exception as e:
response["status"] = "ko"
response["errors"].append(f"{e}")
app.logger.error(f"{e}")
continue
response["n_docs"] = n_docs
msg = f"{n_docs} docs updated in {nat_trends_coll.name}"
app.logger.warning(msg)
response["msg"] = msg
return response
def update_national_collection():
"""Update national collection"""
response = {"status": "ko", "n_inserted_docs": 0, "errors": []}
try:
df = load_df(URL_NATIONAL)
df = preprocess_national_df(df)
df['_id'] = df[DATE_KEY]
inserted_ids = []
records_in_db = list(nat_data_coll.find())
if records_in_db:
df_mongo = pd.DataFrame(records_in_db)
common = df.merge(df_mongo, on=[DATE_KEY])
df_to_db = df[(~df[DATE_KEY].isin(common[DATE_KEY]))]
if not df_to_db.empty:
new_records = df_to_db.to_dict(orient='records')
r = nat_data_coll.insert_many(new_records, ordered=True)
inserted_ids.extend(r.inserted_ids)
response["n_inserted_docs"] = len(inserted_ids)
response["inserted_ids"] = inserted_ids
else:
msg = f"Filling empty {nat_data_coll.name}"
app.logger.warning(msg)
r = nat_data_coll.insert_many(df.to_dict(orient='records'))
inserted_ids.extend(r.inserted_ids)
response["n_inserted_docs"] = len(inserted_ids)
response["msg"] = msg
msg = f"{len(inserted_ids)} docs updated in {nat_data_coll.name}"
app.logger.warning(msg)
response["status"] = "ok"
except Exception as e:
response["errors"], response["msg"] = f"{e}", f"{e}"
return response
def create_national_series_collection():
"""Drop and recreate national series data collection"""
df = pd.read_csv(URL_NATIONAL, parse_dates=[DATE_KEY])
df.drop(columns=COLUMNS_TO_DROP, inplace=True)
df_national_augmented = preprocess_national_df(df)
national_series = build_national_series(df_national_augmented)
try:
app.logger.info("Creating national series collection")
nat_series_coll.drop()
nat_series_coll.insert_one(national_series)
except Exception as e:
app.logger.error(e)
def create_national_collection():
"""Drop and recreate national data collection"""
df = pd.read_csv(URL_NATIONAL, parse_dates=[DATE_KEY])
df.drop(columns=COLUMNS_TO_DROP, inplace=True)
df_national_augmented = preprocess_national_df(df)
national_records = df_national_augmented.to_dict(orient='records')
try:
app.logger.info("Creating national collection")
nat_data_coll.drop()
nat_data_coll.insert_many(national_records, ordered=True)
except Exception as e:
app.logger.error(e)
def update_national_series_collection():
"""Update national series collection"""
response = {"status": "ko", "updated": False, "errors": []}
df = load_df(URL_NATIONAL)
df = preprocess_national_df(df)
df['_id'] = df[DATE_KEY]
national_series = build_national_series(df)
cursor = nat_series_coll.find({})
try:
doc = cursor.next()
mongo_id = doc['_id']
_filter, update = {'_id': mongo_id}, {"$set": national_series}
r = nat_series_coll.update_one(_filter, update, upsert=True)
msg = f"Updated {nat_series_coll.name}"
app.logger.warning(msg)
response["status"], response["updated"] = "ok", r.acknowledged
except StopIteration:
r = nat_series_coll.insert_one(national_series)
msg = f"Filled empty collection {nat_series_coll.name}"
response["status"], response["updated"] = "ok", r.acknowledged
response["msg"] = msg
return response