def update_provincial_breakdown_collection(): """Update provincial breakdown""" n_docs = 0 response = {"status": "ko", "regions": [], "updated": False, "errors": []} updated, msg = False, "" try: df = load_df(URL_PROVINCIAL) pattern = "|".join(PROVINCES) df = df[df[PROVINCE_KEY].str.contains(pattern)] df = preprocess_provincial_df(df) breakdowns = build_provincial_breakdowns(df) for b in breakdowns: _filter = {REGION_KEY: b[REGION_KEY]} update = {"$set": b} res = prov_bdown_coll.update_one(_filter, update, upsert=True) if res.modified_count: n_docs += 1 response["regions"].append(b[REGION_KEY]) response["n_docs"] = n_docs response["updated"] = res.acknowledged updated = True msg = f"Updated {n_docs} docs in {prov_bdown_coll.name}" if not updated: msg = f"Nothing to update in {prov_bdown_coll.name}" app.logger.warning(msg) response["status"], response["msg"] = "ok", msg app.logger.warning(msg) except Exception as e: response["errors"].append(f"{e}") app.logger.error(f"{e}") return response
def update_provincial_collection(): """Update provincial data collection""" response = {"status": "ko", "updated": False, "errors": [], "msg": ""} inserted_ids = [] try: df = load_df(URL_PROVINCIAL) df = preprocess_provincial_df(df) latest_dt = df[DATE_KEY].max() cursor = prov_data_coll.find().sort(DATE_KEY, -1).limit(1) latest_dt_db = next(cursor)[DATE_KEY] if latest_dt.date() == latest_dt_db.date(): msg = "DB up-to-date" app.logger.warning(msg) else: msg = f"Latest data missing in {prov_data_coll.name}! Updating..." app.logger.warning(msg) df = df[df[DATE_KEY] > latest_dt_db] new_records = df.to_dict(orient='records') r = prov_data_coll.insert_many(new_records, ordered=True) inserted_ids.extend(r.inserted_ids) msg = f"{len(inserted_ids)} docs updated in {prov_data_coll.name}" response["updated"] = True response["status"], response["msg"] = "ok", msg app.logger.warning(msg) except Exception as e: err = f"{e}" app.logger.error(err) response["errors"].append(err) response["n_docs"] = len(inserted_ids) return response
def create_provincial_trends_collection(): """Create provincial trends data collection""" df = pd.read_csv(URL_PROVINCIAL, parse_dates=[DATE_KEY]) df.drop(columns=COLUMNS_TO_DROP, inplace=True) df_provincial_augmented = preprocess_provincial_df(df) provincial_trends = build_provincial_trends(df_provincial_augmented) try: app.logger.info("Creating provincial trends collection") prov_trends_coll.drop() prov_trends_coll.insert_many(provincial_trends) except Exception as e: app.logger.error(e)
def create_provincial_collections(): """Drop and recreate provincial collection""" df = pd.read_csv(URL_PROVINCIAL, parse_dates=[DATE_KEY]) df.drop(columns=COLUMNS_TO_DROP, inplace=True) df_provincial_augmented = preprocess_provincial_df(df) provincial_records = df_provincial_augmented.to_dict(orient='records') try: app.logger.info("Creating provincial") prov_data_coll.drop() prov_data_coll.insert_many(provincial_records, ordered=True) except Exception as e: app.logger.error(e)
def update_provincial_series_or_trends_collection(coll_type): """Update provincial series or trends collection""" n_docs = 0 response = {"status": "ko", "provs": [], "updated": False, "errors": []} updated, msg = False, "" df = load_df(URL_PROVINCIAL) pattern = "|".join(PROVINCES) df = df[df[PROVINCE_KEY].str.contains(pattern)] df = preprocess_provincial_df(df) if coll_type == "trends": records = build_provincial_trends(df) coll = prov_trends_coll elif coll_type == "series": records = build_provincial_series(df) coll = prov_series_coll else: msg = "Invalid collection type" app.logger.error(msg) response["errors"].append(msg) return response try: for r in records: _filter = {PROVINCE_KEY: r[PROVINCE_KEY]} update = {"$set": r} results = coll.update_one(_filter, update, upsert=True) if results.modified_count: n_docs += 1 response["provs"].append(r[PROVINCE_KEY]) response["n_docs"] = n_docs response["updated"] = results.acknowledged updated = True msg = f"Updated {n_docs} docs in {prov_bdown_coll.name}" if not updated: msg = f"Nothing to update in {prov_bdown_coll.name}" app.logger.warning(msg) response["status"], response["msg"] = "ok", msg except Exception as e: response["errors"].append(f"{e}") app.logger.error(f"{e}") return response