def save_daily_report(self, statics_reports):
statics_reports = StaticsDomain.to_list_dict(statics_reports)
self.statics_repo.insert_many(statics_reports)
logger.debug(
f"statics_reporting done, len_values:{len(statics_reports)} saved_at: {datetime.now()}"
)
return "Done"
def crawl_data(country, link):
case_repo = CaseReportRepository()
death_repo = DeathReportRepository()
crawl_daily_graph = CrawlDailyStatics(link)
result = crawl_daily_graph.execute()
logger.debug(f"country : {country}")
# check cases statistics and save
if result.death_daily is not None and len(result.death_daily) > 0:
items = [_rebuild_item(item, country) for item in result.death_daily]
death_repo.insert_many(items=items)
# check cases statistics and save
if result.case_daily is not None and len(result.case_daily) > 0:
items = [_rebuild_item(item, country) for item in result.case_daily]
case_repo.insert_many(items=items)
return None
def _update_death_daily_db(self, country_report):
if country_report.new_deaths in ["", None, " "]:
return None
last_d_d = self.death_repo.get_last_report(country_report.country)
new_death_cr = NewDeathDomain.from_dict(country_report.to_dict())
new_death_cr.reported_at = new_death_cr.created_at
logger.debug(new_death_cr.country, new_death_cr.reported_at,
new_death_cr.created_at)
if last_d_d is not None and len(last_d_d) > 0:
last_d_d = NewDeathDomain.from_dict(last_d_d[0])
if self._is_need_update_last_report(last_d_d,
new_death_cr.new_deaths,
new_death_cr.reported_at):
return self.death_repo.update_one_value({"_id": last_d_d._id},
new_death_cr.to_dict())
return self.death_repo.insert_one(new_death_cr.to_dict())
def crawl_daily_report(self, country_links: dict):
"""
crawl history of statics from 'Daily Deaths' and 'Daily New Cases' graph.
And save in database. in separated collections.
:param
country_links: dictionary of name of country and links that was crawl by _crawl_countries function.
"""
counter = 0
for country, link in country_links.items():
crawl_daily_graph = self.crawl_graph(link)
result = crawl_daily_graph.execute()
len_death_daily = len_case_daily = 0
if result.death_daily is not None and len(result.death_daily) > 0:
len_death_daily = len(result.death_daily)
items = [
self._rebuild_item(item, country)
for item in result.death_daily
]
self.death_repo.insert_many(items=items)
if result.case_daily is not None and len(result.case_daily) > 0:
len_case_daily = len(result.case_daily)
items = [
self._rebuild_item(item, country)
for item in result.case_daily
]
self.case_repo.insert_many(items=items)
counter += 1
logger.debug(f"{counter},{country}__"
f"D:{len_death_daily},C:{len_case_daily}"
f" saved_at: {datetime.now()}")
return "Done"
def update_daily_db(self, statics_report):
for static_report in statics_report:
self._update_new_case_daily_db(static_report)
self._update_death_daily_db(static_report)
logger.debug(f"update daily:{static_report.country}")
return "done"