def seed_worker(): while True: seed_object = SEED_QUEUE.get() if seed_object is None: break root_url = seed_object.root_url try: seed_object.validate_required_values() res = get_seed_page(root_url) except Exception as e: logging.error(e) SEED_QUEUE.task_done() continue page_content = res.content seed_object.parse_seed_page_content(page_content) seed_object.add_news_to_extraction_queue() SEED_QUEUE.task_done()
recovered=None, posted_date=datetime.utcnow(), ): self.state = state self.country = country self.last_update = parser.parse(last_update) if last_update else last_update self.lat = lat self.lng = lng self.confirmed = confirmed self.deaths = deaths self.recovered = recovered self.posted_date = posted_date if __name__ == "__main__": db_bingcovid.connect() res = get_seed_page(API_URL).json() # whole world wholeWorld = BingCovid( confirmed=res["totalConfirmed"], deaths=res["totalDeaths"], recovered=res["totalRecovered"], ) logging.debug("Inserting whole_world data: {}".format(wholeWorld.__dict__)) db_bingcovid.insert(wholeWorld.__dict__, target_table=DB_TABLE) # Countries for countryData in res["areas"]: currentCountry = BingCovid( confirmed=countryData["totalConfirmed"], deaths=countryData["totalDeaths"],
def addInfo(self): if __name__ == "__main__": db_bingcovid.connect() res = get_seed_page(API_URL).json()