def main(): configs = load_configs() conf_db = configs["database"] conf_param = configs["params"] crawler = Crawler() db = MongoDb(conf_db["address"], conf_db["port"]) for stock_id in conf_param["stock_ids"]: data = crawler.fetch_stock_data(stock_id, conf_param["period"]) db.append_one(data)
def run(self, db_addr, db_port): while True: try: stock_id, period = self.__get_task__() if stock_id == -1: print("No task to do, will retry after 5 seconds") time.sleep(5) continue db = MongoDb(db_addr, db_port) crawler = Crawler() data = crawler.fetch_stock_data(stock_id, period) db.append_one(data) self.__done_task__(stock_id) except socket.error: print("Master is offline, will retry after 5 seconds") time.sleep(5)