def main():
configs = load_configs()
conf_db = configs["database"]
conf_param = configs["params"]
crawler = Crawler()
db = MongoDb(conf_db["address"], conf_db["port"])
for stock_id in conf_param["stock_ids"]:
data = crawler.fetch_stock_data(stock_id, conf_param["period"])
db.append_one(data)
def run(self, db_addr, db_port):
while True:
try:
stock_id, period = self.__get_task__()
if stock_id == -1:
print("No task to do, will retry after 5 seconds")
time.sleep(5)
continue
db = MongoDb(db_addr, db_port)
crawler = Crawler()
data = crawler.fetch_stock_data(stock_id, period)
db.append_one(data)
self.__done_task__(stock_id)
except socket.error:
print("Master is offline, will retry after 5 seconds")
time.sleep(5)
