class Scheduler(object):
def __init__(self):
self.crawler = Crawler()
self.db = Mongo(MONGO_DB)
def main(self):
"""
程序主逻辑函数
:return:
"""
self.db.add(MONGO_COLLECTION_URL, {'url': START_URL})
while self.db.count(MONGO_COLLECTION_URL) > 0:
url = self.db.remove_one(MONGO_COLLECTION_URL)['url']
userinfo, new_urls = self.crawler.main(url)
if userinfo or new_urls:
self.db.add(MONGO_COLLECTION_USERINFO, userinfo)
for new_url in new_urls:
self.db.add(MONGO_COLLECTION_URL, {'url': new_url})
else:
self.db.add(MONGO_COLLECTION_USERINFO, {
'user_url': url,
'declare': '该账户可能已经注销'
})
