def run(self, uids):
params = {"user_id": ",".join(uids)}
data = self.post(params)
if self.error_occurs(data):
for uid in uids:
self.db.set_profile_crawled(uid, 2)
return
self.output(data, uids)
crawled_user_count = len(data)
logging.info("Got %d users", crawled_user_count)
logging.basicConfig(
level=logging.DEBUG, format="%(asctime)s %(levelname)s %(message)s", filename="./log/user_crawler.log", filemode="w"
)
db = DB(logging)
crawler = Crawler(db, logging)
while True:
uids = db.get_crawling_target_user_ids(100)
if len(uids) == 0:
logging.info("All users in DB are crwaled, sleep 1 hour")
time.sleep(60 * 60)
else:
crawler.run(uids)
