class ChanMinner: def __init__(self): self.run = True self.db = ChanDB(celeryconfig.DATABASE) self.logger = logging.getLogger('4 Chan Minner') self.logger.info('Starting 4Chan manner') def start(self): try: url = "https://a.4cdn.org/boards.json" response = urlopen(url).read().decode('utf8') data = json.loads(response) data["mined_time"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") self.db.insert_home(data) while self.run: for b in data["boards"]: time.sleep(10) self.logger.info(b["board"]) board = basc_py4chan.Board(b["board"]) response = urlopen("https://a.4cdn.org/"+b["board"]+"/threads.json").read().decode('utf8') boarddata = {"threads":json.loads(response)} boarddata["mined_time"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") boarddata["board_name"] = b["board"] self.db.insert_board(boarddata) for tid in board.get_all_thread_ids(): if self.db.is_in_mq(str(tid)+":"+str(b["board"])) == False: self.db.add_to_mq(str(tid)+":"+str(b["board"])) mineChan.delay(b["board"], tid) time.sleep(10) except Exception as e: self.logger.error(e) def stop(self): self.run = False