def mineChan(self, board, thread): try: db = ChanDB(celery.conf.DATABASE) url = "https://a.4cdn.org/"+str(board)+"/thread/"+str(thread)+".json" response = urlopen(url).read().decode('utf8') data = json.loads(response) for pp in data["posts"]: pp["mined_time"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") pp["board_name"] = board pp["thread_id"] = thread pp["post_id"] = pp["no"] logger.info(pp) db.insert_post(pp) logger.info("{0} : Attempting to remove: {1} from MQ on mongo".format(datetime.now().strftime("%c"), str(thread)+":"+str(board))) db.remove_from_mq(str(thread)+":"+str(board)) except Exception as e: # self.retry(exc=e) logger.error(e) raise e