Esempio n. 1
0
def mineChan(self, board, thread):
    try:
        db = ChanDB(celery.conf.DATABASE)
        url = "https://a.4cdn.org/"+str(board)+"/thread/"+str(thread)+".json"
        response = urlopen(url).read().decode('utf8')
        data = json.loads(response)

        for pp in data["posts"]:
            pp["mined_time"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
            pp["board_name"] = board
            pp["thread_id"] = thread
            pp["post_id"] = pp["no"]
            logger.info(pp)
            db.insert_post(pp)
        logger.info("{0} : Attempting to remove: {1} from MQ on mongo".format(datetime.now().strftime("%c"), str(thread)+":"+str(board)))
        db.remove_from_mq(str(thread)+":"+str(board))
    except Exception as e:
    #        self.retry(exc=e)
        logger.error(e)
        raise e