def do_question(question_id): if not CommonOper.is_exist("question_id", question_id): question_url = question_url_format.format(question_id) jsl_log.info("开始爬取url:{}".format(question_url)) crawl_question_and_answer(question_url) else: jsl_log.info("question id:{}已存在,跳过".format(question_id))
def do_people(people_id): if not CommonOper.is_exist("people_id", people_id): people_url = people_url_format.format(people_id) jsl_log.info("开始爬取url:{}".format(people_url)) crawl_people(people_url) else: jsl_log.info("people id:{}已存在,跳过".format(people_id))
def task_filter(task_type, param): if task_type == 'question': if not CommonOper.is_exist("question_id", param): app.send_task('tasks.question.do_question', args=(param, ), queue='question_queue', routing_key='question') else: jsl_log.info("相关question已存在,question_id:{}".format(param)) elif task_type == 'people': if not CommonOper.is_exist("people_id", param): app.send_task('tasks.people.do_people', args=(param, ), queue='people_queue', routing_key='people') else: jsl_log.info("相关people已存在,people_id:{}".format(param))
def do_answer_comment(comment_id): answer_comment_url = answer_comment_url_format.format(comment_id) jsl_log.info("开始爬取url:{}".format(answer_comment_url)) crawl_answer_comment(answer_comment_url)
def session_commit(*args, **kwargs): try: func(*args, **kwargs) except Exception as e: jsl_log.info("DB operation error,here are details :{}".format(e)) db_session.rollback()
def do_follow(follower_id, page_num): follows_url = follows_url_format.format(follower_id, page_num) jsl_log.info("开始爬取url:{}".format(follows_url)) crawl_follows(follows_url)