def do_question(question_id):
if not CommonOper.is_exist("question_id", question_id):
question_url = question_url_format.format(question_id)
jsl_log.info("开始爬取url:{}".format(question_url))
crawl_question_and_answer(question_url)
else:
jsl_log.info("question id:{}已存在，跳过".format(question_id))
def do_people(people_id):
if not CommonOper.is_exist("people_id", people_id):
people_url = people_url_format.format(people_id)
jsl_log.info("开始爬取url:{}".format(people_url))
crawl_people(people_url)
else:
jsl_log.info("people id:{}已存在，跳过".format(people_id))
def task_filter(task_type, param):
if task_type == 'question':
if not CommonOper.is_exist("question_id", param):
app.send_task('tasks.question.do_question',
args=(param, ),
queue='question_queue',
routing_key='question')
else:
jsl_log.info("相关question已存在,question_id:{}".format(param))
elif task_type == 'people':
if not CommonOper.is_exist("people_id", param):
app.send_task('tasks.people.do_people',
args=(param, ),
queue='people_queue',
routing_key='people')
else:
jsl_log.info("相关people已存在,people_id:{}".format(param))
def do_answer_comment(comment_id):
answer_comment_url = answer_comment_url_format.format(comment_id)
jsl_log.info("开始爬取url:{}".format(answer_comment_url))
crawl_answer_comment(answer_comment_url)
def session_commit(*args, **kwargs):
try:
func(*args, **kwargs)
except Exception as e:
jsl_log.info("DB operation error,here are details :{}".format(e))
db_session.rollback()
def do_follow(follower_id, page_num):
follows_url = follows_url_format.format(follower_id, page_num)
jsl_log.info("开始爬取url:{}".format(follows_url))
crawl_follows(follows_url)