def do_question(question_id):
if not CommonOper.is_exist("question_id", question_id):
question_url = question_url_format.format(question_id)
jsl_log.info("开始爬取url:{}".format(question_url))
crawl_question_and_answer(question_url)
else:
jsl_log.info("question id:{}已存在，跳过".format(question_id))
def do_people(people_id):
if not CommonOper.is_exist("people_id", people_id):
people_url = people_url_format.format(people_id)
jsl_log.info("开始爬取url:{}".format(people_url))
crawl_people(people_url)
else:
jsl_log.info("people id:{}已存在，跳过".format(people_id))
def task_filter(task_type, param):
if task_type == 'question':
if not CommonOper.is_exist("question_id", param):
app.send_task('tasks.question.do_question',
args=(param, ),
queue='question_queue',
routing_key='question')
else:
jsl_log.info("相关question已存在,question_id:{}".format(param))
elif task_type == 'people':
if not CommonOper.is_exist("people_id", param):
app.send_task('tasks.people.do_people',
args=(param, ),
queue='people_queue',
routing_key='people')
else:
jsl_log.info("相关people已存在,people_id:{}".format(param))