def excute_keyword_task():
infos = KeywordDao.get_enable()
logger.info('本次搜索关键词个数:' + str(len(infos)))
for info in infos:
kw = info.name
logger.info('分发账号爬取任务:' + kw)
app.send_task('spider.task.keyword.keyword_task',
args=[kw],
queue='search_keyword_queue')
def excute_crawl_history_task():
infos = WeChatUserDao.get_enable_and_history_not_crawled()
logger.info('本次爬取账号个数:' + str(len(infos)))
for info in infos:
account = info.alias
if (account == None or account == ''):
account = info.nickname
logger.info('分发账号爬取任务:' + account)
app.send_task('spider.task.gather.history_task',
args=[account],
queue='crawl_history_queue')
def excute_login_task():
infos = LoginUserDao.get_enable()
print(infos)
# 每次登陆前清楚所有堆积的登录任务
Cookies.check_login_task()
logger.info('本轮模拟登陆开始')
for info in infos:
app.send_task('spider.task.login.login_task',
args=(info.name, info.password),
queue='login_queue')
# routing_key='for_login')
time.sleep(10)
def excute_user_crawl_task(search_url):
app.send_task('spider.task.keyword.user_list_crawl_task',
args=[search_url],
queue='user_list_crawl_queue')
def test_task(request):
res = app.send_task('spider.task.tasks.add', args=(2, 2), queue='add')
# res = add(2,2);
return HttpResponse(res.get(timeout=1))
def excute_wechat_crawl_task(url):
app.send_task('spider.task.wechat_crawl.wechat_crawl_task', args=[url], queue='wechat_crawl_queue')
def excute_wechat_user_crawl_task(wechat_biz):
app.send_task('spider.task.wechat_crawl.wechat_user_crawl_task', args=[wechat_biz],
queue='wechat_user_crawl_queue')
def excute_crawl_site_2level_page_task(site_id, site_name, url, deep):
app.send_task('spider.task.news_crawl.crawl_site_2level_page_task',
args=[site_id, site_name, url, deep],
queue='crawl_site_2level_page_queue')
def excute_crawl_news_site_task(site_id, site_name):
app.send_task('spider.task.news_crawl.crawl_site_task',
args=[site_id, site_name],
queue='crawl_news_site_queue')
def excute_send_remaind_email_task(email_type):
app.send_task('spider.task.send_email.send_remaind_email_task',
args=[email_type],
queue='send_remaind_email_queue')
def test_task(request):
res = app.send_task('spider.task.tasks.add', args=(2, 0), queue='add')
# res = add(2,2);
return HttpResponse("ssssss")