def setUp(self): keys = redis_instance.keys('*print_numbers*') if keys: redis_instance.delete(*keys) keys = redis_instance.keys('*return_what_you_put*') if keys: redis_instance.delete( *redis_instance.keys('*return_what_you_put*'))
def tearDown(self): keys = redis_instance.keys('*print_numbers*') if keys: redis_instance.delete(*redis_instance.keys('*print_numbers*')) keys = redis_instance.keys('*return_what_you_put*') if keys: redis_instance.delete( *redis_instance.keys('*return_what_you_put*'))
def setUp(self): keys = redis_instance.keys('*incr_then_return_test_number*') if keys: redis_instance.delete(*keys) keys = redis_instance.keys('*return_what_you_put*') if keys: redis_instance.delete( *redis_instance.keys('*return_what_you_put*'))
def crawl_lagou_data_task(): """爬取拉勾数据任务""" # 清除抓取记录 keys = redis_instance.keys('crawled_company_jobs*') if keys: redis_instance.delete(*keys) crawl_lagou_city_data_task.delay() # 目前只抓取这几个城市 全国:0, 北京:2 上海:3 杭州:6 深圳:215 广州:213 成都:252 # lagou_city_ids = [2, 3, 6, 215, 213, 252] lagou_city_ids = [0] # lagou_finance_stage_ids = [1, 2, 3, 4, 5, 6, 7, 8] lagou_finance_stage_ids = [0] # lagou_industry_ids = [24, 25, 33, 27, 29, 45, 31, 28, 47, 34, 35, 43, 32, 41, 26, 48, 38, 49, 10594] lagou_industry_ids = [0] # 爬取公司数据 for industry_id in lagou_industry_ids: for city_id in lagou_city_ids: for finance_stage_id in lagou_finance_stage_ids: crawl_lagou_company_data_task.delay(city_id=city_id, finance_stage_id=finance_stage_id, industry_id=industry_id)