def crawl_lagou_company_data_task(city_id, finance_stage_id, industry_id):
"""爬取拉勾公司数据任务"""
companies_pagination = crawlers.get_companies_pagination_from_lagou(city_id=city_id,
finance_stage_id=finance_stage_id,
industry_id=industry_id)
for page_no in companies_pagination.iter_pages:
company_dicts = crawlers.get_companies_from_lagou(city_id=city_id,
finance_stage_id=finance_stage_id,
industry_id=industry_id,
page_no=page_no)
if not company_dicts:
break
for company_dict in company_dicts:
crawlers.clean_lagou_company_data(company_dict)
utils.convert.convert_dict_field_to_constants(company_dict)
industries = company_dict.pop('industries')
city_name = company_dict.pop('city_name')
city_ctl.insert_city_if_not_exist(city_name)
company_dict['city_id'] = city_ctl.get_city_id_by_name(city_name)
company = CompanyModel.get_one(filter_by={'lagou_company_id': company_dict.lagou_company_id})
if company:
CompanyModel.update_by_pk(pk=company.id, values=company_dict)
else:
company_id = CompanyModel.add(**company_dict)
for industry in industries:
industry_ctl.insert_industry_if_not_exist(name=industry)
industry_id = industry_ctl.get_industry_id_by_name(name=industry)
CompanyIndustryModel.add(industry_id=industry_id, company_id=company_id)
crawl_lagou_job_data_task.delay(company_dict.lagou_company_id)
def crawl_lagou_job_data_task(lagou_company_id):
"""爬取拉勾职位数据任务"""
# 过滤本轮已经爬取过职位的公司
if not redis_instance.setnx(constants.CRAWLED_COMPANY_JOBS_REDIS_KEY.format(lagou_company_id=lagou_company_id), 1):
return
jobs_pagination = crawlers.get_jobs_pagination_from_lagou(lagou_company_id=lagou_company_id,
job_type=constants.LagouJobType.technology)
for page_no in jobs_pagination.iter_pages:
job_dicts = crawlers.get_jobs_from_lagou(lagou_company_id=lagou_company_id,
job_type=constants.LagouJobType.technology,
page_no=page_no)
if not job_dicts:
break
for job_dict in job_dicts:
crawlers.clean_lagou_job_data(job_dict)
utils.convert.convert_dict_field_to_constants(job_dict)
keywords = job_dict.pop('keywords')
city_name = job_dict.pop('city_name')
city_ctl.insert_city_if_not_exist(city_name)
job_dict['city_id'] = city_ctl.get_city_id_by_name(city_name)
company = CompanyModel.get_one(filter_by={'lagou_company_id': lagou_company_id})
job_dict['company_id'] = company.id
job = JobModel.get_one(filter_by={'lagou_job_id': job_dict.lagou_job_id})
if job:
JobModel.update_by_pk(pk=job.id, values=job_dict)
else:
job_id = JobModel.add(**job_dict)
for keyword in keywords:
keyword_ctl.insert_keyword_if_not_exist(name=keyword)
keyword_id = keyword_ctl.get_keyword_id_by_name(name=keyword)
JobKeywordModel.add(keyword_id=keyword_id, job_id=job_id)
def convert_lagou_job_data(job_dict):
if job_dict.nature not in constants.JOB_NATURE_DICT:
logger.error(
'{} not in constants.JOB_NATURE_DICT, lagou job id is {}'.format(
job_dict.nature, job_dict.lagou_job_id))
job_dict.nature = constants.JOB_NATURE_DICT.get(
job_dict.nature, constants.JOB_NATURE_DICT['unknown'])
if job_dict.work_year not in constants.WORK_YEARS_REQUEST_DICT:
logger.error(
'{} not in constants.WORK_YEARS_REQUEST_DICT, lagou job id is {}'.
format(job_dict.work_year, job_dict.lagou_job_id))
job_dict.work_year = constants.WORK_YEARS_REQUEST_DICT.get(
job_dict.work_year, constants.WORK_YEARS_REQUEST_DICT['unknown'])
if job_dict.education not in constants.EDUCATION_REQUEST_DICT:
logger.error(
'{} not in constants.EDUCATION_REQUEST_DICT, lagou job id is {}'.
format(job_dict.education, job_dict.lagou_job_id))
job_dict.education = constants.EDUCATION_REQUEST_DICT.get(
job_dict.education, constants.EDUCATION_REQUEST_DICT['unknown'])
city_ctl.insert_city_if_not_exist(job_dict.city)
job_dict.city_id = city_ctl.get_city_id_by_name(job_dict.city)
def test_get_city_id_by_name(self):
city_id = city_ctl.get_city_id_by_name(name='北京')
self.assertEqual(city_id, 2)
with self.assertRaises(ValueError):
city_ctl.get_city_id_by_name(name='通利福尼亚')
