def crawl_lagou_company_data_task(city_id, finance_stage_id, industry_id):
"""爬取拉勾公司数据任务"""
companies_pagination = crawlers.get_companies_pagination_from_lagou(city_id=city_id,
finance_stage_id=finance_stage_id,
industry_id=industry_id)
for page_no in companies_pagination.iter_pages:
company_dicts = crawlers.get_companies_from_lagou(city_id=city_id,
finance_stage_id=finance_stage_id,
industry_id=industry_id,
page_no=page_no)
if not company_dicts:
break
for company_dict in company_dicts:
crawlers.clean_lagou_company_data(company_dict)
utils.convert.convert_dict_field_to_constants(company_dict)
industries = company_dict.pop('industries')
city_name = company_dict.pop('city_name')
city_ctl.insert_city_if_not_exist(city_name)
company_dict['city_id'] = city_ctl.get_city_id_by_name(city_name)
company = CompanyModel.get_one(filter_by={'lagou_company_id': company_dict.lagou_company_id})
if company:
CompanyModel.update_by_pk(pk=company.id, values=company_dict)
else:
company_id = CompanyModel.add(**company_dict)
for industry in industries:
industry_ctl.insert_industry_if_not_exist(name=industry)
industry_id = industry_ctl.get_industry_id_by_name(name=industry)
CompanyIndustryModel.add(industry_id=industry_id, company_id=company_id)
crawl_lagou_job_data_task.delay(company_dict.lagou_company_id)
def crawl_lagou_company_data_suites(city_id, finance_stage_id, industry_id):
companies_pagination = lagou_companies_scripts.crawl_lagou_companies_pagination(
city_id=city_id,
finance_stage_id=finance_stage_id,
industry_id=industry_id)
for page_no in companies_pagination.iter_pages:
company_dicts = lagou_companies_scripts.crawl_lagou_companies(
city_id=city_id,
finance_stage_id=finance_stage_id,
industry_id=industry_id,
page_no=page_no)
if not company_dicts:
break
for company_dict in company_dicts:
if not company_dict.is_exist:
lagou_companies_scripts.clean_lagou_company_data(company_dict)
lagou_companies_scripts.convert_lagou_company_data(
company_dict)
industries = company_dict.pop('industries')
advantage = company_dict.pop('advantage')
introduce = company_dict.pop('introduce')
company_dict.pop('city')
company_id = CompanyModel.add(**company_dict)
CompanyExtraModel.add(introduce=introduce,
company_id=company_id,
advantage=advantage)
for industry in industries:
industry_ctl.insert_industry_if_not_exist(name=industry)
industry_id = industry_ctl.get_industry_id_by_name(
name=industry)
CompanyIndustryModel.add(industry_id=industry_id,
company_id=company_id)
crawl_lagou_job_data_suites(company_dict.lagou_company_id)
def add_company(values):
return CompanyModel.add(**values)