def crawl_lagou_company_data_task(city_id, finance_stage_id, industry_id): """爬取拉勾公司数据任务""" companies_pagination = crawlers.get_companies_pagination_from_lagou(city_id=city_id, finance_stage_id=finance_stage_id, industry_id=industry_id) for page_no in companies_pagination.iter_pages: company_dicts = crawlers.get_companies_from_lagou(city_id=city_id, finance_stage_id=finance_stage_id, industry_id=industry_id, page_no=page_no) if not company_dicts: break for company_dict in company_dicts: crawlers.clean_lagou_company_data(company_dict) utils.convert.convert_dict_field_to_constants(company_dict) industries = company_dict.pop('industries') city_name = company_dict.pop('city_name') city_ctl.insert_city_if_not_exist(city_name) company_dict['city_id'] = city_ctl.get_city_id_by_name(city_name) company = CompanyModel.get_one(filter_by={'lagou_company_id': company_dict.lagou_company_id}) if company: CompanyModel.update_by_pk(pk=company.id, values=company_dict) else: company_id = CompanyModel.add(**company_dict) for industry in industries: industry_ctl.insert_industry_if_not_exist(name=industry) industry_id = industry_ctl.get_industry_id_by_name(name=industry) CompanyIndustryModel.add(industry_id=industry_id, company_id=company_id) crawl_lagou_job_data_task.delay(company_dict.lagou_company_id)
def crawl_lagou_job_data_task(lagou_company_id): """爬取拉勾职位数据任务""" # 过滤本轮已经爬取过职位的公司 if not redis_instance.setnx(constants.CRAWLED_COMPANY_JOBS_REDIS_KEY.format(lagou_company_id=lagou_company_id), 1): return jobs_pagination = crawlers.get_jobs_pagination_from_lagou(lagou_company_id=lagou_company_id, job_type=constants.LagouJobType.technology) for page_no in jobs_pagination.iter_pages: job_dicts = crawlers.get_jobs_from_lagou(lagou_company_id=lagou_company_id, job_type=constants.LagouJobType.technology, page_no=page_no) if not job_dicts: break for job_dict in job_dicts: crawlers.clean_lagou_job_data(job_dict) utils.convert.convert_dict_field_to_constants(job_dict) keywords = job_dict.pop('keywords') city_name = job_dict.pop('city_name') city_ctl.insert_city_if_not_exist(city_name) job_dict['city_id'] = city_ctl.get_city_id_by_name(city_name) company = CompanyModel.get_one(filter_by={'lagou_company_id': lagou_company_id}) job_dict['company_id'] = company.id job = JobModel.get_one(filter_by={'lagou_job_id': job_dict.lagou_job_id}) if job: JobModel.update_by_pk(pk=job.id, values=job_dict) else: job_id = JobModel.add(**job_dict) for keyword in keywords: keyword_ctl.insert_keyword_if_not_exist(name=keyword) keyword_id = keyword_ctl.get_keyword_id_by_name(name=keyword) JobKeywordModel.add(keyword_id=keyword_id, job_id=job_id)
def convert_lagou_job_data(job_dict): if job_dict.nature not in constants.JOB_NATURE_DICT: logger.error( '{} not in constants.JOB_NATURE_DICT, lagou job id is {}'.format( job_dict.nature, job_dict.lagou_job_id)) job_dict.nature = constants.JOB_NATURE_DICT.get( job_dict.nature, constants.JOB_NATURE_DICT['unknown']) if job_dict.work_year not in constants.WORK_YEARS_REQUEST_DICT: logger.error( '{} not in constants.WORK_YEARS_REQUEST_DICT, lagou job id is {}'. format(job_dict.work_year, job_dict.lagou_job_id)) job_dict.work_year = constants.WORK_YEARS_REQUEST_DICT.get( job_dict.work_year, constants.WORK_YEARS_REQUEST_DICT['unknown']) if job_dict.education not in constants.EDUCATION_REQUEST_DICT: logger.error( '{} not in constants.EDUCATION_REQUEST_DICT, lagou job id is {}'. format(job_dict.education, job_dict.lagou_job_id)) job_dict.education = constants.EDUCATION_REQUEST_DICT.get( job_dict.education, constants.EDUCATION_REQUEST_DICT['unknown']) city_ctl.insert_city_if_not_exist(job_dict.city) job_dict.city_id = city_ctl.get_city_id_by_name(job_dict.city)
def test_get_city_id_by_name(self): city_id = city_ctl.get_city_id_by_name(name='北京') self.assertEqual(city_id, 2) with self.assertRaises(ValueError): city_ctl.get_city_id_by_name(name='通利福尼亚')