def test_get_one(self):
# test get by filter
job = JobModel.get_one(filter=(JobModel.id == 4789)).dict()
self.assertDictEqual(job, test_job_data)
job = JobModel.get_one(filter=and_(JobModel.id == 4789, JobModel.created_at > 0)).dict()
self.assertDictEqual(job, test_job_data)
# test get by filter
job = JobModel.get_one(filter_by={'id': 4789}).dict()
self.assertDictEqual(job, test_job_data)
job = JobModel.get_one(filter_by={'id': 4789}, filter=(JobModel.work_year == 1)).dict()
self.assertDictEqual(job, test_job_data)
def crawl_lagou_job_data_task(lagou_company_id):
"""爬取拉勾职位数据任务"""
# 过滤本轮已经爬取过职位的公司
if not redis_instance.setnx(constants.CRAWLED_COMPANY_JOBS_REDIS_KEY.format(lagou_company_id=lagou_company_id), 1):
return
jobs_pagination = crawlers.get_jobs_pagination_from_lagou(lagou_company_id=lagou_company_id,
job_type=constants.LagouJobType.technology)
for page_no in jobs_pagination.iter_pages:
job_dicts = crawlers.get_jobs_from_lagou(lagou_company_id=lagou_company_id,
job_type=constants.LagouJobType.technology,
page_no=page_no)
if not job_dicts:
break
for job_dict in job_dicts:
crawlers.clean_lagou_job_data(job_dict)
utils.convert.convert_dict_field_to_constants(job_dict)
keywords = job_dict.pop('keywords')
city_name = job_dict.pop('city_name')
city_ctl.insert_city_if_not_exist(city_name)
job_dict['city_id'] = city_ctl.get_city_id_by_name(city_name)
company = CompanyModel.get_one(filter_by={'lagou_company_id': lagou_company_id})
job_dict['company_id'] = company.id
job = JobModel.get_one(filter_by={'lagou_job_id': job_dict.lagou_job_id})
if job:
JobModel.update_by_pk(pk=job.id, values=job_dict)
else:
job_id = JobModel.add(**job_dict)
for keyword in keywords:
keyword_ctl.insert_keyword_if_not_exist(name=keyword)
keyword_id = keyword_ctl.get_keyword_id_by_name(name=keyword)
JobKeywordModel.add(keyword_id=keyword_id, job_id=job_id)
def test_get_one(self):
job = JobModel.get_one(filter_by={'id': 1})
self.assertDictEqual(job.dict(), test_job_dict)
job = JobModel.get_one(filter=(JobModel.id == 1))
self.assertDictEqual(job.dict(), test_job_dict)