def crawl_lagou_city_data_suites():
city_dicts = lagou_cites_scripts.crawl_lagou_cites()
for city_dict in city_dicts:
if CityModel.is_exist(pk=city_dict.id):
CityModel.update_by_pk(pk=city_dict.id, values=city_dict)
else:
CityModel.add(**city_dict)
def crawl_lagou_city_data_task():
"""爬取拉勾城市数据任务"""
city_dicts = crawlers.get_cites_from_lagou()
for city_dict in city_dicts:
if CityModel.is_exist(filter_by={'id': city_dict.id}):
CityModel.update_by_pk(pk=city_dict.id, values=city_dict)
else:
CityModel.add(**city_dict)
def test_insert_city_if_not_exist(self):
city_id = city_ctl.insert_city_if_not_exist('湛江')
self.assertTrue(city_id > 0)
city = CityModel.get_by_pk(pk=city_id)
self.assertEqual(city.name, '湛江')
self.assertIsNone(city_ctl.insert_city_if_not_exist('湛江'))
def test_batch_add(self):
# 插入了其他的类实例
init_jobs_count = JobModel.count()
model_instances = [
CityModel(name='你好'),
JobModel(title='招聘资深前端工程师',
city_id=1,
company_id=2,
lg_job_id=100056),
JobModel(title='招聘资深中端工程师',
city_id=1,
company_id=2,
lg_job_id=100055),
]
with self.assertRaises(ValueError):
JobModel.batch_add(model_instances)
self.assertEqual(JobModel.count(), init_jobs_count)
model_instances = [
JobModel(title='招聘资深前端工程师',
city_id=1,
company_id=2,
lg_job_id=100056),
JobModel(title='招聘资深中端工程师',
city_id=1,
company_id=2,
lg_job_id=100055),
]
JobModel.batch_add(model_instances)
self.assertEqual(JobModel.count(), init_jobs_count + 2)