def test_get_finance_stage_statistic(self): test_jobs_model = [JobModel(company_id=1), JobModel(company_id=2), JobModel(company_id=3)] finance_stage_statistic = keyword_statistic_ctl.get_finance_stage_statistic(test_jobs_model) self.assertDictEqual(finance_stage_statistic, { '未融资': 2, 'A轮': 1, })
def crawl_lagou_job_data_task(lagou_company_id): """爬取拉勾职位数据任务""" # 过滤本轮已经爬取过职位的公司 if not redis_instance.setnx(constants.CRAWLED_COMPANY_JOBS_REDIS_KEY.format(lagou_company_id=lagou_company_id), 1): return jobs_pagination = crawlers.get_jobs_pagination_from_lagou(lagou_company_id=lagou_company_id, job_type=constants.LagouJobType.technology) for page_no in jobs_pagination.iter_pages: job_dicts = crawlers.get_jobs_from_lagou(lagou_company_id=lagou_company_id, job_type=constants.LagouJobType.technology, page_no=page_no) if not job_dicts: break for job_dict in job_dicts: crawlers.clean_lagou_job_data(job_dict) utils.convert.convert_dict_field_to_constants(job_dict) keywords = job_dict.pop('keywords') city_name = job_dict.pop('city_name') city_ctl.insert_city_if_not_exist(city_name) job_dict['city_id'] = city_ctl.get_city_id_by_name(city_name) company = CompanyModel.get_one(filter_by={'lagou_company_id': lagou_company_id}) job_dict['company_id'] = company.id job = JobModel.get_one(filter_by={'lagou_job_id': job_dict.lagou_job_id}) if job: JobModel.update_by_pk(pk=job.id, values=job_dict) else: job_id = JobModel.add(**job_dict) for keyword in keywords: keyword_ctl.insert_keyword_if_not_exist(name=keyword) keyword_id = keyword_ctl.get_keyword_id_by_name(name=keyword) JobKeywordModel.add(keyword_id=keyword_id, job_id=job_id)
def test_get_work_years_statistic(self): test_jobs_model = [JobModel(work_year=WORK_YEARS_REQUEST_DICT['应届毕业生']), JobModel(work_year=WORK_YEARS_REQUEST_DICT['应届毕业生']), JobModel(work_year=WORK_YEARS_REQUEST_DICT['1-3年'])] work_years_statistic = keyword_statistic_ctl.get_work_years_statistic(test_jobs_model) self.assertDictEqual(work_years_statistic, { '应届毕业生': 2, '1-3年': 1, })
def test_get_educations_statistic(self): test_jobs_model = [JobModel(education=EDUCATION_REQUEST_DICT['大专']), JobModel(education=EDUCATION_REQUEST_DICT['本科']), JobModel(education=EDUCATION_REQUEST_DICT['本科'])] educations_statistic = keyword_statistic_ctl.get_educations_statistic(test_jobs_model) self.assertDictEqual(educations_statistic, { '本科': 2, '大专': 1, })
def test_count(self): job_quantitys = JobModel.count() self.assertEqual(job_quantitys, 2) # test count filter_by job_quantitys = JobModel.count(filter_by={'id': 4789}) self.assertEqual(job_quantitys, 1) job_quantitys = JobModel.count(filter_by={'id': 1}) self.assertEqual(job_quantitys, 0)
def test_get_city_jobs_count_statistic(self): test_jobs_model = [ JobModel(city_id=2), JobModel(city_id=2), JobModel(city_id=2), JobModel(city_id=2), JobModel(city_id=3), JobModel(city_id=3), JobModel(city_id=3), JobModel(city_id=4), JobModel(city_id=4) ] sorted_city_jobs_count_statistic = keyword_statistic_ctl.get_city_jobs_count_statistic( test_jobs_model) self.assertDictEqual(sorted_city_jobs_count_statistic, { '北京': 4, '上海': 3, '广州': 2, }) sorted_city_jobs_count_statistic = keyword_statistic_ctl.get_city_jobs_count_statistic( test_jobs_model, 2) self.assertDictEqual(sorted_city_jobs_count_statistic, { '北京': 4, '上海': 3 })
def test_count(self): jobs_count = JobModel.count() self.assertEqual(jobs_count, 3) jobs_count = JobModel.count(filter_by={'city_id': 4}) self.assertEqual(jobs_count, 2) jobs_count = JobModel.count( filter=(and_(JobModel.city_id == 4, JobModel.company_id == 3))) self.assertEqual(jobs_count, 1) jobs_count = JobModel.count(filter=(JobModel.id == 1)) self.assertEqual(jobs_count, 1)
def test_get_one(self): # test get by filter job = JobModel.get_one(filter=(JobModel.id == 4789)).dict() self.assertDictEqual(job, test_job_data) job = JobModel.get_one(filter=and_(JobModel.id == 4789, JobModel.created_at > 0)).dict() self.assertDictEqual(job, test_job_data) # test get by filter job = JobModel.get_one(filter_by={'id': 4789}).dict() self.assertDictEqual(job, test_job_data) job = JobModel.get_one(filter_by={'id': 4789}, filter=(JobModel.work_year == 1)).dict() self.assertDictEqual(job, test_job_data)
def test_execute_sql_string(self): job_rows = JobModel.execute_sql_string('select id, title from job where id = :id', {'id': 4789}) self.assertEqual(len(job_rows), 1) self.assertEqual(job_rows[0][0], 4789) self.assertEqual(job_rows[0][1], u'Android开发工程师') job_rows = JobModel.execute_sql_string('select id, title from job') self.assertEqual(len(job_rows), 2) self.assertEqual(job_rows[0][0], 4789) self.assertEqual(job_rows[0][1], u'Android开发工程师') affect_rows = JobModel.execute_sql_string("update job set title = '测试' where id = :id", {'id': 4789}) self.assertEqual(affect_rows, 1) job = JobModel.get_by_pk(pk=4789) self.assertEqual(job.title, u'测试')
def test_get_salary_statistic(self): test_jobs_model = [JobModel(salary='5k-9k'), JobModel(salary='10-15k'), JobModel(salary='15k-20k'), JobModel(salary='16-18k'), JobModel(salary='20k-30k'), JobModel(salary='30k-35k'), JobModel(salary='20k以上'), JobModel(salary='60k-100k'), JobModel(salary='40k-42k')] salary_statistic = keyword_statistic_ctl.get_salary_statistic(test_jobs_model) self.assertDictEqual(salary_statistic, { '10k及以下': 2, '11k-20k': 5, '21k-35k': 3, '36k-60k': 2, '61k以上': 1, })
def crawl_lagou_job_data_suites(lagou_company_id): jobs_pagination = lagou_jobs_scripts.crawl_lagou_jobs_pagination( lagou_company_id=lagou_company_id, job_type=constants.LagouJobType.technology) for page_no in jobs_pagination.iter_pages: job_dicts = lagou_jobs_scripts.crawl_lagou_jobs( lagou_company_id=lagou_company_id, job_type=constants.LagouJobType.technology, page_no=page_no) if not job_dicts: break for job_dict in job_dicts: if not job_dict.is_exist: lagou_jobs_scripts.clean_lagou_job_data(job_dict) lagou_jobs_scripts.convert_lagou_job_data(job_dict) company = CompanyModel.get_one( filter_by={'lagou_company_id': lagou_company_id}) job_dict['company_id'] = company.id keywords = job_dict.pop('keywords') advantage = job_dict.pop('advantage') description = job_dict.pop('description') job_dict.pop('city') job_id = JobModel.add(**job_dict) JobExtraModel.add(advantage=advantage, description=description, job_id=job_id) for keyword in keywords: keyword_ctl.insert_keyword_if_not_exist(name=keyword) keyword_id = keyword_ctl.get_keyword_id_by_name( name=keyword) JobKeywordModel.add(keyword_id=keyword_id, job_id=job_id)
def test_execute_sql_string(self): job_rows = JobModel.execute_sql_string( 'SELECT id, title FROM job WHERE id = :id', {'id': 1}) self.assertEqual(len(job_rows), 1) self.assertEqual(job_rows[0][0], 1) self.assertEqual(job_rows[0][1], u'高级前端开发工程师') job_rows = JobModel.execute_sql_string('SELECT id, title FROM job') self.assertEqual(len(job_rows), 3) self.assertEqual(job_rows[0][0], 1) self.assertEqual(job_rows[0][1], u'高级前端开发工程师') affect_rows = JobModel.execute_sql_string( "UPDATE job SET title = '测试' WHERE id = :id", {'id': 1}) self.assertEqual(affect_rows, 1) job = JobModel.get_by_pk(pk=1) self.assertEqual(job.title, u'测试')
def test_update(self): init_job_data_dict = JobModel.get_by_pk(pk=4789).dict() to_update_data_dict = dict(title=u'后端吃饭工程师', work_year=2, city_id=1, company_id=11, department='飞天面条神教招聘', salary='20k-30k', education=2, description=u'日常工作:吃饭!') affect_rows = JobModel.update(filter_by={'id': 4789}, values=to_update_data_dict) self.assertEqual(affect_rows, 1) # 更新后预期的结果 init_job_data_dict.update(**to_update_data_dict) expected_job_data_dict = init_job_data_dict init_updated_at = init_job_data_dict.pop('updated_at') new_job_data_dict = JobModel.get_by_pk(pk=4789).dict() self.assertDictContainsSubset(expected_job_data_dict, new_job_data_dict) self.assertGreater(new_job_data_dict.updated_at, init_updated_at) # 其他记录不受影响 self.assertEqual(JobModel.get_by_pk(pk=6814).title, u'web前端') # 批量更改 affect_rows = JobModel.update(filter_by={'company_id': 11}, values={'title': '测试'}) self.assertEqual(affect_rows, 2) self.assertEqual(JobModel.get_by_pk(pk=6814).title, u'测试') self.assertEqual(JobModel.get_by_pk(pk=4789).title, u'测试')
def test_add(self): to_add_data_dict = dict( lg_job_id=10004, city_id=3, company_id=1, title='Python 开发工程师', work_year=5, department='吖吖项目组', salary='15k-35k', education=2, nature=1, description='职位介绍D', advantage='16薪,工作居住证,六十八险一金,双休', ) job_id = JobModel.add(**to_add_data_dict) self.assertTrue(job_id > 0) job = JobModel.get_by_pk(pk=job_id) self.assertDictContainsSubset(to_add_data_dict, job.dict())
def test_add(self): to_add_data_dict = dict(title=u'后端吃饭工程师', work_year=2, city_id=1, company_id=1, department='飞天面条神教招聘', salary='20k-30k', education=2, description=u'日常工作:吃饭!', advantage='饭管饱, 管够', nature=0) job_id = JobModel.add(**to_add_data_dict) self.assertTrue(job_id > 0) job = JobModel.get_by_pk(pk=job_id) self.assertDictContainsSubset(to_add_data_dict, job.dict()) self.assertGreater(job.created_at, 0) self.assertGreater(job.updated_at, 0)
def test_list(self): # test list jobs = JobModel.list() self.assertEqual(len(jobs), 3) self.assertDictEqual(jobs[0].dict(), test_job_dict) # test list limit jobs = JobModel.list(limit=1) self.assertEqual(len(jobs), 1) # test list offset jobs = JobModel.list(offset=1) self.assertEqual(len(jobs), 2) # test list filter_by jobs = JobModel.list(filter_by={'id': 1}) self.assertEqual(len(jobs), 1) self.assertEqual(jobs[0].dict(), test_job_dict)
def test_list(self): # test list jobs = JobModel.list() self.assertEqual(len(jobs), 2) self.assertDictEqual(jobs[0].dict(), test_job_data) # test list limit jobs = JobModel.list(limit=1) self.assertEqual(len(jobs), 1) # test list offset jobs = JobModel.list(offset=1) self.assertEqual(len(jobs), 1) # test list filter_by jobs = JobModel.list(filter_by={'id': 4789}) self.assertEqual(len(jobs), 1) self.assertEqual(jobs[0].dict(), test_job_data) jobs = JobModel.list(filter_by={'id': 1}) self.assertEqual(len(jobs), 0) # test list order_by jobs = JobModel.list(order_by=JobModel.id.desc()) self.assertEqual(jobs[1].dict(), test_job_data)
def test_update(self): init_job_data_dict = JobModel.get_by_pk(pk=1).dict() to_update_data_dict = dict(title=u'后端吃饭工程师', work_year=1, city_id=1, company_id=1, department='飞天面条神教招聘', salary='20k-32k', education=2, description=u'日常工作:吃饭!') affect_rows = JobModel.update(filter_by={'id': 1}, values=to_update_data_dict) self.assertEqual(affect_rows, 1) # 更新后预期的结果 init_job_data_dict.update(**to_update_data_dict) predictive_job_data_dict = init_job_data_dict init_updated_at = init_job_data_dict.pop('updated_at') new_job_data_dict = JobModel.get_by_pk(pk=1).dict() self.assertDictContainsSubset(predictive_job_data_dict, new_job_data_dict) self.assertGreater(new_job_data_dict.updated_at, init_updated_at) # 其他记录不受影响 self.assertEqual(JobModel.get_by_pk(pk=2).title, u'前端开发工程师') # 批量更改 affect_rows = JobModel.update(filter_by={'city_id': 4}, values={'title': '测试'}) self.assertEqual(affect_rows, 2) jobs = JobModel.list(filter_by={'city_id': 4}) self.assertTrue(all([job.title == u'测试' for job in jobs]))
def test_batch_add(self): # 插入了其他的类实例 init_jobs_count = JobModel.count() model_instances = [ CityModel(name='你好'), JobModel(title='招聘资深前端工程师', city_id=1, company_id=2, lg_job_id=100056), JobModel(title='招聘资深中端工程师', city_id=1, company_id=2, lg_job_id=100055), ] with self.assertRaises(ValueError): JobModel.batch_add(model_instances) self.assertEqual(JobModel.count(), init_jobs_count) model_instances = [ JobModel(title='招聘资深前端工程师', city_id=1, company_id=2, lg_job_id=100056), JobModel(title='招聘资深中端工程师', city_id=1, company_id=2, lg_job_id=100055), ] JobModel.batch_add(model_instances) self.assertEqual(JobModel.count(), init_jobs_count + 2)
def test_is_exist(self): is_exist = JobModel.is_exist(filter=(JobModel.id == 1)) self.assertEqual(is_exist, True)
def test_model_to_dict(self): job = JobModel.get_by_pk(pk=4789).dict() self.assertTrue(isinstance(job, dict)) self.assertDictEqual(job, test_job_data)
def test_get_by_pk(self): job = JobModel.get_by_pk(pk=4789) self.assertDictEqual(job.dict(), test_job_data)
def test_get_one(self): job = JobModel.get_one(filter_by={'id': 1}) self.assertDictEqual(job.dict(), test_job_dict) job = JobModel.get_one(filter=(JobModel.id == 1)) self.assertDictEqual(job.dict(), test_job_dict)
def test_update_by_pk(self): affect_rows = JobModel.update_by_pk(pk=6814, values={'title': '你好啊啊'}) self.assertEqual(affect_rows, 1) self.assertEqual(JobModel.get_by_pk(pk=6814).title, u'你好啊啊')