def file2job(file, zhinengleibie, province): job = Job() job.zhinengleibie = zhinengleibie if province == '深圳': job.province = '广东' else: job.province = province job.job_id = path.split(file)[-1].replace(".html", "") content = "" try: with open(file, mode='r', encoding='gbk') as f: content = f.read() f.close() except UnicodeDecodeError: print("UnicodeDecodeError") return None soup = BeautifulSoup(content, "html.parser") title_tag = soup.select_one('title') if not title_tag: return None job.page_title = title_tag.text if '异地招聘' in job.page_title: job.province = '异地招聘' job.city = '异地招聘' #职业 start #首先判断职业,如果职业不是程序员,直接pass job.career = zhinengleibie #职业 end #20-40万/年 #'1-1.5万/月' #10万以上/月 #100万以上/年 #3-4.5千/月 #17元/小时 salary_tag = soup.select_one('.cn strong') if not salary_tag: return None salary_string = salary_tag.text #零时工,不统计 job.get_salary(salary_string) if job.monthly_salary == -1: return None job.title = soup.find("h1").text.strip() if any(key in job.title for key in title_key_blacklist): return None #'深圳-福田区|5-7年经验|本科|招1人|04-01发布' job.job_summary = soup.select_one(".msg").text.replace('\xa0', '').replace( ' ', '').strip() #print(basic_info) infos = job.job_summary.split('|') #first location job.city = infos[0].split('-')[0] #remove the first one - location infos = infos[1:] for info in infos: if '经验' in info and not job.check_working_experience(): job.get_working_experience(info) #学历 if not job.check_edu(): job.get_edu(info) #人数 if '招' in info and '人' in info: headcount_string = info.replace('招', '').replace('人', '') if headcount_string == '若干': job.headcount = 5 else: job.headcount = int(headcount_string) if info.endswith('发布'): #date date_string = "2020-" + info.replace("发布", '') job.publish_date = datetime.strptime(date_string, '%Y-%m-%d') weekday = job.publish_date.weekday() job.published_on_weekend = weekday > 4 #language if '英语' in info or '英文' in info: job.english = True if '日语' in info or '日文' in info: job.japanese = True #tags tags = [tag.text for tag in soup.select('.sp4')] job.job_tags = ','.join(tags) job.get_tags(tags) h2_span = soup.select_one('h2 span') job.job_description = h2_span.parent.find_next('div').text.strip() job_description_lower = job.job_description.lower() job_description_lower = job.title + " " + job_description_lower job.get_programming_languages(job_description_lower) \ .get_databases(job_description_lower) \ .get_big_data_stats(job_description_lower) #english and japanese if '英语' in job_description_lower or '英文' in job_description_lower: job.english = True #如果招聘信息本身都是英语写的,那么肯定要求英语 if is_article_english(job_description_lower): job.english = True if '日语' in job_description_lower or '日文' in job_description_lower: job.japanese = True #手机程序员并不单独归类,而是用smart_phone属性表示 #手机应用开发工程师 if 'iso' in job_description_lower or 'iphone' in job_description_lower: job.phone_iso = True job.phone_app = True if 'android' in job_description_lower: job.phone_android = True job.phone_app = True company_title_tag = soup.select_one('.com_name') if not company_title_tag: company_title_tag = soup.select_one('.catn') company_title = company_title_tag.text.strip() #black named companies if company_title in company_blacklist: return None job.company_link = company_title_tag.attrs['href'] job.company_id = re.match('.*(co\d*).html', job.company_link).group(1) #996 #朝九晚五,周末双休 双休 不加班 if '朝九晚五' in job.job_description \ or '朝九晚六' in job.job_description \ or '双休' in job.job_description \ or '不加班' in job.job_description: job._996_no = True if '朝九晚九' in job.job_description: job._996_yes = True if job.tag_rest_two_days: job._996_no = True if job.published_on_weekend: job._996_yes = True return job
def file2job(file, zhinengleibie, province): job=Job() job.zhinengleibie=zhinengleibie if province=='深圳': job.province='广东' else: job.province=province job.job_id=path.split(file)[-1].replace(".html","") if job.job_id in ['110455749','77612262','107681687']: return None #page title #print(file) content="" try: with open(file, mode='r',encoding='gbk') as f: content=f.read() f.close() except UnicodeDecodeError: print("UnicodeDecodeError") return None soup=BeautifulSoup(content, "html.parser") title_tag=soup.select_one('title') if not title_tag: return None job.page_title=title_tag.text if '异地招聘' in job.page_title: return None #职业 start #首先判断职业,如果职业不是程序员,直接pass zhineng_tag=soup.find('span',{'class':'label'},text='职能类别:') if not zhineng_tag: return None #职业 end #20-40万/年 #'1-1.5万/月' #10万以上/月 #100万以上/年 #3-4.5千/月 #17元/小时 salary_tag=soup.select_one('.cn strong') if not salary_tag: return None salary_string=salary_tag.text #零时工,不统计 job.get_salary(salary_string) if job.monthly_salary==-1: return None job.title=soup.find("h1").text.strip() if any(key in job.title for key in title_key_blacklist): return None #'深圳-福田区|5-7年经验|本科|招1人|04-01发布' job.job_summary=soup.select_one(".msg").text.replace('\xa0','').replace(' ','').strip() #print(basic_info) infos=job.job_summary.split('|') #first location job.city=infos[0].split('-')[0] #remove the first one - location infos=infos[1:] for info in infos: if '经验' in info and not job.check_working_experience(): job.get_working_experience(info) #学历 if not job.check_edu(): job.get_edu(info) #人数 if '招' in info and '人' in info: headcount_string=info.replace('招','').replace('人','') if headcount_string=='若干': headcount_string='5' job.headcount=int(headcount_string) if info.endswith('发布'): #date date_string="2020-"+info.replace("发布",'') job.publish_date=datetime.strptime(date_string, '%Y-%m-%d') weekday=job.publish_date.weekday() job.published_on_weekend=weekday>4 #language if '英语' in info or '英文' in info: job.english=True if '日语' in info or '日文' in info: job.japanese=True #tags tags=[tag.text for tag in soup.select('.sp4')] job.job_tags=','.join(tags) job.get_tags(tags) h2_span=soup.select_one('h2 span') job.job_description=h2_span.parent.find_next('div').text.strip() job_description_lower=job.job_description.lower() job_description_lower=job.title+" "+job_description_lower job.get_programming_languages(job_description_lower) \ .get_databases(job_description_lower) \ .get_big_data_stats(job_description_lower) \ .get_machine_learning_stats(job_description_lower) #如果招聘信息本身都是英语写的,那么肯定要求英语 if is_article_english(job_description_lower): job.english=True #<span class="bname">公司信息</span> company_info_tag=soup.find('span',text='公司信息') if company_info_tag: job.company_description=company_info_tag.parent.find_next('div').text.replace('\xa0',' ').strip() company_title_tag=soup.select_one('.com_name') if not company_title_tag: company_title_tag=soup.select_one('.catn') job.company_title=company_title_tag.text.strip() #['民营公司', '150-500人', '服装/纺织/皮革'] company_tags=[p.text.strip() for p in soup.select('.com_tag .at')] if len(company_tags)>0: job.get_company_type(company_tags[0]) if job.company_type=='': company_link=company_title_tag.attrs['href'] company_tags=get_company_tags(company_link) for tag in company_tags: if job.get_company_type(tag).check_company_type(): break if job.company_type=='': return None job.get_company_size(company_tags[1]) if not job.check_company_size(): company_link=company_title_tag.attrs['href'] company_tags=get_company_tags(company_link) for tag in company_tags: if job.get_company_size(tag).check_company_size(): break #计算机/互联网/通信/电子 industry_tags=[p.text.strip() for p in soup.select('.com_tag .at a') if not p.text==''] if len(industry_tags)==0: company_link=soup.select_one('.com_name').attrs['href'] industry_tags=get_company_tags(company_link) for industry_tag in industry_tags: job.get_industry(industry_tag) if job.company_title in ['系统集成有限责任公司','博彦科技股份有限公司']: job.industry='computer' if job.company_title=='软件与服务中心': job.industry='trade' if job.company_title== '中核集团技术经济总院': job.industry='energy' #black named companies if job.company_title in company_blacklist: return None # if not job.check_industry(): # raise Exception("no industry") return job
def file2job(file, city): job = Job() setattr(job, "city_" + city, True) job.job_id = path.split(file)[-1].replace(".html", "") #print(file) content = "" with open(file, mode='r', encoding='gbk') as f: content = f.read() f.close() os.rename(file, file.replace("51jobs", "51jobs_back")) soup = BeautifulSoup(content, "html.parser") #职业 start #首先判断职业,如果职业不是程序员,直接pass #career=soup.find('span',{'class':'label'},text='职能类别:').find_next('a').text.strip() careers = [ a_tag.text.strip() for a_tag in soup.find('span', { 'class': 'label' }, text='职能类别:').parent.find_all('a') ] for career in careers: if (career in [ '软件工程师', '高级软件工程师', 'ERP技术开发', '互联网软件开发工程师', '多媒体/游戏开发工程师', '手机应用开发工程师', 'WEB前端工程师', '脚本开发工程师', '语音/视频/图形开发工程师' ]): job.career_software_engineer = True if career == '算法工程师': job.career_algorithm = True if career in ['系统架构设计师', '网站架构设计师']: job.career_architect = True is_developer = job.career_software_engineer or job.career_algorithm or job.career_architect if not is_developer: return None #职业 end #20-40万/年 #'1-1.5万/月' #10万以上/月 #100万以上/年 #3-4.5千/月 #17元/小时 salary_tag = soup.select_one('.cn strong') if not salary_tag: return None salary_string = salary_tag.text #零时工,不统计 job.get_salary(salary_string) if job.monthly_salary == -1: return None job.title = soup.find("h1").text.strip() #'深圳-福田区|5-7年经验|本科|招1人|04-01发布' basic_info = soup.select_one(".msg").text.replace('\xa0', '').replace(' ', '').strip() #print(basic_info) infos = basic_info.split('|') #remove the first one - location infos = infos[1:] for info in infos: if '经验' in info and not job.check_working_experience(): job.get_working_experience(info) #学历 if not job.check_edu(): job.get_edu(info) if info.endswith('发布'): #date date_string = "2019-" + info.replace("发布", '') job.publish_date = datetime.strptime(date_string, '%Y-%m-%d') weekday = job.publish_date.weekday() job.published_on_weekend = weekday > 4 #language if '英语' in info or '英文' in info: job.english = True if '日语' in info or '日文' in info: job.japanese = True #tags tags = [tag.text for tag in soup.select('.sp4')] job.get_tags(tags) h2_span = soup.select_one('h2 span') job.job_description = h2_span.parent.find_next('div').text.strip() job_description_lower = job.job_description.lower() job_description_lower = job.title + " " + job_description_lower #年龄歧视 job.ageism = '岁' in job.job_description #继续判断是不是架构师 if '架构师' in job_description_lower: job.career_algorithm = True #继续判断是不是算法工程师 if 'tensorflow' in job_description_lower \ or 'keras' in job_description_lower \ or 'caffe' in job_description_lower \ or 'pytorch' in job_description_lower \ or '机器学习' in job_description_lower \ or 'nlp' in job_description_lower \ or '自然语言处理' in job_description_lower \ or '算法工程师' in job_description_lower \ or 'sklearn' in job_description_lower \ or '深度学习' in job_description_lower \ or '图像识别' in job_description_lower: job.career_algorithm = True if job.career_algorithm or job.career_architect: job.career_software_engineer = True job.get_programming_languages(job_description_lower).get_databases( job_description_lower) #english and japanese if '英语' in job_description_lower or '英文' in job_description_lower: job.english = True #如果招聘信息本身都是英语写的,那么肯定要求英语 if is_article_english(job_description_lower): job.english = True if '日语' in job_description_lower or '日文' in job_description_lower: job.japanese = True #手机程序员并不单独归类,而是用smart_phone属性表示 #手机应用开发工程师 if '手机应用开发工程师' in careers: job.phone_app = True if 'iso' in job_description_lower or 'iphone' in job_description_lower: job.phone_iso = True job.phone_app = True if 'android' in job_description_lower: job.phone_android = True job.phone_app = True #<span class="bname">公司信息</span> job.company_description = soup.find( 'span', text='公司信息').parent.find_next('div').text.replace('\xa0', ' ').strip() job.company_title = soup.select_one('.com_name').text.strip() #['民营公司', '150-500人', '服装/纺织/皮革'] company_tags = [p.text.strip() for p in soup.select('.com_tag .at')] job.get_company_type(company_tags[0]) if not job.check_company_type(): company_link = soup.select_one('.com_name').attrs['href'] company_tags = get_company_tags(company_link) for tag in company_tags: if job.get_company_type(tag).check_company_type(): break if not job.check_company_type(): return None job.get_company_size(company_tags[1]) if not job.check_company_size(): company_link = soup.select_one('.com_name').attrs['href'] company_tags = get_company_tags(company_link) for tag in company_tags: if job.get_company_size(tag).check_company_size(): break #计算机/互联网/通信/电子 industry_tags = [ p.text.strip() for p in soup.select('.com_tag .at a') if not p.text == '' ] if len(industry_tags) == 0: company_link = soup.select_one('.com_name').attrs['href'] industry_tags = get_company_tags(company_link) for industry_tag in industry_tags: job.get_industry(industry_tag) if job.company_title in ['系统集成有限责任公司', '博彦科技股份有限公司']: job.industry_computer = True if job.company_title == '软件与服务中心': job.industry_trade = True if job.company_title == '中核集团技术经济总院': job.industry_energy = True # if not job.check_industry(): # raise Exception("no industry") #996 #朝九晚五,周末双休 双休 不加班 if '朝九晚五' in job.job_description \ or '朝九晚六' in job.job_description \ or '双休' in job.job_description \ or '不加班' in job.job_description: job.non_996 = True if '朝九晚九' in job.job_description: job.icu_996 = True if job.tag_rest_two_days: job.non_996 = True if in_996_list(job.company_title): job.icu_996 = True if in_non_996_list(job.company_title): job.non_996 = True if job.published_on_weekend: job.icu_996 = True return job
def file2job(file, zhinengleibie, province): job = Job() job.zhinengleibie = zhinengleibie job.province = province job.job_id = path.split(file)[-1].replace(".html", "") if job.job_id in ['110455749', '77612262', '107681687']: return None #page title #print(file) content = "" try: with open(file, mode='r', encoding='gbk') as f: content = f.read() f.close() except UnicodeDecodeError: print("UnicodeDecodeError") return None soup = BeautifulSoup(content, "html.parser") job.page_title = soup.select_one('title').text if '异地招聘' in job.page_title: return None #职业 start #首先判断职业,如果职业不是程序员,直接pass #career=soup.find('span',{'class':'label'},text='职能类别:').find_next('a').text.strip() zhineng_tag = soup.find('span', {'class': 'label'}, text='职能类别:') if not zhineng_tag: return None careers = [ a_tag.text.strip() for a_tag in zhineng_tag.parent.find_all('a') ] for career in careers: if (career in [ '软件工程师', '高级软件工程师', 'ERP技术开发', '互联网软件开发工程师', '多媒体/游戏开发工程师', '手机应用开发工程师', 'WEB前端工程师', '脚本开发工程师', '语音/视频/图形开发工程师' ]): job.career = '一般程序员' if career == '算法工程师': job.career = '算法工程师' if career in ['系统架构设计师', '网站架构设计师']: job.career = '系统架构师' if '爬虫' in job.title: job.career = '爬虫工程师' if '生物信息' in job.title: job.career = '生物信息工程师' if job.career == '': return None #职业 end #20-40万/年 #'1-1.5万/月' #10万以上/月 #100万以上/年 #3-4.5千/月 #17元/小时 salary_tag = soup.select_one('.cn strong') if not salary_tag: return None salary_string = salary_tag.text #零时工,不统计 job.get_salary(salary_string) if job.monthly_salary == -1: return None job.title = soup.find("h1").text.strip() if any(key in job.title for key in [ '安全工程师', 'seo', '测试', '前端', '信息工程师', '运维', '经理', '嵌入式', '讲师', '老师', '负责人', '合伙人', '计算机技术员', '主任', '总监', 'cto', '需求工程师', '需求分析', '系统集成工程师', '系统工程师', '系统分析师', '计算机辅助设计', 'DBA', '实施', '售前', '售后', '数据库' ]): return None job_title_lower = job.title.lower() if '专家' in job_title_lower: job.expert_expert = False if 'blockchain' in job_title_lower or '区块链' in job_title_lower: job.expert_blockchain = False if 'adas' in job_title_lower: job.expert_adas = False if '嵌入式' in job_title_lower: job.expert_embed = False if 'gis' in job_title_lower: job.expert_gis = False if '架构师' in job_title_lower: job.career = '系统架构师' if '算法工程师' in job_title_lower: job.career = '算法工程师' #'深圳-福田区|5-7年经验|本科|招1人|04-01发布' job.job_summary = soup.select_one(".msg").text.replace('\xa0', '').replace( ' ', '').strip() #print(basic_info) infos = job.job_summary.split('|') #first location job.city = infos[0].split('-')[0] #remove the first one - location infos = infos[1:] for info in infos: if '经验' in info and not job.check_working_experience(): job.get_working_experience(info) #学历 if not job.check_edu(): job.get_edu(info) #人数 if '招' in info and '人' in info: headcount_string = info.replace('招', '').replace('人', '') if headcount_string == '若干': headcount_string = '5' job.headcount = int(headcount_string) if info.endswith('发布'): #date date_string = "2019-" + info.replace("发布", '') job.publish_date = datetime.strptime(date_string, '%Y-%m-%d') weekday = job.publish_date.weekday() job.published_on_weekend = weekday > 4 #language if '英语' in info or '英文' in info: job.english = True if '日语' in info or '日文' in info: job.japanese = True #tags tags = [tag.text for tag in soup.select('.sp4')] job.job_tags = ','.join(tags) job.get_tags(tags) h2_span = soup.select_one('h2 span') job.job_description = h2_span.parent.find_next('div').text.strip() job_description_lower = job.job_description.lower() job_description_lower = job.title + " " + job_description_lower #年龄歧视 job.ageism = '岁' in job.job_description #继续判断是不是架构师 #if '架构师' in job_description_lower: # job.career='系统架构师' #继续判断是不是算法工程师 if 'tensorflow' in job_description_lower \ or 'keras' in job_description_lower \ or 'caffe' in job_description_lower \ or 'pytorch' in job_description_lower \ or '机器学习' in job_description_lower \ or 'nlp' in job_description_lower \ or '自然语言处理' in job_description_lower \ or '算法工程师' in job_description_lower \ or 'sklearn' in job_description_lower \ or '深度学习' in job_description_lower \ or '图像识别' in job_description_lower: job.career = '算法工程师' job.get_programming_languages(job_description_lower).get_databases( job_description_lower) #english and japanese if '英语' in job_description_lower or '英文' in job_description_lower: job.english = True #如果招聘信息本身都是英语写的,那么肯定要求英语 if is_article_english(job_description_lower): job.english = True if '日语' in job_description_lower or '日文' in job_description_lower: job.japanese = True #手机程序员并不单独归类,而是用smart_phone属性表示 #手机应用开发工程师 if '手机应用开发工程师' in careers: job.phone_app = True if 'iso' in job_description_lower or 'iphone' in job_description_lower: job.phone_iso = True job.phone_app = True if 'android' in job_description_lower: job.phone_android = True job.phone_app = True #<span class="bname">公司信息</span> job.company_description = soup.find( 'span', text='公司信息').parent.find_next('div').text.replace('\xa0', ' ').strip() job.company_title = soup.select_one('.com_name').text.strip() #['民营公司', '150-500人', '服装/纺织/皮革'] company_tags = [p.text.strip() for p in soup.select('.com_tag .at')] job.get_company_type(company_tags[0]) if job.company_type == '': company_link = soup.select_one('.com_name').attrs['href'] company_tags = get_company_tags(company_link) for tag in company_tags: if job.get_company_type(tag).check_company_type(): break if job.company_type == '': return None job.get_company_size(company_tags[1]) if not job.check_company_size(): company_link = soup.select_one('.com_name').attrs['href'] company_tags = get_company_tags(company_link) for tag in company_tags: if job.get_company_size(tag).check_company_size(): break #计算机/互联网/通信/电子 industry_tags = [ p.text.strip() for p in soup.select('.com_tag .at a') if not p.text == '' ] if len(industry_tags) == 0: company_link = soup.select_one('.com_name').attrs['href'] industry_tags = get_company_tags(company_link) for industry_tag in industry_tags: job.get_industry(industry_tag) if job.company_title in ['系统集成有限责任公司', '博彦科技股份有限公司']: job.industry = 'computer' if job.company_title == '软件与服务中心': job.industry = 'trade' if job.company_title == '中核集团技术经济总院': job.industry = 'energy' # if not job.check_industry(): # raise Exception("no industry") #996 #朝九晚五,周末双休 双休 不加班 if '朝九晚五' in job.job_description \ or '朝九晚六' in job.job_description \ or '双休' in job.job_description \ or '不加班' in job.job_description: job._996_no = True if '朝九晚九' in job.job_description: job._996_yes = True if job.tag_rest_two_days: job._996_no = True if in_996_list(job.company_title): job._996_yes = True if in__996_no_list(job.company_title): job._996_no = True if job.published_on_weekend: job._996_yes = True if job.company_title in ['青岛云指针软件有限公司']: job._996_no = False job._996_yes = False return job
def file2job(file, zhinengleibie, province): job=Job() job.zhinengleibie=zhinengleibie if province=='深圳': job.province='广东' else: job.province=province job.job_id=path.split(file)[-1].replace(".html","") if job.job_id in ['110455749','77612262','107681687']: return None #page title #print(file) content="" try: with open(file, mode='r',encoding='gbk') as f: content=f.read() f.close() except UnicodeDecodeError: print("UnicodeDecodeError") return None soup=BeautifulSoup(content, "html.parser") title_tag=soup.select_one('title') if not title_tag: return None job.page_title=title_tag.text if '异地招聘' in job.page_title: return None #职业 start #首先判断职业,如果职业不是程序员,直接pass #career=soup.find('span',{'class':'label'},text='职能类别:').find_next('a').text.strip() zhineng_tag=soup.find('span',{'class':'label'},text='职能类别:') if not zhineng_tag: return None careers=[a_tag.text.strip() for a_tag in zhineng_tag.parent.find_all('a')] for career in careers: if (career in ['软件工程师','高级软件工程师','ERP技术开发','互联网软件开发工程师','多媒体/游戏开发工程师','手机应用开发工程师','WEB前端工程师','脚本开发工程师','语音/视频/图形开发工程师']): job.career='一般程序员' if career in ['算法工程师','机器学习工程师','深度学习工程师','图像算法工程师','图像处理工程师','语音识别工程师','图像识别工程师','机器视觉工程师','自然语言处理(NLP)']: job.career='算法工程师' if career in ['系统架构设计师','网站架构设计师']: job.career='系统架构师' #职业 end #20-40万/年 #'1-1.5万/月' #10万以上/月 #100万以上/年 #3-4.5千/月 #17元/小时 salary_tag=soup.select_one('.cn strong') if not salary_tag: return None salary_string=salary_tag.text #零时工,不统计 job.get_salary(salary_string) if job.monthly_salary==-1: return None job.title=soup.find("h1").text.strip() if '爬虫' in job.title: job.career='爬虫工程师' if '生物信息' in job.title: job.career='生物信息工程师' if job.career=='': return None if any(key in job.title for key in title_key_blacklist): return None job_title_lower=job.title.lower() if '专家' in job_title_lower: job.expert_expert=True if 'blockchain' in job_title_lower or '区块链' in job_title_lower: job.expert_blockchain=True if 'adas' in job_title_lower: job.expert_adas=True if '嵌入式' in job_title_lower: job.expert_embed=True if 'gis' in job_title_lower: job.expert_gis=True if '架构师' in job_title_lower: job.career='系统架构师' if '算法工程师' in job_title_lower: job.career='算法工程师' if '数据分析' in job_title_lower: job.career='数据分析' if '图像算法' in job_title_lower: job.career='图像算法' #'深圳-福田区|5-7年经验|本科|招1人|04-01发布' job.job_summary=soup.select_one(".msg").text.replace('\xa0','').replace(' ','').strip() #print(basic_info) infos=job.job_summary.split('|') #first location job.city=infos[0].split('-')[0] #remove the first one - location infos=infos[1:] for info in infos: if '经验' in info and not job.check_working_experience(): job.get_working_experience(info) #学历 if not job.check_edu(): job.get_edu(info) #人数 if '招' in info and '人' in info: headcount_string=info.replace('招','').replace('人','') if headcount_string=='若干': headcount_string='5' job.headcount=int(headcount_string) if info.endswith('发布'): #date date_string="2020-"+info.replace("发布",'') job.publish_date=datetime.strptime(date_string, '%Y-%m-%d') weekday=job.publish_date.weekday() job.published_on_weekend=weekday>4 #language if '英语' in info or '英文' in info: job.english=True if '日语' in info or '日文' in info: job.japanese=True #tags tags=[tag.text for tag in soup.select('.sp4')] job.job_tags=','.join(tags) job.get_tags(tags) h2_span=soup.select_one('h2 span') job.job_description=h2_span.parent.find_next('div').text.strip() job_description_lower=job.job_description.lower() job_description_lower=job.title+" "+job_description_lower #年龄歧视 job.ageism='岁' in job.job_description if '机器学习' in job.title or '深度学习' in job.title or '推荐系统' in job.title \ or '推荐算法' in job.title or '图像识别' in job.title or '人工智能' in job.title \ or 'nlp' in job.title.lower() or '自然语言' in job.title or 'aml' in job.title \ or 'AI' in job.title or '数据科学家' in job.title or 'data scientist' in job.title.lower() \ or '知识图谱' in job.title \ or job.zhinengleibie in ('机器学习工程师','深度学习工程师'): job.career='机器学习' if 'dsp' in job_description_lower: job.career='DSP' if 'slam' in job_description_lower: job.career='SLAM' if 'CT重建' in job.title: job.career='CT重建' if '大数据' in job.title: job.career='大数据' if 'FPGA' in job.title: job.career='FPGA' if '信号处理' in job.title: job.career='信号处理' if '架构师' in job.title: job.career='系统架构师' if '视觉' in job.title: job.career='视觉软件工程师' if '三维重建' in job.title: job.career='视觉软件工程师' if '规划算法' in job.title: job.career='规划算法工程师' if '遥感' in job.title: job.career='遥感' if '光学算法' in job.title: job.career='光学算法工程师' if '机器人' in job.title or 'ROS' in job.title: job.career='机器人' if '爬虫' in job.title: job.career='爬虫工程师' if 'ADAS' in job.title: job.career='adas' if 'GIS' in job.title: job.career='gis' if 'CAE' in job.title: job.career='cae' if 'ETL' in job.title: job.career='etl' if 'unity3d' in job.title.lower() or 'u3d' in job.title.lower(): job.career='Unity3d' job.get_programming_languages(job_description_lower) \ .get_databases(job_description_lower) \ .get_big_data_stats(job_description_lower) \ .get_machine_learning_stats(job_description_lower) #english and japanese if '英语' in job_description_lower or '英文' in job_description_lower: job.english=True #如果招聘信息本身都是英语写的,那么肯定要求英语 if is_article_english(job_description_lower): job.english=True if '日语' in job_description_lower or '日文' in job_description_lower: job.japanese=True #手机程序员并不单独归类,而是用smart_phone属性表示 #手机应用开发工程师 if '手机应用开发工程师' in careers: job.phone_app=True if 'iso' in job_description_lower or 'iphone' in job_description_lower: job.phone_iso=True job.phone_app=True if 'android' in job_description_lower: job.phone_android=True job.phone_app=True #<span class="bname">公司信息</span> company_info_tag=soup.find('span',text='公司信息') if company_info_tag: job.company_description=company_info_tag.parent.find_next('div').text.replace('\xa0',' ').strip() company_title_tag=soup.select_one('.com_name') if not company_title_tag: company_title_tag=soup.select_one('.catn') job.company_title=company_title_tag.text.strip() #['民营公司', '150-500人', '服装/纺织/皮革'] company_tags=[p.text.strip() for p in soup.select('.com_tag .at')] if len(company_tags)>0: job.get_company_type(company_tags[0]) if job.company_type=='': company_link=company_title_tag.attrs['href'] company_tags=get_company_tags(company_link) for tag in company_tags: if job.get_company_type(tag).check_company_type(): break if job.company_type=='': return None job.get_company_size(company_tags[1]) if not job.check_company_size(): company_link=company_title_tag.attrs['href'] company_tags=get_company_tags(company_link) for tag in company_tags: if job.get_company_size(tag).check_company_size(): break #计算机/互联网/通信/电子 industry_tags=[p.text.strip() for p in soup.select('.com_tag .at a') if not p.text==''] if len(industry_tags)==0: company_link=soup.select_one('.com_name').attrs['href'] industry_tags=get_company_tags(company_link) for industry_tag in industry_tags: job.get_industry(industry_tag) if job.company_title in ['系统集成有限责任公司','博彦科技股份有限公司']: job.industry='computer' if job.company_title=='软件与服务中心': job.industry='trade' if job.company_title== '中核集团技术经济总院': job.industry='energy' #black named companies if job.company_title in company_blacklist: return None # if not job.check_industry(): # raise Exception("no industry") #996 #朝九晚五,周末双休 双休 不加班 if '朝九晚五' in job.job_description \ or '朝九晚六' in job.job_description \ or '双休' in job.job_description \ or '不加班' in job.job_description: job._996_no=True if '朝九晚九' in job.job_description: job._996_yes=True if job.tag_rest_two_days: job._996_no=True if job.published_on_weekend: job._996_yes=True if job.company_title in ['青岛云指针软件有限公司']: job._996_no=False job._996_yes=False return job