def read_job_from_html(skill, html_file): """ read job info from downloaded html file :param html_file: contains job info, but sometime the contents are empty. """ html = read_all(html_file) soup = make_soup(html) detail = soup.find('dl', 'job_detail') # in some rare cases, e.g. the job is closed already, then the job info is missing. if not detail: return None job = Job() job.job_id = int(soup.find('input', {'id': 'jobid'})['value']) job.skill_tag = skill log('*** JOB ***') title = detail.find('h1') log(title['title']) log(title.div.text) job.title = title['title'] job.dept = title.div.text log('') request = detail.find('dd', 'job_request') main_features = [] for s in request.stripped_strings: f = s.strip().lstrip(u'职位诱惑 : ').lstrip(u'发布时间:').rstrip(u'发布') log(f) main_features.append(f) assert len(main_features) == 7 job.salary = main_features[0] job.city = main_features[1] job.experience = main_features[2] job.education = main_features[3] job.full_time = main_features[4] == u'全职' job.benefits = main_features[5] job.published_date = get_published_date(main_features[6], created_on(html_file)) log('') desc_html = [] desc = detail.find('dd', 'job_bt').find_all('p') for bt in desc: desc_html.append(unicode(bt)) job.desc = ''.join(desc_html) log(job.desc) log('\n*** COMPANY ***\n') company = Company() comp = soup.find('dl', 'job_company') url = comp.dt.a['href'] pat = re.compile(r'(?P<comp_id>\d+)') m = re.search(pat, url) log(url) company.comp_id = int(m.group('comp_id')) job.comp_id = company.comp_id log(comp.dt.a.img['src']) log(comp.dt.a.div.h2.text.split()[0]) company.logo = comp.dt.a.img['src'] company.name = comp.dt.a.div.h2.text.split()[0] log('') comp_features = comp.dd features = [] for li in comp_features.ul.find_all('li'): for ls in li.stripped_strings: features.append(ls) log(''.join(features)) if len(features) == 6: company.domain = features[1] company.size = features[3] company.url = features[5] else: print(u'features ex: ' + html_file) log('') stage_h = comp_features.h4 stage_tags = stage_h.find_next_sibling('ul').find_all('li') stage = [] for li in stage_tags: for ls in li.stripped_strings: stage.append(ls) log('\t'.join(stage)) if len(stage) % 2 == 0: for i in xrange(0, len(stage), 2): if stage[i] == u'目前阶段': company.cur_stage = stage[i + 1] elif stage[i] == u'投资机构': company.investor = stage[i + 1] else: print(u'stages ex: ' + html_file) log('') # address if comp_features.div: log(comp_features.div.text) company.address = comp_features.div.text return job, company