def Get_html(url):
req = requests.get(url)
if req.status_code != 200:
log.error("网址（%s）无法访问，状态码：%d" % (url, req.status_code))
return None
html = req.content
return html
def Get_html(url):
req = requests.get(url)
if req.status_code != 200:
log.error("网址（%s）无法访问，状态码：%d" % (url, req.status_code))
return None
html=req.content
return html
def get_html_t(url, coding="utf-8"):
response = requests.get(url)
if response.status_code != 200:
log.error("网址（%s）无法访问，状态码：%d" % (url, response.status_code))
content = str(response.content)
r = re.findall("charset=(\w+)\"", content)
if r:
coding = r[0].lower()
response.encoding = coding
return response.text
def get_html_t(url,coding="utf-8"):
response = requests.get(url)
if response.status_code != 200:
log.error("网址（%s）无法访问，状态码：%d" % (url, response.status_code))
content=str(response.content)
r=re.findall("charset=(\w+)\"",content)
if r:
coding=r[0].lower()
response.encoding=coding
return response.text
def __send_poster(open_id, poster_url):
pic_url = html_change_to_png(poster_url)
if pic_url is None:
# __send_message_use_api(open_id, u'生成个人海报失败，请向我们反馈！')
log.error(u'生成个人海报失败，用户openid：%s' % open_id)
return None
content = "<img style='width: 200px;' src='%s'>" % pic_url
__send_message_use_api(open_id, content)
# 告诉微信自动回复系统名片海报已经更新
tell_api_build_card(open_id, pic_url)
def Get_html(url):
"""
获取html
:param url:
:return:
"""
req = requests.get(url)
if req.status_code != 200:
log.error("网址（%s）无法访问，状态码：%d" % (url, req.status_code))
return None
html=req.content.decode('gb2312','ignore').encode('utf8') #解码并编码为utf8
return html
def tell_api_build_card(open_id, pic_url):
"""
告诉API接口，我们生成了一张海报
:return:
"""
r = requests.post('http://wechatsys.lingnanchuangye.com/set_key_data/frefer45g4re514re', data={
'key': 'poster_image_' + open_id,
'value': pic_url
})
if r.status_code != 200:
log.error(u'通知微信公众号生成了新海报操作失败，用户openid：%s' % open_id)
raise Exception('error!')
def tell_api_build_card(open_id, pic_url):
"""
告诉API接口，我们生成了一张海报
:return:
"""
r = requests.post(
'http://wechatsys.lingnanchuangye.com/set_key_data/frefer45g4re514re',
data={
'key': 'poster_image_' + open_id,
'value': pic_url
})
if r.status_code != 200:
log.error(u'通知微信公众号生成了新海报操作失败，用户openid：%s' % open_id)
raise Exception('error!')
def get_all_page_of_job():
"""
获取每一页的招聘信息
:return:
"""
global first_url
response = requests.get(first_url)
if response.status_code != 200:
log.error("网址（%s）无法访问，状态码：%d" % (first_url, response.status_code))
return None
response.encoding = 'gbk'
objs = get_tile_and_link_lists(response.text)
objs = list(map(handle_job_message, objs))
return objs
def get_all_page_of_job():
"""
获取每一页的招聘信息
:return:
"""
global first_url
response = requests.get(first_url)
if response.status_code != 200:
log.error("网址（%s）无法访问，状态码：%d" % (first_url, response.status_code))
return None
response.encoding = 'gbk'
objs = get_tile_and_link_lists(response.text)
objs = list(map(handle_job_message, objs))
return objs
def handle_job_message(obj):
"""
处理兼职信息
:param obj:
:return:
"""
tools.sleep_some_time()
response = requests.get(obj['web_url'])
if response.status_code != 200:
log.error("网址（%s）无法访问，状态码：%d" % (obj['web_url'], response.status_code))
return obj
obj['web_html'] = response.content
obj['company'] = tools.get_company_name(obj['web_html'])
obj['position'] = tools.get_work_position(obj['web_html'])
obj['work_city'] = tools.get_work_citys(obj['web_html'])
return obj
def handle_job_message(obj):
"""
处理兼职信息
:param obj:
:return:
"""
tools.sleep_some_time()
response = requests.get(obj['web_url'])
if response.status_code != 200:
log.error("网址（%s）无法访问，状态码：%d" % (obj['web_url'], response.status_code))
return obj
obj['web_html'] = response.content
obj['company'] = tools.get_company_name(obj['web_html'])
obj['position'] = tools.get_work_position(obj['web_html'])
obj['work_city'] = tools.get_work_citys(obj['web_html'])
return obj
def get_all_page_html():
"""
获取若干页
:return:
"""
global first_html_url
# 获取第一页信息
response = requests.get(first_html_url)
if response.status_code != 200:
return None
html = response.content
if not html:
return None
messages = get_message_title_and_url_list(html)
page = 2
form_data = get_search_form(html)
while page < 50:
form_data['__EVENTTARGET'] = 'ctl00$cph_content_temp$DataPager1$ctl01$ctl0%d' % (page - 1)
response = requests.post(first_html_url, form_data)
if response.status_code != 200:
log.error("网址（%s）无法访问，状态码：%d" % (first_html_url, response.status_code))
page += 1
continue
# 更新form隐藏字段
html = response.content
form_data = get_search_form(html)
temp_messages = get_message_title_and_url_list(html)
messages.extend(temp_messages)
if not temp_messages or len(temp_messages) < 10:
break
page += 1
messages = list(map(handle_job_message, messages))
return messages
def get_all_page_of_work():
"""
获取所有页
:return:
"""
global web_url
page = 1
messages = []
while page < 6:
url = web_url + str(page)
response = requests.get(url)
if response.status_code != 200:
log.error("网址（%s）无法访问，状态码：%d" % (url, response.status_code))
continue
objs = get_message_title_and_url_list(response.content)
messages.extend(objs)
if len(objs) < 15:
break
page += 1
messages = list(map(handle_job_message, messages))
return messages
def get_all_jobs():
"""
获取最近的招聘信息
:return:
"""
global message_url
messages = []
page = 1
while page <= 5:
url = message_url % page
response = requests.get(url)
if response.status_code != 200:
log.error("网址（%s）无法访问，状态码：%d" % (url, response.status_code))
continue
response.encoding = 'gbk'
objs = get_title_and_link_list(response.text)
messages.extend(objs)
if len(objs) < 18:
break
page += 1
messages = list(map(handle_job_message, messages))
return messages
def get_all_page_of_work():
"""
获取所有页
:return:
"""
global web_url
page = 1
messages = []
while page < 6:
url = web_url + str(page)
response = requests.get(url)
if response.status_code != 200:
log.error("网址（%s）无法访问，状态码：%d" % (url, response.status_code))
continue
objs = get_message_title_and_url_list(response.content)
messages.extend(objs)
if len(objs) < 15:
break
page += 1
messages = list(map(handle_job_message, messages))
return messages
def add_a_job(job_title, job_company, job_url, job_city, job_message_source,
job_position, job_release_time, web_html):
"""
添加一条招聘信息
:param job_title: 工作标题
:param job_company: 公司名字
:param job_url: 跳转的页面地址
:param job_city: 工作地点
:param job_message_source: 消息来源
:param job_position: 工作职位
:param job_release_time: 工作发布时间
:param web_html: 抓取的网页html
:return:
"""
if type(job_city) == list:
job_city = '#'.join(job_city)
if type(job_position) == list:
job_position = '#'.join(job_position)
token = hashlib.md5(job_company.encode("UTF-8")).hexdigest()
job_release_time = tools.get_real_time(job_release_time)
sql = """
insert into jobs(title, company, position, web_url, work_city, message_source, job_type,
authentication, status, web_html, release_time, token, create_time)
values(%s, %s, %s, %s, %s, %s, 0, 0, 0, %s, %s, %s, now());
"""
try:
insert_id = db_lib.insert(sql, [
job_title, job_company, job_position, job_url, job_city,
job_message_source, web_html, job_release_time, token
])
add_company(job_company)
return insert_id
except IntegrityError:
return 0
except Exception as error:
log.error("写入数据库失败（错误类型：%s）， 信息地址：%s" % (str(error), job_url))
return -1
def get_all_page_of_job():
"""
获取最近的招聘信息
:return:
"""
global message_url
messages = []
page = 1
while page <= 1:
url = message_url % (page - 1) * 20
response = requests.get(url)
if response.status_code != 200:
log.error("网址（%s）无法访问，状态码：%d" % (url, response.status_code))
continue
objs = get_title_and_link_list(response.content)
messages.extend(objs)
if len(objs) < 20:
break
page += 1
messages = list(map(handle_job_message, messages))
return messages
def get_all_page_of_job():
"""
获取最近的招聘信息
:return:
"""
global message_url
messages = []
page = 1
while page <= 1:
url = message_url % (page - 1) * 20
response = requests.get(url)
if response.status_code != 200:
log.error("网址（%s）无法访问，状态码：%d" % (url, response.status_code))
continue
objs = get_title_and_link_list(response.content)
messages.extend(objs)
if len(objs) < 20:
break
page += 1
messages = list(map(handle_job_message, messages))
return messages
def get_html(url):
response = requests.get(url)
if response.status_code != 200:
log.error("网址（%s）无法访问，状态码：%d" % (url, response.status_code))
return response.content
def get_html(url):
response = requests.get(url)
if response.status_code != 200:
log.error("网址（%s）无法访问，状态码：%d" % (url, response.status_code))
return response.content