def verify(self):
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36"
}
with requests.get(url='http://124.172.189.180:5010/get/') as ip:
proxies = {"http": ip.text}
with requests.session() as session:
session.headers = headers
session.proxies = proxies
with session.get(
url=
f'http://gat.sc.gov.cn/wsga/query/car/captcha?d={str(int(time.time() * 1000))}'
) as img:
# with open('sc.png', "wb+")as f:
# f.write(img.content)
cjy = Chaojiying_Client()
result = cjy.PostPic(img.content, 1902)
if result.get("err_no") == -1005:
print(result.get("err_str"))
return result.get("err_str")
else:
yzm = result.get("pic_str")
# print(yzm)
return session, yzm
def get_capture(self, meta):
url = 'http://www.ezdrving.com/wxjswy/portalLogin/verify.htm'
headers = {"Cookie": meta["jssionid"] + ";"}
rs = requests.get(url=url, headers=headers)
# with open("yzm.jpeg", "wb+") as f:
# f.write(rs.content)
cjy = Chaojiying_Client()
# im = open('yzm.jpeg', 'rb').read()
im = rs.content
result = cjy.PostPic(im, 1902)
if result.get("err_no") == -1005:
print(result.get("err_str"))
return result.get("err_str")
else:
yzm = result.get("pic_str")
# yzm = input("请输入验证码:")
meta["yzm"] = yzm
return self.parse_data(meta)
def chaoji():
chaojiying = Chaojiying_Client('账号', '密码', '软件ID') # 用户中心>>软件ID 生成一个替换 96001
im = open('a.jpg', 'rb').read() # 本地图片文件路径 来替换 a.jpg 有时WIN系统须要//
datas = chaojiying.PostPic(im, 1902) # 1902 验证码类型 官方网站>>价格体系 3.4+版 print 后要加()
ocr = datas['pic_str']
# print datas
print(ocr)
return ocr
def login(self):
index_url = 'https://jxtwap.cwddd.com/v1/index/index.html'
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36",
}
cookie = requests.get(
url=index_url,
headers=headers).history[0].headers.get("Set-Cookie").split(";")[0]
# print(cookie)
headers["Cookie"] = cookie
v_code = requests.get(
url='http://cwapp.cwddd.com/v9.0/wap/public/verify',
headers=headers)
# with open("login.png", "wb+") as f:
# f.write(v_code.content)
cjy = Chaojiying_Client()
result = cjy.PostPic(v_code.content, 1902)
if result.get("err_no") == -1005:
print(result.get("err_str"))
return result.get("err_str")
else:
yzm = result.get("pic_str")
login_url = 'http://cwapp.cwddd.com/v9.0/wap/public/docache'
headers["X-Requested-With"] = "XMLHttpRequest"
data = {
"username": "******",
"password": "******",
"verify": yzm,
}
logined = requests.post(url=login_url, headers=headers, data=data)
# print(logined.text)
if logined.json()["status"]:
return cookie
else:
return self.login()
def verify_code(self):
index_url = 'http://www.zjsgat.gov.cn:8087/was/phone/carIllegalQuery.jsp'
with requests.Session() as session:
session.headers.update({
"User-Agent":
f"Mozilla/5.0 (Linux; Android {randint(2,10)}.0; LND-AL30 Build/HONORLND-AL30; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.126 MQQBrowser/6.2 TBS/044611 Mobile Safari/537.36 MMWEBID/472 MicroMessenger/7.0.4.1420(0x27000439) Process/tools NetType/WIFI Language/zh_CN",
})
with session.get(index_url) as index:
# print(index.text)
pattern = re.compile(r'.*?tblname=(.*?)"')
tblname = re.findall(pattern, index.text)[0]
# print(tblname)
capture_url = 'http://www.zjsgat.gov.cn:8087/was/Kaptcha.jpg?0'
with session.get(capture_url) as capture:
# with open('zj.png', "wb+") as img:
# img.write(capture.content)
cjy = Chaojiying_Client()
result = cjy.PostPic(capture.content, 1902)
if result.get("err_no") == -1005:
print(result.get("err_str"))
return result.get("err_str")
else:
yzm = result.get("pic_str")
# yzm = input("请输入验证码:")
verify_url = 'http://www.zjsgat.gov.cn:8087/was/portals/checkManyYzm.jsp'
with session.post(url=verify_url, data={"randValue": yzm}) as rs:
# print(rs.text)
if rs.json()["result"] == "Y":
# rs = {"session": session, "tblname": tblname, "yzm": yzm}
return session, tblname, yzm
else:
return self.verify_code()
def start_requests(self):
'''
override该入口方法，以selenium模拟登录后拿到cookie交给scrapy的request使用
'''
# 从文件中读取cookie
cookies = None
try:
if os.path.exists('./cookies/lagou.cookie'):
cookies = pickle.load(open('./cookies/lagou.cookie', 'rb'))
except:
pass
if not cookies:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
options = Options()
options.add_argument("--disable-extensions")
# options.add_argument('window-size=1280x800')
# options.add_experimental_option("debuggerAddress", "127.0.0.1:9222")
# options.binary_location = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
browser = webdriver.Chrome(executable_path='./chromedriver',
chrome_options=options)
from selenium.webdriver.common.action_chains import ActionChains
action = ActionChains(browser)
browser.get('https://passport.lagou.com/login/login.html')
browser.find_element_by_css_selector(
'div[data-view="passwordLogin"] div[data-propertyname="username"] .input.input_white'
).send_keys(LAGOU_USERNAME)
browser.find_element_by_css_selector(
'div[data-view="passwordLogin"] div[data-propertyname="password"] .input.input_white'
).send_keys(LAGOU_PASSWORD)
browser.find_element_by_css_selector(
'div[data-view="passwordLogin"] .sense_login_password input[type="submit"]'
).click()
import time
time.sleep(3)
is_login = False
while not is_login:
try:
if browser.find_element_by_css_selector('.unick'):
is_login = True
except:
pass
try:
captcha_element = browser.find_element_by_css_selector(
'.geetest_window>.geetest_item:first-child>.geetest_item_wrap>img'
)
except:
captcha_element = None
if captcha_element:
if action._actions:
action._actions = []
img_url = captcha_element.get_attribute('src')
import ssl
ssl._create_default_https_context = ssl._create_unverified_context
import urllib
urllib.request.urlretrieve(img_url,
filename='lagou_validate.jpeg')
# import requests
# html = requests.get(img_url)
# fh = open('lagou_validate.jpeg', 'wb')
# fh.write(html.content)
# fh.close()
from tools.chaojiying import Chaojiying_Client
from settings import (chaojiying_username,
chaojiying_password,
chaojiying_app_id)
chaojiying = Chaojiying_Client(chaojiying_username,
chaojiying_password,
chaojiying_app_id)
im = open('lagou_validate.jpeg', 'rb').read()
pos_obj = chaojiying.PostPic(im, 9004)
if pos_obj['err_no'] == 0 or pos_obj['err_str'] == 'OK':
pic_str = pos_obj['pic_str']
positions = pic_str.split('|')
from ScrapyDemo.utils.common import get_coordinate
for item in positions:
point = item.split(',')
el_index = get_coordinate(point)
browser.find_element_by_css_selector(
'.geetest_window>.geetest_item:nth-child({})'.
format(el_index)).click()
browser.find_element_by_css_selector(
'.geetest_commit').click()
time.sleep(5)
cookies = browser.get_cookies()
# 写入cookie到文件中
pickle.dump(cookies, open('./cookies/lagou.cookie', 'wb'))
cookie_dict = {}
for cookie in cookies:
cookie_dict[cookie['name']] = cookie['value']
for url in self.start_urls:
yield scrapy.Request(url, dont_filter=True, cookies=cookie_dict)
async def get_data(self, info):
cookie = self.login()
headers = {
"Content-Type": "application/x-www-form-urlencoded",
"User-Agent":
"Mozilla/5.0 (Linux; Android 8.0.0; LND-AL30 Build/HONORLND-AL30; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/68.0.3440.91 Mobile Safari/537.36",
"Cookie": cookie,
"X-Requested-With": "com.cwddd.jxtmobile"
}
verify_code = requests.get(
url='https://jxtwap.cwddd.com/v1/Public/verify', headers=headers)
# with open("cw.png", "wb+") as f:
# f.write(verify_code.content)
cjy = Chaojiying_Client()
result = cjy.PostPic(verify_code.content, 1902)
if result.get("err_no") == -1005:
print(result.get("err_str"))
return result.get("err_str")
else:
yzm = result.get("pic_str")
# yzm = input("请输入验证码:")
data = {
"hpzl": info["hpzl"],
"hphm": info["hphm"],
"code": info["cjh"],
"verify": yzm,
}
rs = requests.post(url='https://jxtwap.cwddd.com/v1/index/search.html',
headers=headers,
data=data)
try:
if not rs.json()["status"]:
return await self.get_data(info)
except Exception as e:
html = etree.HTML(rs.text)
divs = html.xpath('//div[@id="tab1"]/div')[:-1]
# print(len(divs))
rs = {"code": 200, "msg": "成功！", "result": []}
for x in divs:
single = {}
# print(x.xpath('div/text()'))
fkkf = x.xpath('div/text()')[0].strip()
# print(fkkf)
single["fkje"] = int(
fkkf.split(" | ")[0].replace("罚款", "").replace("元", ""))
single["wfjfs"] = int(
fkkf.split(" | ")[1].replace("记", "").replace("分", ""))
detail = x.xpath('div/p/text()')
# print(detail)
single["wfsj"] = detail[1]
single["wfdz"] = detail[3]
single["wfxwzt"] = detail[-1]
rs["result"].append(single)
return rs