def get_cookie(self):
payload = {
"name": "13269431992",
"pwd": "abcd1234",
"tijiao.x": 7,
"tijiao.y": 9,
"checkbox": "on"
}
session = r.session()
session.cookies = cookiejar.LWPCookieJar(
filename=os.path.join(base_dir, "src/cookies.txt"))
session.post(self.login_url, data=payload, headers=header)
# session.cookies.save()
session.close()
cookie = session.cookies
return cookie
def make_session(user_agent=DEFAULT_UA, ssl_verify=None, max_retries=0,
cookie_file=None, cookiejar=None,
http_user=None, http_password=None):
"""
Creates a :py:class:`requests.Session` object for the connection.
It is possible to save the session data by specifying either ``cookiejar``
or ``cookie_file`` arguments. If ``cookiejar`` is present, ``cookie_file``
is ignored.
:param str user_agent: string sent as ``User-Agent`` header to the web server
:param bool ssl_verify: if ``True``, the SSL certificate will be verified
:param int max_retries:
Maximum number of retries for each connection. Applies only to
failed DNS lookups, socket connections and connection timeouts, never
to requests where data has made it to the server.
:param str cookie_file: path to a :py:class:`cookielib.FileCookieJar` file
:param cookiejar: an existing :py:class:`cookielib.CookieJar` object
:returns: :py:class:`requests.Session` object
"""
session = requests.Session()
if cookiejar is not None:
session.cookies = cookiejar
elif cookie_file is not None:
session.cookies = cookielib.LWPCookieJar(cookie_file)
try:
session.cookies.load()
except (cookielib.LoadError, FileNotFoundError):
session.cookies.save()
session.cookies.load()
_auth = None
if http_user is not None and http_password is not None:
_auth = (http_user, http_password)
session.headers.update({"user-agent": user_agent})
session.auth = _auth
session.params.update({"format": "json"})
session.verify = ssl_verify
# granular control over requests' retries: https://stackoverflow.com/a/35504626
retries = Retry(total=max_retries, backoff_factor=1, status_forcelist=[500, 502, 503, 504])
adapter = requests.adapters.HTTPAdapter(max_retries=retries)
session.mount("https://", adapter)
session.mount("http://", adapter)
return session
def get_cookie(self, netloc, ua, timeout):
try:
headers = {'User-Agent': ua}
req = urllib2.Request(netloc)
_add_request_header(req, headers)
try: response = urllib2.urlopen(req, timeout=int(timeout))
except HTTPError as response:
result = response.read(5242880)
try: encoding = response.headers["Content-Encoding"]
except: encoding = None
if encoding == 'gzip': result = gzip.GzipFile(fileobj=BytesIO(result)).read()
jschl = re.findall(r'name\s*=\s*["\']jschl_vc["\']\s*value\s*=\s*["\'](.+?)["\']/>', result, re.I)[0]
init = re.findall(r'setTimeout\(function\(\){\s*.*?.*:(.*?)};', result, re.I)[-1]
builder = re.findall(r"challenge-form\'\);\s*(.*)a.v", result, re.I)[0]
decryptVal = self.parseJSString(init)
lines = builder.split(';')
for line in lines:
if len(line) > 0 and '=' in line:
sections = line.split('=')
line_val = self.parseJSString(sections[1])
decryptVal = int(eval(str(decryptVal) + sections[0][-1] + str(line_val)))
answer = decryptVal + len(urlparse(netloc).netloc)
query = '%s/cdn-cgi/l/chk_jschl?jschl_vc=%s&jschl_answer=%s' % (netloc, jschl, answer)
if 'type="hidden" name="pass"' in result:
passval = re.findall(r'name\s*=\s*["\']pass["\']\s*value\s*=\s*["\'](.*?)["\']', result, re.I)[0]
query = '%s/cdn-cgi/l/chk_jschl?pass=%s&jschl_vc=%s&jschl_answer=%s' % (netloc, quote_plus(passval), jschl, answer)
sleep(6)
cookies = cookiejar.LWPCookieJar()
handlers = [urllib2.HTTPHandler(), urllib2.HTTPSHandler(), urllib2.HTTPCookieProcessor(cookies)]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
try:
req = urllib2.Request(query)
_add_request_header(req, headers)
response = urllib2.urlopen(req, timeout=int(timeout))
except: pass
cookie = '; '.join(['%s=%s' % (i.name, i.value) for i in cookies])
if 'cf_clearance' in cookie: self.cookie = cookie
except:
from resources.lib.modules import log_utils
log_utils.error()
def api_article(registrationnum):
url = "http://mis.mptransport.org/MPLogin/eSewa/VehicleSearch.aspx"
br = mechanize.Browser()
# Cookie Jar
cj = cookielib.LWPCookieJar()
br.set_cookiejar(cj)
# Browser options
br.set_handle_equiv(True)
br.set_handle_gzip(True)
br.set_handle_redirect(True)
br.set_handle_referer(True)
br.set_handle_robots(False)
# Follows refresh 0 but not hangs on refresh > 0
br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
# User-Agent (this is cheating, ok?)
br.addheaders = [
('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')]
# br.set_handle_robots(False)
br.open(url)
br.select_form(name='aspnetForm')
br["ctl00$ContentPlaceHolder1$txtRegNo"] = (registrationnum)
res = br.submit()
content = res.read()
soup = BeautifulSoup(content, 'html.parser')
table = soup.find(
"table", {"border": "1", "id": "ctl00_ContentPlaceHolder1_grvSearchSummary"})
for row in table.findAll('tr', {'class': 'GridItem'}):
col = row.findAll('td')
a = col[2].find('a').string
b = col[14].string
c = col[5].string
d = col[6].string
e = col[8].string
data = {
'registration_num': a,
'model': b,
'owner_name': c,
'rto_name': d,
'regis_date': e
}
# response = jsonify(data)
# print(data
print(json.dumps(data, indent=1))
def setup_browser():
'''Create the browser and configure it properly.'''
browser = mechanize.Browser()
cookiejar = cookielib.LWPCookieJar()
browser.set_cookiejar(cookiejar)
browser.set_handle_equiv(True)
browser.set_handle_gzip(True)
browser.set_handle_redirect(True)
browser.set_handle_referer(True)
browser.set_handle_robots(False)
browser.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(),
max_time=1)
browser.addheaders = [(
'User-agent',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'
)]
return browser
def send_msg(content):
session.cookies = cookielib.LWPCookieJar(filename='/root/weibo/weibo.txt')
session.cookies.load() # 载入cookies
if not islogin(): # 没有登录
login('用户名', '密码')
st = get_st() # 获取st
headers = {
'Host': 'm.weibo.cn',
'User-Agent': User_Agent,
'Accept': 'application/json, text/plain, */*',
'Accept-Language':
'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
'Accept-Encoding': 'gzip, deflate, br',
'X-Requested-With': 'XMLHttpRequest',
'MWeibo-Pwa': '1',
'Content-Type': 'application/x-www-form-urlencoded',
'X-XSRF-TOKEN': st,
'Origin': 'https://m.weibo.cn',
'DNT': '1',
'Connection': 'keep-alive',
'Referer': 'https://m.weibo.cn/compose/',
'TE': 'Trailers',
'Pragma': 'no-cache',
'Cache-Control': 'no-cache'
}
body = {
'content': content,
'st': st, # 10 分钟的有效期
'_spr': 'screen:1600x900'
}
url = 'https://m.weibo.cn/api/statuses/update'
resp = session.post(url, data=body, headers=headers)
if resp.status_code == 200:
try:
data = resp.json()
status = data['ok']
if status == 1:
print(time.strftime('%Y-%m-%d %H:%M:%S '), '发送成功')
else:
serverq('微博发送失败')
except:
serverq('微博发送失败')
else:
serverq('微博发送失败')
def create(self):
self.gog_username = None
self.gog_password = None
self.done = None
self.gamedata = GameData()
# bypass the hardcoded "Netscape HTTP Cookie File" check
cookiejar.MozillaCookieJar.magic_re = r'.*'
self.COOKIES_FILENAME = r'gog-cookies.dat'
self.global_cookies = cookiejar.LWPCookieJar(self.COOKIES_FILENAME)
self.cookieproc = HTTPCookieProcessor(self.global_cookies)
self.opener = build_opener(self.cookieproc)
self.loader = self.add(npyscreen.SliderPercent,
name="Loading", max_height=3)
self.loader.editable = False
def run(self):
di_acpw = eval(open('acpw.txt', 'r', encoding='gbk').read())
nft = 1
for account in di_acpw:
name = di_acpw[account][0]
pwd = di_acpw[account][1]
print('\n--------------------------------------------------------')
print(' 正在读取第 {} 个用户!'.format(nft))
print('--------------------------------------------------------\n')
print('当前用户: {}\n手机号码: {}'.format(name, account))
# 判断一起沃目录是否存在
if not os.path.exists(os.getcwd() + '\\一起沃\\'):
os.makedirs(os.getcwd() + '\\一起沃\\')
cookies_name = os.getcwd() + '\\一起沃\\' + account
# 更新cookies
self.session.cookies = cookiejar.LWPCookieJar(
filename=cookies_name)
try: # 尝试加载cookies
self.session.cookies.load(ignore_discard=True)
except: # 无法加载cookies时,采用账号密码登录
open(cookies_name, 'w+')
while not self.login_status(account):
self.first_login(account, pwd)
try:
# 签到
self.signin_click()
# 拆红包
# redpacket_click()
# 目前信息
self.curinfo()
except:
self.first_login(account, pwd)
# 签到
self.signin_click()
# 拆红包
# redpacket_click()
# 目前信息
self.curinfo()
# 保存cookies文件
self.session.cookies.save()
nft += 1
def __init__(self, headers_info: dict, form_info: dict, meta: dict):
"""
实例化
:param headers_info: HTTP 头信息
:param form_info: 表单信息
:param meta: 其他信息
"""
self.com_headers = headers_info.get('com_headers', {})
self.req_headers = headers_info.get('req_headers', {})
self.res_headers = headers_info.get('res_headers', {})
self.form_data = form_info
self.meta = meta
self.session = requests.Session()
self.session.headers = dict(self.com_headers, **self.req_headers)
self.session.cookies = cookiejar.LWPCookieJar(filename='./cookies.txt')
def enableCookie(self, enableProxy=False):
#self.cookiejar = cookielib.LWPCookieJar() # 建立COOKIE //python 3.6 改成 cookiejar了
self.cookiejar = cookiejar.LWPCookieJar() # 建立COOKIE
#cookie_support = urllib2.HTTPCookieProcessor(self.cookiejar) //urllib2 在 python3.x 中被改为urllib.request
#//利用urllib库中的request的HTTPCookieProcessor对象来创建cookie处理器
cookie_support = urllib.request.HTTPCookieProcessor(self.cookiejar)
if enableProxy: #//这里enableProxy=False，应该不执行if语句
#proxy_support = urllib2.ProxyHandler({'http': 'http://122.96.59.107:843'}) # 使用代理
proxy_support = urllib.request.ProxyHandler({'http': 'http://122.96.59.107:843'}) # 使用代理
#opener = urllib2.build_opener(proxy_support, cookie_support, urllib2.HTTPHandler)
opener = urllib.request.build_opener(proxy_support, cookie_support, urllib.request.HTTPHandler)
self.logger.info("Proxy enable.")
else:
#opener = urllib2.build_opener(cookie_support, urllib2.HTTPHandler)
opener = urllib.request.build_opener(cookie_support, urllib.request.HTTPHandler) #//通过handler来构建opener
#urllib2.install_opener(opener)
urllib.request.install_opener(opener)
def __init__(self):
# self.request_URL='https://www.zhihu.com/login/phone_num'
self.session = requests.Session()
self.session.cookies = cookiejar.LWPCookieJar(filename='cookies.txt')
self.headers = {
'Host':
'www.zhihu.com',
'Referer':
'https://www.zhihu.com',
'User-Agent':
'Mozilla/5.0 (Macintosh; Inter Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87'
}
try:
print(self.session.cookies)
self.session.cookies.load(ignore_discard=True)
except FileNotFoundError:
print('There is no cookies!')
def initcookies(self):
''' cookiejar to selenium driver'''
self.get(BASE_URL)
cookies = cookiejar.LWPCookieJar(filename=cookiespath)
cookies.load(ignore_discard=True)
if os.path.exists(cookiespath):
for cookie in cookies:
cookie_dict = {'name': cookie.name, 'value': cookie.value}
if cookie.expires:
cookie_dict['expiry'] = cookie.expires
if cookie.path_specified:
cookie_dict['path'] = cookie.path
if cookie.domain:
cookie_dict['domain'] = cookie.domain
if cookie.secure:
cookie_dict['secure'] = cookie.secure
self.driver.add_cookie(cookie_dict)
def __init__(self, user, password, cookie_path):
super(WeiboLogin, self).__init__()
self.user = user
self.password = password
self.session = requests.Session()
self.cookie_path = cookie_path
self.session.cookies = cookielib.LWPCookieJar(
filename=self.cookie_path)
try:
self.session.cookies.load(ignore_discard=True)
#pass
except IOError:
print('Cookie未加载！')
print(self.session.cookies)
self.index_url = "http://weibo.com/login.php"
self.session.get(self.index_url, headers=headers, timeout=2)
self.postdata = dict()
def __init__(self, timeout=60, cookie_path='.'):
self.headers = {
'Accept': '*/*',
'Accept-Encoding': 'gzip,deflate,sdch',
'Accept-Language': 'zh-CN,zh;q=0.8,gl;q=0.6,zh-TW;q=0.4',
'Connection': 'keep-alive',
'Content-Type': 'application/x-www-form-urlencoded',
'Host': 'music.163.com',
'Referer': 'https://music.163.com/weapi/v6/playlist/detail?csrf_token=726d365e78adbf07f0982703abb776eb',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
}
self.session = requests.Session()
self.session.headers.update(self.headers)
self.session.cookies = cookiejar.LWPCookieJar(cookie_path)
self.download_session = requests.Session()
self.timeout = timeout
self.ep = Encrypyed()
def __init__(self, user, password, proxy):
super(WeiboLogin, self).__init__()
self.user = user
self.password = password
self.session = requests.Session()
self.session.proxies = {
"http": proxy,
"https": proxy,
}
# self.cookie_path = cookie_path
self.session.cookies = cookielib.LWPCookieJar()
self.index_url = "http://weibo.com/login.php"
self.session.get(self.index_url,
headers=headers,
timeout=(5, 5),
verify=False)
self.postdata = dict()
class AutoLogin():
headers = {
"Host":"www.zhihu.com",
"Referer":"https://www.zhihu.com/",
"User-Agent":"'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:22.0) Gecko/20100101 Firefox/22.0'}"
}
session = requests.session()
session.cookies = cookiejar.LWPCookieJar(filename='cookie.txt')
try:
print(session.cookies)
session.cookies.load(ignore_discard = True)
except:
print("no cookie info")
def getXsrf():
response = session.get("https://www.zhihu.com",headers = headers)
soup = BeautifulSoup(response.content,"html.parser")
xsrf = soup.find("input",attrs={"name":"_xsrf"}).get("value")
return xsrf
#获取验证码
def getCaptcha():
t = str(int(time.time()*1000))
captchaUrl = 'https://www.zhihu.com/captcha.gif?r='+t+"&type=login"
r = session.get(captchaUrl,hearders=hearders)
with open('captcha.jpg','wb') as f:
f.write(r.content)
captcha = input("验证码:")
return captha
def doLogin(email,password):
loginUrl = "https://www.zhihu.com/login/email"
data = {
'email':email,
'password':password,
'_xsrf':getXsrf(),
'captcha':getCaptcha(),
'remember_me':'true'
}
response = session.post(loginUrl,data=data,headers=headers)
loginCode = response.json()
print(loginCode['msg'])
for i in session.cookies:
print(i)
session.cookie.save()
def __init__(self, settings_folder, site, locale, logging_prefix, numresults, cookiestxt, cookiestxt_file, us_uhd):
self.logging_prefix = logging_prefix
self.site_url = site
self.locale = locale
self.numResults = numresults
self.locale_suffix = self.locale.split('_')[1].lower()
self.client_id = str(uuid.uuid1())
self.device_id = self.client_id.replace("-", "")
self.us_uhd = us_uhd
if self.locale_suffix == 'gb':
self.api_url = 'https://disco-api.' + self.site_url
self.realm = 'questuk'
self.site_headers = {'x-disco-params': 'realm='+self.realm}
elif self.locale_suffix == 'us':
self.api_url = 'https://us1-prod-direct.' + self.site_url
self.realm = 'go'
self.site_headers = {'x-disco-params': 'realm=go,siteLookupKey=dplus_us', 'x-disco-client': 'WEB:UNKNOWN:dplus_us:0.0.1'}
elif self.locale_suffix == 'in':
self.api_url = 'https://ap2-prod-direct.' + self.site_url
self.realm = 'dplusindia'
self.site_headers = {'x-disco-params': 'realm=dplusindia', 'x-disco-client': 'WEB:UNKNOWN:dplus-india:prod'}
else:
self.api_url = 'https://disco-api.' + self.site_url
self.realm = 'dplay' + self.locale_suffix
self.site_headers = {'x-disco-params': 'realm='+self.realm}
self.http_session = requests.Session()
self.settings_folder = settings_folder
self.tempdir = os.path.join(settings_folder, 'tmp')
self.unwanted_menu_items = ('Hae mukaan', 'Info', 'Tablå', 'Live TV', 'TV-guide')
if not os.path.exists(self.tempdir):
os.makedirs(self.tempdir)
# If cookiestxt setting is true use users cookies file
if cookiestxt:
self.cookie_jar = cookielib.MozillaCookieJar(cookiestxt_file)
else:
self.cookie_jar = cookielib.LWPCookieJar(os.path.join(self.settings_folder, 'cookie_file'))
try:
self.cookie_jar.load(ignore_discard=True, ignore_expires=True)
except IOError:
pass
self.http_session.cookies = self.cookie_jar
def __init__(self):
self._session = requests.session()
self._session.verify = False
self._session.headers = {
"Host":
"www.zhihu.com",
"Referer":
"https://www.zhihu.com/",
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36'
' (KHTML, like Gecko) Chrome/56.0.2924.87',
}
self._session.cookies = cookiejar.LWPCookieJar(
filename=cookie_filename)
try:
self._session.cookies.load(ignore_discard=True)
except:
pass
def __init__(self):
'''
cookie = cookiejar.CookieJar()
cookieProc = request.HTTPCookieProcessor( cookie )
opener = request.build_opener( cookieProc )
request.install_opener( opener )
'''
cj = cookiejar.LWPCookieJar()
cookie_support = request.HTTPCookieProcessor(cj)
for item in cj:
print('Name1 = ' + item.name)
print('Value2 = ' + item.value)
opener = request.build_opener(cookie_support, request.HTTPHandler)
#proxy_support = request.ProxyHandler({'http':'http://120.193.146.97:843'})
#opener = request.build_opener(proxy_support, cookie_support, request.HTTPHandler)
request.install_opener(opener)
self.req = request
def __init__(self):
# url
self.loginUrl = 'https://github.com/login'
self.postUrl = 'https://github.com/session'
self.profileUrl = 'https://github.com/settings/profile'
# 设置请求头
self.headers = {
'Referer': 'https://github.com/',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36',
'Host': 'github.com'
}
# 设置session
self.session = requests.session()
# 生成github_cookie文件
self.session.cookies = cookielib.LWPCookieJar(filename='github_cookie')
def __init__(self, settings_folder, country, debug=False):
self.debug = debug
self.country = country
self.settings_folder = settings_folder
self.cookie_jar = cookielib.LWPCookieJar(os.path.join(self.settings_folder, 'cookie_file'))
self.tempdir = os.path.join(settings_folder, 'tmp')
if not os.path.exists(self.tempdir):
os.makedirs(self.tempdir)
self.deviceid_file = os.path.join(settings_folder, 'deviceId')
self.http_session = requests.Session()
self.device_key = 'xdk-%s' % self.country
self.base_url = 'https://content.viaplay.{0}/{1}'.format(self.country, self.device_key)
self.login_api = 'https://login.viaplay.%s/api' % self.country
try:
self.cookie_jar.load(ignore_discard=True, ignore_expires=True)
except IOError:
pass
self.http_session.cookies = self.cookie_jar
def __init__(self, account, password, cookie_file='zhihu_cookie.txt'):
self.session = requests.session()
self.session.headers = self.headers
self.cookie_file = cookie_file
self.session.cookies = cookielib.LWPCookieJar(filename=self.cookie_file)
self.username = account
self.password = password
self.check_count = 0
try:
self.session.cookies.load(ignore_discard=True, ignore_expires=True)
print('cookie信息加载成功')
except FileNotFoundError:
print("cookie信息加载失败")
self.login()
else:
if not self.check_login():
print('cookie登录失败，即将重新登录')
self.login()
def __init__(self):
# Creating Browser
self.br = mechanize.Browser()
# Cookie Jar
cj = cookielib.LWPCookieJar()
self.br.set_cookiejar(cj)
# Browser options
self.br.set_handle_equiv(True)
self.br.set_handle_gzip(True)
self.br.set_handle_redirect(True)
self.br.set_handle_referer(True)
self.br.set_handle_robots(False)
self.br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(),
max_time=1)
self.br.addheaders = [('User-agent', 'Chrome')]
def __init__(self, user, password, cookie_path, topic):
super(WeiboLogin, self).__init__()
self.user = user
self.password = password
self.session = requests.Session()
self.cookie_path = cookie_path
self.session.cookies = cookielib.LWPCookieJar(
filename=self.cookie_path)
self.index_url = "http://weibo.com/login.php"
self.session.get(self.index_url, headers=headers, timeout=2)
self.Mheaders = {}
self.postdata = dict()
self.now_user = 0
self.now_url = '' # 记录当前访问的url
self.now_topic = 0
self.now_page = 1 # 记录当前页面
self.topic = topic
def __init__(self, session):
if not session:
requests.adapters.DEFAULT_RETRIES = 5
self.session = requests.Session()
self.session.cookies = cookielib.LWPCookieJar(filename='cookie')
self.session.keep_alive = False
try:
self.session.cookies.load(ignore_discard=True)
except:
print('Cookie 未能加载')
finally:
pass
else:
self.__session = session
# 获取配置
self.config = configparser.ConfigParser()
self.config.read("config.ini")
def __init__(self, info, price1, price2, BInumber, species, library_type,
rRNAremoval_check, password):
super().__init__(info, price1, price2, BInumber, species, library_type,
rRNAremoval_check)
self.kfid = ""
self.client_post = ""
self.password = password
#header and CMS_session save
self.url = "http://cms.novogene.com/"
self.CMS_Session = requests.session()
self.CMS_Session.cookies = cookielib.LWPCookieJar(
filename="CMSCookies.txt")
self.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36"
self.header = {
#"Origin": "http://cms.novogene.com",
"Referer": "http://cms.novogene.com/index.jsp",
"User-Agent": self.UserAgent,
}
def __init__(self, log_print=None):
if log_print:
global print
print = log_print
max_workers = cpu_count() * 4
self.executor = futures.ThreadPoolExecutor(max_workers)
self.executor_video = futures.ThreadPoolExecutor(1)
self.root_path = "~/Downloads/"
self.futures = []
self.session = requests.session()
self.session.cookies = cookielib.LWPCookieJar(filename=self.root_path +
"/_cookie.txt")
self.defaultHeader = {
'upgrade-insecure-requests': "1",
'user-agent':
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36",
'dnt': "1",
'accept':
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
'accept-encoding': "gzip, deflate",
'accept-language': "zh-CN,zh;q=0.9,en;q=0.8,zh-TW;q=0.7,ja;q=0.6",
'cache-control': "no-cache"
}
self.ajaxheaders = {
'upgrade-insecure-requests': "1",
'user-agent':
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36",
'dnt': "1",
'accept-encoding': "gzip, deflate, br",
'x-requested-with': "XMLHttpRequest",
'accept-language': "zh-CN,zh;q=0.9,en;q=0.8,zh-TW;q=0.7,ja;q=0.6",
'cache-control': "no-cache",
'accept': "application/json, text/plain, */*; q=0.01",
}
self.headers = {
'upgrade-insecure-requests':
"1",
'user-agent':
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Safari/537.36",
'dnt':
"1",
'accept':
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"
}
def parse_one(self, response):
# number = response.meta['number']
# search 现场数据
return_data = None
final_time = None
soup = BeautifulSoup(response.text, "html.parser")
scripts = soup.find_all('script')
for script in scripts:
if script.string is not None:
if 'draw_half_line' in script.string:
return_data, final_time = main_extract(script.string)
# else:
# return
# 请求红牌数据
try:
race_events = soup.select('#race_events')[0]
except IndexError:
print('error' + response.meta['number'])
race_events = race_events.select('li')
try:
pai_data = getpai_event(race_events=race_events, max_minute=final_time,
zhudui_name=return_data['shezheng'][0]['name'],
kedui_name=return_data['shezheng'][1]['name'])
except BaseException as e:
raise e
# 请求main data
infor_detail = get_mian(soup=soup)
#请求 四合一数据
siheyi_data = None
session = requests.session()
session.cookies = cookielib.LWPCookieJar(
filename='/home/sunlianjie/PycharmProjects/happy-spiders/scrapy_templates/soccer/soccer/spiders/cookie')
try:
session.cookies.load(ignore_discard=True)
sp_url = 'https://www.dszuqiu.com/race_sp/413968'
re = session.get(sp_url, headers=self.headers,
allow_redirects=False)
siheyi_data = getSiheyi(re.text,race_events)
except BaseException as e:
raise e
def __init__(self, apple_id, password, cookie_directory=None, verify=True):
self.discovery = None
self.client_id = str(uuid.uuid1()).upper()
self.user = {'apple_id': apple_id, 'password': password}
self._home_endpoint = 'https://www.icloud.com'
self._setup_endpoint = 'https://setup.icloud.com/setup/ws/1'
self._base_login_url = '%s/login' % self._setup_endpoint
if cookie_directory:
self._cookie_directory = os.path.expanduser(
os.path.normpath(cookie_directory)
)
else:
self._cookie_directory = os.path.join(
tempfile.gettempdir(),
'pyicloud',
)
self.session = requests.Session()
self.session.verify = verify
self.session.headers.update({
'Origin': self._home_endpoint,
'Referer': '%s/' % self._home_endpoint,
'User-Agent': 'Opera/9.52 (X11; Linux i686; U; en)'
})
cookiejar_path = self._get_cookiejar_path()
self.session.cookies = cookielib.LWPCookieJar(filename=cookiejar_path)
if os.path.exists(cookiejar_path):
try:
self.session.cookies.load()
except cookielib.LoadError:
# Most likely a pickled cookiejar from earlier versions
pass
self.params = {
'clientBuildNumber': '14E45',
'clientId': self.client_id,
}
self.authenticate()
class RequestUtil(object):
header = {
'Accept':
'*/*',
'Accept-Encoding':
'gzip,deflate,sdch',
'Accept-Language':
'zh-CN,zh;q=0.8,gl;q=0.6,zh-TW;q=0.4',
'Connection':
'keep-alive',
'Content-Type':
'application/x-www-form-urlencoded',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0'
# 'Host': ,
# 'Referer': ,
}
session = requests.session()
session.cookies = cookiejar.LWPCookieJar(globalVar.BASE_PATH +
".cookies/" + globalVar.OJ_NAME)
try:
session.cookies.load(ignore_discard=True, ignore_expires=True)
except:
pass
# 解析验证码，当然只支持简单的
@staticmethod
def parseVerifyCode(url):
image = Image.open(BytesIO(RequestUtil.session.get(url).content))
vcode = pytesseract.image_to_string(image)
return vcode
@staticmethod
def doPost(url, data):
return RequestUtil.session.post(url,
headers=RequestUtil.header,
data=data)
@staticmethod
def doGet(url, param=''):
return RequestUtil.session.get(url,
headers=RequestUtil.header,
params=param)
