def __init__(self):
self.headers = {
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
"Accept-Encoding":
"gzip, deflate",
"Accept-Language":
"zh-CN,zh;q=0.9",
"Cache-Control":
"no-cache",
"Connection":
"keep-alive",
"Host":
"adminweb.adminrichboxbox.com",
"Pragma":
"no-cache",
"Referer":
"https://adminweb.adminrichboxbox.com/",
"Upgrade-Insecure-Requests":
1,
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36",
"X-ADMIN- TOKEN":
"eyJsb2NhbGUiOiJ6aF9DTiIsImFsZyI6IkhTNTEyIn0.eyJzdWIiOiJZaW5nIiwiZXhwIjoxNTY2ODgyMzg1fQ.pYVtcUzyqwvom9OPdJXK9eu_PBiyKgRMcKI-oHWf3UaPx89-87fyJXWhQCZvZUwfNCZlxlLduU8sZYog9pUHRg"
}
# 自定义请求方法，获取cookie
cookie_object = cookiejar.CookieJar()
hanler = urllib.request.HTTPCookieProcessor(cookie_object)
self.opener = urllib.request.build_opener(hanler)
# 定义请求连接
self.loanurl = 'https://adminweb.adminrichboxbox.com/api/api/collection/my?'
# 验证码连接
self.catpch = 'https://adminweb.adminrichboxbox.com/api/auth/captcha?width=120&height=50&serialId=KY1uqphQ'
self.login = '******'
def __collect__(self):
Cookiejar = cookiejar.CookieJar()
opener = urllib_request.build_opener(
urllib_request.HTTPCookieProcessor(Cookiejar))
_header = dict(self.headers.items())
if self.cookie:
_header.update({'Cookie': self.cookie})
req = urllib_request.Request(self.url,
headers=_header,
origin_req_host=self.host)
error_counter = 0
while error_counter < 3:
try:
res = opener.open(req)
break
except:
error_counter += 1
time.sleep(0.5)
else:
# return None, None
raise Exception('UrlNotRespond')
return res, Cookiejar._cookies
def __init__(self, *args, **kwargs):
self.is_debug = kwargs.get('is_debug', False)
self.data_format = kwargs.get('data_format', 'json')
self.ds = DataSerialize(format=self.data_format)
error_info = {'error_reason': '', 'error_code': 0}
self.error_info = kwargs.get('error_info', error_info)
self.is_data_serialize = kwargs.get('is_data_serialize', 0)
if 'is_data_serialize' in kwargs:
self.is_data_serialize = kwargs.get('is_data_serialize', 0)
self.is_send_data_serialize = self.is_data_serialize
self.is_rec_data_deserialize = self.is_data_serialize
else:
self.is_send_data_serialize = kwargs.get('is_send_data_serialize',
0)
self.is_rec_data_deserialize = kwargs.get(
'is_rec_data_deserialize', 0)
headers = {"Content-type": "application/json", "Accept": "text/plain"}
self.headers = kwargs.get('headers', headers)
self.cj_ins = cookielib.CookieJar()
self.opener = urllib.request.build_opener(
urllib.request.HTTPCookieProcessor(self.cj_ins))
urllib.request.install_opener(self.opener)
self.cookies = {}
self.disable_ssl_check = kwargs.get('disable_ssl_check', False)
def cookie_test():
# 1.代码登录
#1.1 登录的网址
login_url = "https://www.yaozh.com/login"
# 1.2 登录的参数
login_form_data = {
"username":"******",
"pwd":"qwer1234",
"formhash":"CE3ADF28C5",
"backurl":"https%3A%2F%2Fwww.yaozh.com%2F"
}
# 1.3 发送登录请求
cookie_jar = cookiejar.CookieJar()
# 定义有添加 cookie功能的处理器
cookie_handler = urllib.request.HTTPCookieProcessor(cookie_jar)
# 根据处理器生成 opener
openner = urllib.request.build_opener(cookie_handler)
# 带着参数 发送post请求
headers = {
"User-Agent": "Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0; HTC; Titan)"
}
login_form_data_url = parse.urlencode(login_form_data).encode("UTF-8")
# post 参数必须是 bytes
login_request = urllib.request.Request(login_url,headers=headers,data=login_form_data_url)
# 如果登录成功《cookiejar自动保存cookie
openner.open(login_request)
# 2.代码带着cookie去访问 个人中心
center_url = "https://www.yaozh.com/member/"
center_request = urllib.request.Request(center_url,headers=headers)
response = openner.open(center_url)
# bytes -> str
data = response.read().decode()
with open("cookiejar.html", "w", encoding="UTF-8")as t:
t.write(data)
def login(self, phone_num, password):
url = 'https://www.zhihu.com/#signin'
res = request.urlopen(url)
if res.getcode() == 200:
content = res.read().decode('utf-8')
(xsrf, captcha_type) = get_sign_info(content)
print("xsrf: %s, captcha_type: %s" % (xsrf, captcha_type))
header = {
'Host': 'www.zhihu.com',
'Origin': 'https://www.zhihu.com',
'Referer': 'https://www.zhihu.com/',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36',
'X-Requested-With': 'XMLHttpRequest',
}
post_data = {
'_xsrf': xsrf,
'password': password,
'captcha_type': captcha_type,
'phone_num': phone_num
}
url = 'https://www.zhihu.com/login/phone_num'
cj = cookiejar.CookieJar()
pro = request.HTTPCookieProcessor(cj)
opener = request.build_opener(pro)
request.install_opener(opener)
data = urllib.parse.urlencode(post_data).encode(encoding='utf_8')
res = request.Request(url, data=data, headers=header)
res = request.urlopen(res)
if res.getcode() == 200:
print("recode:%s,reinfo:%s" % (res.getcode(), res.info()))
return res.read().decode('utf-8')
def SinaWeibo_GetCookies(username, password):
sso_url = "https://passport.weibo.cn/sso/login"
login_data = urllib.parse.urlencode([
('username', username),
('password', password),
('entry', 'mweibo'),
('client_id', ''),
('savestate', '1'),
('ec', ''),
]).encode(encoding='UTF-8')
req = urllib.request.Request(sso_url)
req.add_header('Origin', 'https://passport.weibo.cn')
req.add_header(
'User-Agent',
'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.14 (KHTML, like Gecko) Chrome/10.0.601.0 Safari/534.14'
)
req.add_header(
'Referer',
'https://passport.weibo.cn/signin/login?entry=mweibo&res=wel&wm=3349&r=http%3A%2F%2Fm.weibo.cn%2F'
)
weibo_cookies = cookiejar.CookieJar()
handler = urllib.request.HTTPCookieProcessor(weibo_cookies)
opener = urllib.request.build_opener(handler)
opener.open(req, data=login_data)
cookie = dict()
if weibo_cookies._cookies.__contains__(".weibo.cn"):
logging.info("获取密码成功：" + str(username))
weibo_cn_cookiejar = weibo_cookies._cookies[".weibo.cn"]["/"]
cookie['SCF'] = weibo_cn_cookiejar['SCF'].value
cookie['SSOLoginState'] = weibo_cn_cookiejar['SSOLoginState'].value
cookie['SUB'] = weibo_cn_cookiejar['SUB'].value
cookie['SUHB'] = weibo_cn_cookiejar['SUHB'].value
else:
logger.info("获取账号:" + str(username) +
" 的cookie失败，原因：1. 账号或密码错误。 2. 微博登录次数过多，可以换网络登录或过4小时再登录！")
return cookie
def __init__(self):
self.get_proxy()
self.login_post_values = None
self.email = "*****@*****.**"
self.password = "******"
self.cj = cookiejar.CookieJar()
self.proxies = {
"http": 'http://{}:{}@{}'.format('squid_user', 'Urun2017', '137.175.62.130:65432')
}
# self.proxy_support = ProxyHandler(self.proxies)
self.handler = request.HTTPCookieProcessor(self.cj)
auth_handler = request.HTTPBasicAuthHandler()
auth_handler.add_password(realm=None,
uri='http://137.175.62.130:65432',
user='******',
passwd="Urun2017"
)
self.opener = request.build_opener(auth_handler, request.HTTPBasicAuthHandler)
self.opener.add_handler(self.handler)
self.headers1 = {
"User-Agent": "Mozilla / 5.0(Windows NT 6.1;Win64;x64) AppleWebKit / 537.36(KHTML, likeGecko) "
"Chrome / 69.0.3497.81Safari / 537.36"
}
self.post_data_params = None
def cookies_crawl():
#2.登录的网址
login_url = "https://www.yaozh.com/login/"
#3.登录的数据
login_form_data = {
"username": "******",
"pwd": "as751286012",
"formhash": "6E1A90D3A4",
"backurl": "https%3A%2F%2Fwww.yaozh.com%2F"
}
#4.发送登录请求POST
cook_jar = cookiejar.CookieJar()
#定义有添加的 cook 功能的处理器
cook_handler = urllib.request.HTTPCookieProcessor(cook_jar)
#根据处理器创建opener
opener = urllib.request.build_opener(cook_handler)
#带参数的 发送请求头
#添加请求头
header = {
"User-Agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"
}
# 出现ssl错误
ssl._create_default_https_context = ssl._create_unverified_context
str_data = urllib.parse.urlencode(login_form_data).encode('utf-8')
login_request = urllib.request.Request(login_url,headers=header,data=str_data)
#如果登录成功，cookjar自动保存cookie
opener.open(login_request)
#代码带着cookie去访问
center_url = 'https://www.yaozh.com/member/'
center_data = urllib.request.Request(center_url,headers=header)
response = opener.open(center_data).read().decode("utf-8")
with open("11crawl_cookie.html","w",encoding='utf-8')as f:
f.write(response)
def __init__(self):
self.headers = {
"Host": "adminweb.pinjamkilat.co.id",
"User-Agent":
"Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/60.0",
"Accept": "*/*",
"Accept-Language":
"zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2",
"Accept-Encoding": "gzip, deflate, br",
"Referer": "https://adminweb.pinjamkilat.co.id/collection/my",
"origin": "https://adminweb.pinjamkilat.co.id",
"Connection": "keep-alive"
}
#自定义请求方法，获取cookie
cookie_object = cookiejar.CookieJar()
hanler = urllib.request.HTTPCookieProcessor(cookie_object)
self.opener = urllib.request.build_opener(hanler)
#定义请求 历史案件：https://adminweb.pinjamkilat.co.id/api/api/review/loanAppAssignedHistory?offset=0&limit=10
self.baseurl = 'https://adminweb.pinjamkilat.co.id/api/api/review/loanAppAssignedHistory?'
self.loadUrl = 'https://adminweb.pinjamkilat.co.id/api/api/review/applyHistory?'
#验证码连接
self.catpch = 'https://adminweb.pinjamkilat.co.id/api/auth/captcha?width=120&height=50&serialId=KY1uqphQ'
self.login = '******'
def downloadHttp(self, resource_info, retry_count=3, proxy=None, data=None):
headers = {
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.100 Safari/537.36"
}
destination_filename = self.set_filename(resource_info.output_filename)
if resource_info.uri is None:
return None
try:
req = urllib.request.Request(resource_info.uri, headers=headers, data=data)
cookie = cookiejar.CookieJar()
cookie_process = urllib.request.HTTPCookieProcessor(cookie)
opener = urllib.request.build_opener()
if proxy:
proxies = {urllib.urlparse(resource_info.uri).scheme: proxy}
opener.add_handler(urllib.request.ProxyHandler(proxies))
content = opener.open(req).read()
except urllib.error.URLError as e:
print('downloadHttp download error:', e.reason)
content = None
if retry_count > 0:
if hasattr(e, 'code') and 500 <= e.code < 600:
return self.download(resource_info.uri, retry_count-1, headers, proxy, data)
return content
def get_user_info(uid):
# cookie
cookie = cookiejar.CookieJar()
handler = request.HTTPCookieProcessor(cookie)
opener = request.build_opener(handler)
# url
url = f'https://api.bilibili.com/x/web-interface/card?mid={str(uid)}'
# headers
headers = {
'User-Agent': 'BiliBili-uploaders-fans-analyser Client/0.0.1',
'Referer': '',
}
# POST-need
dic = {}
# encode the data
data = bytes(
parse.urlencode(dic),
encoding='utf-8',
)
# send req to web
req = request.Request( # make request
url=url,
data=data,
headers=headers,
method='GET',
)
response = opener.open(req) # open the page / send the request
# convert json data into python dict
usr_info = json.loads(response.read().decode('utf-8', 'ignore'))
print(f"fans: {usr_info['data']['card']['fans']}")
def ch17():
auth_handler = request.HTTPBasicAuthHandler()
auth_handler.add_password('inflate', 'www.pythonchallenge.com', 'huge',
'file')
jar = cookiejar.CookieJar()
cookie_handler = request.HTTPCookieProcessor(jar)
opener = request.build_opener(auth_handler, cookie_handler)
opener.open('http://www.pythonchallenge.com/pc/def/linkedlist.php')
hint = list(jar)[0].value
print('hint:', hint)
nid = '12345'
cookies = ''
while nid:
url = 'http://www.pythonchallenge.com/pc/def/linkedlist.php?busynothing=%s' % nid
r = opener.open(url)
text = r.read().decode()
s = re.search(r'and the next busynothing is ([0-9]+)', text)
if len(list(jar)) > 0:
cookies += list(jar)[0].value
if not s:
nid = None
print('final:', text)
else:
nid = s.groups()[0]
print(cookies)
bm = request.unquote_to_bytes(cookies.replace('+', ' '))
message = bz2.decompress(bm)
print(message)
phonebook = xmlrpc.client.ServerProxy(
'http://www.pythonchallenge.com/pc/phonebook.php')
phonebook.phone('Leopold')
list(jar)[0].value = 'the flowers are on their way'
print(
opener.open(
'http://www.pythonchallenge.com/pc/stuff/violin.php').read())
return 'balloons'
def __init__(self, email=None, password=None):
print('虾米音乐签到\n')
# Get log file
LOG_DIR = os.path.join(os.path.expanduser("~"), 'log')
if not os.path.isdir(LOG_DIR):
os.mkdir(LOG_DIR)
self.LOG_PATH = os.path.join(LOG_DIR, 'xiami_auto_checkin.log')
self.users = set(self.__get_users())
if not self.users:
return
# Init
myCookie = http.HTTPCookieProcessor(cookie.CookieJar())
self.opener = http.build_opener(myCookie)
http.install_opener(self.opener)
if email and password:
self.__add_user(email, password)
for each_user in self.users:
(email, password) = each_user.split(':')
self.__checkin(email.strip(), password.strip())
def fetch_o2_token(self):
import mechanize
try:
import http.cookiejar as cookie
except ImportError:
import cookie
br = mechanize.Browser()
cj = cookie.CookieJar()
br.set_cookiejar(cj)
br.set_handle_equiv(False)
br.set_handle_robots(False)
br.addheaders = [('authority', 'o2api.waipu.tv'),
('User-agent',
'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 '
'Firefox/3.0.1')]
br.set_handle_redirect(mechanize.HTTPRedirectHandler)
br.open("https://o2api.waipu.tv/api/o2/login/token?redirectUri=https%3A%2F%2Fo2tv.waipu.tv%2F&inWebview=true")
br.select_form("Login")
control = br.form.find_control("IDToken1")
control.value = self.__username
control = br.form.find_control("IDToken2")
control.value = self.__password
response = br.submit()
response_plain = str(response.read())
if response_plain.find("Ihre Eingabe ist ungültig. Falls Sie einen Business Tarif bei") != -1:
# invalid login credentials
return 401
for cookie in cj:
if cookie.name == "user_token":
token = str(cookie.value).strip()
decoded_token = self.decode_token(token)
self._access_token = token
xbmcaddon.Addon().setSetting("access_token", token)
return 200
return -1
def cookie_login(loginurl, values):
""" Open a session
login_url : the login url
values : dictionary containing login form field
"""
global urlOpener
# Enable cookie support for urllib2
mcookiejar = cookiejar.CookieJar()
urlOpener = build_opener(HTTPCookieProcessor(mcookiejar), MultipartPostHandler)
data = urlencode(values)
request = Request(loginurl, data)
url = urlOpener.open(request) # Our mcookiejar automatically receives the cookies
urllib.request.install_opener(urlOpener)
# Make sure we are logged in by checking the presence of the cookie "session_ser".
# (which is the cookie containing the session identifier.)
if not 'session_ser' in [cookie.name for cookie in mcookiejar]:
print("Login failed !")
return False
else:
print("We are logged in !")
return True
def keys_attrs(lkey, oformat, host, ckey, cert, debug=0):
"Contact host for list of key/attributes pairs"
url = '%s/das/keys?view=json' % host
headers = {"Accept": "application/json", "User-Agent": DAS_CLIENT}
req = urllib2.Request(url=url, headers=headers)
if ckey and cert:
ckey = fullpath(ckey)
cert = fullpath(cert)
http_hdlr = HTTPSClientAuthHandler(ckey, cert, debug)
else:
http_hdlr = urllib2.HTTPHandler(debuglevel=debug)
proxy_handler = urllib2.ProxyHandler({})
cookie_jar = cookielib.CookieJar()
cookie_handler = urllib2.HTTPCookieProcessor(cookie_jar)
opener = urllib2.build_opener(http_hdlr, proxy_handler, cookie_handler)
fdesc = opener.open(req)
data = json.load(fdesc)
fdesc.close()
if oformat.lower() == 'json':
if lkey == 'all':
print(json.dumps(data))
else:
print(json.dumps({lkey:data[lkey]}))
return
for key, vdict in data.items():
if lkey == 'all':
pass
elif lkey != key:
continue
print()
print("DAS key:", key)
for attr, examples in vdict.items():
prefix = ' '
print('%s%s' % (prefix, attr))
for item in examples:
print('%s%s%s' % (prefix, prefix, item))
def auto_cookie():
# 1.代码登陆
# 1.1登陆地址
# 1.2登陆信息
login_form_data = {
#可以在页面找到相应的formash和backurl的值
"username": "******",
"pwd": "xin665968",
"formhash": "4A95C39D38",
"backurl": "https%3A%2F%2Fwww.yaozh.com%2F"
}
# data参数需要转码（字典到字符串），并设置为二进制形式
# 因为post请求中的data需要的是二进制类型
str_form_data = urllib.parse.urlencode(login_form_data).encode("utf-8")
# 1.3发送登陆请求
cookie_jar = cookiejar.CookieJar()
# 有添加cookie的处理器
cookie_handler = urllib.request.HTTPCookieProcessor(cookie_jar)
opener = urllib.request.build_opener(cookie_handler)
# 添加请求头并发送请求
login_url = "https://www.yaozh.com/login/"
headers = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36"
}
login_request = urllib.request.Request(login_url, headers=headers, data=str_form_data)
# 登陆请求后，opener会自动保存相应的cookie
login_response = opener.open(login_request)
# 2.利用cookiejar可以获取cookie，进而再去发送请求
url = "https://www.yaozh.com/member"
request = urllib.request.Request(url, headers=headers)
response = opener.open(request)
str_response = response.read().decode("utf-8")
with open("3.html", "w", encoding="utf-8") as f:
f.write(str_response)
def __init__(self,
api_url,
cookie_file=None,
cookiejar=None,
user_agent=DEFAULT_UA,
http_user=None,
http_password=None):
self._api_url = api_url
self._http_user = http_user
self._http_password = http_password
if cookiejar:
self._cj = cookiejar
elif cookie_file:
self._cj = cookielib.LWPCookieJar(cookie_file)
try:
self._cj.load()
except IOError:
self._cj.save()
self._cj.load()
else:
self._cj = cookielib.CookieJar()
self._opener = urllib2.build_opener(
urllib2.HTTPCookieProcessor(self._cj))
self._opener.addheaders = [('User-Agent', user_agent)]
def method5():
login_url = 'http://www.jobbole.com/wp-admin/admin-ajax.php'
user_agent = r'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.94 Safari/537.36'
head = {'User-Agnet': user_agent, 'Connection': 'keep-alive'}
Login_Data = {}
Login_Data['action'] = 'user_login'
Login_Data['redirect_url'] = 'http://www.jobbole.com/'
Login_Data['remember_me'] = '0' #是否一个月内自动登陆
Login_Data['user_login'] = '******' #改成你自己的用户名
Login_Data['user_pass'] = '******' #改成你自己的密码
#使用urlencode方法转换标准格式
logingpostdata = parse.urlencode(Login_Data).encode('utf-8')
#声明一个CookieJar对象实例来保存cookie
cookie = cookiejar.CookieJar()
#利用urllib.request库的HTTPCookieProcessor对象来创建cookie处理器,也就CookieHandler
cookie_support = request.HTTPCookieProcessor(cookie)
#通过CookieHandler创建opener
opener = request.build_opener(cookie_support)
#创建Request对象
req1 = request.Request(url=login_url, data=logingpostdata, headers=head)
# 使用自己创建的opener的open方法
response = opener.open(req1)
html = response.read().decode('utf-8')
# 评论起始编号
comid = '0'
url = "http://coral.qq.com/article/" + vid + "/comment?commentid=" + comid + "&reqnum=20"
headers = {
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
# Accept-Encoding: gzip, deflate #最好不要写或者写成Accept-Encoding: utf-8,gb2312
"Accept-Language": "zh-CN,zh;q=0.9",
"User-Agent":
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.170 Safari/537.36",
"Connection": "keep-alive", #或者"Proxy-Connection": "keep-alive"
"referer": "qq.com" #一般为它的首页
}
cjar = cookiejar.CookieJar()
proxy = request.ProxyHandler({'http': '127.0.0.1:8888'}) #为Fiddler截获调试用
opener = request.build_opener(proxy, request.HTTPHandler,
request.HTTPCookieProcessor(cjar))
headall = []
for key, value in headers.items():
item = (key, value)
headall.append(item)
opener.addheaders = headall
request.install_opener(opener)
# 建立自定义函数，实现自动爬取对应评论网页并返回爬取内容
def craw(vid, comid):
url = "http://coral.qq.com/article/" + vid + "/comment?commentid=" + comid + "&reqnum=20"
# Created by Pycharm. # User: [email protected] # Date : 2018/7/3 # Desc : 模拟登陆百度爬取信息 from urllib import request import urllib from urllib import parse from http import cookiejar import re URL_BAIDU_INDEX = u'http://www.baidu.com/' URL_BAIDU_LOGIN = '******' # 设置用户名、密码 username = '******' password = '******' # 设置cookie，这里cookiejar可自动管理，无需手动指定 cj = cookiejar.CookieJar() opener = request.build_opener(request.HTTPCookieProcessor(cj)) request.install_opener(opener) reqReturn = request.urlopen(URL_BAIDU_INDEX) # 构造登录请求参数，该请求数据是通过抓包获得，对应[url=https://passport.baidu.com/v2/api/?login]https://passport.baidu.com/v2/api/?login[/url]请求 postData = { 'username': '', 'password': '', } postData = urllib.parse.urlencode(postData).encode('utf-8') print(postData) # 发送登录请求 loginRequest = request.Request(URL_BAIDU_LOGIN, postData) print(loginRequest) sendPost = request.urlopen(loginRequest)
def get_gid():
return _get_runntime().call('gid')
def get_callback():
return _get_runntime().call('callback')
if __name__ == '__main__':
s = requests.session()
#使用urlencode方法转换标准格式
#logingpostdata = parse.urlencode(Login_Data).encode('utf-8')
#声明一个CookieJar对象实例来保存cookie
cookies = cookiejar.CookieJar()
#利用urllib.request库的HTTPCookieProcessor对象来创建cookie处理器,也就CookieHandler
cookie_support = request.HTTPCookieProcessor(cookies)
#通过CookieHandler创建opener
opener = request.build_opener(cookie_support)
#创建Request对象
req1 = opener.open(
'http://www.dianping.com'
) # request.Request(url=login_url, data=logingpostdata, headers=head)
cookies.add_cookie_header
# cookies.set_cookie(cookiejar.Cookie(version=0, name='BAIDUCUID', value="pULIljZt3zWFHfjpO3ZACs6tj54zmeXK2Vjaz8BRzhkk67Cvhk6Ho5pPbgV3H5EtA",
# port=None, port_specified=None,
# domain=".baidu.com", domain_specified=None, domain_initial_dot=None,
# path="/", path_specified=None,
# secure=None,
# expires=None,
def test_supports_providing_cookiejar(self):
cookiejar = http_cookiejar.CookieJar()
app = webtest.TestApp(debug_app, cookiejar=cookiejar)
self.assertIs(cookiejar, app.cookiejar)
# User-agent is essentially important for getting page isntead of 403 error
useragent = 'Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101 Goanna/4.8 Firefox/68.0 PaleMoon/29.2.1'
# without any given link it downloads the last page of your submissions. downloading the whole list of submissions and galleries is yet to be added
if len(argv) < 2:
link72 = 'https://www.furaffinity.net/msg/submissions/old@72'
else:
link72 = argv[1]
# you can set other proxies than just tor by replacing addresses and "socks5" to "http" and "https"
proxies = {"https": "socks5://127.0.0.1:9050", "http": "socks5://127.0.0.1:9050"}
# creating two sessions for fa and fa's cdn
sess = requests.Session()
cjar_fa = cookiejar.CookieJar()
get_cookies(cjar_fa, cookie_path, 'furaffinity')
sess.headers['User-Agent'] = useragent
sess.cookies = cjar_fa
sess_cdn = requests.Session()
cjar_cdn = cookiejar.CookieJar()
get_cookies(cjar_cdn, cookie_path, 'facdn')
sess_cdn.headers['User-Agent'] = useragent
sess_cdn.cookies = cjar_cdn
# open and parse start page
resp = sess.get(link72, proxies=proxies)
soup = BeautifulSoup(resp.text, 'lxml')
pattern = r'<a href="(.+?)">Download<\/a>'
def __init__(self):
self.LOGIN_URL = 'https://secure.nicovideo.jp/secure/login?site=niconico'
self.LIVE_API_URL = 'http://watch.live.nicovideo.jp/api/'
self.cookies = cookielib.CookieJar()
cjhdr = urllib.request.HTTPCookieProcessor(self.cookies)
self.opener = build_opener(cjhdr)
def request(self, host, handler, request_body, verbose=False):
with GetUrlCookieFile(self.orig_host,
not verbose) as (cookiefile, proxy):
# Python doesn't understand cookies with the #HttpOnly_ prefix
# Since we're only using them for HTTP, copy the file temporarily,
# stripping those prefixes away.
if cookiefile:
tmpcookiefile = tempfile.NamedTemporaryFile()
try:
with open(cookiefile) as f:
for line in f:
if line.startswith("#HttpOnly_"):
line = line[len("#HttpOnly_"):]
tmpcookiefile.write(line)
tmpcookiefile.flush()
cookiejar = cookielib.MozillaCookieJar(tmpcookiefile.name)
cookiejar.load()
finally:
tmpcookiefile.close()
else:
cookiejar = cookielib.CookieJar()
proxyhandler = urllib.request.ProxyHandler
if proxy:
proxyhandler = urllib.request.ProxyHandler({
"http": proxy,
"https": proxy
})
opener = urllib.request.build_opener(
urllib.request.HTTPCookieProcessor(cookiejar), proxyhandler)
url = urllib.parse.urljoin(self.orig_host, handler)
parse_results = urllib.parse.urlparse(url)
scheme = parse_results.scheme
if scheme == 'persistent-http':
scheme = 'http'
if scheme == 'persistent-https':
# If we're proxying through persistent-https, use http. The
# proxy itself will do the https.
if proxy:
scheme = 'http'
else:
scheme = 'https'
# Parse out any authentication information using the base class
host, extra_headers, _ = self.get_host_info(parse_results.netloc)
url = urllib.parse.urlunparse(
(scheme, host, parse_results.path, parse_results.params,
parse_results.query, parse_results.fragment))
request = urllib.request.Request(url, request_body)
if extra_headers is not None:
for (name, header) in extra_headers:
request.add_header(name, header)
request.add_header('Content-Type', 'text/xml')
try:
response = opener.open(request)
except urllib.error.HTTPError as e:
if e.code == 501:
# We may have been redirected through a login process
# but our POST turned into a GET. Retry.
response = opener.open(request)
else:
raise
p, u = xmlrpc.client.getparser()
while 1:
data = response.read(1024)
if not data:
break
p.feed(data)
p.close()
return u.close()
def cookie_jar(self):
jar = cookiejar.CookieJar()
if self.cookies:
for cookie in pickle.loads(self.cookies):
jar.set_cookie(cookie)
return jar
red('Error: This is not an atcoder directory.')
sys.exit(0)
try:
cookies = cookielib.MozillaCookieJar(cookies_path)
cookies.load()
except FileNotFoundError:
red('Error: Load atcoder.jp cookies again.')
cookies = None
if cookies is not None:
# internal_cookies = cookies#._cookies
try:
cookies._cookies['atcoder.jp']
except KeyError:
red('Error: Load atcoder.jp cookies again.')
cj = cookielib.CookieJar()
for cookie in cookies:
if cookie.domain.find('atcoder.jp') != -1:
cj.set_cookie(cookie)
# print(cookie.name, cookie.value, cookie.domain)
cookies = cj
java = False
cpp = False
get_languages()
# print('java', java, 'cpp', cpp)
if args[0] == 'run':
run()
elif args[0] == 'test':
test()
#
url1 = "https://www.renren.com/969250852/newsfeed/specialfocus"
url2 = "http://www.renren.com/getFeedOptions"
formdata = {
"devid": "46a9a729705a54a0c1071d0005198a58",
"devtype": "se",
"metatype": "1",
"method": "Meta.upload",
"qid": "3069526447",
"rtick": "84210",
"v": "10.0.1581.0",
"is_encrypt": "1",
"is_compress": "0",
"meta_count": "12",
"orgsize": "1614",
"sign": "b1ee4b5e4bc731d80fe32ee08b5f487a"
}
formdata = parse.urlencode(formdata).encode()
cj = jar.CookieJar()
handler = request.HTTPCookieProcessor(cj)
opener = request.build_opener(handler)
reques = request.Request(url2, headers=header)
requ2 = opener.open(reques, data=formdata)
print(requ2.getheaders())
#
requ1 = request.Request(url1, headers=header)
requ = opener.open(requ1)
#
with open("C:\\Users\\Administrator\Desktop\\renren.html", "wb") as fo:
fo.write(requ.read())
def __init__(self):
cookie_object = cookiejar.CookieJar()
handler = request.HTTPCookieProcessor(cookie_object)
self.opener = request.build_opener(handler)
