'Chrome/64.0.3282.119 Safari/537.36 ' head = { 'User-Agnet': user_agent, 'Accept': 'text/javascript, text/html, application/xml, text/xml, */*', 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'zh-CN,zh;q=0.9', 'Cache-Control': 'no-cache', 'Host': 'www.zdic.net', 'Pragma': 'no-cache', 'Referer': 'http://www.zdic.net/c/cybs/', 'X-Prototype-Version': '1.5.0', 'X-Requested-With': 'XMLHttpRequest', 'Connection': 'keep-alive' } cookie = cookiejar.CookieJar() cookie_support = request.HTTPCookieProcessor(cookie) opener = request.build_opener(cookie_support) req2 = request.Request(url="http://www.zdic.net/c/cybs/", headers=head, method='GET') res2 = opener.open(req2) # f.writelines("--- %s" % url) # f.write("\n") bushou1 = read() for value in bushou1.values(): if type(value) != list: continue for bso in value: bs = parse.quote(bso) bs_url = url % bs
def __init__(self): self.index_url = "http://bkjw.sxu.edu.cn/" self.class_url = "http://bkjw.sxu.edu.cn/ZNPK/KBFB_RoomSel.aspx" self.score_url = "http://bkjw.sxu.edu.cn/_data/login.aspx" self.cookie = cookiejar.CookieJar() self.handler = rq.HTTPCookieProcessor(self.cookie) self.opener = rq.build_opener(self.handler) self.header = { "Host": "bkjw.sxu.edu.cn", "Origin": "http://bkjw.sxu.edu.cn", "Content-Type": "application/x-www-form-urlencoded", "Referer": "http://bkjw.sxu.edu.cn/_data/login.aspx", "Upgrade-Insecure-Requests": "1", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36" } self.header2 = { "Host": "bkjw.sxu.edu.cn", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", "Accept-Language": "zh-CN,zh;q=0.8", "Accept-Encoding": "gzip, deflate", "Referer": "http://bkjw.sxu.edu.cn/xscj/Stu_MyScore.aspx", "Content-Type": "application/x-www-form-urlencoded", "Content-Length": "76", "Cookie": "", "Connection": "keep-alive", "Upgrade-Insecure-Requests": "1" } self.jxl_list = ["101", "105"] self.js_list_101 = [ "1010101", "1010102", "1010103", "1010104", "1010105", "1010106", "1010107", "1010108", "1010109", "1010110", "1010111", "1010112", "1010113", "1010114", "1010115", "1010201", "1010202", "1010203", "1010204", "1010205", "1010206", "1010207", "1010208", "1010301", "1010302", "1010303", "1010304", "1010305", "1010306", "1010307", "1010308", "1010401", "1010402", "1010501", "1010502", "1010503", "1010504", "1010505", "1010506", "1010507", "1010508", "1010509", "1010510", "1010511" ] self.jxl_list_105 = [ '1050101', '1050102', '1050103', '1050104', '1050105', '1050106', '1050107', '1050108', '1050109', '1050110', '1050111', '1050112', '1050113', '1050114', '1050115', '1050116', '1050201', '1050202', '1050203', '1050204', '1050205', '1050206', '1050207', '1050208', '1050209', '1050211', '1050212', '1050213', '1050214', '1050215', '1050216', '1050217', '1050218', '1050301', '1050302', '1050303', '1050304', '1050305', '1050306', '1050307', '1050308', '1050309', '1050310', '1050311', '1050312', '1050313', '1050314', '1050315', '1050316', '1050317', '1050401', '1050402', '1050403', '1050404', '1050405', '1050406', '1050407', '1050408', '1050409', '1050501', '1050502', '1050503', '1050504', '1050505' ] self.client = AipOcr(self.APP_ID, self.API_KEY, self.SECRET_KEY)
# 将cookie信息保存到本地文件 from urllib import request from http.cookiejar import MozillaCookieJar # # 指定保存到cookie.txt文件中 cookiejar = MozillaCookieJar('cookie.txt') handler = request.HTTPCookieProcessor(cookiejar) opener = request.build_opener(handler) resp = opener.open('http://www.baidu.com') # ignore_discard=True 将过期的cookie信息也保存 cookiejar.save(ignore_discard=True) # ###################################### # 从本地文件中读取cookie信息 cookiejar2 = MozillaCookieJar('cookie.txt') # 导入cookie(包含过期的) cookiejar2.load(ignore_discard=True) handler2 = request.HTTPCookieProcessor(cookiejar2) opener2 = request.build_opener(handler2) # 循环打印cookie信息 for cookie in cookiejar2: print(cookie)
def login(): login_url = "http://passport.58.com/58/login/pc/dologin" data = { "username": "******", "password": "******", "fingerprint": "NzKMrMCut4BeOMt0kF1fCeF2m_gm-gLn", "callback": "successFun", "token": "hxf0vp9ivOjNWAp2czg6GOFsAIe8FiZV", "source": "58-default-pc", "path": "https%3A%2F%2Fwh.58.com%2F%3Fpts%3D1557475516428", "domain": "58.com", "finger2": "zh-CN|24|1|4|1366_768|1299_741|-480|1|1|1|undefined|undefined|unknown|Linux", "psdk-d": "jsdk", "psdk-v": "1.0.0" } headers = { "Host": "passport.58.com", "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:66.0) Gecko/20100101 Firefox/66.0", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2", "Accept-Encoding": "gzip, deflate, br", "Referer": "https://passport.58.com/login/?path=https%3A//wh.58.com/&PGTID=0d100000-0009-edcb-b00f-a62b0d78da0d&ClickID=2", "Content-Type": "application/x-www-form-urlencoded", "Content-Length": len(data), "Connection": "keep-alive", #"Cookie": "58home=wh; id58=c5/njVzVHSEKW46eAyYkAg==; city=wh; 58tj_uuid=47345ffc-ffb7-489d-8706-4915075d259a; new_session=1; new_uv=1; utm_source=; spm=; init_refer=https%253A%252F%252Fwww.baidu.com%252Flink%253Furl%253Ds7gvFQHJlUM8AU57tCeR3fOCp5UFqCA5iyYH6lFxv4u%2526wd%253D%2526eqid%253Da393001900004208000000055cd51d1c; als=0; ppStore_fingerprint=4B1708A666D99535DA1C9EA8B24188572F897C40B1C0FB4D; finger_session=NzKMrMCut4BeOMt0kF1fCeF2m_gm-gLn; xxzl_deviceid=mUBgb7V5YnX3Rv1jagzZ2GTj9Q0snQpFczwIRQZVYy%2BewrHdJRGxVAIK1JaIG218", "Upgrade-Insecure-Requests": "1", "TE": "Trailers" } data = parse.urlencode(data).encode("utf-8") print(data["password"]) f = r"58city_cookie.txt" cookie_handler = cookiejar.MozillaCookieJar(f) http_handler = request.HTTPCookieProcessor(cookie_handler) opener = request.build_opener(http_handler) req = request.Request(login_url, data=data, headers=headers) try: rsp = opener.open(req) cookie_handler.save(f, ignore_discard=True, ignore_expires=True) html = rsp.read().decode("utf-8") print(html) print(type(html)) except Exception as e: print(e)
from http.cookiejar import MozillaCookieJar from urllib import request mozila = MozillaCookieJar('cookie.txt') handler = request.HTTPCookieProcessor(mozila) opener = request.build_opener(handler) url = 'http://www.httpbin.org/cookies/set?username=donghao' headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36' } req = request.Request(url=url, headers=headers) resp = opener.open(req) print(resp.read()) mozila.load(ignore_discard=True) for cookie in mozila: print(cookie) # mozila.save(ignore_discard=True,ignore_expires=True) #ignore_discard把即将过期的cookie信息保存
def new_opener(self): self.opener = urlr.build_opener(urlr.HTTPCookieProcessor(self.cookie))
def getCookie(): cookie = http.cookiejar.CookieJar() handler = request.HTTPCookieProcessor(cookie) opener = request.build_opener(handler) return cookie
if os.path.exists(session_filename): json_data = json.loads(open(session_filename, 'rb').read()) for window in json_data.get('windows', []): for cookie in window.get('cookies', []): c = cookie.Cookie(0, cookie.get('name', ''), cookie.get('value', ''), None, False, cookie.get('host', ''), cookie.get('host', '').startswith('.'), cookie.get('host', '').startswith('.'), cookie.get('path', ''), False, False, str(int(time.time()) + 3600 * 24 * 7), False, None, None, {}) cj.set_cookie(c) else: print('Session filename does not exist:', session_filename) return cj session_filename = "C:/Users/elvyn/AppData/Local/Google/Chrome/User Data/Default" cj = load_gc_sessions(session_filename) processor = request.HTTPCookieProcessor(cj) opener = request.build_opener(processor) html = opener.open(URL).read() data = parse_form(html) data['email'] = EMAIL data['password'] = PSW encoded_data = parse.urlencode(data).encode('utf-8') req = request.Request(URL, encoded_data) response = request.urlopen(req) print(response.geturl())
"Mozilla/5.0 (Windows NT 6.1; rv2.0.1) Gecko/20100101 Firefox/4.0.1", "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11", "Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11" ] etree = html.etree userAgent = random.choice(ua_list) # 要爬取的关键词,中文编码出错,待解决 Img_Name = '美女' urlPre = "http://www.ivsky.com/search.php?q=" + Img_Name + "&PageNo=" # 声明一个CookieJar对象实例来保存cookie cookie = cookiejar.CookieJar() # 利用urllib.request库的HTTPCookieProcessor对象来创建cookie处理器,也就CookieHandler cookieHandler = urllib2.HTTPCookieProcessor(cookie) # 设置代理,创建ProxyHandler httpProxyHandler = urllib2.ProxyHandler({"https": "111.231.91.104:8888"}) # 创建opener opener = urllib2.build_opener(httpProxyHandler, cookieHandler) # 安装opener urllib2.install_opener(opener) # 构造图片页数 pageCount = 1 flag = True while flag: urlPre = quote(urlPre, safe=string.printable) requestPre = urllib2.Request(url=urlPre + str(pageCount)) requestPre.add_header('User-Agent', userAgent) # 使用自己安装好的opener responsePre = urllib2.urlopen(requestPre)
def ff_post_blocking(username=None, password=None, content=None, ua_name=None, proxy_kwargs=None): assert username is not None assert password is not None assert content is not None if ua_name is None: ua_name = urllib_request_helper.DEFAULT_USER_AGENT_NAME ff_login_url = url.urljoin(FF_HTTPS_URL, 'account/login?v=2') ff_share_url = url.urljoin(FF_HTTP_URL, 'a/share') cookies = cookiejar.CookieJar() opener = request.build_opener( request.HTTPCookieProcessor(cookiejar=cookies), ) # *** PHASE: get params for auth *** resp = urllib_request_helper.ext_open( opener, ff_login_url, headers=(('User-Agent', ua_name), ), timeout=urllib_request_helper.DEFAULT_TIMEOUT, proxy_kwargs=proxy_kwargs, ) if resp.getcode() != 200 or resp.geturl() != ff_login_url: raise FfError('phase: get params for auth: http-error') data = resp.read(urllib_request_helper.DEFAULT_RESPONSE_LIMIT).decode( 'utf-8', 'replace') at_nodes = tuple( html_parse.find_tags( html_parse.find_tags( (html_parse.html_parse(data), ), 'form', attrs={ 'method': 'post', 'action': '/account/login?v=2', }, ), 'input', attrs={ 'name': 'at', }, )) if not at_nodes: raise FfError('phase -- get params for auth: not found at_param') at_param = at_nodes[0].attrs.get('value') # *** PHASE: auth *** resp = urllib_request_helper.ext_open( opener, ff_login_url, headers=( ('User-Agent', ua_name), ('Referer', ff_login_url), ), data=url.urlencode({ 'email': username, 'password': password, 'at': at_param, 'next': FF_HTTP_URL, }).encode(), timeout=urllib_request_helper.DEFAULT_TIMEOUT, proxy_kwargs=proxy_kwargs, ) if resp.getcode() != 200 or \ (resp.geturl() != FF_HTTP_URL and resp.geturl() != FF_HTTP_URL + '/'): raise AuthFfError('ff auth error') # *** PHASE: publish *** data = resp.read(urllib_request_helper.DEFAULT_RESPONSE_LIMIT).decode( 'utf-8', 'replace') form_nodes = tuple( html_parse.find_tags( (html_parse.html_parse(data), ), 'form', attrs={ 'method': 'post', 'action': '/a/share', }, )) streams_nodes = tuple( html_parse.find_tags( form_nodes, 'input', attrs={ 'name': 'streams', }, )) at_nodes = tuple( html_parse.find_tags( form_nodes, 'input', attrs={ 'name': 'at', }, )) if not streams_nodes or not at_nodes: raise FfError( 'phase -- publish: not found -- streams_param or/and at_param') streams_param = streams_nodes[0].attrs.get('value') at_param = at_nodes[0].attrs.get('value') resp = urllib_request_helper.ext_open( opener, ff_share_url, headers=( ('User-Agent', ua_name), ('X-Requested-With', 'XMLHttpRequest'), ), data=url.urlencode({ 'title': content, 'streams': '+'.join(( streams_param, 'friends', )), 'at': at_param, '_nano': 1, # what is that? }).encode(), timeout=urllib_request_helper.DEFAULT_TIMEOUT, proxy_kwargs=proxy_kwargs, ) if resp.getcode() != 200 or resp.geturl() != ff_share_url: raise PublishFfError('ff publish error') data = resp.read(urllib_request_helper.DEFAULT_RESPONSE_LIMIT).decode( 'utf-8', 'replace') data_json = json.loads(data) publish_ok = data_json.get('success') if publish_ok is None or not isinstance(publish_ok, bool) or not publish_ok: raise PublishFfError('ff publish error (shared, but not success)') publish_html = str(data_json.get('html', '')) publish_nodes = tuple( html_parse.find_tags( (html_parse.html_parse(publish_html), ), 'a', attrs={'class': 'date'}, )) if not publish_nodes: raise PublishFfError('ff publish error (can not get post_url)') post_url = url.urljoin(FF_HTTP_URL, publish_nodes[0].attrs.get('href')) # *** PHASE: end *** return post_url
import urllib.request as urllib2 from http.cookiejar import CookieJar from html.parser import HTMLParser from bs4 import BeautifulSoup as bs from urllib.parse import urlparse from .authData import authData from functions.splitKeyValue import splitKeyValue encoding = 'utf-8' client_id = 5512115 # scope = 'offline,messages' scope = 'messages' email = authData['email'] password = authData['password'] opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(CookieJar()), urllib2.HTTPRedirectHandler()) def vkAuth(): ########################## GET AUTH FROM TIME ################################## authUrl = "http://oauth.vk.com/oauth/authorize?" + \ "redirect_uri=http://oauth.vk.com/blank.html&response_type=token&" + \ "client_id=%s&scope=%s&display=wap" % (client_id, scope) authResponse = opener.open(authUrl) authForm = bs(authResponse.read()).find('form') ########################## SEND AUTH DATA TIME ################################# authData = {}
import http.cookiejar as hcj import urllib.request as ur # 保存cookie到文件MozillaCookieJar格式 filename = 'cookies.txt' cookie1 = hcj.MozillaCookieJar(filename) handler1 = ur.HTTPCookieProcessor(cookie1) opener1 = ur.build_opener(handler1) response = opener1.open("http://www.baidu.com") cookie1.save(ignore_discard=True, ignore_expires=True) # 保存cookie到文件LWPCookieJar格式 filename = 'cookiesLW.txt' cookie2 = hcj.LWPCookieJar(filename) handler2 = ur.HTTPCookieProcessor(cookie2) opener2 = ur.build_opener(handler2) response = opener2.open("http://www.baidu.com") cookie2.save(ignore_discard=True, ignore_expires=True) # 读取文件中的cookie cookie3 = hcj.LWPCookieJar() cookie3.load('cookiesLW.txt', ignore_expires=True, ignore_discard=True) handler = ur.HTTPCookieProcessor(cookie3) opener = ur.build_opener(handler) response = opener.open("http://www.baidu.com") print(response.read().decode('utf-8')) # 输出获取的cookie # cookie = hcj.CookieJar() # handler = ur.HTTPCookieProcessor(cookie) # opener = ur.build_opener(handler)
def __init__(self): self.cookie = http.cookiejar.CookieJar() self.handler = urllib2.HTTPCookieProcessor(self.cookie) self.opener = urllib2.build_opener(self.handler) urllib2.install_opener(self.opener)
def __init__(self, username, password): #设置初始化状态标志,假设成功 self.status = True #以下为微博登录代码 # 创建cookiejar实例对象 self.cookie = cookiejar.CookieJar() # 根据创建的cookie生成cookie的管理器 self.cookie_handle = request.HTTPCookieProcessor(self.cookie) # 创建http请求管理器 self.http_handle = request.HTTPHandler() # 创建https管理器 self.https_handle = request.HTTPSHandler() # 创建请求管理器,将上面3个管理器作为参数属性 # 有了opener,就可以替代urlopen来获取请求了 self.opener = request.build_opener(self.cookie_handle, self.http_handle, self.https_handle) #负责初次登录 #需要传递用户名和密码,来获取登录的cookie凭证 # 登录url,需要从登录form的action属性中获取 self.url = 'https://passport.weibo.cn/sso/login' # 登录所需要的数据,数据为字典形式, # 此键值需要从form_data中对应的input的name属性中获取 self.form_data = { 'username': username, 'password': password, 'savestate': '1', 'r': 'http://m.weibo.cn/', 'entry': 'mweibo', 'client_id': '', 'ec': '', 'pagerefer': 'https://m.weibo.cn/login?backURL=https%253A%252F%252Fm.weibo.cn%252F', 'mainpageflag': '1' } # 将数据解析成urlencode格式 self.form_data = parse.urlencode(self.form_data) self.req = request.Request(self.url, data=self.form_data.encode('utf-8')) self.req.add_header( 'User-Agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36' ) self.req.add_header('Content-Type', 'application/x-www-form-urlencoded') self.req.add_header( 'Referer', 'https://passport.weibo.cn/signin/login?entry=mweibo&res=wel&wm=3349&r=https%3A%2F%2Fm.weibo.cn%2F' ) self.req.add_header('Sec-Fetch-Mode', 'cors') # 正常是用request.urlopen(),这里用opener.open()发起请求 try: with self.opener.open(self.req) as self.response: print('Status:', self.response.status, self.response.reason) for k, v in self.response.getheaders(): print('%s: %s' % (k, v)) print('Data:', self.response.read().decode('utf-8')) #获取ST值 self.req = request.Request('https://m.weibo.cn/compose') self.req.add_header( 'User-Agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36' ) with self.opener.open(self.req) as self.response: print('Status:', self.response.status, self.response.reason) for k, v in self.response.getheaders(): print('%s: %s' % (k, v)) #print('Data:', response.read().decode('utf-8')) data = self.response.read().decode('utf-8') self.st = re.findall('st:.........', data)[0][5:11] #正则表达式寻找到ST print(self.st) except: self.status = False
# from requests.exceptions import HTTPError from http.cookiejar import CookieJar import json from pprint import pprint # json data pretty format url = "https://m.land.naver.com/map/getRegionList?cortarNo=0000000000&mycortarNo=0000000000" # mycortarNo는 로그인 후 사용하는 듯 try: # req=urllib.request.Request(url, None, {'User-Agent': 'Mozilla/5.0 (X11; Linux i686; G518Rco3Yp0uLV40Lcc9hAzC1BOROTJADjicLjOmlr4=) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36','Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8','Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3','Accept-Encoding': 'gzip, deflate, sdch','Accept-Language': 'en-US,en;q=0.8','Connection': 'keep-alive'}) # User-Agent값이 꼭 있어야 307에러 안 남 req = request.Request(url, None, { 'User-Agent': 'Mozilla/5.0', 'Connection': 'keep-alive' }) cj = CookieJar() opener = request.build_opener(request.HTTPCookieProcessor(cj)) res = opener.open(req) raw_res = res.read().decode('utf8', errors='ignore') data = json.loads(raw_res) # pprint(data) aaa = data.get("result").get("list") for h in aaa: print(h.get("CortarNm")) # pprint(data) res.close() except HTTPError as inst: output = format(inst) print(output)
def li_post_blocking(username=None, password=None, title=None, content=None, tags=None, ua_name=None, proxy_kwargs=None): assert username is not None assert password is not None assert title is not None assert content is not None if tags is None: tags = '' if ua_name is None: ua_name = urllib_request_helper.DEFAULT_USER_AGENT_NAME li_login_url = url.urljoin(LI_HTTPS_URL, 'member.php') li_pda_url = url.urljoin(LI_HTTPS_URL, 'interface/pda/') cookies = cookiejar.CookieJar() opener = request.build_opener( request.HTTPCookieProcessor(cookiejar=cookies), ) # *** PHASE: auth *** resp = urllib_request_helper.ext_open( opener, li_login_url, headers=( ('User-Agent', ua_name), ('Referer', LI_HTTPS_URL), ), data=url.urlencode({ 'username': username, 'password': password, 'charset': 'utf', 'action': 'login', }).encode(), timeout=urllib_request_helper.DEFAULT_TIMEOUT, proxy_kwargs=proxy_kwargs, ) if resp.getcode() != 200 or \ (resp.geturl() != LI_HTTPS_URL and resp.geturl() != LI_HTTPS_URL + '/' and resp.geturl() != url.urljoin(LI_HTTP_URL, 'top/')): raise AuthLiError('li auth error') # *** PHASE: get params *** user_id = cookies._cookies['www.liveinternet.ru']['/']['bbuserid'].value addpost_url = url.urljoin( li_pda_url, '?jid={}&act=addpostform'.format(url.quote_plus(user_id))) blog_url = url.urljoin(LI_HTTPS_URL, 'users/{}/'.format(url.quote_plus(user_id))) # *** PHASE: publishing *** resp = urllib_request_helper.ext_open( opener, addpost_url, headers=( ('User-Agent', ua_name), ('Referer', addpost_url), ), data=url.urlencode({ 'tags': tags.encode('windows-1251', 'replace'), 'postmessage': content.encode('windows-1251', 'replace'), 'postheader': title.encode('windows-1251', 'replace'), 'jid': user_id, 'close_level': '0', 'act': 'addpost', }).encode(), timeout=urllib_request_helper.DEFAULT_TIMEOUT, proxy_kwargs=proxy_kwargs, ) if resp.getcode() != 200 or resp.geturl() != addpost_url: raise PublishLiError('li publishing error') data = resp.read(urllib_request_helper.DEFAULT_RESPONSE_LIMIT).decode( 'windows-1251', 'replace') refresh_nodes = tuple( html_parse.find_tags((html_parse.html_parse(data), ), 'meta', in_attrs={'http-equiv': 'Refresh'})) if not refresh_nodes: raise PublishLiError('li publishing error (may be limit?)') # *** PHASE: get post url *** resp = urllib_request_helper.ext_open( opener, blog_url, headers=(('User-Agent', ua_name), ), timeout=urllib_request_helper.DEFAULT_TIMEOUT, proxy_kwargs=proxy_kwargs, ) data = resp.read(urllib_request_helper.DEFAULT_RESPONSE_LIMIT).decode( 'windows-1251', 'replace') post_link_nodes = tuple( html_parse.find_tags((html_parse.html_parse(data), ), 'a', in_attrs={'class': 'TTL'})) if not post_link_nodes: raise PublishLiError('li publishing error (PHASE: get post url)') post_url = post_link_nodes[0].attrs['href'] # *** END *** return post_url
def get_password_image(cookie): cookie_support = request.HTTPCookieProcessor(cookie) opener = request.build_opener(cookie_support , request.HTTPHandler) opener.open(YKT_TRY_URL) return opener.open(YKT_PASSWORD_PHOTO_URL).read()
def __init__(self): cookoe_object = CookieJar() handler = request.HTTPCookieProcessor(cookoe_object) self.opener = request.build_opener(handler)
from urllib import request, parse from http import cookiejar cookie = cookiejar.CookieJar() cookie_handler = request.HTTPCookieProcessor(cookie) http_handler = request.HTTPHandler() https_handler = request.HTTPSHandler() opener = request.build_opener(http_handler, https_handler, cookie_handler) def login(): url = 'http://passport.zhan.com/UsersLogin/login.html?url=http%3A%2F%2Fi.zhan.com%2F' data = {'username': '******', 'pwd': '73143177da'} data = parse.urlencode(data) req = request.Request(url, data=data.encode()) rsp = opener.open(req) html = rsp.read().decode() with open('test.html', 'w') as f: f.write(html) def getHomepage(): url = 'http://i.zhan.com/'
from config import * from urllib.error import HTTPError, URLError # loads station infomation with open('station.json', 'r', encoding='utf-8') as f: station = load(f) # not warning on unverified http/https ssl._create_default_https_context = ssl._create_unverified_context # create a cookie respoitory header = { 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36' } jar = cookiejar.LWPCookieJar() cookie = request.HTTPCookieProcessor(jar) opener = request.build_opener(cookie) opener.addheaders = [(key, value) for key, value in header.items()] request.install_opener(opener) def post_urls(url, data): data_encode = parse.urlencode(data).encode() req = request.Request(url, data_encode) response = opener.open(req) return response def input_location(): img = np.array(Image.open('img_for_identify.png')) fig = plt.figure(1) plt.imshow(img)