def cookiejar_create(self, cookiejar_file, session):
cookie_jar = LWPCookieJar(cookiejar_file.name)
cookie_jar.set_cookie(
Cookie(0, self.COOKIE_NAME, session, None, False, '', False, True,
'/', True, True, None, None, None, None, {}))
cookie_jar.save()
cookiejar_file.flush()
def cookie_from_str(cookie_str):
from http.cookiejar import split_header_words, LWPCookieJar, LoadError, Cookie, iso2time
import time
cookie_str = cookie_str.split('\n')
cookie = LWPCookieJar()
index = 0
while 1:
line = cookie_str[index]
index += 1
if line == "": break
if not line.startswith(HEADER):
continue
line = line[len(HEADER):].strip()
for data in split_header_words([line]):
name, value = data[0]
standard = {}
rest = {}
for k in BOOLEAN_ATTRS:
standard[k] = False
for k, v in data[1:]:
if k is not None:
lc = k.lower()
else:
lc = None
if (lc in VALUE_ATTRS) or (lc in BOOLEAN_ATTRS):
k = lc
if k in BOOLEAN_ATTRS:
if v is None: v = True
standard[k] = v
elif k in VALUE_ATTRS:
standard[k] = v
else:
rest[k] = v
h = standard.get
expires = h("expires")
discard = h("discard")
if expires is not None:
expires = iso2time(expires)
if expires is None:
discard = True
domain = h("domain")
domain_specified = domain.startswith(".")
c = Cookie(h("version"), name, value,
h("port"), h("port_spec"),
domain, domain_specified, h("domain_dot"),
h("path"), h("path_spec"),
h("secure"),
expires,
discard,
h("comment"),
h("commenturl"),
rest)
cookie.set_cookie(c)
return cookie
def cookiejar_create(self, cookiejar_file, session):
cookie_jar = LWPCookieJar(cookiejar_file.name)
cookie_jar.set_cookie(Cookie(0, self.COOKIE_NAME, session,
None, False,
'', False, True,
'/', True,
True,
None, None, None, None, {}))
cookie_jar.save()
cookiejar_file.flush()
def save_cookie(self, file_name=None, **kwargs):
file_name = file_name or self._cookie_file
lwp_jar = LWPCookieJar()
for item in self._cookie_jar:
args = dict(vars(item).items())
args['rest'] = args['_rest']
del (args['_rest'])
cookie = Cookie(**args)
lwp_jar.set_cookie(cookie)
lwp_jar.save(file_name, **kwargs)
def cookie_from_str(cookie_str):
from http.cookiejar import split_header_words, LWPCookieJar, LoadError, Cookie, iso2time
import time
cookie_str = cookie_str.split('\n')
cookie = LWPCookieJar()
index = 0
while 1:
line = cookie_str[index]
index += 1
if line == "": break
if not line.startswith(HEADER):
continue
line = line[len(HEADER):].strip()
for data in split_header_words([line]):
name, value = data[0]
standard = {}
rest = {}
for k in BOOLEAN_ATTRS:
standard[k] = False
for k, v in data[1:]:
if k is not None:
lc = k.lower()
else:
lc = None
if (lc in VALUE_ATTRS) or (lc in BOOLEAN_ATTRS):
k = lc
if k in BOOLEAN_ATTRS:
if v is None: v = True
standard[k] = v
elif k in VALUE_ATTRS:
standard[k] = v
else:
rest[k] = v
h = standard.get
expires = h("expires")
discard = h("discard")
if expires is not None:
expires = iso2time(expires)
if expires is None:
discard = True
domain = h("domain")
domain_specified = domain.startswith(".")
c = Cookie(h("version"), name, value, h("port"), h("port_spec"),
domain, domain_specified, h("domain_dot"), h("path"),
h("path_spec"), h("secure"), expires, discard,
h("comment"), h("commenturl"), rest)
cookie.set_cookie(c)
return cookie
def requests_session_with_selenium_cookies(driver, config):
"""
Return a Requests library session object initialized with the cookies from Selenium.
We have already logged into Volunteer Connection using selenium; use those cookies to
initialize a Requests session that we will use to download files (Selenium has trouble
intercepting file downloads)
"""
cookies = LWPCookieJar(config.COOKIE_FILE)
selenium_cookies = driver.get_cookies()
for c in selenium_cookies:
log.debug(f"selenium cookie: { c }")
path = c['path']
path_specified = path != None
domain = c['domain']
domain_specified = domain != None
domain_initial_dot = domain_specified and domain[0] == '.'
if 'expiry' in c:
expires = c['expiry'] + 86400 * 365 * 10 # add 10 years to expiry
else:
expires = None
cookie = Cookie(version=0,
name=c['name'],
value=c['value'],
port=None,
port_specified=False,
discard=False,
comment=None,
comment_url=None,
domain=c['domain'],
domain_specified=domain_specified,
domain_initial_dot=domain_initial_dot,
expires=expires,
path=path,
path_specified=path_specified,
rest={'HttpOnly': c['httpOnly']},
secure=c['secure'])
log.debug(f"cookejar cookie: { cookie }\n")
cookies.set_cookie(cookie)
cookies.save(ignore_discard=True, ignore_expires=True)
return cookies
def _initialize_cookies(cookies_path=None) -> LWPCookieJar:
""" 初始化 Cookies
"""
if cookies_path:
jar = LWPCookieJar(cookies_path)
if os.path.isfile(cookies_path):
jar.load()
else:
jar = LWPCookieJar()
for cookie in jar:
if cookie.is_expired():
jar.clear()
break
for k, v in c.BASE_COOKIES.items():
jar.set_cookie(make_cookie(k, v))
if cookies_path:
jar.save()
return jar
class Opener(DefaultOpener):
""" 高级的Opener对象. """
def __init__(self, cookie_path=None):
self.opener = build_opener()
self.cookiejar = LWPCookieJar(cookie_path)
cookie_path and os.path.exists(cookie_path) and self.cookiejar.load()
self.opener.add_handler(HTTPCookieProcessor(self.cookiejar))
self.opener.addheaders = [('User-agent', 'Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko/20120101 Firefox/11.0')]
def set_headers(self, arg={}, **kwargs):
""" 设置headers.
@arg(pairs/dict)
@kwargs(mixed)
"""
if not arg and not kwargs:
return
headers = dict(self.opener.addheaders)
headers.update(arg)
headers.update(kwargs)
self.opener.addheaders = list(headers.items())
def set_cookies(self, *args, **kwargs):
""" 设置cookies.
@args(pairs/dict)
@kwargs(mixed)
"""
for arg in args:
cookie = DefaultCookiePolicy(**dict(arg))
self.cookiejar.set_cookie(cookie)
kwargs and self.cookiejar.set_cookie(DefaultCookiePolicy(**kwargs))
def save_cookies(self, cookie_path=None):
if cookie_path or self.cookiejar.filename:
self.cookiejar.save(cookie_path)
def init_cookie_jar(cookie_file=None,
cookie_t=None,
cookie_y=None,
cookie_euconsent=None):
cookie_jar = LWPCookieJar(
cookie_file) if cookie_file else RequestsCookieJar()
if cookie_file and os.path.exists(cookie_file):
cookie_jar.load(ignore_discard=True)
if cookie_t:
cookie_jar.set_cookie(create_cookie('T', cookie_t))
if cookie_y:
cookie_jar.set_cookie(create_cookie('Y', cookie_y))
if cookie_euconsent:
cookie_jar.set_cookie(create_cookie('EuConsent', cookie_euconsent))
if cookie_file:
cookie_jar.save(ignore_discard=True)
return cookie_jar
class Opener:
def __init__(self, cookie_path=None):
self.opener = build_opener()
self.cookiejar = LWPCookieJar()
self.set_cookie_path(cookie_path)
self.opener.add_handler(HTTPCookieProcessor(self.cookiejar))
self.opener.addheaders = [('User-agent', 'Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko/20120101 Firefox/11.0')]
self.last_request = None
self.last_response = None
@property
def request_info(self):
pass
@property
def response_info(self):
pass
def set_cookie_path(self, path):
self.cookiejar.filename = path
try:
os.path.exists(path) and self.cookiejar.load()
except:
pass
def set_headers(self, arg={}, **kwargs):
if not arg and not kwargs:
return
headers = dict(self.opener.addheaders)
headers.update(arg)
headers.update(kwargs)
self.opener.addheaders = list(headers.items())
def set_cookies(self, *args, **kwargs):
for arg in args:
cookie = DefaultCookiePolicy(**arg)
self.cookiejar.set_cookie(cookie)
kwargs and self.cookiejar.set_cookie(DefaultCookiePolicy(**kwargs))
def save_cookies(self, cookie_path=None):
if cookie_path or self.cookiejar.filename:
self.cookiejar.save(cookie_path)
def urlopen(self, url, param=None, data=None, headers={}, proxies={}, timeout=None, encoding='utf8', errors='strict'):
""" 打开目标链接, 返回一个 HttpResponse对象.
@url(str/Request): 目标链接.
@param(str/dict/pairs tuple): query string.
@data(bytes/str/dict): post data.
@headers(dict): http request headers.
@proxies(dict): 代理, 如:{'http': 'xx.xx.xx.xx:3128', 'https': 'xxx.xxx.xxx.xxx:8080'}.
@timeout(int): http request timeout.
@encoding/errors(str): url编码.
"""
if param:
full_url = isinstance(url, Request) and url.get_full_url() or url
url_parse_dict = urlparse(full_url)._asdict()
query_param = url_parse_dict.get('query') + (isinstance(param, str) and param or urlencode(param, encoding, errors))
url_parse_dict['query'] = query_param
full_url = urlunparse(url_parse_dict.values())
request = Request(full_url)
else:
request = isinstance(url, Request) and url or Request(url)
if data:
if isinstance(data, bytes):
request.data = data
elif isinstance(data, str):
request.data = data.encode(encoding, errors)
else:
request.data = urlencode(data).encode(encoding, errors)
for key, value in headers.items():
request.add_header(key, value)
for proxy_type, proxy_host in proxies.items():
request.set_proxy(proxy_host, proxy_type)
self.last_request = request
self.last_response = self.opener.open(request, timeout=timeout)
return self.last_response
def clear(self):
self.last_request = None
self.last_response = None
def save_cookies(self, cookies):
_cookies = LWPCookieJar(self.filename)
for cookie in cookies:
_cookies.set_cookie(copy.copy(cookie))
_cookies.save()
