def cookiejar_create(self, cookiejar_file, session): cookie_jar = LWPCookieJar(cookiejar_file.name) cookie_jar.set_cookie( Cookie(0, self.COOKIE_NAME, session, None, False, '', False, True, '/', True, True, None, None, None, None, {})) cookie_jar.save() cookiejar_file.flush()
def cookie_from_str(cookie_str): from http.cookiejar import split_header_words, LWPCookieJar, LoadError, Cookie, iso2time import time cookie_str = cookie_str.split('\n') cookie = LWPCookieJar() index = 0 while 1: line = cookie_str[index] index += 1 if line == "": break if not line.startswith(HEADER): continue line = line[len(HEADER):].strip() for data in split_header_words([line]): name, value = data[0] standard = {} rest = {} for k in BOOLEAN_ATTRS: standard[k] = False for k, v in data[1:]: if k is not None: lc = k.lower() else: lc = None if (lc in VALUE_ATTRS) or (lc in BOOLEAN_ATTRS): k = lc if k in BOOLEAN_ATTRS: if v is None: v = True standard[k] = v elif k in VALUE_ATTRS: standard[k] = v else: rest[k] = v h = standard.get expires = h("expires") discard = h("discard") if expires is not None: expires = iso2time(expires) if expires is None: discard = True domain = h("domain") domain_specified = domain.startswith(".") c = Cookie(h("version"), name, value, h("port"), h("port_spec"), domain, domain_specified, h("domain_dot"), h("path"), h("path_spec"), h("secure"), expires, discard, h("comment"), h("commenturl"), rest) cookie.set_cookie(c) return cookie
def cookiejar_create(self, cookiejar_file, session): cookie_jar = LWPCookieJar(cookiejar_file.name) cookie_jar.set_cookie(Cookie(0, self.COOKIE_NAME, session, None, False, '', False, True, '/', True, True, None, None, None, None, {})) cookie_jar.save() cookiejar_file.flush()
def save_cookie(self, file_name=None, **kwargs): file_name = file_name or self._cookie_file lwp_jar = LWPCookieJar() for item in self._cookie_jar: args = dict(vars(item).items()) args['rest'] = args['_rest'] del (args['_rest']) cookie = Cookie(**args) lwp_jar.set_cookie(cookie) lwp_jar.save(file_name, **kwargs)
def requests_session_with_selenium_cookies(driver, config): """ Return a Requests library session object initialized with the cookies from Selenium. We have already logged into Volunteer Connection using selenium; use those cookies to initialize a Requests session that we will use to download files (Selenium has trouble intercepting file downloads) """ cookies = LWPCookieJar(config.COOKIE_FILE) selenium_cookies = driver.get_cookies() for c in selenium_cookies: log.debug(f"selenium cookie: { c }") path = c['path'] path_specified = path != None domain = c['domain'] domain_specified = domain != None domain_initial_dot = domain_specified and domain[0] == '.' if 'expiry' in c: expires = c['expiry'] + 86400 * 365 * 10 # add 10 years to expiry else: expires = None cookie = Cookie(version=0, name=c['name'], value=c['value'], port=None, port_specified=False, discard=False, comment=None, comment_url=None, domain=c['domain'], domain_specified=domain_specified, domain_initial_dot=domain_initial_dot, expires=expires, path=path, path_specified=path_specified, rest={'HttpOnly': c['httpOnly']}, secure=c['secure']) log.debug(f"cookejar cookie: { cookie }\n") cookies.set_cookie(cookie) cookies.save(ignore_discard=True, ignore_expires=True) return cookies
def _initialize_cookies(cookies_path=None) -> LWPCookieJar: """ 初始化 Cookies """ if cookies_path: jar = LWPCookieJar(cookies_path) if os.path.isfile(cookies_path): jar.load() else: jar = LWPCookieJar() for cookie in jar: if cookie.is_expired(): jar.clear() break for k, v in c.BASE_COOKIES.items(): jar.set_cookie(make_cookie(k, v)) if cookies_path: jar.save() return jar
class Opener(DefaultOpener): """ 高级的Opener对象. """ def __init__(self, cookie_path=None): self.opener = build_opener() self.cookiejar = LWPCookieJar(cookie_path) cookie_path and os.path.exists(cookie_path) and self.cookiejar.load() self.opener.add_handler(HTTPCookieProcessor(self.cookiejar)) self.opener.addheaders = [('User-agent', 'Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko/20120101 Firefox/11.0')] def set_headers(self, arg={}, **kwargs): """ 设置headers. @arg(pairs/dict) @kwargs(mixed) """ if not arg and not kwargs: return headers = dict(self.opener.addheaders) headers.update(arg) headers.update(kwargs) self.opener.addheaders = list(headers.items()) def set_cookies(self, *args, **kwargs): """ 设置cookies. @args(pairs/dict) @kwargs(mixed) """ for arg in args: cookie = DefaultCookiePolicy(**dict(arg)) self.cookiejar.set_cookie(cookie) kwargs and self.cookiejar.set_cookie(DefaultCookiePolicy(**kwargs)) def save_cookies(self, cookie_path=None): if cookie_path or self.cookiejar.filename: self.cookiejar.save(cookie_path)
def init_cookie_jar(cookie_file=None, cookie_t=None, cookie_y=None, cookie_euconsent=None): cookie_jar = LWPCookieJar( cookie_file) if cookie_file else RequestsCookieJar() if cookie_file and os.path.exists(cookie_file): cookie_jar.load(ignore_discard=True) if cookie_t: cookie_jar.set_cookie(create_cookie('T', cookie_t)) if cookie_y: cookie_jar.set_cookie(create_cookie('Y', cookie_y)) if cookie_euconsent: cookie_jar.set_cookie(create_cookie('EuConsent', cookie_euconsent)) if cookie_file: cookie_jar.save(ignore_discard=True) return cookie_jar
class Opener: def __init__(self, cookie_path=None): self.opener = build_opener() self.cookiejar = LWPCookieJar() self.set_cookie_path(cookie_path) self.opener.add_handler(HTTPCookieProcessor(self.cookiejar)) self.opener.addheaders = [('User-agent', 'Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko/20120101 Firefox/11.0')] self.last_request = None self.last_response = None @property def request_info(self): pass @property def response_info(self): pass def set_cookie_path(self, path): self.cookiejar.filename = path try: os.path.exists(path) and self.cookiejar.load() except: pass def set_headers(self, arg={}, **kwargs): if not arg and not kwargs: return headers = dict(self.opener.addheaders) headers.update(arg) headers.update(kwargs) self.opener.addheaders = list(headers.items()) def set_cookies(self, *args, **kwargs): for arg in args: cookie = DefaultCookiePolicy(**arg) self.cookiejar.set_cookie(cookie) kwargs and self.cookiejar.set_cookie(DefaultCookiePolicy(**kwargs)) def save_cookies(self, cookie_path=None): if cookie_path or self.cookiejar.filename: self.cookiejar.save(cookie_path) def urlopen(self, url, param=None, data=None, headers={}, proxies={}, timeout=None, encoding='utf8', errors='strict'): """ 打开目标链接, 返回一个 HttpResponse对象. @url(str/Request): 目标链接. @param(str/dict/pairs tuple): query string. @data(bytes/str/dict): post data. @headers(dict): http request headers. @proxies(dict): 代理, 如:{'http': 'xx.xx.xx.xx:3128', 'https': 'xxx.xxx.xxx.xxx:8080'}. @timeout(int): http request timeout. @encoding/errors(str): url编码. """ if param: full_url = isinstance(url, Request) and url.get_full_url() or url url_parse_dict = urlparse(full_url)._asdict() query_param = url_parse_dict.get('query') + (isinstance(param, str) and param or urlencode(param, encoding, errors)) url_parse_dict['query'] = query_param full_url = urlunparse(url_parse_dict.values()) request = Request(full_url) else: request = isinstance(url, Request) and url or Request(url) if data: if isinstance(data, bytes): request.data = data elif isinstance(data, str): request.data = data.encode(encoding, errors) else: request.data = urlencode(data).encode(encoding, errors) for key, value in headers.items(): request.add_header(key, value) for proxy_type, proxy_host in proxies.items(): request.set_proxy(proxy_host, proxy_type) self.last_request = request self.last_response = self.opener.open(request, timeout=timeout) return self.last_response def clear(self): self.last_request = None self.last_response = None
def save_cookies(self, cookies): _cookies = LWPCookieJar(self.filename) for cookie in cookies: _cookies.set_cookie(copy.copy(cookie)) _cookies.save()