def __init__(self, args):
self.cjar = MozillaCookieJar()
self.debug = args.debug
self.user_agent = args.user_agent
self.cookie_file = args.cookie_file
if os.path.exists(self.cookie_file):
try:
self.cjar.load(self.cookie_file, ignore_discard=True)
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
self.cjar = MozillaCookieJar()
if args.debug:
traceback.print_exc()
# Fix from:
# https://stackoverflow.com/questions/19268548/python-ignore-certificate-validation-urllib2
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
self.opener = build_opener(
HTTPSHandler(context=ctx), HTTPCookieProcessor(self.cjar))
def test_authholder_request_simple(auth_holder):
"""Load credentials the first time, hit the network, save credentials."""
# save a cookie to be used
fake_cookiejar = MozillaCookieJar(auth_holder._cookiejar_filepath)
fake_cookiejar.set_cookie(get_cookie())
fake_cookiejar.save()
other_cookie = get_cookie(value="different")
def fake_request(self, method, url, json, headers):
# check it was properly called
assert method == "testmethod"
assert url == "testurl"
assert json == "testbody"
assert headers == {"User-Agent": build_user_agent()}
# check credentials were loaded at this time
assert auth_holder._cookiejar is not None
# modify the credentials, to simulate that a re-auth happened while the request
auth_holder._cookiejar.set_cookie(other_cookie)
return "raw request response"
with patch("macaroonbakery.httpbakery.Client.request", fake_request):
resp = auth_holder.request("testmethod", "testurl", "testbody")
# verify response (the calling checks were done above in fake_request helper)
assert resp == "raw request response"
# check the credentials were saved (that were properly loaded was also check in above's helper)
new_cookiejar = MozillaCookieJar(auth_holder._cookiejar_filepath)
new_cookiejar.load()
assert list(new_cookiejar)[0].value == other_cookie.value
def __init__(self,
proxy=bool(strtobool(environ.get('REQUESTER_PROXY', '0'))),
cookie_filename=None,
caching=None,
user_agent=None,
headers=None,
file_name_with_proxies=path.join(path.dirname(__file__),
'proxies.txt')):
self.opened = None
if cookie_filename:
self.cookie_filename = cookie_filename
if caching is not None:
self.caching = caching
if headers:
self.headers = headers
else:
self.headers = [
('Accept',
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
), ('Accept-Encoding', 'gzip, deflate'),
('Accept-Language', 'ru-ru,ru;q=0.8,en-us;q=0.5,en;q=0.3'),
('Connection', 'keep-alive'),
('User-Agent',
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:10.0.2) Gecko/20100101 Firefox/10.0.2'
if user_agent is None else user_agent)
]
if self.cookie_filename:
makedirs(path.dirname(self.cookie_filename), exist_ok=True)
self.cookiejar = MozillaCookieJar(self.cookie_filename)
if path.exists(self.cookie_filename):
self.cookiejar.load()
else:
self.cookiejar = MozillaCookieJar()
self.http_cookie_processor = urllib.request.HTTPCookieProcessor(
self.cookiejar)
self.opener = urllib.request.build_opener(self.http_cookie_processor)
self.proxer = None
if proxy:
if proxy is True:
if not file_name_with_proxies or not path.exists(
file_name_with_proxies):
raise FileWithProxiesNotFound(
"ERROR: not found '%s' file" % file_name_with_proxies)
self.proxer = proxer(file_name_with_proxies)
proxy = self.proxer.get()
self.print("[proxy]", proxy)
time_response = self.proxer.time_response()
if time_response:
self.print("[average time]", time_response)
self.opener.add_handler(
urllib.request.ProxyHandler({
'http': proxy,
'https': proxy,
}))
self._init_opener_headers = self.headers
def get_cookie():
info_url = "https://i.njtech.edu.cn/index.html"
headers = {"User-Agent": UserAgent().chrome}
request = Request(info_url, headers=headers)
cookie_jar = MozillaCookieJar()
cookie_jar.load("cookie.txt", ignore_discard=True, ignore_expires=True)
handler = MozillaCookieJar(cookie_jar)
opener = build_opener(handler)
response = opener.open(request)
print(response.read().decode())
def init_opener(self):
if self.cookie_filename:
makedirs(path.dirname(self.cookie_filename), exist_ok=True)
self.cookiejar = MozillaCookieJar(self.cookie_filename)
if path.exists(self.cookie_filename):
self.cookiejar.load(ignore_discard=True, ignore_expires=True)
else:
self.cookiejar = MozillaCookieJar()
self.http_cookie_processor = urllib.request.HTTPCookieProcessor(
self.cookiejar)
self.opener = urllib.request.build_opener(self.http_cookie_processor)
self.proxer = None
def get_cookie():
# 创建url
login_url = "*****"
# 构建伪装头
headers = {
"User-Agent": UserAgent().chrome
}
# 构建伪装用户登陆
form_data = {
"user": "******"
"pasword":"*****"
}
f_data = urlencode(form_data).encode()
re = Request(login_url, headers=headers, data=f_data) # 构建请求头
cookie_jar = MozillaCookieJar() # 保存cookie文件
handler = HTTPCookieProcessor(cookie_jar)
opener = build_opener(handler) # 构建opener对象
response = opener.open(re) # 准备返回值
# 保存cookie值到文件
cookie_jar.save("cookie.txt", ignore_expires = True, ignore_discard = True)
def save_cookies_to_file(cookies: "Cookies", path: Path):
jar = MozillaCookieJar(path)
for i in cookies.jar:
jar.set_cookie(i)
if not path.is_file():
path.parent.mkdir(parents=True, exist_ok=True)
jar.save(ignore_discard=True)
def load_cookies_from_file(path: Path) -> Union[MozillaCookieJar, None]:
if path.is_file():
jar = MozillaCookieJar(path)
jar.load(ignore_discard=True)
return jar
else:
return None
def __init__(self):
self.articles = []
self.query = None
self.cjar = MozillaCookieJar()
# If we have a cookie file, load it:
if ScholarConf.COOKIE_JAR_FILE and \
os.path.exists(ScholarConf.COOKIE_JAR_FILE):
try:
self.cjar.load(ScholarConf.COOKIE_JAR_FILE,
ignore_discard=True)
ScholarUtils.log('info', 'loaded cookies file')
except Exception, msg:
ScholarUtils.log('warn',
'could not load cookies file: %s' % msg)
self.cjar = MozillaCookieJar() # Just to be safe
def __init__(self, phone_no=None, pin=None, keyfile=None, locale='de', save_cookies=False):
self.log = get_logger(__name__)
self._locale = locale
self._save_cookies = save_cookies
if not (phone_no and pin):
try:
with open(CREDENTIALS_FILE, 'r') as f:
lines = f.readlines()
self.phone_no = lines[0].strip()
self.pin = lines[1].strip()
except FileNotFoundError:
raise ValueError(f'phone_no and pin must be specified explicitly or via {CREDENTIALS_FILE}')
else:
self.phone_no = phone_no
self.pin = pin
self.keyfile = keyfile if keyfile else KEY_FILE
try:
with open(self.keyfile, 'rb') as f:
self.sk = SigningKey.from_pem(f.read(), hashfunc=hashlib.sha512)
except FileNotFoundError:
pass
self._websession = requests.Session()
self._websession.headers = self._default_headers_web
if self._save_cookies:
self._websession.cookies = MozillaCookieJar(COOKIES_FILE)
def test_authholder_credentials_save_reallysave(auth_holder):
"""Save really do save if cookies changed."""
# create some fake cookies
fake_cookiejar = MozillaCookieJar(auth_holder._cookiejar_filepath)
fake_cookie = get_cookie()
fake_cookiejar.set_cookie(fake_cookie)
fake_cookiejar.save()
# make auth holder to have those credentials loaded, and also load them ourselves for
# later comparison
auth_holder._load_credentials()
with open(auth_holder._cookiejar_filepath, "rb") as fh:
prv_file_content = fh.read()
# set a different credential in the auth_holder (mimickin that the user authenticated
# while doing the request)
other_cookie = get_cookie(value="different")
auth_holder._cookiejar.set_cookie(other_cookie)
# call the tested method and ensure that file changed!
auth_holder._save_credentials_if_changed()
with open(auth_holder._cookiejar_filepath, "rb") as fh:
new_file_content = fh.read()
assert new_file_content != prv_file_content
# call the tested method again, to verify that it was calling save on the cookiejar (and
# not that the file changed as other side effect)
with patch.object(auth_holder._cookiejar, "save") as mock:
auth_holder._save_credentials_if_changed()
assert mock.call_count == 1
def __init__(self):
self.articles = []
self.query = None
self.cjar = MozillaCookieJar()
# If we have a cookie file, load it:
if ScholarConf.COOKIE_JAR_FILE and os.path.exists(ScholarConf.COOKIE_JAR_FILE):
try:
self.cjar.load(ScholarConf.COOKIE_JAR_FILE, ignore_discard=True)
ScholarUtils.log("info", "loaded cookies file")
except Exception as msg:
ScholarUtils.log("warn", "could not load cookies file: %s" % msg)
self.cjar = MozillaCookieJar() # Just to be safe
self.opener = build_opener(HTTPCookieProcessor(self.cjar))
self.settings = None # Last settings object, if any
def urlretrieve(url: str, filename: str, context: ssl.SSLContext, reporthook=None, cookies_path=None):
"""
original source:
https://github.com/python/cpython/blob/
21bee0bd71e1ad270274499f9f58194ebb52e236/Lib/urllib/request.py#L229
Because urlopen also supports context,
I decided to adapt the download function.
"""
url_parsed = urlparse.urlparse(url)
request = urllib.request.Request(url=url, headers=RequestHelper.stdHeader)
if cookies_path is not None:
cookie_jar = MozillaCookieJar(cookies_path)
cookie_jar.load(ignore_discard=True, ignore_expires=True)
cookie_jar.add_cookie_header(request)
with contextlib.closing(urllib.request.urlopen(request, context=context)) as fp:
headers = fp.info()
# Just return the local path and the 'headers' for file://
# URLs. No sense in performing a copy unless requested.
if url_parsed.scheme == 'file' and not filename:
return os.path.normpath(url_parsed.path), headers
if not filename:
raise RuntimeError('No filename specified!')
tfp = open(filename, 'wb')
with tfp:
result = filename, headers
# read overall
read = 0
# 4kb at once
bs = 1024 * 8
blocknum = 0
# guess size
size = int(headers.get('Content-Length', -1))
if reporthook:
reporthook(blocknum, bs, size)
while True:
block = fp.read(bs)
if not block:
break
read += len(block)
tfp.write(block)
blocknum += 1
if reporthook:
reporthook(blocknum, bs, size)
if size >= 0 and read < size:
raise ContentTooShortError('retrieval incomplete: got only %i out of %i bytes' % (read, size), result)
return result
def __init__(self, cookies=None, proxy=None):
"""Initialise a new session for making requests."""
self.session = requests.Session()
self.session.headers = self.__HEADERS
# use proxy if present
if(proxy is not None and isinstance(proxy, str)):
if(proxy.startswith('socks5:')):
print('detected socks5 protocol, please use socks5h',
file=sys.stderr)
print('use proxy', proxy)
self.session.proxies.update({
'http': proxy,
'https': proxy
})
# set the place for chat_db
self.chat_db: Optional[DBChat] = None
cj = MozillaCookieJar(cookies)
if cookies is not None:
# Only attempt to load if the cookie file exists.
if os.path.exists(cookies):
cj.load(ignore_discard=True, ignore_expires=True)
else:
raise CookieError(
"The file '{}' could not be found.".format(cookies))
self.session.cookies = cj
def test_authholder_credentials_load_file_present_ok(auth_holder):
"""Credentials are properly loaded and all internal objects setup ok."""
# create some fake cookies
fake_cookiejar = MozillaCookieJar(auth_holder._cookiejar_filepath)
fake_cookie = get_cookie()
fake_cookiejar.set_cookie(fake_cookie)
fake_cookiejar.save()
auth_holder._load_credentials()
# check credentials
loaded_cookies = list(auth_holder._cookiejar)
assert len(loaded_cookies) == 1
assert (
loaded_cookies[0].value == fake_cookie.value
) # compare the value as no __eq__ in Cookie
assert isinstance(auth_holder._cookiejar, MozillaCookieJar)
assert auth_holder._old_cookies == list(auth_holder._cookiejar)
# check other internal objects
assert isinstance(auth_holder._client, httpbakery.Client)
assert list(auth_holder._client.cookies)[0].value == fake_cookie.value
(im,) = auth_holder._client._interaction_methods
assert isinstance(im, httpbakery.WebBrowserInteractor)
assert im._open_web_browser == visit_page_with_browser
def __init__(self,
mobile,
password=None,
status='0',
cachefile='Fetion.cache',
cookiesfile=''):
'''登录状态：
在线：400 隐身：0 忙碌：600 离开：100
'''
if cachefile:
self.cache = Cache(cachefile)
if not cookiesfile:
cookiesfile = '%s.cookies' % mobile
cookiejar = MozillaCookieJar(filename=cookiesfile)
if not os.path.isfile(cookiesfile):
open(cookiesfile, 'w').write(MozillaCookieJar.header)
cookiejar.load(filename=cookiesfile)
cookie_processor = HTTPCookieProcessor(cookiejar)
self.opener = build_opener(cookie_processor, HTTPHandler)
self.mobile, self.password = mobile, password
if not self.alive():
self._login()
cookiejar.save()
self.changestatus(status)
def __init__(self):
self.logger = logging.getLogger('ghscan')
self.logger.setLevel(logging.DEBUG)
fh = logging.FileHandler(get_or_create_node('ghscan.log'))
fh.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
self.logger.addHandler(fh)
self.logger.debug('Creating an instance of GitGubApi class')
self.cj = MozillaCookieJar()
self.cookie_file_name = get_or_create_node()
self.need_auth = 1
try:
self.cj.load(self.cookie_file_name)
self.need_auth = 0
self.logger.debug('Auth: no auth needed')
except LoadError as e:
pass
self.headers = {
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36'
'(KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36',
}
def would_decorate_driver_with_cookies(driver, cookie_file):
jar = MozillaCookieJar(cookie_file)
jar.load()
decorate = compose(driver.add_cookie, jar_cookie_to_webdriver_cookie)
return map(decorate, jar)
def get_cookies_from_ff(db_filename):
con = sqlite.connect(db_filename)
con.execute("pragma journal_mode=WAL")
cur = con.cursor()
cur.execute(
"select host, path, isSecure, expiry, name, value from moz_cookies"
)
container = []
while True:
try:
row = cur.fetchone()
except:
continue
if not row:
break
if not row[4].startswith('chkSlider'): # FIXME: this is a dirty fix
container.append(row)
con.close()
ftstr = ["FALSE", "TRUE"]
s = StringIO()
s.write("""\
# Netscape HTTP Cookie File
# http://www.netscape.com/newsref/std/cookie_spec.html
# This is a generated file! Do not edit.
""")
for item in container:
v = "%s\t%s\t%s\t%s\t%s\t%s\t%s\n" % (
item[0], ftstr[item[0].startswith('.')], item[1],
ftstr[item[2]], item[3], item[4], item[5])
s.write(v)
s.seek(0)
cookie_jar = MozillaCookieJar()
cookie_jar._really_load(s, '', True, True)
return cookie_jar
def __init__(self, **kwargs):
headers = kwargs.get('headers')
if headers is None:
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36',
'Accept-Language': 'en-US, en'
}
# Set params for use later on
# self.params = kwargs
# Begin session
self.session = requests.Session()
self.session.headers = headers
# Set cookies if present
cookies = kwargs.get('cookies')
cj = MozillaCookieJar(cookies)
if cookies: # is not None
# Only attempt to load if the cookie file exists.
if os.path.exists(cookies):
cj.load(ignore_discard=True, ignore_expires=True)
else:
raise CookieError(
'The file "{}" could not be found.'.format(cookies))
self.session.cookies = cj
def __init__(self, username, password, **kwargs):
"""
init params
:param username: your username
:type username: str
:param password: your password
:type password: str
"""
# path setup
self._path = os.getcwd()
self._cookies_path = os.path.join(os.getcwd(), 'cookies_data')
self._data_path = os.path.join(os.getcwd(), 'freenom_data')
# user setup
self.username = username
self.password = password
# request setup
self.headers = {
'Host': 'my.freenom.com',
'Referer': 'https://my.freenom.com/clientarea.php',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36'
}
self.session = requests.session()
self.session.headers = self.headers
self.token = ''
# cookies setup
cookies = MozillaCookieJar(filename=self._cookies_path)
if os.path.isfile(self._cookies_path):
cookies.load(self._cookies_path, ignore_discard=True, ignore_expires=True)
self.session.cookies = cookies
# option setup --dev
self.timeout = kwargs.get('timeout', 22)
self.saveHtml = kwargs.get('saveHtml', False)
self._RequireData()
def post_URL(self, url: str, data: {str: str} = None, cookie_jar_path: str = None):
"""
Sends a POST request to a specific URL, including saving of cookies in cookie jar.
@param url: The url to which the request is sent. (the moodle base url is not added to the given URL)
@param data: The optional data is added to the POST body.
@param cookie_jar_path: Path to the cookies file.
@return: The resulting response object and the session object.
"""
data_urlencoded = ""
if data is not None:
data_urlencoded = RequestHelper.recursive_urlencode(data)
session = requests.Session()
if cookie_jar_path is not None:
session.cookies = MozillaCookieJar(cookie_jar_path)
if os.path.exists(cookie_jar_path):
session.cookies.load(ignore_discard=True, ignore_expires=True)
try:
response = session.post(url, data=data_urlencoded, headers=self.stdHeader, verify=self.verify, timeout=60)
except RequestException as error:
raise ConnectionError("Connection error: %s" % str(error)) from None
if cookie_jar_path is not None:
for cookie in session.cookies:
cookie.expires = 2147483647
session.cookies.save(ignore_discard=True, ignore_expires=True)
return response, session
def __init__(self, login, cookiejar, **kwargs):
super().__init__(**kwargs)
self.keyring = Keyring()
self.json = misc.JsonUtils()
self._login = login
self.kr_key = '{}#{}'.format(self.login_key, self._login)
self._pwd = self.keyring.get_or_set(self.kr_key,
help='uc login password:'******'^path_(.*)', x)
if not m:
continue
val = m.group(1)
url_attr = '{}_url'.format(val)
url = self.build_url(getattr(uc, x))
setattr(self, url_attr, url)
self.session = requests.Session()
self.jar = MozillaCookieJar(cookiejar)
if os.path.isfile(cookiejar):
self.jar.load(ignore_discard=True)
self.session.cookies = self.jar
self.last_token = None
def get_gerrit_rest_api(cookie_jar_path: str,
gerrit_url: str) -> GerritRestAPI:
cookie_jar = MozillaCookieJar(cookie_jar_path)
cookie_jar.load()
auth = HTTPCookieAuth(cookie_jar)
rest = GerritRestAPI(url=gerrit_url, auth=auth)
return rest
def load_cookies(self, path=None):
# Note: to load cookies for a domain, you must
# first navigate to that domain.
self.logger.info('Loading cookies')
if path is None:
path = self.cookies_file
if path.endswith('.json'):
try:
with open(self.cookies_file, 'r') as f:
cookies = json.load(f)
except FileNotFoundError:
return False
for c in cookies:
if isinstance(c.get('expiry'), float):
c['expiry'] = int(c['expiry'])
self.driver.add_cookie(c)
elif path.endswith('.txt'):
# Assume Mozilla/Netscape format
jar = MozillaCookieJar(path)
jar.load()
for c in jar:
cookie = {
k: getattr(c, k)
for k in ['domain', 'name', 'value', 'secure', 'path']
}
if c.expires: cookie['expiry'] = c.expires
self.driver.add_cookie(cookie)
else:
raise Exception('Unrecognized cookie extension')
return True
def login():
global site
if site and site.logged_in:
logger.info('Already logged into API site')
return True
api_creds = get_credentials()
if api_creds is None:
logger.warning('Not creating API site object, no credentials')
return False
cookie_path = '/tmp/cookies.txt'
cookie_jar = MozillaCookieJar(cookie_path)
if os.path.exists(cookie_path):
# Load cookies from file, including session cookies (expirydate=0)
cookie_jar.load(ignore_discard=True, ignore_expires=True)
logger.info('Loaded %d cookies', len(cookie_jar))
connection = requests.Session()
connection.cookies = cookie_jar
site = mwclient.Site('en.wikipedia.org',
clients_useragent=_ua,
pool=connection)
if not site.logged_in:
try:
logger.info('Logging into API site')
site.login(api_creds['user'], api_creds['pass'])
logger.info('Saving cookies')
cookie_jar.save(ignore_discard=True, ignore_expires=True)
except mwclient.errors.LoginError:
logger.exception('Exception logging into Wikipedia')
return False
return True
def get_URL(self, url: str, cookie_jar_path: str = None):
"""
Sends a GET request to a specific URL of the Moodle system, including additional cookies
(cookies are updated after the request)
@param url: The url to which the request is sent. (the moodle base url is not added to the given URL)
@param cookie_jar_path: The optional cookies to add to the request
@return: The resulting Response object.
"""
session = requests.Session()
if cookie_jar_path is not None:
session.cookies = MozillaCookieJar(cookie_jar_path)
if os.path.exists(cookie_jar_path):
session.cookies.load(ignore_discard=True, ignore_expires=True)
try:
response = session.get(url,
headers=self.stdHeader,
verify=self.verify,
timeout=60)
except RequestException as error:
raise ConnectionError("Connection error: %s" %
str(error)) from None
if cookie_jar_path is not None:
session.cookies.save(ignore_discard=True, ignore_expires=True)
return response, session
def __init__(self):
# error message
self.error = None
# establish connection
self.session = build_opener()
# add proxy handler if needed
if config['proxy']:
if any(config['proxies'].values()):
self.session.add_handler(ProxyHandler(config['proxies']))
logger.debug("Proxy is set!")
else:
self.error = "Proxy enabled, but not set!"
# change user-agent
self.session.addheaders = [('User-Agent', config['ua'])]
# load local cookies
mcj = MozillaCookieJar()
try:
mcj.load(FILE_C, ignore_discard=True)
if 'uid' in [cookie.name for cookie in mcj]:
# if cookie.expires < int(time.time())
logger.info("Local cookies is loaded")
self.session.add_handler(HTTPCookieProcessor(mcj))
else:
logger.info("Local cookies expired or bad")
logger.debug(f"That we have: {[cookie for cookie in mcj]}")
mcj.clear()
self.login(mcj)
except FileNotFoundError:
self.login(mcj)
def saveCookie_Demo():
cookiejar = MozillaCookieJar('./Other/cookie.txt')
handler = request.HTTPCookieProcessor(cookiejar)
opener = request.build_opener(handler)
resp = opener.open('http://httpbin.org/cookies/set?course=suofeiya')
# 第一个参数代表如果Cookie信息是一种页面响应完立即过期的，则可以将其保存下来
cookiejar.save(ignore_discard=True)
def save_cookies(self):
self.driver.get(self.config.url_accounts_no_form)
driver_cookies = self.driver.get_cookies()
cookie_jar = MozillaCookieJar()
for driver_cookie in driver_cookies:
http_cookie = self.get_cookie(driver_cookie)
cookie_jar.set_cookie(http_cookie)
save_cookie_jar(self.config, cookie_jar)
