def create_cookie(name, value, domain, httponly=None, **kwargs):
"""Creates `cookielib.Cookie` instance"""
if domain == 'localhost':
domain = ''
config = dict(
name=name,
value=value,
version=0,
port=None,
domain=domain,
path='/',
secure=False,
expires=None,
discard=True,
comment=None,
comment_url=None,
rfc2109=False,
rest={'HttpOnly': httponly},
)
for key in kwargs:
if key not in config:
raise GrabMisuseError('Function `create_cookie` does not accept '
'`%s` argument' % key)
config.update(**kwargs)
config['rest']['HttpOnly'] = httponly
config['port_specified'] = bool(config['port'])
config['domain_specified'] = bool(config['domain'])
config['domain_initial_dot'] = (config['domain'] or '').startswith('.')
config['path_specified'] = bool(config['path'])
return Cookie(**config)
def har_to_cookie(har_cookie):
"""
Convert a cookie dict in HAR format to a Cookie instance.
>>> har_cookie = {
... "name": "TestCookie",
... "value": "Cookie Value",
... "path": "/foo",
... "domain": "www.janodvarko.cz",
... "expires": "2009-07-24T19:20:30Z",
... "httpOnly": True,
... "secure": True,
... "comment": "this is a test"
... }
>>> cookie = har_to_cookie(har_cookie)
>>> cookie.name
'TestCookie'
>>> cookie.value
'Cookie Value'
>>> cookie.port
>>> cookie.domain
'www.janodvarko.cz'
>>> cookie.path
'/foo'
>>> cookie.secure
True
>>> cookie.expires
1248463230
>>> cookie.comment
'this is a test'
>>> cookie.get_nonstandard_attr('HttpOnly')
True
"""
expires_timestamp = None
if har_cookie.get('expires'):
expires = time.strptime(har_cookie['expires'], "%Y-%m-%dT%H:%M:%SZ")
expires_timestamp = calendar.timegm(expires)
kwargs = dict(
version=har_cookie.get('version') or 0,
name=har_cookie['name'],
value=har_cookie['value'],
port=None,
domain=har_cookie.get('domain', ''),
path=har_cookie.get('path', '/'),
secure=har_cookie.get('secure', False),
expires=expires_timestamp,
discard=False,
comment=har_cookie.get('comment'),
comment_url=bool(har_cookie.get('comment')),
rest={'HttpOnly': har_cookie.get('httpOnly')},
rfc2109=False,
)
kwargs['port_specified'] = bool(kwargs['port'])
kwargs['domain_specified'] = bool(kwargs['domain'])
kwargs['domain_initial_dot'] = kwargs['domain'].startswith('.')
kwargs['path_specified'] = bool(kwargs['path'])
return Cookie(**kwargs)
def get_stream(self, media):
url = "https://mf.svc.nhl.com/ws/media/mf/v2.4/stream"
event_id = media["eventId"]
if not self.session_key:
logger.info("getting session key")
params = {
"eventId": event_id,
"format": "json",
"platform": "WEB_MEDIAPLAYER",
"subject": "NHLTV",
"_": "1538708097285"
}
res = self.session.get(url, params=params)
j = res.json()
logger.trace(
json.dumps(j, sort_keys=True, indent=4,
separators=(',', ': ')))
self.session_key = j["session_key"]
self.save()
params = {
"contentId": media["mediaPlaybackId"],
"playbackScenario": "HTTP_CLOUD_WIRED_WEB",
"sessionKey": self.session_key,
"auth": "response",
"platform": "WEB_MEDIAPLAYER",
"_": "1538708097285"
}
res = self.session.get(url, params=params)
j = res.json()
logger.trace(
json.dumps(j, sort_keys=True, indent=4, separators=(',', ': ')))
try:
media_auth = next(x["attributeValue"]
for x in j["session_info"]["sessionAttributes"]
if x["attributeName"] == "mediaAuth_v2")
except KeyError:
raise StreamSessionException(
f"No stream found for event {event_id}")
self.cookies.set_cookie(
Cookie(0, 'mediaAuth_v2', media_auth, '80', '80', '.nhl.com', None,
None, '/', True, False, 4102444800, None, None, None, {}), )
stream = Stream(j["user_verified_event"][0]["user_verified_content"][0]
["user_verified_media_item"][0])
return stream
def make_cookie(name, value, domain):
'''
Makes a cookie with provided name and value.
'''
return Cookie(version=0,
name=name,
value=value,
port=None,
port_specified=False,
domain=domain,
domain_specified=True,
domain_initial_dot=False,
path="/",
path_specified=True,
secure=False,
expires=None,
discard=False,
comment=None,
comment_url=None,
rest=None)
def create_cookie(self, cookie_dict):
"""Creates a cookie from the given `cookie_dict`.
:param cookie_dict: A dictionary with two keys: 'name' and 'value'. The values of these\
keys are the name of the cookie and its value, respectively.
The keys 'path', 'domain', 'secure', 'expiry' can also be set to values.\
These have the respective meanings as defined in `RFC6265 <http://tools.ietf.org/html/rfc6265#section-5.2>`
"""
name = ascii_as_bytes_or_str(cookie_dict['name'])
value = ascii_as_bytes_or_str(cookie_dict['value'])
path = ascii_as_bytes_or_str(cookie_dict.get('path', ''))
path_set = path != ''
domain = ascii_as_bytes_or_str(cookie_dict.get('domain', ''))
domain_set = domain != ''
secure = cookie_dict.get('secure', False)
expires = cookie_dict.get('expiry', None)
cookie = Cookie(0, name, value, None, False, domain, domain_set, None,
path, path_set, secure, expires, None, None, None,
None)
self.testapp.cookiejar.set_cookie(cookie)
def add_cookie(self, key, value):
value = smart_text(value)
cookie = Cookie(None, key, value, None, False, '', False, None, '',
False, None, None, None, None, None, None)
self._cookie_jar.set_cookie(cookie)
def __init__(self,
url,
cookie_file=None,
username=None,
password=None,
api_token=None,
agent=None,
session=None,
disable_proxy=False,
auth_callback=None,
otp_token_callback=None,
verify_ssl=True,
save_cookies=True,
ext_auth_cookies=None):
if not url.endswith('/'):
url += '/'
self.url = url + 'api/'
self.save_cookies = save_cookies
self.ext_auth_cookies = ext_auth_cookies
if self.save_cookies:
self.cookie_jar, self.cookie_file = create_cookie_jar(
cookie_file=cookie_file)
try:
self.cookie_jar.load(ignore_expires=True)
except IOError:
pass
else:
self.cookie_jar = CookieJar()
self.cookie_file = None
if self.ext_auth_cookies:
try:
self.cookie_jar.load(ext_auth_cookies, ignore_expires=True)
except IOError as e:
logging.critical(
'There was an error while loading a '
'cookie file: %s', e)
pass
# Get the cookie domain from the url. If the domain
# does not contain a '.' (e.g. 'localhost'), we assume
# it is a local domain and suffix it (See RFC 2109).
parsed_url = urlparse(url)
self.domain = parsed_url[1].partition(':')[0] # Remove Port.
if self.domain.count('.') < 1:
self.domain = '%s.local' % self.domain
if session:
cookie = Cookie(version=0,
name=RB_COOKIE_NAME,
value=session,
port=None,
port_specified=False,
domain=self.domain,
domain_specified=True,
domain_initial_dot=True,
path=parsed_url[2],
path_specified=True,
secure=False,
expires=None,
discard=False,
comment=None,
comment_url=None,
rest={'HttpOnly': None})
self.cookie_jar.set_cookie(cookie)
if self.save_cookies:
self.cookie_jar.save()
if username:
# If the username parameter is given, we have to clear the session
# cookie manually or it will override the username:password
# combination retrieved from the authentication callback.
try:
self.cookie_jar.clear(self.domain, parsed_url[2],
RB_COOKIE_NAME)
except KeyError:
pass
# Set up the HTTP libraries to support all of the features we need.
password_mgr = ReviewBoardHTTPPasswordMgr(self.url, username, password,
api_token, auth_callback,
otp_token_callback)
self.preset_auth_handler = PresetHTTPAuthHandler(
self.url, password_mgr)
handlers = []
if not verify_ssl:
context = ssl._create_unverified_context()
handlers.append(HTTPSHandler(context=context))
if disable_proxy:
handlers.append(ProxyHandler({}))
handlers += [
HTTPCookieProcessor(self.cookie_jar),
ReviewBoardHTTPBasicAuthHandler(password_mgr),
HTTPDigestAuthHandler(password_mgr),
self.preset_auth_handler,
ReviewBoardHTTPErrorProcessor(),
]
if agent:
self.agent = agent
else:
self.agent = ('RBTools/' + get_package_version()).encode('utf-8')
opener = build_opener(*handlers)
opener.addheaders = [
(str('User-agent'), str(self.agent)),
]
install_opener(opener)
self._cache = None
self._urlopen = urlopen
def __init__(self,
url,
cookie_file=None,
username=None,
password=None,
api_token=None,
agent=None,
session=None,
disable_proxy=False,
auth_callback=None,
otp_token_callback=None):
self.url = url
if not self.url.endswith('/'):
self.url += '/'
self.url = self.url + 'api/'
self.cookie_jar, self.cookie_file = create_cookie_jar(
cookie_file=cookie_file)
try:
self.cookie_jar.load(ignore_expires=True)
except IOError:
pass
if session:
parsed_url = urlparse(url)
# Get the cookie domain from the url. If the domain
# does not contain a '.' (e.g. 'localhost'), we assume
# it is a local domain and suffix it (See RFC 2109).
domain = parsed_url[1].partition(':')[0] # Remove Port.
if domain.count('.') < 1:
domain = '%s.local' % domain
cookie = Cookie(version=0,
name=RB_COOKIE_NAME,
value=session,
port=None,
port_specified=False,
domain=domain,
domain_specified=True,
domain_initial_dot=True,
path=parsed_url[2],
path_specified=True,
secure=False,
expires=None,
discard=False,
comment=None,
comment_url=None,
rest={'HttpOnly': None})
self.cookie_jar.set_cookie(cookie)
self.cookie_jar.save()
# Set up the HTTP libraries to support all of the features we need.
password_mgr = ReviewBoardHTTPPasswordMgr(self.url, username, password,
api_token, auth_callback,
otp_token_callback)
self.preset_auth_handler = PresetHTTPAuthHandler(
self.url, password_mgr)
handlers = []
if disable_proxy:
handlers.append(ProxyHandler({}))
handlers += [
HTTPCookieProcessor(self.cookie_jar),
ReviewBoardHTTPBasicAuthHandler(password_mgr),
HTTPDigestAuthHandler(password_mgr),
self.preset_auth_handler,
ReviewBoardHTTPErrorProcessor(),
]
if agent:
self.agent = agent
else:
self.agent = ('RBTools/' + get_package_version()).encode('utf-8')
opener = build_opener(*handlers)
opener.addheaders = [
(b'User-agent', self.agent),
]
install_opener(opener)
self._cache = APICache()
def hook(response, *args, **kwargs):
''' Requests hooks system, this is the hook for the response.
'''
status_401 = (response.status_code == 401 and
response.headers.get('WWW-Authenticate') == 'Macaroon')
if not status_401 and response.status_code != 407:
return response
if response.headers.get('Content-Type') != 'application/json':
return response
try:
error = response.json()
except:
raise BakeryException('unable to read discharge error response')
if error.get('Code') != ERR_DISCHARGE_REQUIRED:
return response
Retry.count += 1
if Retry.count > MAX_DISCHARGE_RETRIES:
raise BakeryException('too many discharges')
info = error.get('Info')
if not isinstance(info, dict):
raise BakeryException(
'unable to read info in discharge error response')
serialized_macaroon = info.get('Macaroon')
if not isinstance(serialized_macaroon, dict):
raise BakeryException(
'unable to read macaroon in discharge error response')
macaroon = utils.deserialize(serialized_macaroon)
discharges = discharge_all(macaroon, visit_page, jar, key)
encoded_discharges = map(utils.serialize_macaroon_string, discharges)
macaroons = '[' + ','.join(encoded_discharges) + ']'
all_macaroons = base64.urlsafe_b64encode(
macaroons.encode('utf-8')).decode('ascii')
full_path = urljoin(response.url, info['MacaroonPath'])
parsed_url = urlparse(full_path)
if info and info.get('CookieNameSuffix'):
name = 'macaroon-' + info['CookieNameSuffix']
else:
name = 'macaroon-' + discharges[0].signature
domain = parsed_url.hostname or parsed_url.netloc
port = str(parsed_url.port) if parsed_url.port is not None else None
secure = parsed_url.scheme == 'https'
cookie = Cookie(version=0,
name=name,
value=all_macaroons,
port=port,
port_specified=port is not None,
domain=domain,
domain_specified=True,
domain_initial_dot=False,
path=parsed_url.path,
path_specified=True,
secure=secure,
expires=None,
discard=False,
comment=None,
comment_url=None,
rest=None,
rfc2109=False)
jar.set_cookie(cookie)
# Replace the private _cookies from req as it is a copy of
# the original cookie jar passed into the requests method and we need
# to set the cookie for this request.
req._cookies = jar
req.headers.pop('Cookie', None)
req.prepare_cookies(req._cookies)
req.headers['Bakery-Protocol-Version'] = '1'
with requests.Session() as s:
return s.send(req)
def _really_load(self, f, filename, ignore_discard, ignore_expires):
"""
This function is required to monkey patch MozillaCookieJar's _really_load
function which does not understand the curl format cookie file created
by ecp-cookie-init. It patches the code so that #HttpOnly_ get loaded.
https://bugs.python.org/issue2190
https://bugs.python.org/file37625/httponly.patch
"""
now = time.time()
magic = f.readline()
if not re.search(self.magic_re, magic):
f.close()
raise LoadError(
"%r does not look like a Netscape format cookies file" % filename)
try:
while 1:
line = f.readline()
if line == "": break
# last field may be absent, so keep any trailing tab
if line.endswith("\n"): line = line[:-1]
sline = line.strip()
# support HttpOnly cookies (as stored by curl or old Firefox).
if sline.startswith("#HttpOnly_"):
line = sline[10:]
# skip comments and blank lines XXX what is $ for?
elif (sline.startswith(("#", "$")) or sline == ""):
continue
domain, domain_specified, path, secure, expires, name, value = \
line.split("\t")
secure = (secure == "TRUE")
domain_specified = (domain_specified == "TRUE")
if name == "":
# cookies.txt regards 'Set-Cookie: foo' as a cookie
# with no name, whereas cookielib regards it as a
# cookie with no value.
name = value
value = None
initial_dot = domain.startswith(".")
assert domain_specified == initial_dot
discard = False
if expires == "":
expires = None
discard = True
# assume path_specified is false
c = Cookie(0, name, value, None, False, domain, domain_specified,
initial_dot, path, False, secure, expires, discard,
None, None, {})
if not ignore_discard and c.discard:
continue
if not ignore_expires and c.is_expired(now):
continue
self.set_cookie(c)
except IOError:
raise
except Exception:
_warn_unhandled_exception()
raise LoadError("invalid Netscape format cookies file %r: %r" %
(filename, line))
def _really_load(self, f, filename, ignore_discard, ignore_expires):
"""
This function is required to monkey patch MozillaCookieJar's _really_load
function which does not understand the curl format cookie file created
by ecp-cookie-init. It patches the code so that #HttpOnly_ get loaded.
https://bugs.python.org/issue2190
https://bugs.python.org/file37625/httponly.patch
"""
now = time.time()
magic = f.readline()
if not re.search(self.magic_re, magic):
f.close()
raise LoadError(
"%r does not look like a Netscape format cookies file" %
filename)
try:
while 1:
line = f.readline()
if line == "": break
# last field may be absent, so keep any trailing tab
if line.endswith("\n"): line = line[:-1]
sline = line.strip()
# support HttpOnly cookies (as stored by curl or old Firefox).
if sline.startswith("#HttpOnly_"):
line = sline[10:]
# skip comments and blank lines XXX what is $ for?
elif (sline.startswith(("#", "$")) or sline == ""):
continue
domain, domain_specified, path, secure, expires, name, value = \
line.split("\t")
secure = (secure == "TRUE")
domain_specified = (domain_specified == "TRUE")
if name == "":
# cookies.txt regards 'Set-Cookie: foo' as a cookie
# with no name, whereas cookielib regards it as a
# cookie with no value.
name = value
value = None
initial_dot = domain.startswith(".")
assert domain_specified == initial_dot
discard = False
if expires == "":
expires = None
discard = True
# assume path_specified is false
c = Cookie(0, name, value,
None, False,
domain, domain_specified, initial_dot,
path, False,
secure,
expires,
discard,
None,
None,
{})
if not ignore_discard and c.discard:
continue
if not ignore_expires and c.is_expired(now):
continue
self.set_cookie(c)
except IOError:
raise
except Exception:
_warn_unhandled_exception()
raise LoadError("invalid Netscape format cookies file %r: %r" %
(filename, line))
def downloadpage(url, post=None, headers=None, timeout=None, follow_redirects=True, cookies=True, replace_headers=False,
add_referer=False, only_headers=False, bypass_cloudflare=True, bypass_testcookie=True, no_decode=False,
method=None, cache=CACHE_ENABLED, cache_expiration=CACHE_EXPIRATION):
"""
Descarga una página web y devuelve los resultados
:type url: str
:type post: dict, str
:type headers: dict, list
:type timeout: int
:type follow_redirects: bool
:type cookies: bool, dict
:type replace_headers: bool
:type add_referer: bool
:type only_headers: bool
:type bypass_cloudflare: bool
:type cache: bool
:type cache_expiration: timedelta
:return: Resultado
"""
arguments = locals().copy()
if cache:
try:
cache_key = '|'.join(["%s:%s" %(k,v) for k,v in sorted(arguments.items(),key= lambda x: x[0]) if v]).encode()
cache_key = CACHE_PREFIX + hashlib.sha1(cache_key).hexdigest()
cacheado = CACHE.get(cache_key)
if cacheado:
return HTTPResponse(cacheado)
except:
pass
response = {}
# Post tipo dict
if type(post) == dict:
post = urllib_parse.urlencode(post)
# Url quote
url = urllib_parse.quote(url, safe="%/:=&?~#+!$,;'@()*[]")
# Headers por defecto, si no se especifica nada
request_headers = default_headers.copy()
# Headers pasados como parametros
if headers is not None:
if not replace_headers:
request_headers.update(dict(headers))
else:
request_headers = dict(headers)
# Referer
if add_referer:
request_headers["Referer"] = "/".join(url.split("/")[:3])
#logger("Headers:")
#logger(request_headers, 'info')
# Handlers
handlers = list()
handlers.append(HTTPHandler(debuglevel=False))
handlers.append(HTTPSHandler(debuglevel=False))
handlers.append(urllib_request.HTTPBasicAuthHandler())
# No redirects
if not follow_redirects:
handlers.append(NoRedirectHandler())
else:
handlers.append(HTTPRedirectHandler())
# Dict con cookies para la sesión
if type(cookies) == dict:
for name, value in cookies.items():
if not type(value) == dict:
value = {'value': value}
ck = Cookie(
version=0,
name=name,
value=value.get('value', ''),
port=None,
port_specified=False,
domain=value.get('domain', urllib_parse.urlparse(url)[1]),
domain_specified=False,
domain_initial_dot=False,
path=value.get('path', '/'),
path_specified=True,
secure=False,
expires=value.get('expires', time.time() + 3600 * 24),
discard=True,
comment=None,
comment_url=None,
rest={'HttpOnly': None},
rfc2109=False
)
cj.set_cookie(ck)
if cookies:
handlers.append(urllib_request.HTTPCookieProcessor(cj))
# Opener
opener = urllib_request.build_opener(*handlers)
# Contador
inicio = time.time()
# Request
req = Request(url, six.ensure_binary(post) if post else None, request_headers, method=method)
try:
#logger("Realizando Peticion")
handle = opener.open(req, timeout=timeout)
#logger('Peticion realizada')
except HTTPError as handle:
#logger('Peticion realizada con error')
response["sucess"] = False
response["code"] = handle.code
response["error"] = handle.__dict__.get("reason", str(handle))
response["headers"] = dict(handle.headers.items())
response['cookies'] = get_cookies(urllib_parse.urlparse(url)[1])
if not only_headers:
#logger('Descargando datos...')
response["data"] = handle.read()
else:
response["data"] = b""
response["time"] = time.time() - inicio
response["url"] = handle.geturl()
except Exception as e:
#logger('Peticion NO realizada')
response["sucess"] = False
response["code"] = e.__dict__.get("errno", e.__dict__.get("code", str(e)))
response["error"] = e.__dict__.get("reason", str(e))
response["headers"] = {}
response['cookies'] = get_cookies(urllib_parse.urlparse(url)[1])
response["data"] = b""
response["time"] = time.time() - inicio
response["url"] = url
else:
response["sucess"] = True
response["code"] = handle.code
response["error"] = None
response["headers"] = dict(handle.headers.items())
response['cookies'] = get_cookies(urllib_parse.urlparse(url)[1])
if not only_headers:
#logger('Descargando datos...')
response["data"] = handle.read()
else:
response["data"] = b""
response["time"] = time.time() - inicio
response["url"] = handle.geturl()
response['headers'] = dict([(k.lower(), v) for k, v in response['headers'].items()])
#logger("Terminado en %.2f segundos" % (response["time"]))
#logger("url: %s" % url)
#logger("Response sucess : %s" % (response["sucess"]))
#logger("Response code : %s" % (response["code"]))
#logger("Response error : %s" % (response["error"]))
#logger("Response cookies : %s" % (response["cookies"]))
#logger("Response data length: %s" % (len(response["data"])))
#logger("Response headers:")
#logger(response['headers'])
# Guardamos las cookies
if cookies:
save_cookies()
# Gzip
if response["headers"].get('content-encoding') == 'gzip':
response["data"] = gzip.GzipFile(fileobj=BytesIO(response["data"])).read()
# Binarios no se codifican ni se comprueba cloudflare, etc...
if not is_binary(response):
response['data'] = six.ensure_str(response['data'], errors='replace')
if not no_decode:
response["data"] = six.ensure_str(HTMLParser().unescape(
six.ensure_text(response['data'], errors='replace')
))
# Anti TestCookie
if bypass_testcookie:
if 'document.cookie="__test="+toHex(slowAES.decrypt(c,2,a,b))+"' in response['data']:
a = re.findall('a=toNumbers\("([^"]+)"\)', response['data'])[0].decode("HEX")
b = re.findall('b=toNumbers\("([^"]+)"\)', response['data'])[0].decode("HEX")
c = re.findall('c=toNumbers\("([^"]+)"\)', response['data'])[0].decode("HEX")
arguments['bypass_testcookie'] = False
if not type(arguments['cookies']) == dict:
arguments['cookies'] = {'__test': ii11.new(a, ii11.MODE_CBC, b).decrypt(c).encode("HEX")}
else:
arguments['cookies']['__test'] = ii11.new(a, ii11.MODE_CBC, b).decrypt(c).encode("HEX")
response = downloadpage(**arguments).__dict__
# Anti Cloudflare
if bypass_cloudflare:
response = retry_if_cloudflare(response, arguments)
if cache:
CACHE.set(cache_key, response, expiration=cache_expiration)
return HTTPResponse(response)