def parse_authorization_header(header):
""" Parse requests authorization header into list.
Raise ValueError if some problem occurs. """
# digest is marked as part of header and causes problem
# parsing, so remove its
if not header.startswith("Digest "):
raise ValueError("Header do not start with Digest")
header = header[len("Digest ") :]
# Convert the auth params to a dict
items = urllib2.parse_http_list(header)
params = urllib2.parse_keqv_list(items)
required = ["username", "realm", "nonce", "uri", "response"]
for field in required:
if not params.has_key(field):
raise ValueError("Required field %s not found" % field)
# check for qop companions (sect. 3.2.2)
if params.has_key("qop") and not params.has_key("cnonce") and params.has_key("cn"):
raise ValueError("qop sent without cnonce and cn")
return params
def getYoutubeMovie(url):
try:
conn = urllib2.urlopen(url)
encoding = conn.headers.getparam('charset')
content = conn.read().decode(encoding)
#get available streams
s = re.findall(r'"url_encoded_fmt_stream_map": ?"([^"]+)"', content)
print s
if s and len(s):
s = s[0].split(',')
values = {}
for stream in s:
stream = stream.replace('\\u0026', '&')
stream = urllib2.parse_keqv_list(stream.split('&'))
values[stream.get('itag') or "0"] = stream
itags = values.keys()
sorted(itags, reverse=True)
print itags
link = None
for itag in itags:
z = values[itag]
if itag == '84' or itag == '82' or itag == '38' or itag == '37' or itag == '22' or itag == '18':
try:
link = urllib.unquote(z['url'] + '&signature=%s' % z['sig'])
except:
link = urllib.unquote(z['url'])
return link
except Exception as e:
print e
return None
def parse_authorization_header(header):
""" Parse requests authorization header into list.
Raise ValueError if some problem occurs. """
# digest is marked as part of header and causes problem
# parsing, so remove its
if not header.startswith('Digest '):
raise ValueError("Header do not start with Digest")
header = header[len('Digest '):]
# Convert the auth params to a dict
items = urllib2.parse_http_list(header)
params = urllib2.parse_keqv_list(items)
required = ["username", "realm", "nonce", "uri", "response"]
for field in required:
if not params.has_key(field):
raise ValueError("Required field %s not found" % field)
# check for qop companions (sect. 3.2.2)
if params.has_key(
"qop") and not params.has_key("cnonce") and params.has_key("cn"):
raise ValueError("qop sent without cnonce and cn")
return params
def parse_authorization_header(value):
"""Parse the Authenticate header
Returns nothing on failure, opts hash on success with type='basic' or 'digest'
and other params.
<http://nullege.com/codes/search/werkzeug.http.parse_authorization_header>
<http://stackoverflow.com/questions/1349367/parse-an-http-request-authorization-header-with-python>
<http://bugs.python.org/file34041/0001-Add-an-authorization-header-to-the-initial-request.patch>
"""
try:
(auth_type, auth_info) = value.split(' ', 1)
auth_type = auth_type.lower()
except ValueError as e:
return
if (auth_type == 'basic'):
try:
(username, password) = base64.b64decode(auth_info).split(':', 1)
except Exception as e:
return
return {'type':'basic', 'username': username, 'password': password}
elif (auth_type == 'digest'):
auth_map = urllib2.parse_keqv_list(urllib2.parse_http_list(auth_info))
print auth_map
for key in 'username', 'realm', 'nonce', 'uri', 'response':
if not key in auth_map:
return
if 'qop' in auth_map:
if not auth_map.get('nc') or not auth_map.get('cnonce'):
return
auth_map['type']='digest'
return auth_map
else:
# unknown auth type
return
def parse_oauth_header(header):
type, rest = header.split(" ", 1)
if type != "OAuth":
raise ValueError("Authorization is not of OAuth type")
return urllib2.parse_keqv_list(urllib2.parse_http_list(rest))
def parse_401_response_headers(response_headers):
"""
Parse the headers from a 401 response into a dictionary that contains the information
necessary to retrieve a token.
Example:
Www-Authenticate: Bearer realm="https://auth.docker.io/token",
service="registry.docker.io",scope="repository:library/nginx:pull,push"
"""
auth_header = response_headers.get("www-authenticate")
if auth_header is None:
raise IOError(
"401 responses are expected to contain authentication information")
auth_header = auth_header[len("Bearer "):]
# The remaining string consists of comma separated key=value pairs
# according to RFC 2617
try:
items = request.parse_http_list(auth_header)
return request.parse_keqv_list(items)
except ValueError as e:
six.raise_from(
IOError(
"401 responses are expected to contain authentication information"
),
e,
)
def youtube_video(url):
try:
conn = urllib2.urlopen(url)
encoding = conn.headers.getparam("charset")
content = conn.read().decode(encoding)
s = re.findall(r'"url_encoded_fmt_stream_map": ?"([^"]+)"', content)
if s:
import HTMLParser
s = s[0].split(",")
s = [a.replace("\\u0026", "&") for a in s]
s = [urllib2.parse_keqv_list(a.split("&")) for a in s]
n = re.findall(r"<title>(.+) - YouTube</title>", content)
s, n = (s or [], HTMLParser.HTMLParser().unescape(n[0]))
for z in s:
if z["itag"] == "18":
if "mp4" in z["type"]:
ext = ".mp4"
elif "flv" in z["type"]:
ext = ".flv"
try:
link = urllib.unquote(z["url"] + "&signature=%s" % z["sig"])
except:
link = urllib.unquote(z["url"])
return link
except:
return False
def compose(self, digest=None, basic=None, username=None, password=None,
challenge=None, path=None, method=None):
assert username and password
if basic or not challenge:
assert not digest
userpass = "******" % (username.strip(), password.strip())
return "Basic %s" % userpass.encode('base64').strip()
assert challenge and not basic
path = path or "/"
(_, realm) = challenge.split('realm="')
(realm, _) = realm.split('"', 1)
auth = urllib2.AbstractDigestAuthHandler()
auth.add_password(realm, path, username, password)
(token, challenge) = challenge.split(' ', 1)
chal = urllib2.parse_keqv_list(urllib2.parse_http_list(challenge))
class FakeRequest(object):
def get_full_url(self):
return path
def has_data(self):
return False
def get_method(self):
return method or "GET"
get_selector = get_full_url
retval = "Digest %s" % auth.get_authorization(FakeRequest(), chal)
return (retval,)
def youtube_video(url):
try:
conn = urllib2.urlopen(url)
encoding = conn.headers.getparam('charset')
content = conn.read().decode(encoding)
s = re.findall(r'"url_encoded_fmt_stream_map": ?"([^"]+)"', content)
if s:
import HTMLParser
s = s[0].split(',')
s = [a.replace('\\u0026', '&') for a in s]
s = [urllib2.parse_keqv_list(a.split('&')) for a in s]
n = re.findall(r'<title>(.+) - YouTube</title>', content)
s, n = (s or [], HTMLParser.HTMLParser().unescape(n[0]))
for z in s:
if z['itag'] == '18':
if 'mp4' in z['type']:
ext = '.mp4'
elif 'flv' in z['type']:
ext = '.flv'
try:
link = urllib.unquote(z['url'] +
'&signature=%s' % z['sig'])
except:
link = urllib.unquote(z['url'])
return link
except:
return False
def registry_get(self, api):
url = "https://%s/v2/%s" % (self.registry, api)
response = get(
url,
auth=(self.user, self.password),
headers={
"Accept":
'application/vnd.docker.distribution.manifest.v2+json'
},
verify=self.verify)
if response.status_code == 401:
challenge = response.headers['Www-Authenticate']
if challenge.startswith("Bearer "):
challenge = challenge[7:]
opts = urllib2.parse_keqv_list(urllib2.parse_http_list(challenge))
authresp = get(
"{realm}?service={service}&scope={scope}".format(**opts),
auth=(self.user, self.password),
verify=self.verify)
if authresp.ok:
token = authresp.json()['token']
response = get(url,
headers={'Authorization': 'Bearer %s' % token},
verify=self.verify)
else:
raise TaskError(
"problem authenticating with docker registry: [%s] %s" %
(authresp.status_code, authresp.content))
return response
def _parseHeader(self, authheader, request):
n = 7 # n = len("Digest ")
try:
authheader = authheader[n:].strip()
items = urllib2.parse_http_list(authheader)
request.env['__DIGEST_PARAMS__'] = urllib2.parse_keqv_list(items)
except Exception, e:
request.env['__DIGEST_PARAMS__'] = {}
def _parseHeader(self, authheader, request):
n = 7 # n = len("Digest ")
try:
authheader = authheader[n:].strip()
items = urllib2.parse_http_list(authheader)
request.env['__DIGEST_PARAMS__'] = urllib2.parse_keqv_list(items)
except Exception, e:
request.env['__DIGEST_PARAMS__'] = {}
def parse_authorization_header(authorization_header):
"""Parse an OAuth authorization header into a list of 2-tuples"""
auth_scheme = 'OAuth '
if authorization_header.startswith(auth_scheme):
authorization_header = authorization_header.replace(auth_scheme, '', 1)
items = urllib2.parse_http_list(authorization_header)
try:
return urllib2.parse_keqv_list(items).items()
except ValueError:
raise ValueError('Malformed authorization header')
def _parse(self, authorization):
scheme, rest = authorization.split(None, 1)
args = urllib2.parse_keqv_list(urllib2.parse_http_list(rest))
challengeType = {
'basic': BasicChallenge,
'digest': DigestChallenge,
}.get(scheme.lower())
if challengeType is None:
return "", None
return scheme.lower(), challengeType(**args)
def __init__(self, auth_header, http_method, debug=False):
self.http_method = http_method
self.debug = debug
scheme, params = auth_header.split(" ", 1)
self.scheme = scheme.lower()
if self.scheme != 'digest':
raise ValueError('Authorization scheme is not "Digest"')
self.auth_header = auth_header
# make a dict of the params
items = parse_http_list(params)
paramsd = parse_keqv_list(items)
self.realm = paramsd.get('realm')
self.username = paramsd.get('username')
self.nonce = paramsd.get('nonce')
self.uri = paramsd.get('uri')
self.method = paramsd.get('method')
self.response = paramsd.get('response') # the response digest
self.algorithm = paramsd.get('algorithm', 'MD5')
self.cnonce = paramsd.get('cnonce')
self.opaque = paramsd.get('opaque')
self.qop = paramsd.get('qop') # qop
self.nc = paramsd.get('nc') # nonce count
# perform some correctness checks
if self.algorithm not in valid_algorithms:
raise ValueError(
self.errmsg("Unsupported value for algorithm: '%s'" %
self.algorithm))
has_reqd = self.username and \
self.realm and \
self.nonce and \
self.uri and \
self.response
if not has_reqd:
raise ValueError(
self.errmsg("Not all required parameters are present."))
if self.qop:
if self.qop not in valid_qops:
raise ValueError(
self.errmsg("Unsupported value for qop: '%s'" % self.qop))
if not (self.cnonce and self.nc):
raise ValueError(
self.errmsg(
"If qop is sent then cnonce and nc MUST be present"))
else:
if self.cnonce or self.nc:
raise ValueError(
self.errmsg(
"If qop is not sent, neither cnonce nor nc can be present"
))
def _digest_login_callback(self, response):
self._update_session_cookie(response)
if response.code == 200:
self._authenticated = True
self._alive = True
print response.headers
print 'yay'
return
elif response.code == 401:
print 'meh'
else:
print 'boo'
return
#If 401
if not 'WWW-Authenticate' in response.headers:
raise InterfaceError()
if not 'Digest' in response.headers['WWW-Authenticate']:
raise InterfaceError()
param_string = response.headers.get('WWW-Authenticate').partition(
'Digest')[2]
param_list = parse_http_list(param_string)
params = parse_keqv_list(param_list)
path = urlsplit(
response.effective_url) #asterisk throws query into hash
digest_path = "%s?%s" % (path.path, path.query)
digest_cnonce = uuid.uuid1().hex
digest_nc = '%08d' % self._digest_nc
digest_qop = "auth"
digest_response = md5(":".join([
md5(":".join(
(self._username, params['realm'], self._secret))).hexdigest(),
params['nonce'], digest_nc, digest_cnonce, digest_qop,
md5(":".join(('GET', digest_path))).hexdigest()
])).hexdigest()
headers = {}
headers[
'Authorization'] = 'Digest username="******", realm="%s", nonce="%s", uri="%s", cnonce="%s", nc=%s, qop="%s", response="%s", opaque="%s", algorithm="%s"' % (
self._username, params['realm'], params['nonce'], digest_path,
digest_cnonce, digest_nc, digest_qop, digest_response,
params['opaque'], params['algorithm'])
self._digest_nc += 1
self.command(self._authentication_request, headers=headers)
def retry_http_digest_auth(self, req, auth):
token, challenge = auth.split(' ', 1)
chal = urllib2.parse_keqv_list(urllib2.parse_http_list(challenge))
auth = self.get_authorization(req, chal)
if auth:
auth_val = 'X-Digest %s' % auth
if req.headers.get(self.auth_header, None) == auth_val:
return None
req.add_unredirected_header(self.auth_header, auth_val)
resp = self.parent.open(req, timeout=req.timeout)
return resp
def retry_http_digest_auth(self, req, resp, host, auth):
_token, challenge = auth.split(' ', 1)
chal = urllib2.parse_keqv_list(urllib2.parse_http_list(challenge))
auth = self.get_authorization(req, chal)
if auth:
auth_val = 'Digest %s' % auth
if req.headers.get(self.auth_header, None) == auth_val:
return None
req.add_unredirected_header(self.auth_header, auth_val)
return 'request', req
return None
def retry_http_digest_auth(self, req, auth):
token, challenge = auth.split(' ', 1)
chal = parse_keqv_list(parse_http_list(challenge))
auth = self.get_authorization(req, chal)
if auth:
auth_val = 'Digest %s' % auth
if req.headers.get(self.auth_header, None) == auth_val:
return None
newreq = copy.copy(req)
newreq.add_unredirected_header(self.auth_header, auth_val)
newreq.visit = False
return self.parent.open(newreq)
def yt_get_all_url_maps_name(url):
conn = urllib2.urlopen(url)
encoding = conn.headers.getparam('charset')
content = conn.read().decode(encoding)
s = re.findall(r'"url_encoded_fmt_stream_map": "([^"]+)"', content)
if s:
s = s[0].split(',')
s = [a.replace('\\u0026', '&') for a in s]
s = [urllib2.parse_keqv_list(a.split('&')) for a in s]
n = re.findall(r'<title>(.+) - YouTube</title>', content)
return (s or [], HTMLParser.HTMLParser().unescape(n[0]))
def retry_http_digest_auth(self, req, auth):
token, challenge = auth.split(' ', 1)
chal = urllib2.parse_keqv_list(urllib2.parse_http_list(challenge))
auth = self.get_authorization(req, chal)
if auth:
auth_val = 'X-Digest %s' % auth
if req.headers.get(self.auth_header, None) == auth_val:
return None
req.add_unredirected_header(self.auth_header, auth_val)
resp = self.parent.open(req, timeout=req.timeout)
return resp
def retry_http_digest_auth(self, req, resp, host, auth):
_token, challenge = auth.split(" ", 1)
chal = urllib2.parse_keqv_list(urllib2.parse_http_list(challenge))
auth = self.get_authorization(req, chal)
if auth:
auth_val = "Digest %s" % auth
if req.headers.get(self.auth_header, None) == auth_val:
return None
req.add_unredirected_header(self.auth_header, auth_val)
return "request", req
return None
def retry_http_digest_auth(self, req, auth):
token, challenge = auth.split(' ', 1)
chal = parse_keqv_list(parse_http_list(challenge))
auth = self.get_authorization(req, chal)
if auth:
auth_val = 'Digest %s' % auth
if req.headers.get(self.auth_header, None) == auth_val:
return None
newreq = copy.copy(req)
newreq.add_unredirected_header(self.auth_header, auth_val)
newreq.visit = False
return self.parent.open(newreq)
def _authenticate_quay(self, headers):
"""
Attempt to perform an authentication with registry's authentication server.
Once authentication is complete, add the token to the Session object.
Specifics can be found at https://docs.docker.com/registry/spec/auth/token/
Args:
headers (dict):
Headers of the 401 response received from the registry.
Raises:
RegistryAuthError:
When there's an issue with the authentication procedure.
"""
if "WWW-Authenticate" not in headers:
raise RegistryAuthError(
"'WWW-Authenticate' is not in the 401 response's header. "
"Authentication cannot continue.")
if "Bearer " not in headers["WWW-Authenticate"]:
raise RegistryAuthError(
"Different than the Bearer authentication type was requested. "
"Only Bearer is supported.")
# parse header to get a dictionary
params = request.parse_keqv_list(
request.parse_http_list(
headers["WWW-Authenticate"][len("Bearer "):]) # noqa: E203
)
host = params.pop("realm")
session = requests.Session()
retry = Retry(
total=3,
read=3,
connect=3,
backoff_factor=2,
status_forcelist=set(range(500, 512)),
)
adapter = requests.adapters.HTTPAdapter(max_retries=retry)
session.mount("http://", adapter)
session.mount("https://", adapter)
# Make an authentication request to the specified realm with the provided REST parameters.
# Basic username + password authentication is expected.
r = session.get(host,
params=params,
auth=(self.username, self.password),
timeout=10)
r.raise_for_status()
if "token" not in r.json():
raise RegistryAuthError(
"Authentication server response doesn't contain a token.")
self.session.set_auth_token(r.json()["token"])
def authorized(self):
tcs = self.server.test_case_server
auth_header = self.headers.get(tcs.auth_header_recv, None)
if auth_header is None:
return False
scheme, auth = auth_header.split(None, 1)
if scheme.lower() == tcs.auth_scheme:
auth_dict = urllib2.parse_keqv_list(urllib2.parse_http_list(auth))
return tcs.digest_authorized(auth_dict, self.command)
return False
def authorized(self):
tcs = self.server.test_case_server
auth_header = self.headers.get(tcs.auth_header_recv, None)
if auth_header is None:
return False
scheme, auth = auth_header.split(None, 1)
if scheme.lower() == tcs.auth_scheme:
auth_dict = urllib2.parse_keqv_list(urllib2.parse_http_list(auth))
return tcs.digest_authorized(auth_dict, self.command)
return False
def yt_get_all_url_maps_name(url):
conn = urllib2.urlopen(url)
encoding = conn.headers.getparam('charset')
content = conn.read().decode(encoding)
s = re.findall(r'"url_encoded_fmt_stream_map": "([^"]+)"', content)
if s:
s = s[0].split(',')
s = [a.replace('\\u0026', '&') for a in s]
s = [urllib2.parse_keqv_list(a.split('&')) for a in s]
n = re.findall(r'<title>(.+) - YouTube</title>', content)
return (s or [],
HTMLParser.HTMLParser().unescape(n[0]))
def createAuthObject(authHeader):
"""
Returns the authentication mechanism, or None if not implemented.
"""
authType, challenge = authHeader.split(' ', 1)
_authType_lower = authType.lower()
challenge = urllib2.parse_keqv_list(urllib2.parse_http_list(challenge))
assert _authType_lower in ('digest', 'ssl'), \
repr(_authType_lower)
# "basic" authentication is not supported
return DigestAuthentication(challenge) if _authType_lower == 'digest' \
else None
def _parse(self, authorization):
try:
scheme, rest = authorization.split(None, 1)
except ValueError:
# Probably "negotiate", which we don't support
scheme = authorization
rest = ""
args = urllib2.parse_keqv_list(urllib2.parse_http_list(rest))
challengeType = {
'basic': BasicChallenge,
'digest': DigestChallenge,
}.get(scheme.lower())
if challengeType is None:
return "", None
return scheme.lower(), challengeType(**args)
def _parse(self, authorization):
try:
scheme, rest = authorization.split(None, 1)
except ValueError:
# Probably "negotiate", which we don't support
scheme = authorization
rest = ""
args = urllib2.parse_keqv_list(urllib2.parse_http_list(rest))
challengeType = {
'basic': BasicChallenge,
'digest': DigestChallenge,
}.get(scheme.lower())
if challengeType is None:
return "", None
return scheme.lower(), challengeType(**args)
def __init__(self, auth_header, http_method, debug=False):
self.http_method = http_method
self.debug = debug
scheme, params = auth_header.split(" ", 1)
self.scheme = scheme.lower()
if self.scheme != 'digest':
raise ValueError('Authorization scheme is not "Digest"')
self.auth_header = auth_header
# make a dict of the params
items = parse_http_list(params)
paramsd = parse_keqv_list(items)
self.realm = paramsd.get('realm')
self.username = paramsd.get('username')
self.nonce = paramsd.get('nonce')
self.uri = paramsd.get('uri')
self.method = paramsd.get('method')
self.response = paramsd.get('response') # the response digest
self.algorithm = paramsd.get('algorithm', 'MD5')
self.cnonce = paramsd.get('cnonce')
self.opaque = paramsd.get('opaque')
self.qop = paramsd.get('qop') # qop
self.nc = paramsd.get('nc') # nonce count
# perform some correctness checks
if self.algorithm not in valid_algorithms:
raise ValueError(self.errmsg("Unsupported value for algorithm: '%s'" % self.algorithm))
has_reqd = self.username and \
self.realm and \
self.nonce and \
self.uri and \
self.response
if not has_reqd:
raise ValueError(self.errmsg("Not all required parameters are present."))
if self.qop:
if self.qop not in valid_qops:
raise ValueError(self.errmsg("Unsupported value for qop: '%s'" % self.qop))
if not (self.cnonce and self.nc):
raise ValueError(self.errmsg("If qop is sent then cnonce and nc MUST be present"))
else:
if self.cnonce or self.nc:
raise ValueError(self.errmsg("If qop is not sent, neither cnonce nor nc can be present"))
def dr(self, url, expected=(), user=None, password=None):
user = user or self.user
password = password or self.password
response = self.get(url,
auth=(user, password),
expected=expected + (401, ))
if response.status_code == 401:
challenge = response.headers['Www-Authenticate']
if challenge.startswith("Bearer "):
challenge = challenge[7:]
opts = urllib2.parse_keqv_list(urllib2.parse_http_list(challenge))
token = self.get(
"{realm}?service={service}&scope={scope}".format(**opts),
auth=(user, password)).json()['token']
response = self.get(url,
headers={'Authorization': 'Bearer %s' % token},
expected=expected)
return response
def _parseDigestAuthorization (auth_params):
# Convert the auth params to a dict
items = urllib2.parse_http_list (auth_params)
params = urllib2.parse_keqv_list (items)
# Now validate the params
# Check for required parameters
required = ["username", "realm", "nonce", "uri", "response"]
for k in required:
if not params.has_key(k):
return None
# If qop is sent then cnonce and cn MUST be present
if params.has_key("qop") and not params.has_key("cnonce") \
and params.has_key("cn"):
return None
return params
def get_proxy_authorization_line(auth_type, auth_details, method, url, proxy_username, proxy_password):
if auth_type.lower() == 'basic':
response = base64.encodestring('%s:%s' % (proxy_username, proxy_password)).strip()
elif auth_type.lower() == 'digest':
class Passwd:
def __init__(self, user, passwd):
self.user, self.passwd = user, passwd
def add_password(self, user, passwd):
pass
def find_user_password(self, realm, url):
return (self.user, self.passwd)
def get_full_url(self):
return ''
class DummyRequest:
def __init__(self, method, url):
self.method, self.url = method, url
def get_method(self):
return self.method
def get_selector(self):
return self.url
def get_full_url(self):
return self.url
def has_data(self):
return False
digest_auth_handler = urllib2.AbstractDigestAuthHandler(passwd=Passwd(proxy_username or '', proxy_password or ''))
chal = urllib2.parse_keqv_list(urllib2.parse_http_list(auth_details))
response = digest_auth_handler.get_authorization(DummyRequest(method, url), chal)
else:
raise ValueError('Invalid proxy-authenticate line %r' % auth_type)
return 'Proxy-Authorization: %s %s' % (auth_type, response)
def http_error_401(self, req, fp, code, msg, headers):
host = urllib2.urlparse.urlparse(req.get_full_url())[1]
authreq = headers.get('www-authenticate', None)
if authreq == None: return None
authreq = authreq.split(' ', 1)
if authreq[0].lower() != 'basic': return None
chal = urllib2.parse_keqv_list(urllib2.parse_http_list(authreq[1]))
realm = chal['realm']
self.auth = (host, realm)
self.retried += 1
if self.retried >= 3:
self.f.delkey(proto="pass", host=host, realm=realm, role="client")
self.f.start(proto="pass", host=host, realm=realm, role="client")
pw = self.f.read().replace(' ', ':', 1)
val = 'Basic %s' % base64.b64encode(pw).strip()
if req.headers.get('Authorization', None) == val: return None
req.add_header('Authorization', val)
result = self.parent.open(req)
self.retried = 0
return result
def _parse_auth_info(auth_info):
method, info_str = auth_info.split(' ', 1)
if method != "Digest":
raise ProviderError("Unknown authentication method: %s" % method)
items = parse_http_list(info_str)
info = parse_keqv_list(items)
try:
qop = info["qop"]
realm = info["realm"]
nonce = info["nonce"]
except KeyError as e:
raise ProviderError("Authentication request missing required key: %s" %
e)
algorithm = info.get("algorithm", "MD5")
if algorithm != "MD5":
raise ProviderError("Unsupported digest algorithm: %s" % algorithm)
if "auth" not in qop.split(","):
raise ProviderError("Unsupported quality-of-protection: %s" % qop)
return realm, nonce, "auth", algorithm
def _parse_auth_info(auth_info):
method, info_str = auth_info.split(' ', 1)
if method != "Digest":
raise ProviderError("Unknown authentication method: %s" % method)
items = parse_http_list(info_str)
info = parse_keqv_list(items)
try:
qop = info["qop"]
realm = info["realm"]
nonce = info["nonce"]
except KeyError as e:
raise ProviderError(
"Authentication request missing required key: %s" % e)
algorithm = info.get("algorithm", "MD5")
if algorithm != "MD5":
raise ProviderError("Unsupported digest algorithm: %s" % algorithm)
if "auth" not in qop.split(","):
raise ProviderError("Unsupported quality-of-protection: %s" % qop)
return realm, nonce, "auth", algorithm
def http_error_401(self, req, fp, code, msg, headers):
host = urllib2.urlparse.urlparse(req.get_full_url())[1]
authreq = headers.get('www-authenticate', None)
if authreq == None: return None
authreq = authreq.split(' ', 1)
if authreq[0].lower() != 'basic': return None
chal = urllib2.parse_keqv_list(urllib2.parse_http_list(authreq[1]))
realm = chal['realm']
self.auth = (host, realm)
self.retried += 1
if self.retried >= 3:
self.f.delkey(proto="pass", host=host, realm=realm, role="client")
self.f.start(proto="pass", host=host, realm=realm, role="client")
pw = self.f.read().replace(' ', ':', 1)
val = 'Basic %s' % base64.b64encode(pw).strip()
if req.headers.get('Authorization', None) == val: return None
req.add_header('Authorization', val)
result = self.parent.open(req)
self.retried = 0
return result
def http_error_401(self, req, fp, code, msg, headers):
self.retried += 1
host = urllib2.urlparse.urlparse(req.get_full_url())[1]
authreq = headers.get('www-authenticate', None)
if authreq == None: return None
authreq = authreq.split(' ', 1)
if authreq[0].lower() != 'digest': return None
chal = urllib2.parse_keqv_list(urllib2.parse_http_list(authreq[1]))
realm = chal['realm']
nonce = chal['nonce']
if self.retried >= 6:
self.f.delkey(proto="httpdigest", realm=realm, host=host)
self.f.start(proto="httpdigest", role="client", realm=realm, host=host)
self.f.write(nonce + ' ' + req.get_method() + ' ' + req.get_selector())
resp = self.f.read()
user = self.f.attr()["user"]
self.f.close()
val = 'Digest username="******", realm="%s", nonce="%s", uri="%s", response="%s", algorithm=MD5' % (user, realm, nonce, req.get_selector(), resp)
if req.headers.get('Authorization', None) == val: return None
req.add_unredirected_header('Authorization', val)
result = self.parent.open(req)
self.retried = 0
return result
def youtube_video(url):
try:
conn = urllib2.urlopen(url)
encoding = conn.headers.getparam('charset')
content = conn.read().decode(encoding)
s = re.findall(r'"url_encoded_fmt_stream_map": ?"([^"]+)"', content)
if s:
import HTMLParser
s = s[0].split(',')
s = [a.replace('\\u0026', '&') for a in s]
s = [urllib2.parse_keqv_list(a.split('&')) for a in s]
n = re.findall(r'<title>(.+) - YouTube</title>', content)
s, n = (s or [], HTMLParser.HTMLParser().unescape(n[0]))
for z in s:
if z['itag'] == '18':
if 'mp4' in z['type']:
ext = '.mp4'
elif 'flv' in z['type']:
ext = '.flv'
try: link = urllib.unquote(z['url'] + '&signature=%s' % z['sig'])
except: link = urllib.unquote(z['url'])
return link
except: return False
def _parseDigestAuthorization (auth_params):
# Convert the auth params to a dict
items = parse_http_list(auth_params)
params = parse_keqv_list(items)
# Now validate the params
# Check for required parameters
required = ["username", "realm", "nonce", "uri", "response"]
for k in required:
if k not in params:
return None
# If qop is sent then cnonce and nc MUST be present
if "qop" in params and not ("cnonce" in params \
and "nc" in params):
return None
# If qop is not sent, neither cnonce nor nc can be present
if ("cnonce" in params or "nc" in params) and \
"qop" not in params:
return None
return params
def http_error_401(self, req, fp, code, msg, headers):
self.retried += 1
host = urllib2.urlparse.urlparse(req.get_full_url())[1]
authreq = headers.get('www-authenticate', None)
if authreq == None: return None
authreq = authreq.split(' ', 1)
if authreq[0].lower() != 'digest': return None
chal = urllib2.parse_keqv_list(urllib2.parse_http_list(authreq[1]))
realm = chal['realm']
nonce = chal['nonce']
if self.retried >= 6:
self.f.delkey(proto="httpdigest", realm=realm, host=host)
self.f.start(proto="httpdigest", role="client", realm=realm, host=host)
self.f.write(nonce + ' ' + req.get_method() + ' ' + req.get_selector())
resp = self.f.read()
user = self.f.attr()["user"]
self.f.close()
val = 'Digest username="******", realm="%s", nonce="%s", uri="%s", response="%s", algorithm=MD5' % (
user, realm, nonce, req.get_selector(), resp)
if req.headers.get('Authorization', None) == val: return None
req.add_unredirected_header('Authorization', val)
result = self.parent.open(req)
self.retried = 0
return result
def _parseDigestAuthorization(auth_params):
# Convert the auth params to a dict
items = parse_http_list(auth_params)
params = parse_keqv_list(items)
# Now validate the params
# Check for required parameters
required = ["username", "realm", "nonce", "uri", "response"]
for k in required:
if k not in params:
return None
# If qop is sent then cnonce and nc MUST be present
if "qop" in params and not ("cnonce" in params \
and "nc" in params):
return None
# If qop is not sent, neither cnonce nor nc can be present
if ("cnonce" in params or "nc" in params) and \
"qop" not in params:
return None
return params
def parse_keqv_list(l):
"""A unicode-safe version of urllib2.parse_keqv_list"""
# With Python 2.6, parse_http_list handles unicode fine
return urllib2.parse_keqv_list(l)
def build_digest_response(self, fields, username, password):
"""
Takes a Proxy-Authenticate: Digest header and creates a response
header
:param fields:
The string portion of the Proxy-Authenticate header after
"Digest "
:param username:
The username to use for the response
:param password:
The password to use for the response
:return:
None if invalid Proxy-Authenticate header, otherwise the
string of fields for the Proxy-Authorization: Digest header
"""
fields = parse_keqv_list(parse_http_list(fields))
realm = fields.get('realm')
nonce = fields.get('nonce')
qop = fields.get('qop')
algorithm = fields.get('algorithm')
if algorithm:
algorithm = algorithm.lower()
opaque = fields.get('opaque')
if algorithm in ('md5', None):
def md5hash(string):
return hashlib.md5(string).hexdigest()
_hash = md5hash
elif algorithm == 'sha':
def sha1hash(string):
return hashlib.sha1(string).hexdigest()
_hash = sha1hash
else:
return None
host_port = '%s:%s' % self.url_info
a1 = '%s:%s:%s' % (username, realm, password)
a2 = 'CONNECT:%s' % host_port
ha1 = _hash(a1)
ha2 = _hash(a2)
if qop is None:
response = _hash('%s:%s:%s' % (ha1, nonce, ha2))
elif qop == 'auth':
nc = '00000001'
cnonce = _hash(os.urandom(8))[:8]
response = _hash('%s:%s:%s:%s:%s:%s' %
(ha1, nonce, nc, cnonce, qop, ha2))
else:
return None
resp_fields = OrderedDict()
resp_fields['username'] = username
resp_fields['realm'] = realm
resp_fields['nonce'] = nonce
resp_fields['response'] = response
resp_fields['uri'] = host_port
if algorithm:
resp_fields['algorithm'] = algorithm
if qop == 'auth':
resp_fields['nc'] = nc
resp_fields['cnonce'] = cnonce
resp_fields['qop'] = qop
if opaque:
resp_fields['opaque'] = opaque
return ', '.join(
['%s="%s"' % (field, resp_fields[field]) for field in resp_fields])
def parse_keqv_list(l):
"""A unicode-safe version of urllib2.parse_keqv_list"""
# With Python 2.6, parse_http_list handles unicode fine
return urllib2.parse_keqv_list(l)
def parse_digest_challenge(self, challenge_string):
return urllib2.parse_keqv_list(
urllib2.parse_http_list(challenge_string))
except:
mode = urllib.unquote_plus(params[mode])
try:
if mode is None:
mode = str(params[mode])
if len(mode) < 1:
mode = str(params[mode])
except:
pass
try:
page = str(params['page'])
except:
page = urllib.unquote_plus(params["page"])
except:
try:
params = urllib2.parse_keqv_list(sys.argv[2])
except:
displayRootMenu()
if mode == None or url == None or len(url) < 1:
displayRootMenu()
elif mode == 'loadVideoList':
displayCatMenu()
elif mode == 'scrapeVideoListCat':
indexCatVideos(url, name)
elif mode == 'scrapeVideoList':
indexVideos(urlpath=basejoin(base_url, str(url +'/'+ page)), page="page{0}".format(page))
elif mode == 'playVideo':
playVideo(url, name, thumb)
#setView(True)
def build_digest_response(self, fields, username, password):
"""
Takes a Proxy-Authenticate: Digest header and creates a response
header
:param fields:
The string portion of the Proxy-Authenticate header after
"Digest "
:param username:
The username to use for the response
:param password:
The password to use for the response
:return:
None if invalid Proxy-Authenticate header, otherwise the
string of fields for the Proxy-Authorization: Digest header
"""
fields = parse_keqv_list(parse_http_list(fields))
realm = fields.get('realm')
nonce = fields.get('nonce')
qop = fields.get('qop')
algorithm = fields.get('algorithm')
if algorithm:
algorithm = algorithm.lower()
opaque = fields.get('opaque')
if algorithm in ('md5', None):
def md5hash(string):
return hashlib.md5(string).hexdigest()
_hash = md5hash
elif algorithm == 'sha':
def sha1hash(string):
return hashlib.sha1(string).hexdigest()
_hash = sha1hash
else:
return None
host_port = '%s:%s' % self.url_info
a1 = '%s:%s:%s' % (username, realm, password)
a2 = 'CONNECT:%s' % host_port
ha1 = _hash(a1)
ha2 = _hash(a2)
if qop is None:
response = _hash('%s:%s:%s' % (ha1, nonce, ha2))
elif qop == 'auth':
nc = '00000001'
cnonce = _hash(os.urandom(8))[:8]
response = _hash('%s:%s:%s:%s:%s:%s' % (ha1, nonce, nc, cnonce, qop, ha2))
else:
return None
resp_fields = OrderedDict()
resp_fields['username'] = username
resp_fields['realm'] = realm
resp_fields['nonce'] = nonce
resp_fields['response'] = response
resp_fields['uri'] = host_port
if algorithm:
resp_fields['algorithm'] = algorithm
if qop == 'auth':
resp_fields['nc'] = nc
resp_fields['cnonce'] = cnonce
resp_fields['qop'] = qop
if opaque:
resp_fields['opaque'] = opaque
return ', '.join(['%s="%s"' % (field, resp_fields[field]) for field in resp_fields])
def http_open(self, req):
(code, msg, headers, data, timestamp) = self._load(req.get_full_url())
# some info needed to process everything
cache_control = parse_http_list(headers.get('cache-control', ()))
cache_control += parse_http_list(headers.get('pragma', ()))
cc_list = [x for x in cache_control if '=' not in x]
cc_values = parse_keqv_list([x for x in cache_control if '=' in x])
cache_age = time.time() - timestamp
# list in a simple way what to do when
if req.get_header(
'Morss'
) == 'from_304': # for whatever reason, we need an uppercase
# we're just in the middle of a dirty trick, use cache
pass
elif self.force_min == -2:
if code is not None:
# already in cache, perfect, use cache
pass
else:
headers['Morss'] = 'from_cache'
resp = addinfourl(BytesIO(), headers, req.get_full_url(), 409)
resp.msg = 'Conflict'
return resp
elif code is None:
# cache empty, refresh
return None
elif self.force_min == -1:
# force use cache
pass
elif self.force_min == 0:
# force refresh
return None
elif self.force_min is None and (
'no-cache' in cc_list or 'no-store' in cc_list or
('private' in cc_list and not self.private)):
# kindly follow web servers indications, refresh
return None
elif 'max-age' in cc_values and int(cc_values['max-age']) > cache_age:
# server says it's still fine (and we trust him, if not, use force_min=0), use cache
pass
elif self.force_min is not None and self.force_min > cache_age:
# still recent enough for us, use cache
pass
else:
# according to the www, we have to refresh when nothing is said
return None
# return the cache as a response
headers[
'morss'] = 'from_cache' # TODO delete the morss header from incoming pages, to avoid websites messing up with us
resp = addinfourl(BytesIO(data), headers, req.get_full_url(), code)
resp.msg = msg
return resp
def parse_keqv_list(l):
"""A unicode-safe version of urllib2.parse_keqv_list"""
encoded_list = [u.encode('utf-8') for u in l]
encoded_parsed = urllib2.parse_keqv_list(encoded_list)
return dict((k.decode('utf-8'), v.decode('utf-8'))
for k, v in encoded_parsed.items())
def parse_digest_challenge(self, challenge_string):
return urllib2.parse_keqv_list(urllib2.parse_http_list(challenge_string))
def http_digest_auth(a, uri, user, pw, method):
token, challenge = a.split(' ', 1)
chal = urllib2.parse_keqv_list(urllib2.parse_http_list(challenge))
a = get_authorization(chal, uri, user, pw, method)
if a: return 'Digest %s' % a
def putfile(f, uri, username=None, password=None):
"""HTTP PUT the file f to uri, with optional auth data."""
host, port, path = parseuri(uri)
redirect = set([301, 302, 307])
authenticate = set([401])
okay = set([200, 201, 204])
authorized = False
authorization = None
tries = 0
while True:
# Attempt to HTTP PUT the data
h = httplib.HTTPConnection(host, port)
h.putrequest('PUT', path)
h.putheader('User-Agent', 'put.py/1.0')
h.putheader('Connection', 'keep-alive')
h.putheader('Transfer-Encoding', 'chunked')
h.putheader('Expect', '100-continue')
h.putheader('Accept', '*/*')
if authorization:
h.putheader('Authorization', authorization)
h.endheaders()
# Chunked transfer encoding
# Cf. 'All HTTP/1.1 applications MUST be able to receive and
# decode the "chunked" transfer-coding'
# - http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html
while True:
bytes = f.read(1024)
if not bytes: break
length = len(bytes)
# h.send('%X\r\n' % length)
h.send(bytes)
# h.send('0\r\n\r\n')
resp = h.getresponse()
status = resp.status # an int
# Got a response, now decide how to act upon it
if status in redirect:
location = resp.getheader('Location')
uri = urlparse.urljoin(uri, location)
host, port, path = parseuri(uri)
# We may have to authenticate again
if authorization:
authorization = None
elif status in authenticate:
# If we've done this already, break
if authorization:
# barf("Going around in authentication circles")
barf("Authentication failed")
if not (username and password):
barf("Need a username and password to authenticate with")
# Get the scheme: Basic or Digest?
wwwauth = resp.msg['www-authenticate'] # We may need this again
wauth = wwwauth.lstrip(' \t') # Hence use wauth not wwwauth here
wauth = wwwauth.replace('\t', ' ')
i = wauth.index(' ')
scheme = wauth[:i].lower()
if scheme in set(['basic', 'digest']):
if verbose:
msg = "Performing %s Authentication..." % scheme.capitalize(
)
print >> sys.stderr, msg
else:
barf("Unknown authentication scheme: %s" % scheme)
if scheme == 'basic':
import base64
userpass = username + ':' + password
userpass = base64.encodestring(userpass).strip()
authorized, authorization = True, 'Basic ' + userpass
elif scheme == 'digest':
if verbose:
msg = "uses fragile, undocumented features in urllib2"
print >> sys.stderr, "Warning! Digest Auth %s" % msg
import urllib2 # See warning above
passwd = type(
'Password', (object, ), {
'find_user_password':
lambda self, *args: (username, password),
'add_password':
lambda self, *args: None
})()
req = type(
'Request', (object, ), {
'get_full_url': lambda self: uri,
'has_data': lambda self: None,
'get_method': lambda self: 'PUT',
'get_selector': lambda self: path
})()
# Cf. urllib2.AbstractDigestAuthHandler.retry_http_digest_auth
auth = urllib2.AbstractDigestAuthHandler(passwd)
token, challenge = wwwauth.split(' ', 1)
chal = urllib2.parse_keqv_list(
urllib2.parse_http_list(challenge))
userpass = auth.get_authorization(req, chal)
authorized, authorization = True, 'Digest ' + userpass
elif status in okay:
if (username and password) and (not authorized):
msg = "Warning! The supplied username and password went unused"
print >> sys.stderr, msg
if verbose:
resultLine = "Success! Resource %s"
statuses = {200: 'modified', 201: 'created', 204: 'modified'}
print resultLine % statuses[status]
statusLine = "Response-Status: %s %s"
print statusLine % (status, resp.reason)
body = resp.read(58)
body = body.rstrip('\r\n')
body = body.encode('string_escape')
if len(body) >= 58:
body = body[:57] + '[...]'
bodyLine = 'Response-Body: "%s"'
print bodyLine % body
break
# @@ raise PutError, do the catching in main?
else:
barf('Got "%s %s"' % (status, resp.reason))
tries += 1
if tries >= 50:
barf("Too many redirects")
return status, resp
def putfile(f, uri, username=None, password=None):
"""HTTP PUT the file f to uri, with optional auth data."""
host, port, path = parseuri(uri)
redirect = set([301, 302, 307])
authenticate = set([401])
okay = set([200, 201, 204])
authorized = False
authorization = None
tries = 0
while True:
# Attempt to HTTP PUT the data
h = httplib.HTTPConnection(host, port)
h.putrequest('PUT', path)
h.putheader('User-Agent', 'put.py/1.0')
h.putheader('Connection', 'keep-alive')
h.putheader('Transfer-Encoding', 'chunked')
h.putheader('Expect', '100-continue')
h.putheader('Accept', '*/*')
if authorization:
h.putheader('Authorization', authorization)
h.endheaders()
# Chunked transfer encoding
# Cf. 'All HTTP/1.1 applications MUST be able to receive and
# decode the "chunked" transfer-coding'
# - http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html
while True:
bytes = f.read(1024)
if not bytes: break
length = len(bytes)
#h.send('%X\r\n' % length)
h.send(bytes)
#h.send('0\r\n\r\n')
resp = h.getresponse()
status = resp.status # an int
# Got a response, now decide how to act upon it
if status in redirect:
location = resp.getheader('Location')
uri = urlparse.urljoin(uri, location)
host, port, path = parseuri(uri)
# We may have to authenticate again
if authorization:
authorization = None
elif status in authenticate:
# If we've done this already, break
if authorization:
# barf("Going around in authentication circles")
barf("Authentication failed")
if not (username and password):
barf("Need a username and password to authenticate with")
# Get the scheme: Basic or Digest?
wwwauth = resp.msg['www-authenticate'] # We may need this again
wauth = wwwauth.lstrip(' \t') # Hence use wauth not wwwauth here
wauth = wwwauth.replace('\t', ' ')
i = wauth.index(' ')
scheme = wauth[:i].lower()
if scheme in set(['basic', 'digest']):
if verbose:
msg = "Performing %s Authentication..." % scheme.capitalize()
print >> sys.stderr, msg
else: barf("Unknown authentication scheme: %s" % scheme)
if scheme == 'basic':
import base64
userpass = username + ':' + password
userpass = base64.encodestring(userpass).strip()
authorized, authorization = True, 'Basic ' + userpass
elif scheme == 'digest':
if verbose:
msg = "uses fragile, undocumented features in urllib2"
print >> sys.stderr, "Warning! Digest Auth %s" % msg
import urllib2 # See warning above
passwd = type('Password', (object,), {
'find_user_password': lambda self, *args: (username, password),
'add_password': lambda self, *args: None
})()
req = type('Request', (object,), {
'get_full_url': lambda self: uri,
'has_data': lambda self: None,
'get_method': lambda self: 'PUT',
'get_selector': lambda self: path
})()
# Cf. urllib2.AbstractDigestAuthHandler.retry_http_digest_auth
auth = urllib2.AbstractDigestAuthHandler(passwd)
token, challenge = wwwauth.split(' ', 1)
chal = urllib2.parse_keqv_list(urllib2.parse_http_list(challenge))
userpass = auth.get_authorization(req, chal)
authorized, authorization = True, 'Digest ' + userpass
elif status in okay:
if (username and password) and (not authorized):
msg = "Warning! The supplied username and password went unused"
print >> sys.stderr, msg
if verbose:
resultLine = "Success! Resource %s"
statuses = {200: 'modified', 201: 'created', 204: 'modified'}
print resultLine % statuses[status]
statusLine = "Response-Status: %s %s"
print statusLine % (status, resp.reason)
body = resp.read(58)
body = body.rstrip('\r\n')
body = body.encode('string_escape')
if len(body) >= 58:
body = body[:57] + '[...]'
bodyLine = 'Response-Body: "%s"'
print bodyLine % body
break
# @@ raise PutError, do the catching in main?
else: barf('Got "%s %s"' % (status, resp.reason))
tries += 1
if tries >= 50:
barf("Too many redirects")
return status, resp
def http_open(self, req):
# Reminder of how/when this function is called by urllib2:
# If 'None' is returned, try your chance with the next-available handler
# If a 'resp' is returned, stop there, and proceed with 'http_response'
(code, msg, headers, data, timestamp) = self.load(req.get_full_url())
# some info needed to process everything
cache_control = parse_http_list(headers.get('cache-control', ()))
cache_control += parse_http_list(headers.get('pragma', ()))
cc_list = [x for x in cache_control if '=' not in x]
cc_values = parse_keqv_list([x for x in cache_control if '=' in x])
cache_age = time.time() - timestamp
# list in a simple way what to do when
if self.force_min == -2:
if code is not None:
# already in cache, perfect, use cache
return self.cached_response(req)
else:
# raise an error, via urllib handlers
resp = addinfourl(BytesIO(), headers, req.get_full_url(), 409)
resp.msg = 'Conflict'
return resp
elif code is None:
# cache empty, refresh
return None
elif self.force_min == -1:
# force use cache
return self.cached_response(req)
elif self.force_min == 0:
# force refresh
return None
elif code == 301 and cache_age < 7*24*3600:
# "301 Moved Permanently" has to be cached...as long as we want
# (awesome HTTP specs), let's say a week (why not?). Use force_min=0
# if you want to bypass this (needed for a proper refresh)
return self.cached_response(req)
elif (self.force_min is None or self.force_min > 0) and ('no-cache' in cc_list or 'no-store' in cc_list or ('private' in cc_list and not self.private_cache)):
# kindly follow web servers indications, refresh
# if the same settings are used all along, this section shouldn't be
# of any use, since the page woudln't be cached in the first place
# the check is only performed "just in case"
return None
elif 'max-age' in cc_values and int(cc_values['max-age']) > cache_age:
# server says it's still fine (and we trust him, if not, use force_min=0), use cache
return self.cached_response(req)
elif self.force_min is not None and self.force_min > cache_age:
# still recent enough for us, use cache
return self.cached_response(req)
else:
# according to the www, we have to refresh when nothing is said
return None
