def parse_authorization_header(header): """ Parse requests authorization header into list. Raise ValueError if some problem occurs. """ # digest is marked as part of header and causes problem # parsing, so remove its if not header.startswith("Digest "): raise ValueError("Header do not start with Digest") header = header[len("Digest ") :] # Convert the auth params to a dict items = urllib2.parse_http_list(header) params = urllib2.parse_keqv_list(items) required = ["username", "realm", "nonce", "uri", "response"] for field in required: if not params.has_key(field): raise ValueError("Required field %s not found" % field) # check for qop companions (sect. 3.2.2) if params.has_key("qop") and not params.has_key("cnonce") and params.has_key("cn"): raise ValueError("qop sent without cnonce and cn") return params
def getYoutubeMovie(url): try: conn = urllib2.urlopen(url) encoding = conn.headers.getparam('charset') content = conn.read().decode(encoding) #get available streams s = re.findall(r'"url_encoded_fmt_stream_map": ?"([^"]+)"', content) print s if s and len(s): s = s[0].split(',') values = {} for stream in s: stream = stream.replace('\\u0026', '&') stream = urllib2.parse_keqv_list(stream.split('&')) values[stream.get('itag') or "0"] = stream itags = values.keys() sorted(itags, reverse=True) print itags link = None for itag in itags: z = values[itag] if itag == '84' or itag == '82' or itag == '38' or itag == '37' or itag == '22' or itag == '18': try: link = urllib.unquote(z['url'] + '&signature=%s' % z['sig']) except: link = urllib.unquote(z['url']) return link except Exception as e: print e return None
def parse_authorization_header(header): """ Parse requests authorization header into list. Raise ValueError if some problem occurs. """ # digest is marked as part of header and causes problem # parsing, so remove its if not header.startswith('Digest '): raise ValueError("Header do not start with Digest") header = header[len('Digest '):] # Convert the auth params to a dict items = urllib2.parse_http_list(header) params = urllib2.parse_keqv_list(items) required = ["username", "realm", "nonce", "uri", "response"] for field in required: if not params.has_key(field): raise ValueError("Required field %s not found" % field) # check for qop companions (sect. 3.2.2) if params.has_key( "qop") and not params.has_key("cnonce") and params.has_key("cn"): raise ValueError("qop sent without cnonce and cn") return params
def parse_authorization_header(value): """Parse the Authenticate header Returns nothing on failure, opts hash on success with type='basic' or 'digest' and other params. <http://nullege.com/codes/search/werkzeug.http.parse_authorization_header> <http://stackoverflow.com/questions/1349367/parse-an-http-request-authorization-header-with-python> <http://bugs.python.org/file34041/0001-Add-an-authorization-header-to-the-initial-request.patch> """ try: (auth_type, auth_info) = value.split(' ', 1) auth_type = auth_type.lower() except ValueError as e: return if (auth_type == 'basic'): try: (username, password) = base64.b64decode(auth_info).split(':', 1) except Exception as e: return return {'type':'basic', 'username': username, 'password': password} elif (auth_type == 'digest'): auth_map = urllib2.parse_keqv_list(urllib2.parse_http_list(auth_info)) print auth_map for key in 'username', 'realm', 'nonce', 'uri', 'response': if not key in auth_map: return if 'qop' in auth_map: if not auth_map.get('nc') or not auth_map.get('cnonce'): return auth_map['type']='digest' return auth_map else: # unknown auth type return
def parse_oauth_header(header): type, rest = header.split(" ", 1) if type != "OAuth": raise ValueError("Authorization is not of OAuth type") return urllib2.parse_keqv_list(urllib2.parse_http_list(rest))
def parse_401_response_headers(response_headers): """ Parse the headers from a 401 response into a dictionary that contains the information necessary to retrieve a token. Example: Www-Authenticate: Bearer realm="https://auth.docker.io/token", service="registry.docker.io",scope="repository:library/nginx:pull,push" """ auth_header = response_headers.get("www-authenticate") if auth_header is None: raise IOError( "401 responses are expected to contain authentication information") auth_header = auth_header[len("Bearer "):] # The remaining string consists of comma separated key=value pairs # according to RFC 2617 try: items = request.parse_http_list(auth_header) return request.parse_keqv_list(items) except ValueError as e: six.raise_from( IOError( "401 responses are expected to contain authentication information" ), e, )
def youtube_video(url): try: conn = urllib2.urlopen(url) encoding = conn.headers.getparam("charset") content = conn.read().decode(encoding) s = re.findall(r'"url_encoded_fmt_stream_map": ?"([^"]+)"', content) if s: import HTMLParser s = s[0].split(",") s = [a.replace("\\u0026", "&") for a in s] s = [urllib2.parse_keqv_list(a.split("&")) for a in s] n = re.findall(r"<title>(.+) - YouTube</title>", content) s, n = (s or [], HTMLParser.HTMLParser().unescape(n[0])) for z in s: if z["itag"] == "18": if "mp4" in z["type"]: ext = ".mp4" elif "flv" in z["type"]: ext = ".flv" try: link = urllib.unquote(z["url"] + "&signature=%s" % z["sig"]) except: link = urllib.unquote(z["url"]) return link except: return False
def compose(self, digest=None, basic=None, username=None, password=None, challenge=None, path=None, method=None): assert username and password if basic or not challenge: assert not digest userpass = "******" % (username.strip(), password.strip()) return "Basic %s" % userpass.encode('base64').strip() assert challenge and not basic path = path or "/" (_, realm) = challenge.split('realm="') (realm, _) = realm.split('"', 1) auth = urllib2.AbstractDigestAuthHandler() auth.add_password(realm, path, username, password) (token, challenge) = challenge.split(' ', 1) chal = urllib2.parse_keqv_list(urllib2.parse_http_list(challenge)) class FakeRequest(object): def get_full_url(self): return path def has_data(self): return False def get_method(self): return method or "GET" get_selector = get_full_url retval = "Digest %s" % auth.get_authorization(FakeRequest(), chal) return (retval,)
def youtube_video(url): try: conn = urllib2.urlopen(url) encoding = conn.headers.getparam('charset') content = conn.read().decode(encoding) s = re.findall(r'"url_encoded_fmt_stream_map": ?"([^"]+)"', content) if s: import HTMLParser s = s[0].split(',') s = [a.replace('\\u0026', '&') for a in s] s = [urllib2.parse_keqv_list(a.split('&')) for a in s] n = re.findall(r'<title>(.+) - YouTube</title>', content) s, n = (s or [], HTMLParser.HTMLParser().unescape(n[0])) for z in s: if z['itag'] == '18': if 'mp4' in z['type']: ext = '.mp4' elif 'flv' in z['type']: ext = '.flv' try: link = urllib.unquote(z['url'] + '&signature=%s' % z['sig']) except: link = urllib.unquote(z['url']) return link except: return False
def registry_get(self, api): url = "https://%s/v2/%s" % (self.registry, api) response = get( url, auth=(self.user, self.password), headers={ "Accept": 'application/vnd.docker.distribution.manifest.v2+json' }, verify=self.verify) if response.status_code == 401: challenge = response.headers['Www-Authenticate'] if challenge.startswith("Bearer "): challenge = challenge[7:] opts = urllib2.parse_keqv_list(urllib2.parse_http_list(challenge)) authresp = get( "{realm}?service={service}&scope={scope}".format(**opts), auth=(self.user, self.password), verify=self.verify) if authresp.ok: token = authresp.json()['token'] response = get(url, headers={'Authorization': 'Bearer %s' % token}, verify=self.verify) else: raise TaskError( "problem authenticating with docker registry: [%s] %s" % (authresp.status_code, authresp.content)) return response
def _parseHeader(self, authheader, request): n = 7 # n = len("Digest ") try: authheader = authheader[n:].strip() items = urllib2.parse_http_list(authheader) request.env['__DIGEST_PARAMS__'] = urllib2.parse_keqv_list(items) except Exception, e: request.env['__DIGEST_PARAMS__'] = {}
def parse_authorization_header(authorization_header): """Parse an OAuth authorization header into a list of 2-tuples""" auth_scheme = 'OAuth ' if authorization_header.startswith(auth_scheme): authorization_header = authorization_header.replace(auth_scheme, '', 1) items = urllib2.parse_http_list(authorization_header) try: return urllib2.parse_keqv_list(items).items() except ValueError: raise ValueError('Malformed authorization header')
def _parse(self, authorization): scheme, rest = authorization.split(None, 1) args = urllib2.parse_keqv_list(urllib2.parse_http_list(rest)) challengeType = { 'basic': BasicChallenge, 'digest': DigestChallenge, }.get(scheme.lower()) if challengeType is None: return "", None return scheme.lower(), challengeType(**args)
def __init__(self, auth_header, http_method, debug=False): self.http_method = http_method self.debug = debug scheme, params = auth_header.split(" ", 1) self.scheme = scheme.lower() if self.scheme != 'digest': raise ValueError('Authorization scheme is not "Digest"') self.auth_header = auth_header # make a dict of the params items = parse_http_list(params) paramsd = parse_keqv_list(items) self.realm = paramsd.get('realm') self.username = paramsd.get('username') self.nonce = paramsd.get('nonce') self.uri = paramsd.get('uri') self.method = paramsd.get('method') self.response = paramsd.get('response') # the response digest self.algorithm = paramsd.get('algorithm', 'MD5') self.cnonce = paramsd.get('cnonce') self.opaque = paramsd.get('opaque') self.qop = paramsd.get('qop') # qop self.nc = paramsd.get('nc') # nonce count # perform some correctness checks if self.algorithm not in valid_algorithms: raise ValueError( self.errmsg("Unsupported value for algorithm: '%s'" % self.algorithm)) has_reqd = self.username and \ self.realm and \ self.nonce and \ self.uri and \ self.response if not has_reqd: raise ValueError( self.errmsg("Not all required parameters are present.")) if self.qop: if self.qop not in valid_qops: raise ValueError( self.errmsg("Unsupported value for qop: '%s'" % self.qop)) if not (self.cnonce and self.nc): raise ValueError( self.errmsg( "If qop is sent then cnonce and nc MUST be present")) else: if self.cnonce or self.nc: raise ValueError( self.errmsg( "If qop is not sent, neither cnonce nor nc can be present" ))
def _digest_login_callback(self, response): self._update_session_cookie(response) if response.code == 200: self._authenticated = True self._alive = True print response.headers print 'yay' return elif response.code == 401: print 'meh' else: print 'boo' return #If 401 if not 'WWW-Authenticate' in response.headers: raise InterfaceError() if not 'Digest' in response.headers['WWW-Authenticate']: raise InterfaceError() param_string = response.headers.get('WWW-Authenticate').partition( 'Digest')[2] param_list = parse_http_list(param_string) params = parse_keqv_list(param_list) path = urlsplit( response.effective_url) #asterisk throws query into hash digest_path = "%s?%s" % (path.path, path.query) digest_cnonce = uuid.uuid1().hex digest_nc = '%08d' % self._digest_nc digest_qop = "auth" digest_response = md5(":".join([ md5(":".join( (self._username, params['realm'], self._secret))).hexdigest(), params['nonce'], digest_nc, digest_cnonce, digest_qop, md5(":".join(('GET', digest_path))).hexdigest() ])).hexdigest() headers = {} headers[ 'Authorization'] = 'Digest username="******", realm="%s", nonce="%s", uri="%s", cnonce="%s", nc=%s, qop="%s", response="%s", opaque="%s", algorithm="%s"' % ( self._username, params['realm'], params['nonce'], digest_path, digest_cnonce, digest_nc, digest_qop, digest_response, params['opaque'], params['algorithm']) self._digest_nc += 1 self.command(self._authentication_request, headers=headers)
def retry_http_digest_auth(self, req, auth): token, challenge = auth.split(' ', 1) chal = urllib2.parse_keqv_list(urllib2.parse_http_list(challenge)) auth = self.get_authorization(req, chal) if auth: auth_val = 'X-Digest %s' % auth if req.headers.get(self.auth_header, None) == auth_val: return None req.add_unredirected_header(self.auth_header, auth_val) resp = self.parent.open(req, timeout=req.timeout) return resp
def retry_http_digest_auth(self, req, resp, host, auth): _token, challenge = auth.split(' ', 1) chal = urllib2.parse_keqv_list(urllib2.parse_http_list(challenge)) auth = self.get_authorization(req, chal) if auth: auth_val = 'Digest %s' % auth if req.headers.get(self.auth_header, None) == auth_val: return None req.add_unredirected_header(self.auth_header, auth_val) return 'request', req return None
def retry_http_digest_auth(self, req, auth): token, challenge = auth.split(' ', 1) chal = parse_keqv_list(parse_http_list(challenge)) auth = self.get_authorization(req, chal) if auth: auth_val = 'Digest %s' % auth if req.headers.get(self.auth_header, None) == auth_val: return None newreq = copy.copy(req) newreq.add_unredirected_header(self.auth_header, auth_val) newreq.visit = False return self.parent.open(newreq)
def yt_get_all_url_maps_name(url): conn = urllib2.urlopen(url) encoding = conn.headers.getparam('charset') content = conn.read().decode(encoding) s = re.findall(r'"url_encoded_fmt_stream_map": "([^"]+)"', content) if s: s = s[0].split(',') s = [a.replace('\\u0026', '&') for a in s] s = [urllib2.parse_keqv_list(a.split('&')) for a in s] n = re.findall(r'<title>(.+) - YouTube</title>', content) return (s or [], HTMLParser.HTMLParser().unescape(n[0]))
def retry_http_digest_auth(self, req, resp, host, auth): _token, challenge = auth.split(" ", 1) chal = urllib2.parse_keqv_list(urllib2.parse_http_list(challenge)) auth = self.get_authorization(req, chal) if auth: auth_val = "Digest %s" % auth if req.headers.get(self.auth_header, None) == auth_val: return None req.add_unredirected_header(self.auth_header, auth_val) return "request", req return None
def _authenticate_quay(self, headers): """ Attempt to perform an authentication with registry's authentication server. Once authentication is complete, add the token to the Session object. Specifics can be found at https://docs.docker.com/registry/spec/auth/token/ Args: headers (dict): Headers of the 401 response received from the registry. Raises: RegistryAuthError: When there's an issue with the authentication procedure. """ if "WWW-Authenticate" not in headers: raise RegistryAuthError( "'WWW-Authenticate' is not in the 401 response's header. " "Authentication cannot continue.") if "Bearer " not in headers["WWW-Authenticate"]: raise RegistryAuthError( "Different than the Bearer authentication type was requested. " "Only Bearer is supported.") # parse header to get a dictionary params = request.parse_keqv_list( request.parse_http_list( headers["WWW-Authenticate"][len("Bearer "):]) # noqa: E203 ) host = params.pop("realm") session = requests.Session() retry = Retry( total=3, read=3, connect=3, backoff_factor=2, status_forcelist=set(range(500, 512)), ) adapter = requests.adapters.HTTPAdapter(max_retries=retry) session.mount("http://", adapter) session.mount("https://", adapter) # Make an authentication request to the specified realm with the provided REST parameters. # Basic username + password authentication is expected. r = session.get(host, params=params, auth=(self.username, self.password), timeout=10) r.raise_for_status() if "token" not in r.json(): raise RegistryAuthError( "Authentication server response doesn't contain a token.") self.session.set_auth_token(r.json()["token"])
def authorized(self): tcs = self.server.test_case_server auth_header = self.headers.get(tcs.auth_header_recv, None) if auth_header is None: return False scheme, auth = auth_header.split(None, 1) if scheme.lower() == tcs.auth_scheme: auth_dict = urllib2.parse_keqv_list(urllib2.parse_http_list(auth)) return tcs.digest_authorized(auth_dict, self.command) return False
def createAuthObject(authHeader): """ Returns the authentication mechanism, or None if not implemented. """ authType, challenge = authHeader.split(' ', 1) _authType_lower = authType.lower() challenge = urllib2.parse_keqv_list(urllib2.parse_http_list(challenge)) assert _authType_lower in ('digest', 'ssl'), \ repr(_authType_lower) # "basic" authentication is not supported return DigestAuthentication(challenge) if _authType_lower == 'digest' \ else None
def _parse(self, authorization): try: scheme, rest = authorization.split(None, 1) except ValueError: # Probably "negotiate", which we don't support scheme = authorization rest = "" args = urllib2.parse_keqv_list(urllib2.parse_http_list(rest)) challengeType = { 'basic': BasicChallenge, 'digest': DigestChallenge, }.get(scheme.lower()) if challengeType is None: return "", None return scheme.lower(), challengeType(**args)
def __init__(self, auth_header, http_method, debug=False): self.http_method = http_method self.debug = debug scheme, params = auth_header.split(" ", 1) self.scheme = scheme.lower() if self.scheme != 'digest': raise ValueError('Authorization scheme is not "Digest"') self.auth_header = auth_header # make a dict of the params items = parse_http_list(params) paramsd = parse_keqv_list(items) self.realm = paramsd.get('realm') self.username = paramsd.get('username') self.nonce = paramsd.get('nonce') self.uri = paramsd.get('uri') self.method = paramsd.get('method') self.response = paramsd.get('response') # the response digest self.algorithm = paramsd.get('algorithm', 'MD5') self.cnonce = paramsd.get('cnonce') self.opaque = paramsd.get('opaque') self.qop = paramsd.get('qop') # qop self.nc = paramsd.get('nc') # nonce count # perform some correctness checks if self.algorithm not in valid_algorithms: raise ValueError(self.errmsg("Unsupported value for algorithm: '%s'" % self.algorithm)) has_reqd = self.username and \ self.realm and \ self.nonce and \ self.uri and \ self.response if not has_reqd: raise ValueError(self.errmsg("Not all required parameters are present.")) if self.qop: if self.qop not in valid_qops: raise ValueError(self.errmsg("Unsupported value for qop: '%s'" % self.qop)) if not (self.cnonce and self.nc): raise ValueError(self.errmsg("If qop is sent then cnonce and nc MUST be present")) else: if self.cnonce or self.nc: raise ValueError(self.errmsg("If qop is not sent, neither cnonce nor nc can be present"))
def dr(self, url, expected=(), user=None, password=None): user = user or self.user password = password or self.password response = self.get(url, auth=(user, password), expected=expected + (401, )) if response.status_code == 401: challenge = response.headers['Www-Authenticate'] if challenge.startswith("Bearer "): challenge = challenge[7:] opts = urllib2.parse_keqv_list(urllib2.parse_http_list(challenge)) token = self.get( "{realm}?service={service}&scope={scope}".format(**opts), auth=(user, password)).json()['token'] response = self.get(url, headers={'Authorization': 'Bearer %s' % token}, expected=expected) return response
def _parseDigestAuthorization (auth_params): # Convert the auth params to a dict items = urllib2.parse_http_list (auth_params) params = urllib2.parse_keqv_list (items) # Now validate the params # Check for required parameters required = ["username", "realm", "nonce", "uri", "response"] for k in required: if not params.has_key(k): return None # If qop is sent then cnonce and cn MUST be present if params.has_key("qop") and not params.has_key("cnonce") \ and params.has_key("cn"): return None return params
def get_proxy_authorization_line(auth_type, auth_details, method, url, proxy_username, proxy_password): if auth_type.lower() == 'basic': response = base64.encodestring('%s:%s' % (proxy_username, proxy_password)).strip() elif auth_type.lower() == 'digest': class Passwd: def __init__(self, user, passwd): self.user, self.passwd = user, passwd def add_password(self, user, passwd): pass def find_user_password(self, realm, url): return (self.user, self.passwd) def get_full_url(self): return '' class DummyRequest: def __init__(self, method, url): self.method, self.url = method, url def get_method(self): return self.method def get_selector(self): return self.url def get_full_url(self): return self.url def has_data(self): return False digest_auth_handler = urllib2.AbstractDigestAuthHandler(passwd=Passwd(proxy_username or '', proxy_password or '')) chal = urllib2.parse_keqv_list(urllib2.parse_http_list(auth_details)) response = digest_auth_handler.get_authorization(DummyRequest(method, url), chal) else: raise ValueError('Invalid proxy-authenticate line %r' % auth_type) return 'Proxy-Authorization: %s %s' % (auth_type, response)
def http_error_401(self, req, fp, code, msg, headers): host = urllib2.urlparse.urlparse(req.get_full_url())[1] authreq = headers.get('www-authenticate', None) if authreq == None: return None authreq = authreq.split(' ', 1) if authreq[0].lower() != 'basic': return None chal = urllib2.parse_keqv_list(urllib2.parse_http_list(authreq[1])) realm = chal['realm'] self.auth = (host, realm) self.retried += 1 if self.retried >= 3: self.f.delkey(proto="pass", host=host, realm=realm, role="client") self.f.start(proto="pass", host=host, realm=realm, role="client") pw = self.f.read().replace(' ', ':', 1) val = 'Basic %s' % base64.b64encode(pw).strip() if req.headers.get('Authorization', None) == val: return None req.add_header('Authorization', val) result = self.parent.open(req) self.retried = 0 return result
def _parse_auth_info(auth_info): method, info_str = auth_info.split(' ', 1) if method != "Digest": raise ProviderError("Unknown authentication method: %s" % method) items = parse_http_list(info_str) info = parse_keqv_list(items) try: qop = info["qop"] realm = info["realm"] nonce = info["nonce"] except KeyError as e: raise ProviderError("Authentication request missing required key: %s" % e) algorithm = info.get("algorithm", "MD5") if algorithm != "MD5": raise ProviderError("Unsupported digest algorithm: %s" % algorithm) if "auth" not in qop.split(","): raise ProviderError("Unsupported quality-of-protection: %s" % qop) return realm, nonce, "auth", algorithm
def _parse_auth_info(auth_info): method, info_str = auth_info.split(' ', 1) if method != "Digest": raise ProviderError("Unknown authentication method: %s" % method) items = parse_http_list(info_str) info = parse_keqv_list(items) try: qop = info["qop"] realm = info["realm"] nonce = info["nonce"] except KeyError as e: raise ProviderError( "Authentication request missing required key: %s" % e) algorithm = info.get("algorithm", "MD5") if algorithm != "MD5": raise ProviderError("Unsupported digest algorithm: %s" % algorithm) if "auth" not in qop.split(","): raise ProviderError("Unsupported quality-of-protection: %s" % qop) return realm, nonce, "auth", algorithm
def http_error_401(self, req, fp, code, msg, headers): self.retried += 1 host = urllib2.urlparse.urlparse(req.get_full_url())[1] authreq = headers.get('www-authenticate', None) if authreq == None: return None authreq = authreq.split(' ', 1) if authreq[0].lower() != 'digest': return None chal = urllib2.parse_keqv_list(urllib2.parse_http_list(authreq[1])) realm = chal['realm'] nonce = chal['nonce'] if self.retried >= 6: self.f.delkey(proto="httpdigest", realm=realm, host=host) self.f.start(proto="httpdigest", role="client", realm=realm, host=host) self.f.write(nonce + ' ' + req.get_method() + ' ' + req.get_selector()) resp = self.f.read() user = self.f.attr()["user"] self.f.close() val = 'Digest username="******", realm="%s", nonce="%s", uri="%s", response="%s", algorithm=MD5' % (user, realm, nonce, req.get_selector(), resp) if req.headers.get('Authorization', None) == val: return None req.add_unredirected_header('Authorization', val) result = self.parent.open(req) self.retried = 0 return result
def _parseDigestAuthorization (auth_params): # Convert the auth params to a dict items = parse_http_list(auth_params) params = parse_keqv_list(items) # Now validate the params # Check for required parameters required = ["username", "realm", "nonce", "uri", "response"] for k in required: if k not in params: return None # If qop is sent then cnonce and nc MUST be present if "qop" in params and not ("cnonce" in params \ and "nc" in params): return None # If qop is not sent, neither cnonce nor nc can be present if ("cnonce" in params or "nc" in params) and \ "qop" not in params: return None return params
def http_error_401(self, req, fp, code, msg, headers): self.retried += 1 host = urllib2.urlparse.urlparse(req.get_full_url())[1] authreq = headers.get('www-authenticate', None) if authreq == None: return None authreq = authreq.split(' ', 1) if authreq[0].lower() != 'digest': return None chal = urllib2.parse_keqv_list(urllib2.parse_http_list(authreq[1])) realm = chal['realm'] nonce = chal['nonce'] if self.retried >= 6: self.f.delkey(proto="httpdigest", realm=realm, host=host) self.f.start(proto="httpdigest", role="client", realm=realm, host=host) self.f.write(nonce + ' ' + req.get_method() + ' ' + req.get_selector()) resp = self.f.read() user = self.f.attr()["user"] self.f.close() val = 'Digest username="******", realm="%s", nonce="%s", uri="%s", response="%s", algorithm=MD5' % ( user, realm, nonce, req.get_selector(), resp) if req.headers.get('Authorization', None) == val: return None req.add_unredirected_header('Authorization', val) result = self.parent.open(req) self.retried = 0 return result
def _parseDigestAuthorization(auth_params): # Convert the auth params to a dict items = parse_http_list(auth_params) params = parse_keqv_list(items) # Now validate the params # Check for required parameters required = ["username", "realm", "nonce", "uri", "response"] for k in required: if k not in params: return None # If qop is sent then cnonce and nc MUST be present if "qop" in params and not ("cnonce" in params \ and "nc" in params): return None # If qop is not sent, neither cnonce nor nc can be present if ("cnonce" in params or "nc" in params) and \ "qop" not in params: return None return params
def parse_keqv_list(l): """A unicode-safe version of urllib2.parse_keqv_list""" # With Python 2.6, parse_http_list handles unicode fine return urllib2.parse_keqv_list(l)
def build_digest_response(self, fields, username, password): """ Takes a Proxy-Authenticate: Digest header and creates a response header :param fields: The string portion of the Proxy-Authenticate header after "Digest " :param username: The username to use for the response :param password: The password to use for the response :return: None if invalid Proxy-Authenticate header, otherwise the string of fields for the Proxy-Authorization: Digest header """ fields = parse_keqv_list(parse_http_list(fields)) realm = fields.get('realm') nonce = fields.get('nonce') qop = fields.get('qop') algorithm = fields.get('algorithm') if algorithm: algorithm = algorithm.lower() opaque = fields.get('opaque') if algorithm in ('md5', None): def md5hash(string): return hashlib.md5(string).hexdigest() _hash = md5hash elif algorithm == 'sha': def sha1hash(string): return hashlib.sha1(string).hexdigest() _hash = sha1hash else: return None host_port = '%s:%s' % self.url_info a1 = '%s:%s:%s' % (username, realm, password) a2 = 'CONNECT:%s' % host_port ha1 = _hash(a1) ha2 = _hash(a2) if qop is None: response = _hash('%s:%s:%s' % (ha1, nonce, ha2)) elif qop == 'auth': nc = '00000001' cnonce = _hash(os.urandom(8))[:8] response = _hash('%s:%s:%s:%s:%s:%s' % (ha1, nonce, nc, cnonce, qop, ha2)) else: return None resp_fields = OrderedDict() resp_fields['username'] = username resp_fields['realm'] = realm resp_fields['nonce'] = nonce resp_fields['response'] = response resp_fields['uri'] = host_port if algorithm: resp_fields['algorithm'] = algorithm if qop == 'auth': resp_fields['nc'] = nc resp_fields['cnonce'] = cnonce resp_fields['qop'] = qop if opaque: resp_fields['opaque'] = opaque return ', '.join( ['%s="%s"' % (field, resp_fields[field]) for field in resp_fields])
def parse_digest_challenge(self, challenge_string): return urllib2.parse_keqv_list( urllib2.parse_http_list(challenge_string))
except: mode = urllib.unquote_plus(params[mode]) try: if mode is None: mode = str(params[mode]) if len(mode) < 1: mode = str(params[mode]) except: pass try: page = str(params['page']) except: page = urllib.unquote_plus(params["page"]) except: try: params = urllib2.parse_keqv_list(sys.argv[2]) except: displayRootMenu() if mode == None or url == None or len(url) < 1: displayRootMenu() elif mode == 'loadVideoList': displayCatMenu() elif mode == 'scrapeVideoListCat': indexCatVideos(url, name) elif mode == 'scrapeVideoList': indexVideos(urlpath=basejoin(base_url, str(url +'/'+ page)), page="page{0}".format(page)) elif mode == 'playVideo': playVideo(url, name, thumb) #setView(True)
def build_digest_response(self, fields, username, password): """ Takes a Proxy-Authenticate: Digest header and creates a response header :param fields: The string portion of the Proxy-Authenticate header after "Digest " :param username: The username to use for the response :param password: The password to use for the response :return: None if invalid Proxy-Authenticate header, otherwise the string of fields for the Proxy-Authorization: Digest header """ fields = parse_keqv_list(parse_http_list(fields)) realm = fields.get('realm') nonce = fields.get('nonce') qop = fields.get('qop') algorithm = fields.get('algorithm') if algorithm: algorithm = algorithm.lower() opaque = fields.get('opaque') if algorithm in ('md5', None): def md5hash(string): return hashlib.md5(string).hexdigest() _hash = md5hash elif algorithm == 'sha': def sha1hash(string): return hashlib.sha1(string).hexdigest() _hash = sha1hash else: return None host_port = '%s:%s' % self.url_info a1 = '%s:%s:%s' % (username, realm, password) a2 = 'CONNECT:%s' % host_port ha1 = _hash(a1) ha2 = _hash(a2) if qop is None: response = _hash('%s:%s:%s' % (ha1, nonce, ha2)) elif qop == 'auth': nc = '00000001' cnonce = _hash(os.urandom(8))[:8] response = _hash('%s:%s:%s:%s:%s:%s' % (ha1, nonce, nc, cnonce, qop, ha2)) else: return None resp_fields = OrderedDict() resp_fields['username'] = username resp_fields['realm'] = realm resp_fields['nonce'] = nonce resp_fields['response'] = response resp_fields['uri'] = host_port if algorithm: resp_fields['algorithm'] = algorithm if qop == 'auth': resp_fields['nc'] = nc resp_fields['cnonce'] = cnonce resp_fields['qop'] = qop if opaque: resp_fields['opaque'] = opaque return ', '.join(['%s="%s"' % (field, resp_fields[field]) for field in resp_fields])
def http_open(self, req): (code, msg, headers, data, timestamp) = self._load(req.get_full_url()) # some info needed to process everything cache_control = parse_http_list(headers.get('cache-control', ())) cache_control += parse_http_list(headers.get('pragma', ())) cc_list = [x for x in cache_control if '=' not in x] cc_values = parse_keqv_list([x for x in cache_control if '=' in x]) cache_age = time.time() - timestamp # list in a simple way what to do when if req.get_header( 'Morss' ) == 'from_304': # for whatever reason, we need an uppercase # we're just in the middle of a dirty trick, use cache pass elif self.force_min == -2: if code is not None: # already in cache, perfect, use cache pass else: headers['Morss'] = 'from_cache' resp = addinfourl(BytesIO(), headers, req.get_full_url(), 409) resp.msg = 'Conflict' return resp elif code is None: # cache empty, refresh return None elif self.force_min == -1: # force use cache pass elif self.force_min == 0: # force refresh return None elif self.force_min is None and ( 'no-cache' in cc_list or 'no-store' in cc_list or ('private' in cc_list and not self.private)): # kindly follow web servers indications, refresh return None elif 'max-age' in cc_values and int(cc_values['max-age']) > cache_age: # server says it's still fine (and we trust him, if not, use force_min=0), use cache pass elif self.force_min is not None and self.force_min > cache_age: # still recent enough for us, use cache pass else: # according to the www, we have to refresh when nothing is said return None # return the cache as a response headers[ 'morss'] = 'from_cache' # TODO delete the morss header from incoming pages, to avoid websites messing up with us resp = addinfourl(BytesIO(data), headers, req.get_full_url(), code) resp.msg = msg return resp
def parse_keqv_list(l): """A unicode-safe version of urllib2.parse_keqv_list""" encoded_list = [u.encode('utf-8') for u in l] encoded_parsed = urllib2.parse_keqv_list(encoded_list) return dict((k.decode('utf-8'), v.decode('utf-8')) for k, v in encoded_parsed.items())
def parse_digest_challenge(self, challenge_string): return urllib2.parse_keqv_list(urllib2.parse_http_list(challenge_string))
def http_digest_auth(a, uri, user, pw, method): token, challenge = a.split(' ', 1) chal = urllib2.parse_keqv_list(urllib2.parse_http_list(challenge)) a = get_authorization(chal, uri, user, pw, method) if a: return 'Digest %s' % a
def putfile(f, uri, username=None, password=None): """HTTP PUT the file f to uri, with optional auth data.""" host, port, path = parseuri(uri) redirect = set([301, 302, 307]) authenticate = set([401]) okay = set([200, 201, 204]) authorized = False authorization = None tries = 0 while True: # Attempt to HTTP PUT the data h = httplib.HTTPConnection(host, port) h.putrequest('PUT', path) h.putheader('User-Agent', 'put.py/1.0') h.putheader('Connection', 'keep-alive') h.putheader('Transfer-Encoding', 'chunked') h.putheader('Expect', '100-continue') h.putheader('Accept', '*/*') if authorization: h.putheader('Authorization', authorization) h.endheaders() # Chunked transfer encoding # Cf. 'All HTTP/1.1 applications MUST be able to receive and # decode the "chunked" transfer-coding' # - http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html while True: bytes = f.read(1024) if not bytes: break length = len(bytes) # h.send('%X\r\n' % length) h.send(bytes) # h.send('0\r\n\r\n') resp = h.getresponse() status = resp.status # an int # Got a response, now decide how to act upon it if status in redirect: location = resp.getheader('Location') uri = urlparse.urljoin(uri, location) host, port, path = parseuri(uri) # We may have to authenticate again if authorization: authorization = None elif status in authenticate: # If we've done this already, break if authorization: # barf("Going around in authentication circles") barf("Authentication failed") if not (username and password): barf("Need a username and password to authenticate with") # Get the scheme: Basic or Digest? wwwauth = resp.msg['www-authenticate'] # We may need this again wauth = wwwauth.lstrip(' \t') # Hence use wauth not wwwauth here wauth = wwwauth.replace('\t', ' ') i = wauth.index(' ') scheme = wauth[:i].lower() if scheme in set(['basic', 'digest']): if verbose: msg = "Performing %s Authentication..." % scheme.capitalize( ) print >> sys.stderr, msg else: barf("Unknown authentication scheme: %s" % scheme) if scheme == 'basic': import base64 userpass = username + ':' + password userpass = base64.encodestring(userpass).strip() authorized, authorization = True, 'Basic ' + userpass elif scheme == 'digest': if verbose: msg = "uses fragile, undocumented features in urllib2" print >> sys.stderr, "Warning! Digest Auth %s" % msg import urllib2 # See warning above passwd = type( 'Password', (object, ), { 'find_user_password': lambda self, *args: (username, password), 'add_password': lambda self, *args: None })() req = type( 'Request', (object, ), { 'get_full_url': lambda self: uri, 'has_data': lambda self: None, 'get_method': lambda self: 'PUT', 'get_selector': lambda self: path })() # Cf. urllib2.AbstractDigestAuthHandler.retry_http_digest_auth auth = urllib2.AbstractDigestAuthHandler(passwd) token, challenge = wwwauth.split(' ', 1) chal = urllib2.parse_keqv_list( urllib2.parse_http_list(challenge)) userpass = auth.get_authorization(req, chal) authorized, authorization = True, 'Digest ' + userpass elif status in okay: if (username and password) and (not authorized): msg = "Warning! The supplied username and password went unused" print >> sys.stderr, msg if verbose: resultLine = "Success! Resource %s" statuses = {200: 'modified', 201: 'created', 204: 'modified'} print resultLine % statuses[status] statusLine = "Response-Status: %s %s" print statusLine % (status, resp.reason) body = resp.read(58) body = body.rstrip('\r\n') body = body.encode('string_escape') if len(body) >= 58: body = body[:57] + '[...]' bodyLine = 'Response-Body: "%s"' print bodyLine % body break # @@ raise PutError, do the catching in main? else: barf('Got "%s %s"' % (status, resp.reason)) tries += 1 if tries >= 50: barf("Too many redirects") return status, resp
def putfile(f, uri, username=None, password=None): """HTTP PUT the file f to uri, with optional auth data.""" host, port, path = parseuri(uri) redirect = set([301, 302, 307]) authenticate = set([401]) okay = set([200, 201, 204]) authorized = False authorization = None tries = 0 while True: # Attempt to HTTP PUT the data h = httplib.HTTPConnection(host, port) h.putrequest('PUT', path) h.putheader('User-Agent', 'put.py/1.0') h.putheader('Connection', 'keep-alive') h.putheader('Transfer-Encoding', 'chunked') h.putheader('Expect', '100-continue') h.putheader('Accept', '*/*') if authorization: h.putheader('Authorization', authorization) h.endheaders() # Chunked transfer encoding # Cf. 'All HTTP/1.1 applications MUST be able to receive and # decode the "chunked" transfer-coding' # - http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html while True: bytes = f.read(1024) if not bytes: break length = len(bytes) #h.send('%X\r\n' % length) h.send(bytes) #h.send('0\r\n\r\n') resp = h.getresponse() status = resp.status # an int # Got a response, now decide how to act upon it if status in redirect: location = resp.getheader('Location') uri = urlparse.urljoin(uri, location) host, port, path = parseuri(uri) # We may have to authenticate again if authorization: authorization = None elif status in authenticate: # If we've done this already, break if authorization: # barf("Going around in authentication circles") barf("Authentication failed") if not (username and password): barf("Need a username and password to authenticate with") # Get the scheme: Basic or Digest? wwwauth = resp.msg['www-authenticate'] # We may need this again wauth = wwwauth.lstrip(' \t') # Hence use wauth not wwwauth here wauth = wwwauth.replace('\t', ' ') i = wauth.index(' ') scheme = wauth[:i].lower() if scheme in set(['basic', 'digest']): if verbose: msg = "Performing %s Authentication..." % scheme.capitalize() print >> sys.stderr, msg else: barf("Unknown authentication scheme: %s" % scheme) if scheme == 'basic': import base64 userpass = username + ':' + password userpass = base64.encodestring(userpass).strip() authorized, authorization = True, 'Basic ' + userpass elif scheme == 'digest': if verbose: msg = "uses fragile, undocumented features in urllib2" print >> sys.stderr, "Warning! Digest Auth %s" % msg import urllib2 # See warning above passwd = type('Password', (object,), { 'find_user_password': lambda self, *args: (username, password), 'add_password': lambda self, *args: None })() req = type('Request', (object,), { 'get_full_url': lambda self: uri, 'has_data': lambda self: None, 'get_method': lambda self: 'PUT', 'get_selector': lambda self: path })() # Cf. urllib2.AbstractDigestAuthHandler.retry_http_digest_auth auth = urllib2.AbstractDigestAuthHandler(passwd) token, challenge = wwwauth.split(' ', 1) chal = urllib2.parse_keqv_list(urllib2.parse_http_list(challenge)) userpass = auth.get_authorization(req, chal) authorized, authorization = True, 'Digest ' + userpass elif status in okay: if (username and password) and (not authorized): msg = "Warning! The supplied username and password went unused" print >> sys.stderr, msg if verbose: resultLine = "Success! Resource %s" statuses = {200: 'modified', 201: 'created', 204: 'modified'} print resultLine % statuses[status] statusLine = "Response-Status: %s %s" print statusLine % (status, resp.reason) body = resp.read(58) body = body.rstrip('\r\n') body = body.encode('string_escape') if len(body) >= 58: body = body[:57] + '[...]' bodyLine = 'Response-Body: "%s"' print bodyLine % body break # @@ raise PutError, do the catching in main? else: barf('Got "%s %s"' % (status, resp.reason)) tries += 1 if tries >= 50: barf("Too many redirects") return status, resp
def http_open(self, req): # Reminder of how/when this function is called by urllib2: # If 'None' is returned, try your chance with the next-available handler # If a 'resp' is returned, stop there, and proceed with 'http_response' (code, msg, headers, data, timestamp) = self.load(req.get_full_url()) # some info needed to process everything cache_control = parse_http_list(headers.get('cache-control', ())) cache_control += parse_http_list(headers.get('pragma', ())) cc_list = [x for x in cache_control if '=' not in x] cc_values = parse_keqv_list([x for x in cache_control if '=' in x]) cache_age = time.time() - timestamp # list in a simple way what to do when if self.force_min == -2: if code is not None: # already in cache, perfect, use cache return self.cached_response(req) else: # raise an error, via urllib handlers resp = addinfourl(BytesIO(), headers, req.get_full_url(), 409) resp.msg = 'Conflict' return resp elif code is None: # cache empty, refresh return None elif self.force_min == -1: # force use cache return self.cached_response(req) elif self.force_min == 0: # force refresh return None elif code == 301 and cache_age < 7*24*3600: # "301 Moved Permanently" has to be cached...as long as we want # (awesome HTTP specs), let's say a week (why not?). Use force_min=0 # if you want to bypass this (needed for a proper refresh) return self.cached_response(req) elif (self.force_min is None or self.force_min > 0) and ('no-cache' in cc_list or 'no-store' in cc_list or ('private' in cc_list and not self.private_cache)): # kindly follow web servers indications, refresh # if the same settings are used all along, this section shouldn't be # of any use, since the page woudln't be cached in the first place # the check is only performed "just in case" return None elif 'max-age' in cc_values and int(cc_values['max-age']) > cache_age: # server says it's still fine (and we trust him, if not, use force_min=0), use cache return self.cached_response(req) elif self.force_min is not None and self.force_min > cache_age: # still recent enough for us, use cache return self.cached_response(req) else: # according to the www, we have to refresh when nothing is said return None