def _open_url(self, url): """Open a urllib2 request, handling HTTP authentication, and local files support. """ scheme, netloc, path, params, query, frag = urlparse.urlparse(url) # authentication stuff if scheme in ('http', 'https'): auth, host = urllib2.splituser(netloc) else: auth = None # add index.html automatically for filesystem paths if scheme == 'file': if url.endswith(os.path.sep): url += "index.html" # add authorization headers if auth is provided if auth: auth = "Basic " + \ urlparse.unquote(auth).encode('base64').strip() new_url = urlparse.urlunparse(( scheme, host, path, params, query, frag)) request = urllib2.Request(new_url) request.add_header("Authorization", auth) else: request = urllib2.Request(url) request.add_header('User-Agent', USER_AGENT) try: fp = urllib2.urlopen(request) except (ValueError, httplib.InvalidURL), v: msg = ' '.join([str(arg) for arg in v.args]) raise PackagingPyPIError('%s %s' % (url, msg))
def download(self, source, dest): """ Download an archive file. :param str source: URL pointing to an archive file. :param str dest: Local path location to download archive file to. """ # propogate all exceptions # URLError, OSError, etc proto, netloc, path, params, query, fragment = urlparse.urlparse(source) if proto in ('http', 'https'): auth, barehost = urllib2.splituser(netloc) if auth is not None: source = urlparse.urlunparse((proto, barehost, path, params, query, fragment)) username, password = urllib2.splitpasswd(auth) passman = urllib2.HTTPPasswordMgrWithDefaultRealm() # Realm is set to None in add_password to force the username and password # to be used whatever the realm passman.add_password(None, source, username, password) authhandler = urllib2.HTTPBasicAuthHandler(passman) opener = urllib2.build_opener(authhandler) urllib2.install_opener(opener) response = urllib2.urlopen(source) try: with open(dest, 'w') as dest_file: dest_file.write(response.read()) except Exception as e: if os.path.isfile(dest): os.unlink(dest) raise e
def open_with_auth(url): """Open a urllib2 request, handling HTTP authentication""" scheme, netloc, path, params, query, frag = urlparse.urlparse(url) if scheme in ('http', 'https'): auth, host = urllib2.splituser(netloc) else: auth = None if auth: auth = "Basic " + urllib2.unquote(auth).encode('base64').strip() new_url = urlparse.urlunparse((scheme,host,path,params,query,frag)) request = urllib2.Request(new_url) request.add_header("Authorization", auth) else: request = urllib2.Request(url) request.add_header('User-Agent', user_agent) fp = urllib2.urlopen(request) if auth: # Put authentication info back into request URL if same host, # so that links found on the page will work s2, h2, path2, param2, query2, frag2 = urlparse.urlparse(fp.url) if s2==scheme and h2==host: fp.url = urlparse.urlunparse((s2,netloc,path2,param2,query2,frag2)) return fp
def open_with_auth(url): """ Open a urllib2 request, handling HTTP authentication """ import config scheme, netloc, path, params, query, frag = urlparse.urlparse(url) assert not query auth, host = urllib2.splituser(netloc) if auth: auth = urllib2.unquote(auth).encode('base64').strip() elif 'enthought.com/repo/' in url and 'repo/pypi/eggs/' not in url: auth = config.get('EPD_auth') if auth is None: userpass = config.get('EPD_userpass') if userpass: auth = userpass.encode('base64').strip() if auth: new_url = urlparse.urlunparse((scheme, host, path, params, query, frag)) request = urllib2.Request(new_url) request.add_header("Authorization", "Basic " + auth) logger.debug('Requesting %s with auth' % new_url) else: request = urllib2.Request(url) logger.debug('Requesting %s without auth' % url) request.add_header('User-Agent', 'enstaller/%s' % __version__) return urllib2.urlopen(request)
def get_transport_and_path_from_url(url, config=None, **kwargs): """Obtain a git client from a URL. :param url: URL to open (a unicode string) :param config: Optional config object :param thin_packs: Whether or not thin packs should be retrieved :param report_activity: Optional callback for reporting transport activity. :return: Tuple with client instance and relative path. """ parsed = urlparse.urlparse(url) auth, host = urllib2.splituser(parsed.netloc) if parsed.scheme == 'git': return (TCPGitClient(parsed.hostname, port=parsed.port, **kwargs), parsed.path) elif parsed.scheme in ('git+ssh', 'ssh'): path = parsed.path if path.startswith('/'): path = parsed.path[1:] return SSHGitClient(parsed.hostname, port=parsed.port, username=parsed.username, **kwargs), path elif parsed.scheme in ('http', 'https'): base_url = urlparse.urlunparse(parsed._replace(netloc=host)) return HttpGitClient(base_url, config=config, user=parsed.username, passwd=parsed.password, **kwargs), parsed.path elif parsed.scheme == 'file': return default_local_git_client_cls(**kwargs), parsed.path raise ValueError("unknown scheme '%s'" % parsed.scheme)
def open_with_auth(url): """Open a urllib2 request, handling HTTP authentication""" scheme, netloc, path, params, query, frag = urlparse.urlparse(url) # Double scheme does not raise on Mac OS X as revealed by a # failing test. We would expect "nonnumeric port". Refs #20. if netloc.endswith(':'): raise httplib.InvalidURL("nonnumeric port: ''") if scheme in ('http', 'https'): auth, host = urllib2.splituser(netloc) else: auth = None if auth: auth = "Basic " + _encode_auth(auth) new_url = urlparse.urlunparse((scheme,host,path,params,query,frag)) request = urllib2.Request(new_url) request.add_header("Authorization", auth) else: request = urllib2.Request(url) request.add_header('User-Agent', user_agent) fp = urllib2.urlopen(request) if auth: # Put authentication info back into request URL if same host, # so that links found on the page will work s2, h2, path2, param2, query2, frag2 = urlparse.urlparse(fp.url) if s2==scheme and h2==host: fp.url = urlparse.urlunparse((s2,netloc,path2,param2,query2,frag2)) return fp
def from_parsedurl(cls, parsedurl, **kwargs): auth, host = urllib2.splituser(parsedurl.netloc) password = parsedurl.password username = parsedurl.username # TODO(jelmer): This also strips the username parsedurl = parsedurl._replace(netloc=host) return cls(urlparse.urlunparse(parsedurl), password=password, username=username, **kwargs)
def get_proxy_info(proxystr=""): """ Get proxy config from string or environment variables. If a proxy string is passed in, it overrides whatever might be in the environment variables. Returns dictionary of identified proxy information, or None if proxystr was empty and no config was found fmor os.environ Raises InvalidConfiguration on any configuration error. """ if proxystr == "": # FIXME: We should be supporting http_proxy, HTTP_PROXY variables. proxy_info = { 'host' : os.environ.get('PROXY_HOST', None), 'port' : _convert_port_value(os.environ.get('PROXY_PORT', None)), 'user' : os.environ.get('PROXY_USER', None), 'pass' : os.environ.get('PROXY_PASS', None) } if proxy_info.get("host") is None: return None else: parts = urlparse.urlparse(proxystr) if parts.netloc == "": proxystr = "http://{0}".format(proxystr) parts = urlparse.urlparse(proxystr) _, hostport = urllib2.splituser(parts.netloc) host, _ = urllib2.splitport(hostport) host = urlparse.urlunparse((parts.scheme, host, "", "", "", "")) proxy_info = { 'host' : host, 'port' : _convert_port_value(parts.port), 'user' : parts.username, 'pass' : parts.password, } # If a user was specified, but no password was, prompt for it now. user = proxy_info.get('user', None) if user is not None and len(user) > 0: passwd = proxy_info.get('pass', None) if passwd is None or len(passwd) < 1: import getpass proxy_info['pass'] = getpass.getpass() if proxy_info["host"] is None or len(proxy_info["host"]) < 1: error_msg = ("Invalid proxy configuration '{0}': " "proxy string should be of the form " "'http://host:port' or 'http://host'".format(proxystr)) raise InvalidConfiguration(error_msg) return proxy_info
def open_url(url): """ Open a urllib2 request, handling HTTP authentication """ scheme, netloc, path, params, query, frag = urlparse.urlparse(url) assert not query auth, host = urllib2.splituser(netloc) request = urllib2.Request(url) request.add_header("User-Agent", "IronPkg/%s" % __version__) return urllib2.urlopen(request)
def _getBaseDownloadUrl(self): """ Return the base URL relative to which to download images, removing any user/password information, and using http """ # extract the downloadImage base url from the serverUrl configuration parts = list(urllib2.urlparse.urlparse(self.rbuilderUrl)) parts[1] = urllib2.splituser(parts[1])[1] # FIXME: is this whole ../ business a workaround for splitbrain rBO? parts[2] = parts[2] and parts[2] or "/" return "http://%s%s" % (parts[1], util.normpath(parts[2] + "../")[1:])
def set_site(cls, value): cls._threadlocal.connection = None ShopifyResource._site = cls._threadlocal.site = value if value is not None: host = urlparse.urlsplit(value)[1] auth_info, host = urllib2.splituser(host) if auth_info: user, password = urllib2.splitpasswd(auth_info) if user: cls.user = urllib.unquote(user) if password: cls.password = urllib.unquote(password)
def set_site(cls, value): if value is not None: host = urlparse.urlsplit(value)[1] auth_info, host = urllib2.splituser(host) if auth_info: user, password = urllib2.splitpasswd(auth_info) if user: cls._user = urllib.unquote(user) if password: cls._password = urllib.unquote(password) cls._connection = None cls._site = value
def __init__(self, url): self.url = url self.schema, url = urllib2.splittype(url) host, path = urllib2.splithost(url) userpass, host = urllib2.splituser(host) if userpass: self.user, self.password = urllib2.splitpasswd(userpass) path, self.querystring = urllib.splitquery(path) self.query = self.querystring and self.querystring.split('&') or [] #urllib.splitquery(url) self.host, self.port = urllib2.splitport(host) path, self.tag = urllib2.splittag(path) self.path = path.strip('/')
def _inject_credentials(url, username=None, password=None): """Used by `inject_credentials` decorators to actually do the injecting""" if username and password: scheme, netloc, path, params, query, frag = urlparse.urlparse(url) if scheme in ('http', 'https'): auth_part, host_part = urllib2.splituser(netloc) if not auth_part: # If the URL doesn't have credentials in it already netloc = '%s:%s@%s' % ( urllib.quote(username, ''), urllib.quote(password, ''), host_part, ) url = urlparse.urlunparse((scheme,netloc,path,params,query,frag)) return url
def _parse_site(self, site): """Retrieve the auth information and base url for a site. Args: site: The URL to parse. Returns: A tuple containing (site, username, password). """ proto, host, path, query, fragment = urlparse.urlsplit(site) auth_info, host = urllib2.splituser(host) if not auth_info: user, password = None, None else: user, password = urllib2.splitpasswd(auth_info) new_site = urlparse.urlunparse((proto, host, '', '', '', '')) return (new_site, user, password)
def urlretrieve(url, tmp_path): """Work around Python issue 24599 includig basic auth support """ scheme, netloc, path, params, query, frag = urlparse(url) auth, host = urllib2.splituser(netloc) if auth: url = urlunparse((scheme, host, path, params, query, frag)) req = urllib2.Request(url) base64string = base64.encodestring(auth)[:-1] basic = "Basic " + base64string req.add_header("Authorization", basic) else: req = urllib2.Request(url) url_obj = urllib2.urlopen(req) with open(tmp_path, 'wb') as fp: fp.write(url_obj.read()) return tmp_path, url_obj.info()
def _build_request(url): # Detect basic auth # Adapted from python-feedparser urltype, rest = urllib2.splittype(url) realhost, rest = urllib2.splithost(rest) if realhost: user_passwd, realhost = urllib2.splituser(realhost) if user_passwd: url = '%s://%s%s' % (urltype, realhost, rest) # Start request req = urllib2.Request(url) # Add headers req.add_header('User-Agent', 'SABnzbd+/%s' % sabnzbd.version.__version__) if not any(item in url for item in _BAD_GZ_HOSTS): req.add_header('Accept-encoding', 'gzip') if user_passwd: req.add_header('Authorization', 'Basic ' + user_passwd.encode('base64').strip()) return urllib2.urlopen(req)
def open_with_auth(url): """ open a urllib2 request, handling HTTP authentication """ scheme, netloc, path, params, query, frag = urlparse.urlparse(url) auth, host = urllib2.splituser(netloc) if auth: auth = urllib2.unquote(auth).encode('base64').strip() elif userpass: auth = ('%s:%s' % userpass).encode('base64') if auth: new_url = urlparse.urlunparse((scheme, host, path, params, query, frag)) request = urllib2.Request(new_url) request.add_unredirected_header("Authorization", "Basic " + auth) else: request = urllib2.Request(url) request.add_header('User-Agent', 'enstaller') return urllib2.urlopen(request)
def get_data(self, key): url = self._location(key) scheme, netloc, path, params, query, frag = urlparse.urlparse(url) auth, host = urllib2.splituser(netloc) if auth: auth = urllib2.unquote(auth) elif self.userpass: auth = ('%s:%s' % self.userpass) if auth: new_url = urlparse.urlunparse((scheme, host, path, params, query, frag)) request = urllib2.Request(new_url) request.add_unredirected_header("Authorization", "Basic " + auth.encode('base64').strip()) else: request = urllib2.Request(url) request.add_header('User-Agent', 'enstaller') try: return urllib2.urlopen(request) except urllib2.HTTPError as e: raise KeyError("%s: %s" % (e, url))
def open_with_auth(url): """Open a urllib2 request, handling HTTP authentication""" scheme, netloc, path, params, query, frag = urlparse.urlparse(url) # Double scheme does not raise on Mac OS X as revealed by a # failing test. We would expect "nonnumeric port". Refs #20. if netloc.endswith(':'): raise httplib.InvalidURL("nonnumeric port: ''") if scheme in ('http', 'https'): auth, host = urllib2.splituser(netloc) else: auth = None if auth: auth = "Basic " + _encode_auth(auth) new_url = urlparse.urlunparse( (scheme, host, path, params, query, frag)) request = urllib2.Request(new_url) request.add_header("Authorization", auth) else: request = urllib2.Request(url) request.add_header('User-Agent', user_agent) fp = urllib2.urlopen(request) if auth: # Put authentication info back into request URL if same host, # so that links found on the page will work s2, h2, path2, param2, query2, frag2 = urlparse.urlparse(fp.url) if s2 == scheme and h2 == host: fp.url = urlparse.urlunparse( (s2, netloc, path2, param2, query2, frag2)) return fp
def strip_auth(url): scheme, netloc, path, params, query, frag = urlparse.urlparse(url) if scheme in ('http', 'https'): auth, host = urllib2.splituser(netloc) return urlparse.urlunparse((scheme,host,path,params,query,frag)) return url
def open_with_auth(url): """Open a urllib2 request, handling HTTP authentication""" # Parse the url to determine two things. First, what the scheme for the # URL is -- important because we only try logging in if it is http or # https protocol. Second, try to determine if credentials were embedded # in the URL. scheme, netloc, path, params, query, frag = urlparse.urlparse(url) if scheme in ('http', 'https'): auth, host = urllib2.splituser(netloc) else: auth = None host = None # Do not cache auth for non-web requests. # Prefer any previously cached authorization over anything in the URL # as the user may have already corrected the URL's embedded version. # Note that we cache the authorizations as metadata associated with this # method. # FIXME: For now, we assume a single login per host machine. if not hasattr(open_with_auth, 'auth_cache'): open_with_auth.auth_cache = auth_cache = {} else: auth_cache = open_with_auth.auth_cache if host in auth_cache: auth = auth_cache[host] # Attempt the request in such a way that if we get an authorization # failure, we can give the user a couple chances to provide valid # credentials. if auth: coded_auth = "Basic " + urllib2.unquote(auth).encode('base64').strip() new_url = urlparse.urlunparse((scheme,host,path,params,query,frag)) request = urllib2.Request(new_url) request.add_header("Authorization", coded_auth) else: request = urllib2.Request(url) request.add_header('User-Agent', user_agent) tries_left = 3 while tries_left >= 0: try: fp = urllib2.urlopen(request) if host and auth: auth_cache[host] = auth break except urllib2.HTTPError, e: if e.code == 401: old_user, old_passwd = split_auth(auth) print ("Please enter credentials to access the repository at " "%s:" % host) msg = 'User name' if old_user: msg = msg + ' [%s]: ' % old_user else: msg = msg + ': ' user = raw_input(msg) if len(user) < 1: user = old_user passwd = getpass.getpass("Password: "******"%s:%s" % (user, passwd) coded_auth = "Basic " + urllib2.unquote(auth).encode('base64').strip() new_url = urlparse.urlunparse((scheme,host,path,params,query,frag)) request = urllib2.Request(new_url) request.add_header("Authorization", coded_auth) tries_left -= 1 else: raise e