def _open_url(self, url):
"""Open a urllib2 request, handling HTTP authentication, and local
files support.
"""
scheme, netloc, path, params, query, frag = urlparse.urlparse(url)
# authentication stuff
if scheme in ('http', 'https'):
auth, host = urllib2.splituser(netloc)
else:
auth = None
# add index.html automatically for filesystem paths
if scheme == 'file':
if url.endswith(os.path.sep):
url += "index.html"
# add authorization headers if auth is provided
if auth:
auth = "Basic " + \
urlparse.unquote(auth).encode('base64').strip()
new_url = urlparse.urlunparse((
scheme, host, path, params, query, frag))
request = urllib2.Request(new_url)
request.add_header("Authorization", auth)
else:
request = urllib2.Request(url)
request.add_header('User-Agent', USER_AGENT)
try:
fp = urllib2.urlopen(request)
except (ValueError, httplib.InvalidURL), v:
msg = ' '.join([str(arg) for arg in v.args])
raise PackagingPyPIError('%s %s' % (url, msg))
def download(self, source, dest):
"""
Download an archive file.
:param str source: URL pointing to an archive file.
:param str dest: Local path location to download archive file to.
"""
# propogate all exceptions
# URLError, OSError, etc
proto, netloc, path, params, query, fragment = urlparse.urlparse(source)
if proto in ('http', 'https'):
auth, barehost = urllib2.splituser(netloc)
if auth is not None:
source = urlparse.urlunparse((proto, barehost, path, params, query, fragment))
username, password = urllib2.splitpasswd(auth)
passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
# Realm is set to None in add_password to force the username and password
# to be used whatever the realm
passman.add_password(None, source, username, password)
authhandler = urllib2.HTTPBasicAuthHandler(passman)
opener = urllib2.build_opener(authhandler)
urllib2.install_opener(opener)
response = urllib2.urlopen(source)
try:
with open(dest, 'w') as dest_file:
dest_file.write(response.read())
except Exception as e:
if os.path.isfile(dest):
os.unlink(dest)
raise e
def open_with_auth(url):
"""Open a urllib2 request, handling HTTP authentication"""
scheme, netloc, path, params, query, frag = urlparse.urlparse(url)
if scheme in ('http', 'https'):
auth, host = urllib2.splituser(netloc)
else:
auth = None
if auth:
auth = "Basic " + urllib2.unquote(auth).encode('base64').strip()
new_url = urlparse.urlunparse((scheme,host,path,params,query,frag))
request = urllib2.Request(new_url)
request.add_header("Authorization", auth)
else:
request = urllib2.Request(url)
request.add_header('User-Agent', user_agent)
fp = urllib2.urlopen(request)
if auth:
# Put authentication info back into request URL if same host,
# so that links found on the page will work
s2, h2, path2, param2, query2, frag2 = urlparse.urlparse(fp.url)
if s2==scheme and h2==host:
fp.url = urlparse.urlunparse((s2,netloc,path2,param2,query2,frag2))
return fp
def open_with_auth(url):
"""
Open a urllib2 request, handling HTTP authentication
"""
import config
scheme, netloc, path, params, query, frag = urlparse.urlparse(url)
assert not query
auth, host = urllib2.splituser(netloc)
if auth:
auth = urllib2.unquote(auth).encode('base64').strip()
elif 'enthought.com/repo/' in url and 'repo/pypi/eggs/' not in url:
auth = config.get('EPD_auth')
if auth is None:
userpass = config.get('EPD_userpass')
if userpass:
auth = userpass.encode('base64').strip()
if auth:
new_url = urlparse.urlunparse((scheme, host, path,
params, query, frag))
request = urllib2.Request(new_url)
request.add_header("Authorization", "Basic " + auth)
logger.debug('Requesting %s with auth' % new_url)
else:
request = urllib2.Request(url)
logger.debug('Requesting %s without auth' % url)
request.add_header('User-Agent', 'enstaller/%s' % __version__)
return urllib2.urlopen(request)
def get_transport_and_path_from_url(url, config=None, **kwargs):
"""Obtain a git client from a URL.
:param url: URL to open (a unicode string)
:param config: Optional config object
:param thin_packs: Whether or not thin packs should be retrieved
:param report_activity: Optional callback for reporting transport
activity.
:return: Tuple with client instance and relative path.
"""
parsed = urlparse.urlparse(url)
auth, host = urllib2.splituser(parsed.netloc)
if parsed.scheme == 'git':
return (TCPGitClient(parsed.hostname, port=parsed.port, **kwargs),
parsed.path)
elif parsed.scheme in ('git+ssh', 'ssh'):
path = parsed.path
if path.startswith('/'):
path = parsed.path[1:]
return SSHGitClient(parsed.hostname, port=parsed.port,
username=parsed.username, **kwargs), path
elif parsed.scheme in ('http', 'https'):
base_url = urlparse.urlunparse(parsed._replace(netloc=host))
return HttpGitClient(base_url, config=config, user=parsed.username,
passwd=parsed.password, **kwargs), parsed.path
elif parsed.scheme == 'file':
return default_local_git_client_cls(**kwargs), parsed.path
raise ValueError("unknown scheme '%s'" % parsed.scheme)
def open_with_auth(url):
"""Open a urllib2 request, handling HTTP authentication"""
scheme, netloc, path, params, query, frag = urlparse.urlparse(url)
# Double scheme does not raise on Mac OS X as revealed by a
# failing test. We would expect "nonnumeric port". Refs #20.
if netloc.endswith(':'):
raise httplib.InvalidURL("nonnumeric port: ''")
if scheme in ('http', 'https'):
auth, host = urllib2.splituser(netloc)
else:
auth = None
if auth:
auth = "Basic " + _encode_auth(auth)
new_url = urlparse.urlunparse((scheme,host,path,params,query,frag))
request = urllib2.Request(new_url)
request.add_header("Authorization", auth)
else:
request = urllib2.Request(url)
request.add_header('User-Agent', user_agent)
fp = urllib2.urlopen(request)
if auth:
# Put authentication info back into request URL if same host,
# so that links found on the page will work
s2, h2, path2, param2, query2, frag2 = urlparse.urlparse(fp.url)
if s2==scheme and h2==host:
fp.url = urlparse.urlunparse((s2,netloc,path2,param2,query2,frag2))
return fp
def from_parsedurl(cls, parsedurl, **kwargs):
auth, host = urllib2.splituser(parsedurl.netloc)
password = parsedurl.password
username = parsedurl.username
# TODO(jelmer): This also strips the username
parsedurl = parsedurl._replace(netloc=host)
return cls(urlparse.urlunparse(parsedurl),
password=password, username=username, **kwargs)
def get_proxy_info(proxystr=""):
"""
Get proxy config from string or environment variables.
If a proxy string is passed in, it overrides whatever might be in the
environment variables.
Returns dictionary of identified proxy information, or None if proxystr was
empty and no config was found fmor os.environ
Raises InvalidConfiguration on any configuration error.
"""
if proxystr == "":
# FIXME: We should be supporting http_proxy, HTTP_PROXY variables.
proxy_info = {
'host' : os.environ.get('PROXY_HOST', None),
'port' : _convert_port_value(os.environ.get('PROXY_PORT', None)),
'user' : os.environ.get('PROXY_USER', None),
'pass' : os.environ.get('PROXY_PASS', None)
}
if proxy_info.get("host") is None:
return None
else:
parts = urlparse.urlparse(proxystr)
if parts.netloc == "":
proxystr = "http://{0}".format(proxystr)
parts = urlparse.urlparse(proxystr)
_, hostport = urllib2.splituser(parts.netloc)
host, _ = urllib2.splitport(hostport)
host = urlparse.urlunparse((parts.scheme, host, "", "", "", ""))
proxy_info = {
'host' : host,
'port' : _convert_port_value(parts.port),
'user' : parts.username,
'pass' : parts.password,
}
# If a user was specified, but no password was, prompt for it now.
user = proxy_info.get('user', None)
if user is not None and len(user) > 0:
passwd = proxy_info.get('pass', None)
if passwd is None or len(passwd) < 1:
import getpass
proxy_info['pass'] = getpass.getpass()
if proxy_info["host"] is None or len(proxy_info["host"]) < 1:
error_msg = ("Invalid proxy configuration '{0}': "
"proxy string should be of the form "
"'http://host:port' or 'http://host'".format(proxystr))
raise InvalidConfiguration(error_msg)
return proxy_info
def open_url(url):
"""
Open a urllib2 request, handling HTTP authentication
"""
scheme, netloc, path, params, query, frag = urlparse.urlparse(url)
assert not query
auth, host = urllib2.splituser(netloc)
request = urllib2.Request(url)
request.add_header("User-Agent", "IronPkg/%s" % __version__)
return urllib2.urlopen(request)
def _getBaseDownloadUrl(self):
"""
Return the base URL relative to which to download images,
removing any user/password information, and using http
"""
# extract the downloadImage base url from the serverUrl configuration
parts = list(urllib2.urlparse.urlparse(self.rbuilderUrl))
parts[1] = urllib2.splituser(parts[1])[1]
# FIXME: is this whole ../ business a workaround for splitbrain rBO?
parts[2] = parts[2] and parts[2] or "/"
return "http://%s%s" % (parts[1], util.normpath(parts[2] + "../")[1:])
def set_site(cls, value):
cls._threadlocal.connection = None
ShopifyResource._site = cls._threadlocal.site = value
if value is not None:
host = urlparse.urlsplit(value)[1]
auth_info, host = urllib2.splituser(host)
if auth_info:
user, password = urllib2.splitpasswd(auth_info)
if user:
cls.user = urllib.unquote(user)
if password:
cls.password = urllib.unquote(password)
def set_site(cls, value):
if value is not None:
host = urlparse.urlsplit(value)[1]
auth_info, host = urllib2.splituser(host)
if auth_info:
user, password = urllib2.splitpasswd(auth_info)
if user:
cls._user = urllib.unquote(user)
if password:
cls._password = urllib.unquote(password)
cls._connection = None
cls._site = value
def __init__(self, url):
self.url = url
self.schema, url = urllib2.splittype(url)
host, path = urllib2.splithost(url)
userpass, host = urllib2.splituser(host)
if userpass:
self.user, self.password = urllib2.splitpasswd(userpass)
path, self.querystring = urllib.splitquery(path)
self.query = self.querystring and self.querystring.split('&') or []
#urllib.splitquery(url)
self.host, self.port = urllib2.splitport(host)
path, self.tag = urllib2.splittag(path)
self.path = path.strip('/')
def _inject_credentials(url, username=None, password=None):
"""Used by `inject_credentials` decorators to actually do the injecting"""
if username and password:
scheme, netloc, path, params, query, frag = urlparse.urlparse(url)
if scheme in ('http', 'https'):
auth_part, host_part = urllib2.splituser(netloc)
if not auth_part: # If the URL doesn't have credentials in it already
netloc = '%s:%s@%s' % (
urllib.quote(username, ''),
urllib.quote(password, ''),
host_part,
)
url = urlparse.urlunparse((scheme,netloc,path,params,query,frag))
return url
def _parse_site(self, site):
"""Retrieve the auth information and base url for a site.
Args:
site: The URL to parse.
Returns:
A tuple containing (site, username, password).
"""
proto, host, path, query, fragment = urlparse.urlsplit(site)
auth_info, host = urllib2.splituser(host)
if not auth_info:
user, password = None, None
else:
user, password = urllib2.splitpasswd(auth_info)
new_site = urlparse.urlunparse((proto, host, '', '', '', ''))
return (new_site, user, password)
def _parse_site(self, site):
"""Retrieve the auth information and base url for a site.
Args:
site: The URL to parse.
Returns:
A tuple containing (site, username, password).
"""
proto, host, path, query, fragment = urlparse.urlsplit(site)
auth_info, host = urllib2.splituser(host)
if not auth_info:
user, password = None, None
else:
user, password = urllib2.splitpasswd(auth_info)
new_site = urlparse.urlunparse((proto, host, '', '', '', ''))
return (new_site, user, password)
def urlretrieve(url, tmp_path):
"""Work around Python issue 24599 includig basic auth support
"""
scheme, netloc, path, params, query, frag = urlparse(url)
auth, host = urllib2.splituser(netloc)
if auth:
url = urlunparse((scheme, host, path, params, query, frag))
req = urllib2.Request(url)
base64string = base64.encodestring(auth)[:-1]
basic = "Basic " + base64string
req.add_header("Authorization", basic)
else:
req = urllib2.Request(url)
url_obj = urllib2.urlopen(req)
with open(tmp_path, 'wb') as fp:
fp.write(url_obj.read())
return tmp_path, url_obj.info()
def _build_request(url):
# Detect basic auth
# Adapted from python-feedparser
urltype, rest = urllib2.splittype(url)
realhost, rest = urllib2.splithost(rest)
if realhost:
user_passwd, realhost = urllib2.splituser(realhost)
if user_passwd:
url = '%s://%s%s' % (urltype, realhost, rest)
# Start request
req = urllib2.Request(url)
# Add headers
req.add_header('User-Agent', 'SABnzbd+/%s' % sabnzbd.version.__version__)
if not any(item in url for item in _BAD_GZ_HOSTS):
req.add_header('Accept-encoding', 'gzip')
if user_passwd:
req.add_header('Authorization', 'Basic ' + user_passwd.encode('base64').strip())
return urllib2.urlopen(req)
def _build_request(url):
# Detect basic auth
# Adapted from python-feedparser
urltype, rest = urllib2.splittype(url)
realhost, rest = urllib2.splithost(rest)
if realhost:
user_passwd, realhost = urllib2.splituser(realhost)
if user_passwd:
url = '%s://%s%s' % (urltype, realhost, rest)
# Start request
req = urllib2.Request(url)
# Add headers
req.add_header('User-Agent', 'SABnzbd+/%s' % sabnzbd.version.__version__)
if not any(item in url for item in _BAD_GZ_HOSTS):
req.add_header('Accept-encoding', 'gzip')
if user_passwd:
req.add_header('Authorization', 'Basic ' + user_passwd.encode('base64').strip())
return urllib2.urlopen(req)
def open_with_auth(url):
"""
open a urllib2 request, handling HTTP authentication
"""
scheme, netloc, path, params, query, frag = urlparse.urlparse(url)
auth, host = urllib2.splituser(netloc)
if auth:
auth = urllib2.unquote(auth).encode('base64').strip()
elif userpass:
auth = ('%s:%s' % userpass).encode('base64')
if auth:
new_url = urlparse.urlunparse((scheme, host, path,
params, query, frag))
request = urllib2.Request(new_url)
request.add_unredirected_header("Authorization", "Basic " + auth)
else:
request = urllib2.Request(url)
request.add_header('User-Agent', 'enstaller')
return urllib2.urlopen(request)
def get_data(self, key):
url = self._location(key)
scheme, netloc, path, params, query, frag = urlparse.urlparse(url)
auth, host = urllib2.splituser(netloc)
if auth:
auth = urllib2.unquote(auth)
elif self.userpass:
auth = ('%s:%s' % self.userpass)
if auth:
new_url = urlparse.urlunparse((scheme, host, path,
params, query, frag))
request = urllib2.Request(new_url)
request.add_unredirected_header("Authorization",
"Basic " + auth.encode('base64').strip())
else:
request = urllib2.Request(url)
request.add_header('User-Agent', 'enstaller')
try:
return urllib2.urlopen(request)
except urllib2.HTTPError as e:
raise KeyError("%s: %s" % (e, url))
def open_with_auth(url):
"""Open a urllib2 request, handling HTTP authentication"""
scheme, netloc, path, params, query, frag = urlparse.urlparse(url)
# Double scheme does not raise on Mac OS X as revealed by a
# failing test. We would expect "nonnumeric port". Refs #20.
if netloc.endswith(':'):
raise httplib.InvalidURL("nonnumeric port: ''")
if scheme in ('http', 'https'):
auth, host = urllib2.splituser(netloc)
else:
auth = None
if auth:
auth = "Basic " + _encode_auth(auth)
new_url = urlparse.urlunparse(
(scheme, host, path, params, query, frag))
request = urllib2.Request(new_url)
request.add_header("Authorization", auth)
else:
request = urllib2.Request(url)
request.add_header('User-Agent', user_agent)
fp = urllib2.urlopen(request)
if auth:
# Put authentication info back into request URL if same host,
# so that links found on the page will work
s2, h2, path2, param2, query2, frag2 = urlparse.urlparse(fp.url)
if s2 == scheme and h2 == host:
fp.url = urlparse.urlunparse(
(s2, netloc, path2, param2, query2, frag2))
return fp
def strip_auth(url):
scheme, netloc, path, params, query, frag = urlparse.urlparse(url)
if scheme in ('http', 'https'):
auth, host = urllib2.splituser(netloc)
return urlparse.urlunparse((scheme,host,path,params,query,frag))
return url
def open_with_auth(url):
"""Open a urllib2 request, handling HTTP authentication"""
# Parse the url to determine two things. First, what the scheme for the
# URL is -- important because we only try logging in if it is http or
# https protocol. Second, try to determine if credentials were embedded
# in the URL.
scheme, netloc, path, params, query, frag = urlparse.urlparse(url)
if scheme in ('http', 'https'):
auth, host = urllib2.splituser(netloc)
else:
auth = None
host = None # Do not cache auth for non-web requests.
# Prefer any previously cached authorization over anything in the URL
# as the user may have already corrected the URL's embedded version.
# Note that we cache the authorizations as metadata associated with this
# method.
# FIXME: For now, we assume a single login per host machine.
if not hasattr(open_with_auth, 'auth_cache'):
open_with_auth.auth_cache = auth_cache = {}
else:
auth_cache = open_with_auth.auth_cache
if host in auth_cache:
auth = auth_cache[host]
# Attempt the request in such a way that if we get an authorization
# failure, we can give the user a couple chances to provide valid
# credentials.
if auth:
coded_auth = "Basic " + urllib2.unquote(auth).encode('base64').strip()
new_url = urlparse.urlunparse((scheme,host,path,params,query,frag))
request = urllib2.Request(new_url)
request.add_header("Authorization", coded_auth)
else:
request = urllib2.Request(url)
request.add_header('User-Agent', user_agent)
tries_left = 3
while tries_left >= 0:
try:
fp = urllib2.urlopen(request)
if host and auth:
auth_cache[host] = auth
break
except urllib2.HTTPError, e:
if e.code == 401:
old_user, old_passwd = split_auth(auth)
print ("Please enter credentials to access the repository at "
"%s:" % host)
msg = 'User name'
if old_user:
msg = msg + ' [%s]: ' % old_user
else:
msg = msg + ': '
user = raw_input(msg)
if len(user) < 1:
user = old_user
passwd = getpass.getpass("Password: "******"%s:%s" % (user, passwd)
coded_auth = "Basic " + urllib2.unquote(auth).encode('base64').strip()
new_url = urlparse.urlunparse((scheme,host,path,params,query,frag))
request = urllib2.Request(new_url)
request.add_header("Authorization", coded_auth)
tries_left -= 1
else:
raise e
