def lookup_xml_id(self, url):
"""A helper method for locating a part of a WADL document.
:param url: The URL (with anchor) of the desired part of the
WADL document.
:return: The XML ID corresponding to the anchor.
"""
markup_uri = URI(self.markup_url).ensureNoSlash()
markup_uri.fragment = None
if url.startswith('http'):
# It's an absolute URI.
this_uri = URI(url).ensureNoSlash()
else:
# It's a relative URI.
this_uri = markup_uri.resolve(url)
possible_xml_id = this_uri.fragment
this_uri.fragment = None
if this_uri == markup_uri:
# The URL pointed elsewhere within the same WADL document.
# Return its fragment.
return possible_xml_id
# XXX leonardr 2008-05-28:
# This needs to be implemented eventually for Launchpad so
# that a script using this client can navigate from a WADL
# representation of a non-root resource to its definition at
# the server root.
raise NotImplementedError("Can't look up definition in another "
"url (%s)" % url)
def _normalize_stacked_on_url(self, branch):
"""Normalize and return the stacked-on location of `branch`.
In the common case, `branch` will either be unstacked or stacked on a
relative path, in which case this is very easy: just return the
location.
If `branch` is stacked on the absolute URL of another Launchpad
branch, we normalize this to a relative path (mutating the branch) and
return the relative path.
If `branch` is stacked on some other absolute URL we don't recognise,
we just return that and rely on the `branchChanged` XML-RPC method
recording a complaint in the appropriate place.
"""
stacked_on_url = get_stacked_on_url(branch)
if stacked_on_url is None:
return None
if '://' not in stacked_on_url:
# Assume it's a relative path.
return stacked_on_url
uri = URI(stacked_on_url)
if uri.scheme not in ['http', 'bzr+ssh', 'sftp']:
return stacked_on_url
launchpad_domain = config.vhost.mainsite.hostname
if not uri.underDomain(launchpad_domain):
return stacked_on_url
# We use TransportConfig directly because the branch
# is still locked at this point! We're effectively
# 'borrowing' the lock that is being released.
branch_config = TransportConfig(branch._transport, 'branch.conf')
branch_config.set_option(uri.path, 'stacked_on_location')
return uri.path
def getNonRestrictedURL(self, request):
"""Returns the non-restricted version of the request URL.
The intended use is for determining the equivalent URL on the
production Launchpad instance if a user accidentally ends up
on a restrict_to_team Launchpad instance.
If a non-restricted URL can not be determined, None is returned.
"""
base_host = config.vhost.mainsite.hostname
production_host = config.launchpad.non_restricted_hostname
# If we don't have a production hostname, or it is the same as
# this instance, then we can't provide a nonRestricted URL.
if production_host is None or base_host == production_host:
return None
# Are we under the main site's domain?
uri = URI(request.getURL())
if not uri.host.endswith(base_host):
return None
# Update the hostname, and complete the URL from the request:
new_host = uri.host[:-len(base_host)] + production_host
uri = uri.replace(host=new_host, path=request['PATH_INFO'])
query_string = request.get('QUERY_STRING')
if query_string:
uri = uri.replace(query=query_string)
return str(uri)
def _normalize_stacked_on_url(self, branch):
"""Normalize and return the stacked-on location of `branch`.
In the common case, `branch` will either be unstacked or stacked on a
relative path, in which case this is very easy: just return the
location.
If `branch` is stacked on the absolute URL of another Launchpad
branch, we normalize this to a relative path (mutating the branch) and
return the relative path.
If `branch` is stacked on some other absolute URL we don't recognise,
we just return that and rely on the `branchChanged` XML-RPC method
recording a complaint in the appropriate place.
"""
stacked_on_url = get_stacked_on_url(branch)
if stacked_on_url is None:
return None
if "://" not in stacked_on_url:
# Assume it's a relative path.
return stacked_on_url
uri = URI(stacked_on_url)
if uri.scheme not in ["http", "bzr+ssh", "sftp"]:
return stacked_on_url
launchpad_domain = config.vhost.mainsite.hostname
if not uri.underDomain(launchpad_domain):
return stacked_on_url
# We use TransportConfig directly because the branch
# is still locked at this point! We're effectively
# 'borrowing' the lock that is being released.
branch_config = TransportConfig(branch._transport, "branch.conf")
branch_config.set_option(uri.path, "stacked_on_location")
return uri.path
def _validate(self, value):
# import here to avoid circular import
from lp.services.webapp import canonical_url
from lazr.uri import URI
super(DistroMirrorURIField, self)._validate(value)
uri = URI(self.normalize(value))
# This field is also used when creating new mirrors and in that case
# self.context is not an IDistributionMirror so it doesn't make sense
# to try to get the existing value of the attribute.
if IDistributionMirror.providedBy(self.context):
orig_value = self.get(self.context)
if orig_value is not None and URI(orig_value) == uri:
return # url was not changed
mirror = self.getMirrorByURI(str(uri))
if mirror is not None:
message = _(
'The distribution mirror <a href="${url}">${mirror}</a> '
'is already registered with this URL.',
mapping={
'url': html_escape(canonical_url(mirror)),
'mirror': html_escape(mirror.title)
})
raise LaunchpadValidationError(structured(message))
def test_underDomain_matches_subdomain(self):
# URI.underDomain should return True when asked whether the url is
# under one of its parent domains.
uri = URI('http://code.launchpad.dev/foo')
self.assertTrue(uri.underDomain('code.launchpad.dev'))
self.assertTrue(uri.underDomain('launchpad.dev'))
self.assertTrue(uri.underDomain(''))
def __init__(self, authorizer, service_root, cache=None,
timeout=None, proxy_info=None, version=None,
base_client_name='', max_retries=Browser.MAX_RETRIES):
"""Root access to a lazr.restful API.
:param credentials: The credentials used to access the service.
:param service_root: The URL to the root of the web service.
:type service_root: string
"""
if version is not None:
if service_root[-1] != '/':
service_root += '/'
service_root += str(version)
if service_root[-1] != '/':
service_root += '/'
self._root_uri = URI(service_root)
# Set up data necessary to calculate the User-Agent header.
self._base_client_name = base_client_name
# Get the WADL definition.
self.credentials = authorizer
self._browser = Browser(
self, authorizer, cache, timeout, proxy_info, self._user_agent,
max_retries)
self._wadl = self._browser.get_wadl_application(self._root_uri)
# Get the root resource.
root_resource = self._wadl.get_resource_by_path('')
bound_root = root_resource.bind(
self._browser.get(root_resource), 'application/json')
super(ServiceRoot, self).__init__(None, bound_root)
def test_underDomain_doesnt_match_non_subdomain(self):
# URI.underDomain should return False when asked whether the url is
# under a domain which isn't one of its parents.
uri = URI('http://code.launchpad.dev/foo')
self.assertFalse(uri.underDomain('beta.code.launchpad.dev'))
self.assertFalse(uri.underDomain('google.com'))
self.assertFalse(uri.underDomain('unchpad.dev'))
def test_underDomain_doesnt_match_non_subdomain(self):
# URI.underDomain should return False when asked whether the url is
# under a domain which isn't one of its parents.
uri = URI('http://code.launchpad.dev/foo')
self.assertFalse(uri.underDomain('beta.code.launchpad.dev'))
self.assertFalse(uri.underDomain('google.com'))
self.assertFalse(uri.underDomain('unchpad.dev'))
def test_underDomain_matches_subdomain(self):
# URI.underDomain should return True when asked whether the url is
# under one of its parent domains.
uri = URI('http://code.launchpad.dev/foo')
self.assertTrue(uri.underDomain('code.launchpad.dev'))
self.assertTrue(uri.underDomain('launchpad.dev'))
self.assertTrue(uri.underDomain(''))
def getNonRestrictedURL(self, request):
"""Returns the non-restricted version of the request URL.
The intended use is for determining the equivalent URL on the
production Launchpad instance if a user accidentally ends up
on a restrict_to_team Launchpad instance.
If a non-restricted URL can not be determined, None is returned.
"""
base_host = config.vhost.mainsite.hostname
production_host = config.launchpad.non_restricted_hostname
# If we don't have a production hostname, or it is the same as
# this instance, then we can't provide a nonRestricted URL.
if production_host is None or base_host == production_host:
return None
# Are we under the main site's domain?
uri = URI(request.getURL())
if not uri.host.endswith(base_host):
return None
# Update the hostname, and complete the URL from the request:
new_host = uri.host[:-len(base_host)] + production_host
uri = uri.replace(host=new_host, path=request['PATH_INFO'])
query_string = request.get('QUERY_STRING')
if query_string:
uri = uri.replace(query=query_string)
return str(uri)
def items(self):
"""Return a list of `IBreadcrumb` objects visible in the hierarchy.
The list starts with the breadcrumb closest to the hierarchy root.
"""
breadcrumbs = []
for obj in self.objects:
breadcrumb = IBreadcrumb(obj, None)
if breadcrumb is not None:
breadcrumbs.append(breadcrumb)
host = URI(self.request.getURL()).host
mainhost = allvhosts.configs['mainsite'].hostname
if (len(breadcrumbs) != 0 and
host != mainhost and
self.vhost_breadcrumb):
# We have breadcrumbs and we're not on the mainsite, so we'll
# sneak an extra breadcrumb for the vhost we're on.
vhost = host.split('.')[0]
# Iterate over the context of our breadcrumbs in reverse order and
# for the first one we find an adapter named after the vhost we're
# on, generate an extra breadcrumb and insert it in our list.
for idx, breadcrumb in reversed(list(enumerate(breadcrumbs))):
extra_breadcrumb = queryAdapter(
breadcrumb.context, IBreadcrumb, name=vhost)
if extra_breadcrumb is not None:
breadcrumbs.insert(idx + 1, extra_breadcrumb)
break
if len(breadcrumbs) > 0:
page_crumb = self.makeBreadcrumbForRequestedPage()
if page_crumb:
breadcrumbs.append(page_crumb)
return breadcrumbs
def _init_link_data(self):
if self._initialized:
return
self._initialized = True
self.initialize()
self._check_links()
links_set = set(self.links)
assert not links_set.intersection(self._forbiddenlinknames), (
"The following names may not be links: %s" %
', '.join(self._forbiddenlinknames))
if isinstance(self.context, LaunchpadView):
# It's a navigation menu for a view instead of a db object. Views
# don't have a canonical URL, they use the db object one used as
# the context for that view.
context = self.context.context
else:
context = self.context
self._contexturlobj = URI(canonical_url(context))
if self.enable_only is ALL_LINKS:
self._enable_only_set = links_set
else:
self._enable_only_set = set(self.enable_only)
unknown_links = self._enable_only_set - links_set
if len(unknown_links) > 0:
# There are links named in enable_only that do not exist in
# self.links.
raise AssertionError(
"Links in 'enable_only' not found in 'links': %s" %
', '.join(sorted(unknown_links)))
def test_makeBranchTrailingSlash(self):
"""makeBranch creates a mirrored branch even if the URL ends with /.
"""
uri = URI(self.factory.getUniqueURL())
expected_name = self.popup.getBranchNameFromURL(
str(uri.ensureNoSlash()))
branch = self.popup.makeBranchFromURL(str(uri.ensureSlash()))
self.assertEqual(str(uri.ensureNoSlash()), branch.url)
self.assertEqual(expected_name, branch.name)
def archive_url(self):
"""Return a custom archive url for basic authentication."""
normal_url = URI(self.archive.archive_url)
if self.name:
name = '+' + self.name
else:
name = self.person.name
auth_url = normal_url.replace(userinfo="%s:%s" % (name, self.token))
return str(auth_url)
def test_makeBranchTrailingSlash(self):
"""makeBranch creates a mirrored branch even if the URL ends with /.
"""
uri = URI(self.factory.getUniqueURL())
expected_name = self.popup.getBranchNameFromURL(
str(uri.ensureNoSlash()))
branch = self.popup.makeBranchFromURL(str(uri.ensureSlash()))
self.assertEqual(str(uri.ensureNoSlash()), branch.url)
self.assertEqual(expected_name, branch.name)
def _with_url_query_variable_set(self, url, variable, new_value):
"""A helper method to set a query variable in a URL."""
uri = URI(url)
if uri.query is None:
params = {}
else:
params = cgi.parse_qs(uri.query)
params[variable] = str(new_value)
uri.query = urllib.urlencode(params, True)
return str(uri)
def _with_url_query_variable_set(self, url, variable, new_value):
"""A helper method to set a query variable in a URL."""
uri = URI(url)
if uri.query is None:
params = {}
else:
params = parse_qs(uri.query)
params[variable] = str(new_value)
uri.query = urllib.urlencode(params, True)
return str(uri)
def web_root_for_service_root(service_root):
"""Turn a service root URL into a web root URL.
This is done heuristically, not with a lookup.
"""
service_root = lookup_service_root(service_root)
web_root_uri = URI(service_root)
web_root_uri.path = ""
web_root_uri.host = web_root_uri.host.replace("api.", "", 1)
web_root = str(web_root_uri.ensureSlash())
return web_root
def web_root_for_service_root(service_root):
"""Turn a service root URL into a web root URL.
This is done heuristically, not with a lookup.
"""
service_root = lookup_service_root(service_root)
web_root_uri = URI(service_root)
web_root_uri.path = ""
web_root_uri.host = web_root_uri.host.replace("api.", "", 1)
web_root = str(web_root_uri.ensureSlash())
return web_root
def compose_public_url(scheme, unique_name, suffix=None):
# Accept sftp as a legacy protocol.
accepted_schemes = set(SUPPORTED_SCHEMES)
accepted_schemes.add('sftp')
assert scheme in accepted_schemes, "Unknown scheme: %s" % scheme
host = URI(config.codehosting.supermirror_root).host
# After quoting and encoding, the path should be perfectly
# safe as a plain ASCII string, str() just enforces this
path = '/' + str(urllib.quote(six.ensure_binary(unique_name), safe='/~+'))
if suffix:
path = os.path.join(path, suffix)
return str(URI(scheme=scheme, host=host, path=path))
def _test_one_semaphore_for_each_host(self, mirror1, mirror2, mirror3,
probe_function):
"""Check that we create one semaphore per host when probing the given
mirrors using the given probe_function.
mirror1.base_url and mirror2.base_url must be on the same host while
mirror3.base_url must be on a different one.
The given probe_function must be either probe_cdimage_mirror or
probe_archive_mirror.
"""
request_manager = RequestManager()
mirror1_host = URI(mirror1.base_url).host
mirror2_host = URI(mirror2.base_url).host
mirror3_host = URI(mirror3.base_url).host
probe_function(mirror1, StringIO(), [], logging)
# Since we have a single mirror to probe we need to have a single
# DeferredSemaphore with a limit of PER_HOST_REQUESTS, to ensure we
# don't issue too many simultaneous connections on that host.
self.assertEquals(len(request_manager.host_locks), 1)
multi_lock = request_manager.host_locks[mirror1_host]
self.assertEquals(multi_lock.host_lock.limit, PER_HOST_REQUESTS)
# Note that our multi_lock contains another semaphore to control the
# overall number of requests.
self.assertEquals(multi_lock.overall_lock.limit, OVERALL_REQUESTS)
probe_function(mirror2, StringIO(), [], logging)
# Now we have two mirrors to probe, but they have the same hostname,
# so we'll still have a single semaphore in host_semaphores.
self.assertEquals(mirror2_host, mirror1_host)
self.assertEquals(len(request_manager.host_locks), 1)
multi_lock = request_manager.host_locks[mirror2_host]
self.assertEquals(multi_lock.host_lock.limit, PER_HOST_REQUESTS)
probe_function(mirror3, StringIO(), [], logging)
# This third mirror is on a separate host, so we'll have a second
# semaphore added to host_semaphores.
self.failUnless(mirror3_host != mirror1_host)
self.assertEquals(len(request_manager.host_locks), 2)
multi_lock = request_manager.host_locks[mirror3_host]
self.assertEquals(multi_lock.host_lock.limit, PER_HOST_REQUESTS)
# When using an http_proxy, even though we'll actually connect to the
# proxy, we'll use the mirror's host as the key to find the semaphore
# that should be used
orig_proxy = os.getenv('http_proxy')
os.environ['http_proxy'] = 'http://squid.internal:3128/'
probe_function(mirror3, StringIO(), [], logging)
self.assertEquals(len(request_manager.host_locks), 2)
restore_http_proxy(orig_proxy)
def test_switch_branches(self):
# switch_branches moves a branch to the new location and places a
# branch (with no revisions) stacked on the new branch in the old
# location.
chroot_server = ChrootServer(self.get_transport())
chroot_server.start_server()
self.addCleanup(chroot_server.stop_server)
scheme = chroot_server.get_url().rstrip('/:')
old_branch = FakeBranch(1)
self.get_transport(old_branch.unique_name).create_prefix()
tree = self.make_branch_and_tree(old_branch.unique_name)
# XXX: AaronBentley 2010-08-06 bug=614404: a bzr username is
# required to generate the revision-id.
with override_environ(BZR_EMAIL='*****@*****.**'):
tree.commit(message='.')
new_branch = FakeBranch(2)
switch_branches('.', scheme, old_branch, new_branch)
# Post conditions:
# 1. unstacked branch in new_branch's location
# 2. stacked branch with no revisions in repo at old_branch
# 3. last_revision() the same for two branches
old_location_bzrdir = BzrDir.open(
str(URI(scheme=scheme, host='',
path='/' + old_branch.unique_name)))
new_location_bzrdir = BzrDir.open(
str(URI(scheme=scheme, host='',
path='/' + new_branch.unique_name)))
old_location_branch = old_location_bzrdir.open_branch()
new_location_branch = new_location_bzrdir.open_branch()
# 1. unstacked branch in new_branch's location
self.assertRaises(NotStacked, new_location_branch.get_stacked_on_url)
# 2. stacked branch with no revisions in repo at old_branch
self.assertEqual('/' + new_branch.unique_name,
old_location_branch.get_stacked_on_url())
self.assertEqual(
[],
old_location_bzrdir.open_repository().all_revision_ids())
# 3. last_revision() the same for two branches
self.assertEqual(old_location_branch.last_revision(),
new_location_branch.last_revision())
def list(self, dirname):
"""Download the HTML index at subdir and scrape for URLs.
Returns a list of directory names (links ending with /, or
that result in redirects to themselves ending in /) and
filenames (everything else) that reside underneath the path.
"""
self.log.info("Listing %s" % dirname)
try:
response = self.request("GET", dirname)
try:
soup = BeautifulSoup(response.read())
finally:
response.close()
except (IOError, socket.error) as exc:
raise HTTPWalkerError(str(exc))
base = URI(self.base).resolve(dirname)
# Collect set of URLs that are below the base URL
urls = set()
for anchor in soup("a"):
href = anchor.get("href")
if href is None:
continue
try:
url = base.resolve(href)
except InvalidURIError:
continue
# Only add the URL if it is strictly inside the base URL.
if base.contains(url) and not url.contains(base):
urls.add(url)
dirnames = set()
filenames = set()
for url in urls:
if url.path.endswith(';type=a') or url.path.endswith(';type=i'):
# these links come from Squid's FTP dir listing to
# force either ASCII or binary download and can be
# ignored.
continue
filename = subdir(base.path, url.path)
if self.isDirectory(url.path):
dirnames.add(as_dir(filename))
else:
filenames.add(filename)
return (sorted(dirnames), sorted(filenames))
def list(self, dirname):
"""Download the HTML index at subdir and scrape for URLs.
Returns a list of directory names (links ending with /, or
that result in redirects to themselves ending in /) and
filenames (everything else) that reside underneath the path.
"""
self.log.info("Listing %s" % dirname)
try:
response = self.request("GET", dirname)
try:
soup = BeautifulSoup(response.read())
finally:
response.close()
except (IOError, socket.error) as exc:
raise HTTPWalkerError(str(exc))
base = URI(self.base).resolve(dirname)
# Collect set of URLs that are below the base URL
urls = set()
for anchor in soup("a"):
href = anchor.get("href")
if href is None:
continue
try:
url = base.resolve(href)
except InvalidURIError:
continue
# Only add the URL if it is strictly inside the base URL.
if base.contains(url) and not url.contains(base):
urls.add(url)
dirnames = set()
filenames = set()
for url in urls:
if url.path.endswith(';type=a') or url.path.endswith(';type=i'):
# these links come from Squid's FTP dir listing to
# force either ASCII or binary download and can be
# ignored.
continue
filename = subdir(base.path, url.path)
if self.isDirectory(url.path):
dirnames.add(as_dir(filename))
else:
filenames.add(filename)
return (sorted(dirnames), sorted(filenames))
def _get_binary_sources_list_line(archive, distroarchseries, pocket,
components):
"""Return the correponding binary sources_list line."""
# Encode the private PPA repository password in the
# sources_list line. Note that the buildlog will be
# sanitized to not expose it.
if archive.private:
uri = URI(archive.archive_url)
uri = uri.replace(userinfo="buildd:%s" % archive.buildd_secret)
url = str(uri)
else:
url = archive.archive_url
suite = distroarchseries.distroseries.name + pocketsuffix[pocket]
return 'deb %s %s %s' % (url, suite, ' '.join(components))
def test_abnormal_resolution(self):
# Abnormal URI resolution examples from Section 5.4.2 of RFC 3986:
base = URI('http://a/b/c/d;p?q')
def resolve(relative):
return str(base.resolve(relative))
self.assertEqual(resolve('../../../g'), 'http://a/g')
self.assertEqual(resolve('../../../../g'), 'http://a/g')
self.assertEqual(resolve('/./g'), 'http://a/g')
self.assertEqual(resolve('/../g'), 'http://a/g')
self.assertEqual(resolve('g.'), 'http://a/b/c/g.')
self.assertEqual(resolve('.g'), 'http://a/b/c/.g')
self.assertEqual(resolve('g..'), 'http://a/b/c/g..')
self.assertEqual(resolve('..g'), 'http://a/b/c/..g')
self.assertEqual(resolve('./../g'), 'http://a/b/g')
self.assertEqual(resolve('./g/.'), 'http://a/b/c/g/')
self.assertEqual(resolve('g/./h'), 'http://a/b/c/g/h')
self.assertEqual(resolve('g/../h'), 'http://a/b/c/h')
self.assertEqual(resolve('g;x=1/./y'), 'http://a/b/c/g;x=1/y')
self.assertEqual(resolve('g;x=1/../y'), 'http://a/b/c/y')
self.assertEqual(resolve('g?y/./x'), 'http://a/b/c/g?y/./x')
self.assertEqual(resolve('g?y/../x'), 'http://a/b/c/g?y/../x')
self.assertEqual(resolve('g#s/./x'), 'http://a/b/c/g#s/./x')
self.assertEqual(resolve('g#s/../x'), 'http://a/b/c/g#s/../x')
def get_method_and_path(request):
"""Extract the method of the request and path of the requested file."""
method, ignore, rest = request.partition(' ')
# In the below, the common case is that `first` is the path and `last` is
# the protocol.
first, ignore, last = rest.rpartition(' ')
if first == '':
# HTTP 1.0 requests might omit the HTTP version so we cope with them.
path = last
elif not last.startswith('HTTP'):
# We cope with HTTP 1.0 protocol without HTTP version *and* a
# space in the path (see bug 676489 for example).
path = rest
else:
# This is the common case.
path = first
if path.startswith('http://') or path.startswith('https://'):
try:
uri = URI(path)
path = uri.path
except InvalidURIError:
# The URL is not valid, so we can't extract a path. Let it
# pass through, where it will probably be skipped when no
# download key can be determined.
pass
return method, path
def _init_link_data(self):
if self._initialized:
return
self._initialized = True
self.initialize()
self._check_links()
links_set = set(self.links)
assert not links_set.intersection(self._forbiddenlinknames), (
"The following names may not be links: %s" %
', '.join(self._forbiddenlinknames))
if isinstance(self.context, LaunchpadView):
# It's a navigation menu for a view instead of a db object. Views
# don't have a canonical URL, they use the db object one used as
# the context for that view.
context = self.context.context
else:
context = self.context
self._contexturlobj = URI(canonical_url(context))
if self.enable_only is ALL_LINKS:
self._enable_only_set = links_set
else:
self._enable_only_set = set(self.enable_only)
unknown_links = self._enable_only_set - links_set
if len(unknown_links) > 0:
# There are links named in enable_only that do not exist in
# self.links.
raise AssertionError(
"Links in 'enable_only' not found in 'links': %s" %
', '.join(sorted(unknown_links)))
def make_bugtracker_name(uri):
"""Return a name string for a bug tracker based on a URI.
:param uri: The base URI to be used to identify the bug tracker,
e.g. http://bugs.example.com or mailto:[email protected]
"""
base_uri = URI(uri)
if base_uri.scheme == 'mailto':
if valid_email(base_uri.path):
base_name = base_uri.path.split('@', 1)[0]
else:
raise AssertionError(
'Not a valid email address: %s' % base_uri.path)
elif base_uri.host == 'github.com' and base_uri.path.endswith('/issues'):
repository_id = base_uri.path[:-len('/issues')].lstrip('/')
base_name = 'github-' + repository_id.replace('/', '-').lower()
elif (('gitlab' in base_uri.host or
base_uri.host == 'salsa.debian.org') and
base_uri.path.endswith('/issues')):
repository_id = base_uri.path[:-len('/issues')].lstrip('/')
base_name = '%s-%s' % (
base_uri.host, repository_id.replace('/', '-').lower())
else:
base_name = base_uri.host
return 'auto-%s' % sanitize_name(base_name)
def initLink(self, linkname, request_url=None):
self._init_link_data()
link = self._get_link(linkname)
link.name = linkname
# Set the .enabled attribute of the link to False if it is not
# in enable_only.
if linkname not in self._enable_only_set:
link.enabled = False
# Set the .url attribute of the link, using the menu's context.
if link.site is None:
rootsite = self._contexturlobj.resolve('/')
else:
rootsite = self._rootUrlForSite(link.site)
# Is the target a full URI already?
try:
link.url = URI(link.target)
except InvalidURIError:
if link.target.startswith('/'):
link.url = rootsite.resolve(link.target)
else:
link.url = rootsite.resolve(self._contexturlobj.path).append(
link.target)
# Make the link unlinked if it is a link to the current page.
if request_url is not None:
if request_url.ensureSlash() == link.url.ensureSlash():
link.linked = False
idx = self.links.index(linkname)
link.sort_key = idx
return link
def __init__(self, baseurl):
"""Create a new Roundup instance.
:baseurl: The starting URL for accessing the remote Roundup
bug tracker.
The fields/columns to fetch from the remote bug tracker are
derived based on the host part of the baseurl.
"""
super(Roundup, self).__init__(baseurl)
self.host = URI(self.baseurl).host
self._status_fields = (self._status_fields_map.get(
self.host, ('status', )))
fields = ('title', 'id', 'activity') + self._status_fields
# Roundup is quite particular about URLs, so although several
# of the parameters below seem redundant or irrelevant, they
# are needed for compatibility with the broadest range of
# Roundup instances in the wild. Test before changing them!
self.query_base = [
("@action", "export_csv"),
("@columns", ",".join(fields)),
("@sort", "id"),
("@group", "priority"),
("@filter", "id"),
("@pagesize", "50"),
("@startwith", "0"),
]
def setRequestId(self, request, id):
"""As per CookieClientIdManager.setRequestID, except
we force the domain key on the cookie to be set to allow our
session to be shared between virtual hosts where possible, and
we set the secure key to stop the session key being sent to
insecure URLs like the Librarian.
We also log the referrer url on creation of a new
requestid so we can track where first time users arrive from.
"""
CookieClientIdManager.setRequestId(self, request, id)
cookie = request.response.getCookie(self.namespace)
uri = URI(request.getURL())
# Forbid browsers from exposing it to JS.
cookie['HttpOnly'] = True
# Set secure flag on cookie.
if uri.scheme != 'http':
cookie['secure'] = True
else:
cookie['secure'] = False
# Set domain attribute on cookie if vhosting requires it.
cookie_domain = get_cookie_domain(uri.host)
if cookie_domain is not None:
cookie['domain'] = cookie_domain
def _get_binary_sources_list_line(archive, distroarchseries, pocket,
components):
"""Return the correponding binary sources_list line."""
# Encode the private PPA repository password in the
# sources_list line. Note that the buildlog will be
# sanitized to not expose it.
if archive.private:
uri = URI(archive.archive_url)
uri = uri.replace(
userinfo="buildd:%s" % archive.buildd_secret)
url = str(uri)
else:
url = archive.archive_url
suite = distroarchseries.distroseries.name + pocketsuffix[pocket]
return 'deb %s %s %s' % (url, suite, ' '.join(components))
def _validate(self, value):
"""Ensure the value is a valid URI."""
uri = URI(self.normalize(value))
if self.allowed_schemes and uri.scheme not in self.allowed_schemes:
raise LaunchpadValidationError(
'The URI scheme "%s" is not allowed. Only URIs with '
'the following schemes may be used: %s'
% (uri.scheme, ', '.join(sorted(self.allowed_schemes))))
if not self.allow_userinfo and uri.userinfo is not None:
raise LaunchpadValidationError(
'A username may not be specified in the URI.')
if not self.allow_port and uri.port is not None:
raise LaunchpadValidationError(
'Non-default ports are not allowed.')
if not self.allow_query and uri.query is not None:
raise LaunchpadValidationError(
'URIs with query strings are not allowed.')
if not self.allow_fragment and uri.fragment is not None:
raise LaunchpadValidationError(
'URIs with fragment identifiers are not allowed.')
super(URIField, self)._validate(value)
def getByUrl(self, url):
"""See `IBranchLookup`."""
if url is None:
return None
url = url.rstrip('/')
try:
uri = URI(url)
except InvalidURIError:
return None
path = self.uriToHostingPath(uri)
if path is not None:
branch, trailing = self.getByHostingPath(path)
if branch is not None:
return branch
if uri.scheme == 'lp':
if not self._uriHostAllowed(uri):
return None
try:
return self.getByLPPath(uri.path.lstrip('/'))[0]
except (CannotHaveLinkedBranch, InvalidNamespace,
InvalidProductName, NoSuchBranch, NoSuchPerson,
NoSuchProduct, NoSuchProductSeries, NoSuchDistroSeries,
NoSuchSourcePackageName, NoLinkedBranch):
return None
return Branch.selectOneBy(url=url)
def __init__(self, authorizer, service_root, cache=None,
timeout=None, proxy_info=None, version=None,
base_client_name='', max_retries=Browser.MAX_RETRIES):
"""Root access to a lazr.restful API.
:param credentials: The credentials used to access the service.
:param service_root: The URL to the root of the web service.
:type service_root: string
"""
if version is not None:
if service_root[-1] != '/':
service_root += '/'
service_root += str(version)
if service_root[-1] != '/':
service_root += '/'
self._root_uri = URI(service_root)
# Set up data necessary to calculate the User-Agent header.
self._base_client_name = base_client_name
# Get the WADL definition.
self.credentials = authorizer
self._browser = Browser(
self, authorizer, cache, timeout, proxy_info, self._user_agent,
max_retries)
self._wadl = self._browser.get_wadl_application(self._root_uri)
# Get the root resource.
root_resource = self._wadl.get_resource_by_path('')
bound_root = root_resource.bind(
self._browser.get(root_resource), 'application/json')
super(ServiceRoot, self).__init__(None, bound_root)
def test_normal_resolution(self):
# Normal URI resolution examples from Section 5.4.1 of RFC 3986:
base = URI('http://a/b/c/d;p?q')
def resolve(relative):
return str(base.resolve(relative))
self.assertEqual(resolve('g:h'), 'g:h')
self.assertEqual(resolve('g'), 'http://a/b/c/g')
self.assertEqual(resolve('./g'), 'http://a/b/c/g')
self.assertEqual(resolve('g/'), 'http://a/b/c/g/')
self.assertEqual(resolve('/g'), 'http://a/g')
# The extra slash here comes from normalisation:
self.assertEqual(resolve('//g'), 'http://g/')
self.assertEqual(resolve('?y'), 'http://a/b/c/d;p?y')
self.assertEqual(resolve('g?y'), 'http://a/b/c/g?y')
self.assertEqual(resolve('#s'), 'http://a/b/c/d;p?q#s')
self.assertEqual(resolve('g#s'), 'http://a/b/c/g#s')
self.assertEqual(resolve('g?y#s'), 'http://a/b/c/g?y#s')
self.assertEqual(resolve(';x'), 'http://a/b/c/;x')
self.assertEqual(resolve('g;x'), 'http://a/b/c/g;x')
self.assertEqual(resolve('g;x?y#s'), 'http://a/b/c/g;x?y#s')
self.assertEqual(resolve(''), 'http://a/b/c/d;p?q')
self.assertEqual(resolve('.'), 'http://a/b/c/')
self.assertEqual(resolve('./'), 'http://a/b/c/')
self.assertEqual(resolve('..'), 'http://a/b/')
self.assertEqual(resolve('../'), 'http://a/b/')
self.assertEqual(resolve('../g'), 'http://a/b/g')
self.assertEqual(resolve('../..'), 'http://a/')
self.assertEqual(resolve('../../'), 'http://a/')
self.assertEqual(resolve('../../g'), 'http://a/g')
def makeBranchFromURL(self, url):
"""Make a mirrored branch for `url`.
The product and owner of the branch are derived from information in
the launchbag. The name of the branch is derived from the last segment
of the URL and is guaranteed to be unique for the product.
:param url: The URL to mirror.
:return: An `IBranch`.
"""
# XXX: JonathanLange 2008-12-08 spec=package-branches: This method
# needs to be rewritten to get the sourcepackage and distroseries out
# of the launch bag.
url = unicode(URI(url).ensureNoSlash())
if getUtility(IBranchLookup).getByUrl(url) is not None:
raise AlreadyRegisteredError('Already a branch for %r' % url)
# Make sure the URL is valid.
IBranch['url'].validate(url)
product = self.getProduct()
if product is None:
raise NoProductError("Could not find product in LaunchBag.")
owner = self.getPerson()
name = self.getBranchNameFromURL(url)
namespace = get_branch_namespace(person=owner, product=product)
branch = namespace.createBranchWithPrefix(BranchType.MIRRORED,
name,
owner,
url=url)
branch.requestMirror()
self.request.response.addNotification(
structured('Registered %s' %
BranchFormatterAPI(branch).link(None)))
return branch
def assertResolves(self, lp_url_path, public_branch_path, lp_path=None):
"""Assert that `lp_url_path` resolves to the specified paths.
:param public_branch_path: The path that is accessible over http.
:param lp_path: The short branch alias that will be resolved over
bzr+ssh. The branch alias prefix is prefixed to this path.
If it is not set, the bzr+ssh resolved name will be checked
against the public_branch_path instead.
"""
api = PublicCodehostingAPI(None, None)
results = api.resolve_lp_path(lp_url_path)
if lp_path is None:
ssh_branch_path = public_branch_path
else:
if lp_path.startswith('~'):
ssh_branch_path = lp_path
else:
ssh_branch_path = '%s/%s' % (BRANCH_ALIAS_PREFIX, lp_path)
# This improves the error message if results happens to be a fault.
if isinstance(results, LaunchpadFault):
raise results
for url in results['urls']:
uri = URI(url)
if uri.scheme == 'http':
self.assertEqual('/' + public_branch_path, uri.path)
else:
self.assertEqual('/' + ssh_branch_path, uri.path)
def lp_save(self):
"""Save changes to the entry."""
representation = self._transform_resources_to_links(
self._dirty_attributes)
# If the entry contains an ETag, set the If-Match header
# to that value.
headers = {}
etag = getattr(self, 'http_etag', None)
if etag is not None:
headers['If-Match'] = etag
# PATCH the new representation to the 'self' link. It's possible that
# this will cause the object to be permanently moved. Catch that
# exception and refresh our representation.
response, content = self._root._browser.patch(URI(self.self_link),
representation, headers)
if response.status == 301:
self.lp_refresh(response['location'])
self._dirty_attributes.clear()
content_type = response['content-type']
if response.status == 209 and content_type == self.JSON_MEDIA_TYPE:
# The server sent back a new representation of the object.
# Use it in preference to the existing representation.
new_representation = simplejson.loads(unicode(content))
self._wadl_resource.representation = new_representation
self._wadl_resource.media_type = content_type
def getBranchNameFromURL(self, url):
"""Return a branch name based on `url`.
The name is based on the last path segment of the URL. If there is
already another branch of that name on the product, then we'll try to
find a unique name by appending numbers.
"""
return URI(url).ensureNoSlash().path.split('/')[-1]
def checkOneURL(self, url):
"""See `BranchOpenPolicy.checkOneURL`.
We refuse to mirror from Launchpad or a ssh-like or file URL.
"""
try:
uri = URI(url)
except InvalidURIError:
raise BadUrl(url)
launchpad_domain = config.vhost.mainsite.hostname
if uri.underDomain(launchpad_domain):
raise BadUrl(url)
for hostname in get_blacklisted_hostnames():
if uri.underDomain(hostname):
raise BadUrl(url)
if uri.scheme not in self.allowed_schemes:
raise BadUrl(url)
def get_token_and_login(cls, consumer_name,
service_root=uris.STAGING_SERVICE_ROOT,
cache=None, timeout=None, proxy_info=None,
authorizer_class=AuthorizeRequestTokenWithBrowser,
allow_access_levels=[], max_failed_attempts=3,
version=DEFAULT_VERSION):
"""Get credentials from Launchpad and log into the service root.
This is a convenience method which will open up the user's preferred
web browser and thus should not be used by most applications.
Applications should, instead, use Credentials.get_request_token() to
obtain the authorization URL and
Credentials.exchange_request_token_for_access_token() to obtain the
actual OAuth access token.
This method will negotiate an OAuth access token with the service
provider, but to complete it we will need the user to log into
Launchpad and authorize us, so we'll open the authorization page in
a web browser and ask the user to come back here and tell us when they
finished the authorization process.
:param consumer_name: The consumer name, as appropriate for the
`Consumer` constructor
:type consumer_name: string
:param service_root: The URL to the root of the web service.
:type service_root: string
:return: The web service root
:rtype: `Launchpad`
"""
credentials = Credentials(consumer_name)
service_root = uris.lookup_service_root(service_root)
web_root_uri = URI(service_root)
web_root_uri.path = ""
web_root_uri.host = web_root_uri.host.replace("api.", "", 1)
web_root = str(web_root_uri.ensureSlash())
authorization_json = credentials.get_request_token(
web_root=web_root, token_format=Credentials.DICT_TOKEN_FORMAT)
authorizer = authorizer_class(
web_root, authorization_json['oauth_token_consumer'],
authorization_json['oauth_token'], allow_access_levels,
max_failed_attempts)
authorizer()
credentials.exchange_request_token_for_access_token(web_root)
return cls(credentials, service_root, cache, timeout, proxy_info,
version)
def normalize(self, input):
"""See `IURIField`."""
if input is None:
return input
try:
uri = URI(input)
except InvalidURIError as exc:
raise LaunchpadValidationError(str(exc))
# If there is a policy for whether trailing slashes are
# allowed at the end of the path segment, ensure that the
# URI conforms.
if self.trailing_slash is not None:
if self.trailing_slash:
uri = uri.ensureSlash()
else:
uri = uri.ensureNoSlash()
input = unicode(uri)
return input
def test_uriToHostingPath(self):
"""Ensure uriToHostingPath works.
Only codehosting-based using http, sftp or bzr+ssh URLs will
be handled. If any other URL gets passed (including lp), the return
value will be None.
"""
branch_set = getUtility(IBranchLookup)
uri = URI(config.codehosting.supermirror_root)
uri.path = '/~foo/bar/baz'
# Test valid schemes
uri.scheme = 'http'
self.assertEqual('~foo/bar/baz', branch_set.uriToHostingPath(uri))
uri.scheme = 'sftp'
self.assertEqual('~foo/bar/baz', branch_set.uriToHostingPath(uri))
uri.scheme = 'bzr+ssh'
self.assertEqual('~foo/bar/baz', branch_set.uriToHostingPath(uri))
# Test invalid scheme
uri.scheme = 'ftp'
self.assertIs(None, branch_set.uriToHostingPath(uri))
# Test valid scheme, invalid domain
uri.scheme = 'sftp'
uri.host = 'example.com'
self.assertIs(None, branch_set.uriToHostingPath(uri))
class ServiceRoot(Resource):
"""Entry point to the service. Subclass this for a service-specific client.
:ivar credentials: The credentials instance used to access Launchpad.
"""
# Custom subclasses of Resource to use when
# instantiating resources of a certain WADL type.
RESOURCE_TYPE_CLASSES = {'HostedFile': HostedFile,
'ScalarValue': ScalarValue}
def __init__(self, authorizer, service_root, cache=None,
timeout=None, proxy_info=None, version=None,
base_client_name='', max_retries=Browser.MAX_RETRIES):
"""Root access to a lazr.restful API.
:param credentials: The credentials used to access the service.
:param service_root: The URL to the root of the web service.
:type service_root: string
"""
if version is not None:
if service_root[-1] != '/':
service_root += '/'
service_root += str(version)
if service_root[-1] != '/':
service_root += '/'
self._root_uri = URI(service_root)
# Set up data necessary to calculate the User-Agent header.
self._base_client_name = base_client_name
# Get the WADL definition.
self.credentials = authorizer
self._browser = Browser(
self, authorizer, cache, timeout, proxy_info, self._user_agent,
max_retries)
self._wadl = self._browser.get_wadl_application(self._root_uri)
# Get the root resource.
root_resource = self._wadl.get_resource_by_path('')
bound_root = root_resource.bind(
self._browser.get(root_resource), 'application/json')
super(ServiceRoot, self).__init__(None, bound_root)
@property
def _user_agent(self):
"""The value for the User-Agent header.
This will be something like:
launchpadlib 1.6.1, lazr.restfulclient 1.0.0; application=apport
That is, a string describing lazr.restfulclient and an
optional custom client built on top, and parameters containing
any authorization-specific information that identifies the
user agent (such as the application name).
"""
base_portion = "lazr.restfulclient %s" % __version__
if self._base_client_name != '':
base_portion = self._base_client_name + ' (' + base_portion + ')'
message = Message()
message['User-Agent'] = base_portion
if self.credentials is not None:
user_agent_params = self.credentials.user_agent_params
for key in sorted(user_agent_params):
value = user_agent_params[key]
message.set_param(key, value, 'User-Agent')
return message['User-Agent']
def httpFactory(self, authorizer, cache, timeout, proxy_info):
return RestfulHttp(authorizer, cache, timeout, proxy_info)
def load(self, url):
"""Load a resource given its URL."""
parsed = urlparse(url)
if parsed.scheme == '':
# This is a relative URL. Make it absolute by joining
# it with the service root resource.
if url[:1] == '/':
url = url[1:]
url = self._root_uri.append(url)
document = self._browser.get(url)
try:
representation = simplejson.loads(unicode(document))
except ValueError:
raise ValueError("%s doesn't serve a JSON document." % url)
type_link = representation.get("resource_type_link")
if type_link is None:
raise ValueError("Couldn't determine the resource type of %s."
% url)
resource_type = self._root._wadl.get_resource_type(type_link)
wadl_resource = WadlResource(self._root._wadl, url, resource_type.tag)
return self._create_bound_resource(
self._root, wadl_resource, representation, 'application/json',
representation_needs_processing=False)
def uri_path_replace(url, old, new):
"""Replace a substring of a URL's path."""
parsed = URI(url)
return str(parsed.replace(path=parsed.path.replace(old, new)))
def archive_url(self):
"""Return a custom archive url for basic authentication."""
normal_url = URI(self.archive.archive_url)
auth_url = normal_url.replace(
userinfo="%s:%s" %(self.person.name, self.token))
return str(auth_url)
class MenuBase(UserAttributeCache):
"""Base class for facets and menus."""
implements(IMenuBase)
links = None
extra_attributes = None
enable_only = ALL_LINKS
_baseclassname = 'MenuBase'
_initialized = False
_forbiddenlinknames = set(
['user', 'initialize', 'links', 'enable_only', 'iterlinks',
'initLink', 'updateLink', 'extra_attributes'])
def __init__(self, context):
# The attribute self.context is defined in IMenuBase.
self.context = context
self.request = None
def initialize(self):
"""Override this in subclasses to do initialization."""
pass
def _check_links(self):
assert self.links is not None, (
'Subclasses of %s must provide self.links' % self._baseclassname)
assert isinstance(self.links, (tuple, list)), (
"self.links must be a tuple or list.")
def _buildLink(self, name):
method = getattr(self, name, None)
# Since Zope traversals hides the root cause of an AttributeError,
# an AssertionError is raised explaining what went wrong.
if method is None:
raise AssertionError(
'%r does not define %r method.' % (self, name))
linkdata = method()
# The link need only provide ILinkData. We need an ILink so that
# we can set attributes on it like 'name' and 'url' and 'linked'.
return ILink(linkdata)
def _get_link(self, name):
request = get_current_browser_request()
if request is not None:
# We must not use a weak ref here because if we do so and
# templates do stuff like "context/menu:bugs/foo", then there
# would be no reference to the Link object, which would allow it
# to be garbage collected during the course of the request.
cache = request.annotations.setdefault(MENU_ANNOTATION_KEY, {})
key = (self.__class__, self.context, name)
link = cache.get(key)
if link is None:
link = self._buildLink(name)
cache[key] = link
return link
return self._buildLink(name)
def _rootUrlForSite(self, site):
"""Return the root URL for the given site."""
try:
return URI(allvhosts.configs[site].rooturl)
except KeyError:
raise AssertionError('unknown site', site)
def _init_link_data(self):
if self._initialized:
return
self._initialized = True
self.initialize()
self._check_links()
links_set = set(self.links)
assert not links_set.intersection(self._forbiddenlinknames), (
"The following names may not be links: %s" %
', '.join(self._forbiddenlinknames))
if isinstance(self.context, LaunchpadView):
# It's a navigation menu for a view instead of a db object. Views
# don't have a canonical URL, they use the db object one used as
# the context for that view.
context = self.context.context
else:
context = self.context
self._contexturlobj = URI(canonical_url(context))
if self.enable_only is ALL_LINKS:
self._enable_only_set = links_set
else:
self._enable_only_set = set(self.enable_only)
unknown_links = self._enable_only_set - links_set
if len(unknown_links) > 0:
# There are links named in enable_only that do not exist in
# self.links.
raise AssertionError(
"Links in 'enable_only' not found in 'links': %s" %
', '.join(sorted(unknown_links)))
def initLink(self, linkname, request_url=None):
self._init_link_data()
link = self._get_link(linkname)
link.name = linkname
# Set the .enabled attribute of the link to False if it is not
# in enable_only.
if linkname not in self._enable_only_set:
link.enabled = False
# Set the .url attribute of the link, using the menu's context.
if link.site is None:
rootsite = self._contexturlobj.resolve('/')
else:
rootsite = self._rootUrlForSite(link.site)
# Is the target a full URI already?
try:
link.url = URI(link.target)
except InvalidURIError:
if link.target.startswith('/'):
link.url = rootsite.resolve(link.target)
else:
link.url = rootsite.resolve(self._contexturlobj.path).append(
link.target)
# Make the link unlinked if it is a link to the current page.
if request_url is not None:
if request_url.ensureSlash() == link.url.ensureSlash():
link.linked = False
idx = self.links.index(linkname)
link.sort_key = idx
return link
def updateLink(self, link, request_url, **kwargs):
"""Called each time a link is rendered.
Override to update the link state as required for the given request.
"""
pass
def iterlinks(self, request_url=None, **kwargs):
"""See IMenu."""
self._check_links()
for linkname in self.links:
link = self.initLink(linkname, request_url)
self.updateLink(link, request_url, **kwargs)
yield link