def test_unescape(self):
self.assertEqual('%', urlutils.unescape('%25'))
self.assertEqual(u'\xe5', urlutils.unescape('%C3%A5'))
self.assertRaises(InvalidURL, urlutils.unescape, u'\xe5')
self.assertRaises(InvalidURL, urlutils.unescape, '\xe5')
self.assertRaises(InvalidURL, urlutils.unescape, '%E5')
def test_unescape(self):
self.assertEqual('%', urlutils.unescape('%25'))
self.assertEqual(u'\xe5', urlutils.unescape('%C3%A5'))
self.assertRaises(InvalidURL, urlutils.unescape, u'\xe5')
self.assertRaises(InvalidURL, urlutils.unescape, '\xe5')
self.assertRaises(InvalidURL, urlutils.unescape, '%E5')
def translate_client_path(self, relpath):
# VFS requests are made with escaped paths so the escaping done in
# SmartServerRequest.translate_client_path leads to double escaping.
# Remove it here -- the fact that the result is still escaped means
# that the str() will not fail on valid input.
x = request.SmartServerRequest.translate_client_path(self, relpath)
return str(urlutils.unescape(x))
def translate_client_path(self, relpath):
# VFS requests are made with escaped paths so the escaping done in
# SmartServerRequest.translate_client_path leads to double escaping.
# Remove it here -- the fact that the result is still escaped means
# that the str() will not fail on valid input.
x = request.SmartServerRequest.translate_client_path(self, relpath)
return str(urlutils.unescape(x))
def _remote_path(self, relpath):
"""See ConnectedTransport._remote_path.
user and passwords are not embedded in the path provided to the server.
"""
relative = urlutils.unescape(relpath).encode('utf-8')
path = self._combine_paths(self._path, relative)
return self._unsplit_url(self._unqualified_scheme, None, None,
self._host, self._port, path)
def translate_path(requester):
if not path.startswith('/'):
return faults.InvalidPath(path)
stripped_path = unescape(path.strip('/'))
lookup = lambda l: self.performLookup(requester_id, path, l)
result = get_first_path_result(stripped_path, lookup, None)
if result is None:
raise faults.PathTranslationError(path)
return result
def translatePath(self, requester_id, path):
if not path.startswith('/'):
return faults.InvalidPath(path)
result = get_first_path_result(unescape(path.strip('/')),
lambda l: self.performLookup(requester_id, l), None)
if result is not None:
return result
else:
return faults.PathTranslationError(path)
def translate_path(requester):
if not path.startswith('/'):
return faults.InvalidPath(path)
stripped_path = unescape(path.strip('/'))
lookup = lambda l: self.performLookup(requester_id, path, l)
result = get_first_path_result(stripped_path, lookup, None)
if result is None:
raise faults.PathTranslationError(path)
return result
def _remote_path(self, relpath):
# XXX: It seems that ftplib does not handle Unicode paths
# at the same time, medusa won't handle utf8 paths So if
# we .encode(utf8) here (see ConnectedTransport
# implementation), then we get a Server failure. while
# if we use str(), we get a UnicodeError, and the test
# suite just skips testing UnicodePaths.
relative = str(urlutils.unescape(relpath))
remote_path = self._combine_paths(self._path, relative)
return remote_path
def createBranch(self, requester_id, branch_path):
if not branch_path.startswith('/'):
return faults.InvalidPath(branch_path)
escaped_path = unescape(branch_path.strip('/'))
registrant = self._person_set.get(requester_id)
try:
return self._createBranch(registrant, escaped_path)
except LaunchpadFault as e:
return e
except LaunchpadValidationError as e:
return faults.PermissionDenied(six.ensure_binary(e.args[0]))
def createBranch(self, requester_id, branch_path):
if not branch_path.startswith('/'):
return faults.InvalidPath(branch_path)
escaped_path = unescape(branch_path.strip('/'))
registrant = self._person_set.get(requester_id)
try:
return self._createBranch(registrant, escaped_path)
except LaunchpadFault as e:
return e
except LaunchpadValidationError as e:
msg = e.args[0]
if isinstance(msg, unicode):
msg = msg.encode('utf-8')
return faults.PermissionDenied(msg)
def create_branch(requester):
if not branch_path.startswith('/'):
return faults.InvalidPath(branch_path)
escaped_path = unescape(branch_path.strip('/'))
try:
namespace, branch_name, link_func, path = (
self._getBranchNamespaceExtras(escaped_path, requester))
except ValueError:
return faults.PermissionDenied(
"Cannot create branch at '%s'" % branch_path)
except InvalidNamespace:
return faults.PermissionDenied(
"Cannot create branch at '%s'" % branch_path)
except NoSuchPerson as e:
return faults.NotFound(
"User/team '%s' does not exist." % e.name)
except NoSuchProduct as e:
return faults.NotFound(
"Project '%s' does not exist." % e.name)
except InvalidProductName as e:
return faults.InvalidProductName(escape(e.name))
except NoSuchSourcePackageName as e:
try:
getUtility(ISourcePackageNameSet).new(e.name)
except InvalidName:
return faults.InvalidSourcePackageName(e.name)
return self.createBranch(login_id, branch_path)
except NameLookupFailed as e:
return faults.NotFound(str(e))
try:
branch = namespace.createBranch(
BranchType.HOSTED, branch_name, requester)
except LaunchpadValidationError as e:
msg = e.args[0]
if isinstance(msg, unicode):
msg = msg.encode('utf-8')
return faults.PermissionDenied(msg)
except BranchCreationException as e:
return faults.PermissionDenied(str(e))
if link_func:
try:
link_func(branch)
except Unauthorized:
# We don't want to keep the branch we created.
transaction.abort()
return faults.PermissionDenied(
"Cannot create linked branch at '%s'." % path)
return branch.id
def create_branch(requester):
if not branch_path.startswith('/'):
return faults.InvalidPath(branch_path)
escaped_path = unescape(branch_path.strip('/'))
try:
namespace, branch_name, link_func, path = (
self._getBranchNamespaceExtras(escaped_path, requester))
except ValueError:
return faults.PermissionDenied("Cannot create branch at '%s'" %
branch_path)
except InvalidNamespace:
return faults.PermissionDenied("Cannot create branch at '%s'" %
branch_path)
except NoSuchPerson as e:
return faults.NotFound("User/team '%s' does not exist." %
e.name)
except NoSuchProduct as e:
return faults.NotFound("Project '%s' does not exist." % e.name)
except InvalidProductName as e:
return faults.InvalidProductName(escape(e.name))
except NoSuchSourcePackageName as e:
try:
getUtility(ISourcePackageNameSet).new(e.name)
except InvalidName:
return faults.InvalidSourcePackageName(e.name)
return self.createBranch(login_id, branch_path)
except NameLookupFailed as e:
return faults.NotFound(str(e))
try:
branch = namespace.createBranch(BranchType.HOSTED, branch_name,
requester)
except LaunchpadValidationError as e:
msg = e.args[0]
if isinstance(msg, unicode):
msg = msg.encode('utf-8')
return faults.PermissionDenied(msg)
except BranchCreationException as e:
return faults.PermissionDenied(str(e))
if link_func:
try:
link_func(branch)
except Unauthorized:
# We don't want to keep the branch we created.
transaction.abort()
return faults.PermissionDenied(
"Cannot create linked branch at '%s'." % path)
return branch.id
def _abspath(self, relative_reference):
"""Return a path for use in os calls.
Several assumptions are made:
- relative_reference does not contain '..'
- relative_reference is url escaped.
"""
relative_reference = self._anvilise_path(relative_reference)
if relative_reference in ('.', ''):
# _local_base normally has a trailing slash; strip it so that stat
# on a transport pointing to a symlink reads the link not the
# referent but be careful of / and c:\
return osutils.split(self._local_base)[0]
#return self._local_base + urlutils.unescape(relative_reference)
return urlutils.unescape(relative_reference)
def _format_directory_entries(self, stat_results, filenames):
"""Produce entries suitable for returning from `openDirectory`.
:param stat_results: A list of the results of calling `stat` on each
file in filenames.
:param filenames: The list of filenames to produce entries for.
:return: An iterator of ``(shortname, longname, attributes)``.
"""
for stat_result, filename in zip(stat_results, filenames):
shortname = urlutils.unescape(filename).encode('utf-8')
stat_result = copy(stat_result)
for attribute in ['st_uid', 'st_gid', 'st_mtime', 'st_nlink']:
if getattr(stat_result, attribute, None) is None:
setattr(stat_result, attribute, 0)
longname = lsLine(shortname, stat_result)
attr_dict = self._translate_stat(stat_result)
yield (shortname, longname, attr_dict)
def _remote_path(self, relpath):
"""Return the path to be passed along the sftp protocol for relpath.
:param relpath: is a urlencoded string.
"""
relative = urlutils.unescape(relpath).encode('utf-8')
remote_path = self._combine_paths(self._path, relative)
# the initial slash should be removed from the path, and treated as a
# homedir relative path (the path begins with a double slash if it is
# absolute). see draft-ietf-secsh-scp-sftp-ssh-uri-03.txt
# RBC 20060118 we are not using this as its too user hostile. instead
# we are following lftp and using /~/foo to mean '~/foo'
# vila--20070602 and leave absolute paths begin with a single slash.
if remote_path.startswith('/~/'):
remote_path = remote_path[3:]
elif remote_path == '/~':
remote_path = ''
return remote_path
def _abspath(self, relpath):
"""Generate an internal absolute path."""
relpath = urlutils.unescape(relpath)
if relpath[:1] == '/':
return relpath
cwd_parts = self._cwd.split('/')
rel_parts = relpath.split('/')
r = []
for i in cwd_parts + rel_parts:
if i == '..':
if not r:
raise ValueError("illegal relpath %r under %r" %
(relpath, self._cwd))
r = r[:-1]
elif i == '.' or i == '':
pass
else:
r.append(i)
return '/' + '/'.join(r)
def _abspath(self, relpath):
"""Generate an internal absolute path."""
relpath = urlutils.unescape(relpath)
if relpath[:1] == '/':
return relpath
cwd_parts = self._cwd.split('/')
rel_parts = relpath.split('/')
r = []
for i in cwd_parts + rel_parts:
if i == '..':
if not r:
raise ValueError("illegal relpath %r under %r"
% (relpath, self._cwd))
r = r[:-1]
elif i == '.' or i == '':
pass
else:
r.append(i)
return '/' + '/'.join(r)
def abspath(self, relpath):
"""Return the full url to the given relative URL."""
# TODO: url escape the result. RBC 20060523.
# jam 20060426 Using normpath on the real path, because that ensures
# proper handling of stuff like
relpath = self._anvilise_path(relpath)
path = osutils.normpath(osutils.pathjoin(
self._local_base, urlutils.unescape(relpath)))
# on windows, our _local_base may or may not have a drive specified
# (ie, it may be "/" or "c:/foo").
# If 'relpath' is '/' we *always* get back an abspath without
# the drive letter - but if our transport already has a drive letter,
# we want our abspaths to have a drive letter too - so handle that
# here.
if (sys.platform == "win32" and self._local_base[1:2] == ":"
and path == '/'):
path = self._local_base[:3]
return urlutils.local_path_to_url(path)
def _makeControlTransport(self, default_stack_on, trailing_path=None):
"""Make a transport that points to a control directory.
A control directory is a .bzr directory containing a 'control.conf'
file. This is used to specify configuration for branches created
underneath the directory that contains the control directory.
:param default_stack_on: The default stacked-on branch URL for
branches that respect this control directory. If empty, then
we'll return an empty memory transport.
:return: A read-only `MemoryTransport` containing a working BzrDir,
configured to use the given default stacked-on location.
"""
memory_server = MemoryServer()
memory_server.start_server()
transport = get_transport(memory_server.get_url())
if default_stack_on == "":
return transport
format = BzrDirFormat.get_default_format()
bzrdir = format.initialize_on_transport(transport)
bzrdir.get_config().set_default_stack_on(urlutils.unescape(default_stack_on))
return get_readonly_transport(transport)
def _makeControlTransport(self, default_stack_on, trailing_path=None):
"""Make a transport that points to a control directory.
A control directory is a .bzr directory containing a 'control.conf'
file. This is used to specify configuration for branches created
underneath the directory that contains the control directory.
:param default_stack_on: The default stacked-on branch URL for
branches that respect this control directory. If empty, then
we'll return an empty memory transport.
:return: A read-only `MemoryTransport` containing a working BzrDir,
configured to use the given default stacked-on location.
"""
memory_server = MemoryServer()
memory_server.start_server()
transport = get_transport(memory_server.get_url())
if default_stack_on == '':
return transport
format = BzrDirFormat.get_default_format()
bzrdir = format.initialize_on_transport(transport)
bzrdir.get_config().set_default_stack_on(
urlutils.unescape(default_stack_on))
return get_readonly_transport(transport)
def __call__(self, environ, start_response):
request_is_private = (environ['SERVER_PORT'] == str(
config.codebrowse.private_port))
environ['loggerhead.static.url'] = environ['SCRIPT_NAME']
if environ['PATH_INFO'].startswith('/static/'):
path_info_pop(environ)
return static_app(environ, start_response)
elif environ['PATH_INFO'] == '/favicon.ico':
return favicon_app(environ, start_response)
elif environ['PATH_INFO'] == '/robots.txt':
return robots_app(environ, start_response)
elif not request_is_private:
if environ['PATH_INFO'].startswith('/+login'):
return self._complete_login(environ, start_response)
elif environ['PATH_INFO'].startswith('/+logout'):
return self._logout(environ, start_response)
path = environ['PATH_INFO']
trailingSlashCount = len(path) - len(path.rstrip('/'))
if request_is_private:
# Requests on the private port are internal API requests from
# something that has already performed security checks. As
# such, they get read-only access to everything.
identity_url = LAUNCHPAD_SERVICES
user = LAUNCHPAD_SERVICES
else:
identity_url = environ[self.session_var].get(
'identity_url', LAUNCHPAD_ANONYMOUS)
user = environ[self.session_var].get('user', LAUNCHPAD_ANONYMOUS)
lp_server = get_lp_server(identity_url,
branch_transport=self.get_transport())
lp_server.start_server()
try:
try:
branchfs = self.get_branchfs()
transport_type, info, trail = branchfs.translatePath(
identity_url, urlutils.escape(path))
except xmlrpclib.Fault as f:
if check_fault(f, faults.PathTranslationError):
raise HTTPNotFound()
elif check_fault(f, faults.PermissionDenied):
# If we're not allowed to see the branch...
if environ['wsgi.url_scheme'] != 'https':
# ... the request shouldn't have come in over http, as
# requests for private branches over http should be
# redirected to https by the dynamic rewrite script we
# use (which runs before this code is reached), but
# just in case...
env_copy = environ.copy()
env_copy['wsgi.url_scheme'] = 'https'
raise HTTPMovedPermanently(construct_url(env_copy))
elif user != LAUNCHPAD_ANONYMOUS:
# ... if the user is already logged in and still can't
# see the branch, they lose.
exc = HTTPUnauthorized()
exc.explanation = "You are logged in as %s." % user
raise exc
else:
# ... otherwise, lets give them a chance to log in
# with OpenID.
return self._begin_login(environ, start_response)
else:
raise
if transport_type != BRANCH_TRANSPORT:
raise HTTPNotFound()
trail = urlutils.unescape(trail).encode('utf-8')
trail += trailingSlashCount * '/'
amount_consumed = len(path) - len(trail)
consumed = path[:amount_consumed]
branch_name = consumed.strip('/')
self.log.info('Using branch: %s', branch_name)
if trail and not trail.startswith('/'):
trail = '/' + trail
environ['PATH_INFO'] = trail
environ['SCRIPT_NAME'] += consumed.rstrip('/')
branch_url = lp_server.get_url() + branch_name
branch_link = urlparse.urljoin(config.codebrowse.launchpad_root,
branch_name)
cachepath = os.path.join(config.codebrowse.cachepath,
branch_name[1:])
if not os.path.isdir(cachepath):
os.makedirs(cachepath)
self.log.info('branch_url: %s', branch_url)
private = info['private']
if private:
self.log.info("Branch is private")
else:
self.log.info("Branch is public")
try:
bzr_branch = safe_open(lp_server.get_url().strip(':/'),
branch_url)
except errors.NotBranchError as err:
self.log.warning('Not a branch: %s', err)
raise HTTPNotFound()
bzr_branch.lock_read()
try:
view = BranchWSGIApp(bzr_branch,
branch_name, {'cachepath': cachepath},
self.graph_cache,
branch_link=branch_link,
served_url=None,
private=private)
return view.app(environ, start_response)
finally:
bzr_branch.repository.revisions.clear_cache()
bzr_branch.repository.signatures.clear_cache()
bzr_branch.repository.inventories.clear_cache()
if bzr_branch.repository.chk_bytes is not None:
bzr_branch.repository.chk_bytes.clear_cache()
bzr_branch.repository.texts.clear_cache()
bzr_branch.unlock()
finally:
lp_server.stop_server()
def test_escape_unescape(self):
self.assertEqual(u'\xe5', urlutils.unescape(urlutils.escape(u'\xe5')))
self.assertEqual('%', urlutils.unescape(urlutils.escape('%')))
def test_escape_unescape(self):
self.assertEqual(u'\xe5', urlutils.unescape(urlutils.escape(u'\xe5')))
self.assertEqual('%', urlutils.unescape(urlutils.escape('%')))
def __call__(self, environ, start_response):
environ['loggerhead.static.url'] = environ['SCRIPT_NAME']
if environ['PATH_INFO'].startswith('/static/'):
path_info_pop(environ)
return static_app(environ, start_response)
elif environ['PATH_INFO'] == '/favicon.ico':
return favicon_app(environ, start_response)
elif environ['PATH_INFO'] == '/robots.txt':
return robots_app(environ, start_response)
elif environ['PATH_INFO'].startswith('/+login'):
return self._complete_login(environ, start_response)
elif environ['PATH_INFO'].startswith('/+logout'):
return self._logout(environ, start_response)
path = environ['PATH_INFO']
trailingSlashCount = len(path) - len(path.rstrip('/'))
user = environ[self.session_var].get('user', LAUNCHPAD_ANONYMOUS)
lp_server = get_lp_server(user, branch_transport=self.get_transport())
lp_server.start_server()
try:
try:
transport_type, info, trail = self.branchfs.translatePath(
user, urlutils.escape(path))
except xmlrpclib.Fault as f:
if check_fault(f, faults.PathTranslationError):
raise HTTPNotFound()
elif check_fault(f, faults.PermissionDenied):
# If we're not allowed to see the branch...
if environ['wsgi.url_scheme'] != 'https':
# ... the request shouldn't have come in over http, as
# requests for private branches over http should be
# redirected to https by the dynamic rewrite script we
# use (which runs before this code is reached), but
# just in case...
env_copy = environ.copy()
env_copy['wsgi.url_scheme'] = 'https'
raise HTTPMovedPermanently(construct_url(env_copy))
elif user != LAUNCHPAD_ANONYMOUS:
# ... if the user is already logged in and still can't
# see the branch, they lose.
exc = HTTPUnauthorized()
exc.explanation = "You are logged in as %s." % user
raise exc
else:
# ... otherwise, lets give them a chance to log in
# with OpenID.
return self._begin_login(environ, start_response)
else:
raise
if transport_type != BRANCH_TRANSPORT:
raise HTTPNotFound()
trail = urlutils.unescape(trail).encode('utf-8')
trail += trailingSlashCount * '/'
amount_consumed = len(path) - len(trail)
consumed = path[:amount_consumed]
branch_name = consumed.strip('/')
self.log.info('Using branch: %s', branch_name)
if trail and not trail.startswith('/'):
trail = '/' + trail
environ['PATH_INFO'] = trail
environ['SCRIPT_NAME'] += consumed.rstrip('/')
branch_url = lp_server.get_url() + branch_name
branch_link = urlparse.urljoin(
config.codebrowse.launchpad_root, branch_name)
cachepath = os.path.join(
config.codebrowse.cachepath, branch_name[1:])
if not os.path.isdir(cachepath):
os.makedirs(cachepath)
self.log.info('branch_url: %s', branch_url)
base_api_url = allvhosts.configs['api'].rooturl
branch_api_url = '%s/%s/%s' % (
base_api_url,
'devel',
branch_name,
)
self.log.info('branch_api_url: %s', branch_api_url)
req = urllib2.Request(branch_api_url)
private = False
try:
# We need to determine if the branch is private
response = urllib2.urlopen(req)
except urllib2.HTTPError as response:
code = response.getcode()
if code in (400, 401, 403, 404):
# There are several error codes that imply private data.
# 400 (bad request) is a default error code from the API
# 401 (unauthorized) should never be returned as the
# requests are always from anon. If it is returned
# however, the data is certainly private.
# 403 (forbidden) is obviously private.
# 404 (not found) implies privacy from a private team or
# similar situation, which we hide as not existing rather
# than mark as forbidden.
self.log.info("Branch is private")
private = True
self.log.info(
"Branch state not determined; api error, return code: %s",
code)
response.close()
else:
self.log.info("Branch is public")
response.close()
try:
bzr_branch = safe_open(
lp_server.get_url().strip(':/'), branch_url)
except errors.NotBranchError as err:
self.log.warning('Not a branch: %s', err)
raise HTTPNotFound()
bzr_branch.lock_read()
try:
view = BranchWSGIApp(
bzr_branch, branch_name, {'cachepath': cachepath},
self.graph_cache, branch_link=branch_link,
served_url=None, private=private)
return view.app(environ, start_response)
finally:
bzr_branch.repository.revisions.clear_cache()
bzr_branch.repository.signatures.clear_cache()
bzr_branch.repository.inventories.clear_cache()
if bzr_branch.repository.chk_bytes is not None:
bzr_branch.repository.chk_bytes.clear_cache()
bzr_branch.repository.texts.clear_cache()
bzr_branch.unlock()
finally:
lp_server.stop_server()
def unescape_path(path):
unescaped_path = urlutils.unescape(path)
return unescaped_path.encode('utf-8')
def local_path(path):
if path.startswith("file://"):
return urlutils.local_path_from_url(path)
else:
return urlutils.unescape(path)
