def test_script(self):
branches = [
self.factory.makeProductBranch(),
self.factory.makePersonalBranch(),
self.factory.makePackageBranch(),
]
input_lines = ["/%s/.bzr/README" % branch.unique_name for branch in branches] + [
"/%s/changes" % branch.unique_name for branch in branches
]
expected_lines = [
"file:///var/tmp/bazaar.launchpad.dev/mirrors/%s/.bzr/README" % branch_id_to_path(branch.id)
for branch in branches
] + ["http://localhost:8080/%s/changes" % branch.unique_name for branch in branches]
transaction.commit()
script_file = os.path.join(config.root, "scripts", "branch-rewrite.py")
proc = subprocess.Popen(
[script_file], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0
)
output_lines = []
# For each complete line of input, the script should, without
# buffering, write a complete line of output.
for input_line in input_lines:
proc.stdin.write(input_line + "\n")
output_lines.append(nonblocking_readline(proc.stdout, 60).rstrip("\n"))
# If we create a new branch after the branch-rewrite.py script has
# connected to the database, or edit a branch name that has already
# been rewritten, both are rewritten successfully.
new_branch = self.factory.makeAnyBranch()
edited_branch = removeSecurityProxy(branches[0])
edited_branch.name = self.factory.getUniqueString()
transaction.commit()
new_branch_input = "/%s/.bzr/README" % new_branch.unique_name
expected_lines.append(
"file:///var/tmp/bazaar.launchpad.dev/mirrors/%s/.bzr/README" % branch_id_to_path(new_branch.id)
)
proc.stdin.write(new_branch_input + "\n")
output_lines.append(nonblocking_readline(proc.stdout, 60).rstrip("\n"))
edited_branch_input = "/%s/.bzr/README" % edited_branch.unique_name
expected_lines.append(
"file:///var/tmp/bazaar.launchpad.dev/mirrors/%s/.bzr/README" % branch_id_to_path(edited_branch.id)
)
proc.stdin.write(edited_branch_input + "\n")
output_lines.append(nonblocking_readline(proc.stdout, 60).rstrip("\n"))
os.kill(proc.pid, signal.SIGINT)
err = proc.stderr.read()
# The script produces logging output, but not to stderr.
self.assertEqual("", err)
self.assertEqual(expected_lines, output_lines)
def switch_branches(prefix, scheme, old_db_branch, new_db_branch):
"""Move bzr data from an old to a new branch, leaving old stacked on new.
This function is intended to be used just after Ubuntu is released to
create (at the bzr level) a new trunk branch for a source package for the
next release of the distribution. We move the bzr data to the location
for the new branch and replace the trunk branch for the just released
version with a stacked branch pointing at the new branch.
The procedure is to complicated to be carried out atomically, so if this
function is interrupted things may be a little inconsistent (e.g. there
might be a branch in the old location, but not stacked on the new location
yet). There should be no data loss though.
:param prefix: The non-branch id dependent part of the physical path to
the branches on disk.
:param scheme: The branches should be open-able at a URL of the form
``scheme + :/// + unique_name``.
:param old_db_branch: The branch that currently has the trunk bzr data.
:param old_db_branch: The new trunk branch. This should not have any
presence on disk yet.
"""
# Move .bzr directory from old to new location, crashing through the
# abstraction we usually hide our branch locations behind.
old_underlying_path = os.path.join(
prefix, branch_id_to_path(old_db_branch.id))
new_underlying_path = os.path.join(
prefix, branch_id_to_path(new_db_branch.id))
os.makedirs(new_underlying_path)
os.rename(
os.path.join(old_underlying_path, '.bzr'),
os.path.join(new_underlying_path, '.bzr'))
# Create branch at old location -- we use the "clone('null:')" trick to
# preserve the format. We have to open at the logical, unique_name-based,
# location so that it works to set the stacked on url to '/' + a
# unique_name.
new_location_bzrdir = BzrDir.open(
scheme + ':///' + new_db_branch.unique_name)
old_location_bzrdir = new_location_bzrdir.clone(
scheme + ':///' + old_db_branch.unique_name, revision_id='null:')
# Set the stacked on url for old location.
old_location_branch = old_location_bzrdir.open_branch()
old_location_branch.set_stacked_on_url('/' + new_db_branch.unique_name)
# Pull from new location to old -- this won't actually transfer any
# revisions, just update the last revision pointer.
old_location_branch.pull(new_location_bzrdir.open_branch())
def rewriteLine(self, resource_location):
"""Rewrite 'resource_location' to a more concrete location.
We use the 'translatePath' BranchFileSystemClient method. There are
three cases:
(1) The request is for something within the .bzr directory of a
branch.
In this case we rewrite the request to the location from which
branches are served by ID.
(2) The request is for something within a branch, but not the .bzr
directory.
In this case, we hand the request off to codebrowse.
(3) The branch is not found. Two sub-cases: the request is for a
product control directory or the we don't know how to translate
the path.
In both these cases we return 'NULL' which indicates to Apache
that we don't know how to rewrite the request (and so it should
go on to generate a 404 response).
Other errors are allowed to propagate, on the assumption that the
caller will catch and log them.
"""
# Codebrowse generates references to its images and stylesheets
# starting with "/static", so pass them on unthinkingly.
T = time.time()
# Tell the webapp adapter that we are in a request, so that DB
# statement timeouts will be applied.
set_request_started()
try:
cached = None
if resource_location.startswith('/static/'):
r = self._codebrowse_url(resource_location)
cached = 'N/A'
else:
branch_id, trailing, cached = self._getBranchIdAndTrailingPath(
resource_location)
if branch_id is None:
if resource_location.startswith('/' +
BRANCH_ID_ALIAS_PREFIX):
r = 'NULL'
else:
r = self._codebrowse_url(resource_location)
else:
if trailing.startswith('/.bzr'):
r = urlutils.join(
config.codehosting.internal_branch_by_id_root,
branch_id_to_path(branch_id), trailing[1:])
else:
r = self._codebrowse_url(resource_location)
finally:
clear_request_started()
self.logger.info("%r -> %r (%fs, cache: %s)", resource_location, r,
time.time() - T, cached)
return r
def createBazaarBranch(self,
user,
product,
branch,
creator=None,
branch_root=None):
"""Create a new branch in the database and push our test branch there.
Used to create branches that the test user is not able to create, and
might not even be able to view.
"""
authserver = xmlrpclib.ServerProxy(
config.codehosting.authentication_endpoint)
codehosting_api = xmlrpclib.ServerProxy(
config.codehosting.codehosting_endpoint)
if creator is None:
creator_id = authserver.getUserAndSSHKeys(user)['id']
else:
creator_id = authserver.getUserAndSSHKeys(creator)['id']
if branch_root is None:
branch_root = self.server._mirror_root
branch_id = codehosting_api.createBranch(
creator_id, '/~%s/%s/%s' % (user, product, branch))
branch_url = 'file://' + os.path.abspath(
os.path.join(branch_root, branch_id_to_path(branch_id)))
self.push(self.local_branch_path, branch_url, ['--create-prefix'])
return branch_url
def make_bazaar_branch_and_tree(db_branch):
"""Make a dummy Bazaar branch and working tree from a database Branch."""
assert db_branch.branch_type == BranchType.HOSTED, (
"Can only create branches for HOSTED branches: %r" % db_branch)
branch_dir = os.path.join(config.codehosting.mirrored_branches_root,
branch_id_to_path(db_branch.id))
return create_branch_with_one_revision(branch_dir)
def test_branch(self):
# A branch location is the physical disk directory.
branch = self.factory.makeAnyBranch(branch_type=BranchType.HOSTED)
script = ModifiedBranchesScript('modified-branches', test_args=[])
location = script.branch_location(branch)
path = branch_id_to_path(branch.id)
self.assertEqual(
os.path.join(config.codehosting.mirrored_branches_root, path),
location)
def test_rewriteLine_id_alias_found_dot_bzr(self):
# Requests for /+branch-id/$id/.bzr/... are redirected to where the
# branches are served from by ID if they are public.
rewriter = self.makeRewriter()
branches = [
self.factory.makeProductBranch(),
self.factory.makePersonalBranch(),
self.factory.makePackageBranch(),
]
transaction.commit()
output = [rewriter.rewriteLine("%s/.bzr/README" % branch_id_alias(branch)) for branch in branches]
expected = [
"file:///var/tmp/bazaar.launchpad.dev/mirrors/%s/.bzr/README" % branch_id_to_path(branch.id)
for branch in branches
]
self.assertEqual(expected, output)
def test_rewriteLine_found_dot_bzr(self):
# Requests for /$branch_name/.bzr/... are redirected to where the
# branches are served from by ID.
rewriter = self.makeRewriter()
branches = [
self.factory.makeProductBranch(),
self.factory.makePersonalBranch(),
self.factory.makePackageBranch()
]
transaction.commit()
output = [
rewriter.rewriteLine("/%s/.bzr/README" % branch.unique_name)
for branch in branches
]
expected = [
'file:///var/tmp/bazaar.launchpad.dev/mirrors/%s/.bzr/README' %
branch_id_to_path(branch.id) for branch in branches
]
self.assertEqual(expected, output)
def createBazaarBranch(self, user, product, branch, creator=None,
branch_root=None):
"""Create a new branch in the database and push our test branch there.
Used to create branches that the test user is not able to create, and
might not even be able to view.
"""
authserver = xmlrpclib.ServerProxy(
config.codehosting.authentication_endpoint)
codehosting_api = xmlrpclib.ServerProxy(
config.codehosting.codehosting_endpoint)
if creator is None:
creator_id = authserver.getUserAndSSHKeys(user)['id']
else:
creator_id = authserver.getUserAndSSHKeys(creator)['id']
if branch_root is None:
branch_root = self.server._mirror_root
branch_id = codehosting_api.createBranch(
creator_id, '/~%s/%s/%s' % (user, product, branch))
branch_url = 'file://' + os.path.abspath(
os.path.join(branch_root, branch_id_to_path(branch_id)))
self.push(self.local_branch_path, branch_url, ['--create-prefix'])
return branch_url
def unique_name(self):
return branch_id_to_path(self.id)
def test_script(self):
# The .bzr subdirectory of all public branch types gets mapped
# to the internal by-ID branch store, but everything else, and
# all private branch requests, go through to loggerhead.
bs = [
self.factory.makeProductBranch(),
self.factory.makePersonalBranch(),
self.factory.makePackageBranch()
]
privbranch = removeSecurityProxy(
self.factory.makeProductBranch(
information_type=InformationType.USERDATA))
allbs = bs + [privbranch]
input_lines = (
["/%s/.bzr/README" % branch.unique_name for branch in allbs] +
["/%s/changes" % branch.unique_name for branch in allbs])
expected_lines = (
[
'file:///var/tmp/bazaar.launchpad.dev/mirrors/%s/.bzr/README' %
branch_id_to_path(branch.id) for branch in bs
] +
['http://localhost:8080/%s/.bzr/README' % privbranch.unique_name] +
['http://localhost:8080/%s/changes' % b.unique_name for b in bs] +
['http://localhost:8080/%s/changes' % privbranch.unique_name])
transaction.commit()
script_file = os.path.join(config.root, 'scripts', 'branch-rewrite.py')
proc = subprocess.Popen([script_file],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=0)
output_lines = []
# For each complete line of input, the script should, without
# buffering, write a complete line of output.
for input_line in input_lines:
proc.stdin.write(input_line + '\n')
output_lines.append(
nonblocking_readline(proc.stdout, 60).rstrip('\n'))
# If we create a new branch after the branch-rewrite.py script has
# connected to the database, or edit a branch name that has already
# been rewritten, both are rewritten successfully.
new_branch = self.factory.makeAnyBranch()
edited_branch = removeSecurityProxy(bs[0])
edited_branch.name = self.factory.getUniqueString()
transaction.commit()
new_branch_input = '/%s/.bzr/README' % new_branch.unique_name
expected_lines.append(
'file:///var/tmp/bazaar.launchpad.dev/mirrors/%s/.bzr/README' %
branch_id_to_path(new_branch.id))
proc.stdin.write(new_branch_input + '\n')
output_lines.append(nonblocking_readline(proc.stdout, 60).rstrip('\n'))
edited_branch_input = '/%s/.bzr/README' % edited_branch.unique_name
expected_lines.append(
'file:///var/tmp/bazaar.launchpad.dev/mirrors/%s/.bzr/README' %
branch_id_to_path(edited_branch.id))
proc.stdin.write(edited_branch_input + '\n')
output_lines.append(nonblocking_readline(proc.stdout, 60).rstrip('\n'))
os.kill(proc.pid, signal.SIGINT)
err = proc.stderr.read()
# The script produces logging output, but not to stderr.
self.assertEqual('', err)
self.assertEqual(expected_lines, output_lines)
def make_bazaar_branch_and_tree(db_branch):
"""Make a dummy Bazaar branch and working tree from a database Branch."""
assert db_branch.branch_type == BranchType.HOSTED, "Can only create branches for HOSTED branches: %r" % db_branch
branch_dir = os.path.join(config.codehosting.mirrored_branches_root, branch_id_to_path(db_branch.id))
return create_branch_with_one_revision(branch_dir)
def branch_location(self, branch):
"""Return the branch path for the given branch."""
path = branch_id_to_path(branch.id)
return os.path.join(config.codehosting.mirrored_branches_root, path)
def rewriteLine(self, resource_location):
"""Rewrite 'resource_location' to a more concrete location.
We use the 'translatePath' BranchFileSystemClient method. There are
three cases:
(1) The request is for something within the .bzr directory of a
branch.
In this case we rewrite the request to the location from which
branches are served by ID.
(2) The request is for something within a branch, but not the .bzr
directory.
In this case, we hand the request off to codebrowse.
(3) The branch is not found. Two sub-cases: the request is for a
product control directory or the we don't know how to translate
the path.
In both these cases we return 'NULL' which indicates to Apache
that we don't know how to rewrite the request (and so it should
go on to generate a 404 response).
Other errors are allowed to propagate, on the assumption that the
caller will catch and log them.
"""
# Codebrowse generates references to its images and stylesheets
# starting with "/static", so pass them on unthinkingly.
T = time.time()
# Tell the webapp adapter that we are in a request, so that DB
# statement timeouts will be applied.
set_request_started()
try:
cached = None
if resource_location.startswith('/static/'):
r = self._codebrowse_url(resource_location)
cached = 'N/A'
else:
branch_id, trailing, cached = self._getBranchIdAndTrailingPath(
resource_location)
if branch_id is None:
if resource_location.startswith(
'/' + BRANCH_ID_ALIAS_PREFIX):
r = 'NULL'
else:
r = self._codebrowse_url(resource_location)
else:
if trailing.startswith('/.bzr'):
r = urlutils.join(
config.codehosting.internal_branch_by_id_root,
branch_id_to_path(branch_id), trailing[1:])
else:
r = self._codebrowse_url(resource_location)
finally:
clear_request_started()
self.logger.info(
"%r -> %r (%fs, cache: %s)",
resource_location, r, time.time() - T, cached)
return r
def branch_location(self, branch):
"""Return the branch path for the given branch."""
path = branch_id_to_path(branch.id)
return os.path.join(config.codehosting.mirrored_branches_root, path)
def unique_name(self):
return branch_id_to_path(self.id)
