def transformFallbackLocation(self, branch, url):
"""See `BranchOpenPolicy.transformFallbackLocation`.
For mirrored branches, we stack on whatever the remote branch claims
to stack on, but this URL still needs to be checked.
"""
return urljoin(branch.base, url), True
def transformFallbackLocation(self, branch, url):
"""See `BranchOpenPolicy.transformFallbackLocation`.
For mirrored branches, we stack on whatever the remote branch claims
to stack on, but this URL still needs to be checked.
"""
return urljoin(branch.base, url), True
def _setHead(self, target_url, target_ref):
"""Set HEAD on a remote repository.
This relies on the turnip-set-symbolic-ref extension.
"""
service = "turnip-set-symbolic-ref"
url = urljoin(target_url, service)
headers = {
"Content-Type": "application/x-%s-request" % service,
}
body = pkt_line("HEAD %s" % target_ref) + pkt_line(None)
try:
response = urlfetch(url, method="POST", headers=headers, data=body)
response.raise_for_status()
except Exception as e:
raise GitProtocolError(str(e))
content_type = response.headers.get("Content-Type")
if content_type != ("application/x-%s-result" % service):
raise GitProtocolError("Invalid Content-Type from server: %s" %
content_type)
content = io.BytesIO(response.content)
proto = Protocol(content.read, None)
pkt = proto.read_pkt_line()
if pkt is None:
raise GitProtocolError("Unexpected flush-pkt from server")
elif pkt.rstrip(b"\n") == b"ACK HEAD":
pass
elif pkt.startswith(b"ERR "):
raise GitProtocolError(
pkt[len(b"ERR "):].rstrip(b"\n").decode("UTF-8"))
else:
raise GitProtocolError("Unexpected packet %r from server" % pkt)
def setUp(self):
super(TestPullerMasterIntegration, self).setUp()
self.makeCleanDirectory(config.codehosting.mirrored_branches_root)
self.bzr_tree = self.make_branch_and_tree('src-branch')
url = urljoin(self.serveOverHTTP(), 'src-branch')
self.bzr_tree.commit('rev1')
branch_id = self.factory.makeAnyBranch(
branch_type=BranchType.MIRRORED, url=url).id
self.layer.txn.commit()
self.db_branch = getUtility(IBranchLookup).get(branch_id)
self.client = FakeCodehostingEndpointProxy()
def setUp(self):
super(TestPullerMasterIntegration, self).setUp()
self.makeCleanDirectory(config.codehosting.mirrored_branches_root)
self.bzr_tree = self.make_branch_and_tree('src-branch')
url = urljoin(self.serveOverHTTP(), 'src-branch')
self.bzr_tree.commit('rev1')
branch_id = self.factory.makeAnyBranch(
branch_type=BranchType.MIRRORED, url=url).id
self.layer.txn.commit()
self.db_branch = getUtility(IBranchLookup).get(branch_id)
self.client = FakeCodehostingEndpointProxy()
def makeDirectory(self, directory_name, commit_message=None):
"""Make a directory on the repository."""
if commit_message is None:
commit_message = 'Make %r' % (directory_name,)
ra = self._get_ra(self.get_url())
editor = ra.get_commit_editor({"svn:log": commit_message})
root = editor.open_root()
root.add_directory(directory_name).close()
root.close()
editor.close()
return urljoin(self.get_url(), directory_name)
def makeDirectory(self, directory_name, commit_message=None):
"""Make a directory on the repository."""
if commit_message is None:
commit_message = 'Make %r' % (directory_name,)
ra = self._get_ra(self.get_url())
editor = ra.get_commit_editor({"svn:log": commit_message})
root = editor.open_root()
root.add_directory(directory_name).close()
root.close()
editor.close()
return urljoin(self.get_url(), directory_name)
def push(self,
db_branch_id,
bzr_branch,
required_format,
stacked_on_url=None):
"""Push up `bzr_branch` as the Bazaar branch for `code_import`.
:return: A boolean that is true if the push was non-trivial
(i.e. actually transferred revisions).
"""
self.transport.create_prefix()
target_url = self._getMirrorURL(db_branch_id, push=True)
try:
remote_branch = Branch.open(target_url)
except NotBranchError:
remote_branch = BzrDir.create_branch_and_repo(
target_url, format=required_format)
old_branch = None
else:
if remote_branch.bzrdir.needs_format_conversion(required_format):
# For upgrades, push to a new branch in
# the new format. When done pushing,
# retire the old .bzr directory and rename
# the new one in place.
old_branch = remote_branch
upgrade_url = urljoin(target_url, "backup.bzr")
try:
remote_branch.bzrdir.root_transport.delete_tree(
'backup.bzr')
except NoSuchFile:
pass
remote_branch = BzrDir.create_branch_and_repo(
upgrade_url, format=required_format)
else:
old_branch = None
# This can be done safely, since only modern formats are used to
# import to.
if stacked_on_url is not None:
remote_branch.set_stacked_on_url(stacked_on_url)
pull_result = remote_branch.pull(bzr_branch, overwrite=True)
# Because of the way we do incremental imports, there may be revisions
# in the branch's repo that are not in the ancestry of the branch tip.
# We need to transfer them too.
remote_branch.repository.fetch(bzr_branch.repository)
if old_branch is not None:
# The format has changed; move the new format
# branch in place.
base_transport = old_branch.bzrdir.root_transport
base_transport.delete_tree('.bzr')
base_transport.rename("backup.bzr/.bzr", ".bzr")
base_transport.rmdir("backup.bzr")
return pull_result.old_revid != pull_result.new_revid
def _getMirrorURL(self, db_branch_id, push=False):
"""Return the URL that `db_branch` is stored at."""
base_url = self.transport.base
if push:
# Pulling large branches over sftp is less CPU-intensive, but
# pushing over bzr+ssh seems to be more reliable.
split = urlsplit(base_url)
if split.scheme == 'sftp':
base_url = urlunsplit([
'bzr+ssh', split.netloc, split.path, split.query,
split.fragment
])
return urljoin(base_url, '%08x' % db_branch_id)
def push(self, db_branch_id, bzr_branch, required_format,
stacked_on_url=None):
"""Push up `bzr_branch` as the Bazaar branch for `code_import`.
:return: A boolean that is true if the push was non-trivial
(i.e. actually transferred revisions).
"""
self.transport.create_prefix()
target_url = self._getMirrorURL(db_branch_id)
try:
remote_branch = Branch.open(target_url)
except NotBranchError:
remote_branch = BzrDir.create_branch_and_repo(
target_url, format=required_format)
old_branch = None
else:
if remote_branch.bzrdir.needs_format_conversion(
required_format):
# For upgrades, push to a new branch in
# the new format. When done pushing,
# retire the old .bzr directory and rename
# the new one in place.
old_branch = remote_branch
upgrade_url = urljoin(target_url, "backup.bzr")
try:
remote_branch.bzrdir.root_transport.delete_tree(
'backup.bzr')
except NoSuchFile:
pass
remote_branch = BzrDir.create_branch_and_repo(
upgrade_url, format=required_format)
else:
old_branch = None
# This can be done safely, since only modern formats are used to
# import to.
if stacked_on_url is not None:
remote_branch.set_stacked_on_url(stacked_on_url)
pull_result = remote_branch.pull(bzr_branch, overwrite=True)
# Because of the way we do incremental imports, there may be revisions
# in the branch's repo that are not in the ancestry of the branch tip.
# We need to transfer them too.
remote_branch.repository.fetch(bzr_branch.repository)
if old_branch is not None:
# The format has changed; move the new format
# branch in place.
base_transport = old_branch.bzrdir.root_transport
base_transport.delete_tree('.bzr')
base_transport.rename("backup.bzr/.bzr", ".bzr")
base_transport.rmdir("backup.bzr")
return pull_result.old_revid != pull_result.new_revid
def test_mirror_imported_branch(self):
# Run the puller on a populated imported branch pull queue.
# Create the branch in the database.
db_branch = self.factory.makeAnyBranch(branch_type=BranchType.IMPORTED)
db_branch.requestMirror()
transaction.commit()
# Create the Bazaar branch in the expected location.
branch_url = urljoin(config.launchpad.bzr_imports_root_url,
'%08x' % db_branch.id)
branch = BzrDir.create_branch_convenience(branch_url)
tree = branch.bzrdir.open_workingtree()
tree.commit('rev1')
transaction.commit()
# Run the puller.
command, retcode, output, error = self.runPuller()
self.assertRanSuccessfully(command, retcode, output, error)
self.assertMirrored(db_branch, source_branch=branch)
def test_mirror_imported_branch(self):
# Run the puller on a populated imported branch pull queue.
# Create the branch in the database.
db_branch = self.factory.makeAnyBranch(
branch_type=BranchType.IMPORTED)
db_branch.requestMirror()
transaction.commit()
# Create the Bazaar branch in the expected location.
branch_url = urljoin(
config.launchpad.bzr_imports_root_url, '%08x' % db_branch.id)
branch = BzrDir.create_branch_convenience(branch_url)
tree = branch.bzrdir.open_workingtree()
tree.commit('rev1')
transaction.commit()
# Run the puller.
command, retcode, output, error = self.runPuller()
self.assertRanSuccessfully(command, retcode, output, error)
self.assertMirrored(db_branch, source_branch=branch)
def _doImport(self):
self._logger.info("Starting job.")
try:
self._opener_policy.checkOneURL(self.source_details.url)
except BadUrl as e:
self._logger.info("Invalid URL: %s" % e)
return CodeImportWorkerExitCode.FAILURE_FORBIDDEN
unauth_target_url = urljoin(config.codehosting.git_browse_root,
self.source_details.target_id)
split = urlsplit(unauth_target_url)
target_netloc = ":%s@%s" % (self.source_details.macaroon.serialize(),
split.hostname)
if split.port:
target_netloc += ":%s" % split.port
target_url = urlunsplit(
[split.scheme, target_netloc, split.path, "", ""])
# XXX cjwatson 2016-10-11: Ideally we'd put credentials in a
# credentials store instead. However, git only accepts credentials
# that have both a non-empty username and a non-empty password.
self._logger.info("Getting existing repository from hosting service.")
try:
self._runGit("clone", "--mirror", target_url, "repository")
except subprocess.CalledProcessError as e:
self._logger.info(
"Unable to get existing repository from hosting service: "
"git clone exited %s" % e.returncode)
return CodeImportWorkerExitCode.FAILURE
self._logger.info("Fetching remote repository.")
try:
self._runGit("config", "gc.auto", "0", cwd="repository")
# Remove any stray remote-tracking refs from the last time round.
self._deleteRefs("repository", "refs/remotes/source/**")
self._runGit("remote",
"add",
"source",
self.source_details.url,
cwd="repository")
self._runGit("fetch",
"--prune",
"source",
"+refs/*:refs/*",
cwd="repository")
try:
new_head = self._getHead("repository", "source")
except (subprocess.CalledProcessError, GitProtocolError) as e2:
self._logger.info("Unable to fetch default branch: %s" % e2)
new_head = None
self._runGit("remote", "rm", "source", cwd="repository")
# XXX cjwatson 2016-11-03: For some reason "git remote rm"
# doesn't actually remove the refs.
self._deleteRefs("repository", "refs/remotes/source/**")
except subprocess.CalledProcessError as e:
self._logger.info("Unable to fetch remote repository: %s" % e)
return CodeImportWorkerExitCode.FAILURE_INVALID
self._logger.info("Pushing repository to hosting service.")
try:
if new_head is not None:
# Push the target of HEAD first to ensure that it is always
# available.
self._runGit("push",
target_url,
"+%s:%s" % (new_head, new_head),
cwd="repository")
try:
self._setHead(target_url, new_head)
except GitProtocolError as e:
self._logger.info("Unable to set default branch: %s" % e)
self._runGit("push", "--mirror", target_url, cwd="repository")
except subprocess.CalledProcessError as e:
self._logger.info(
"Unable to push to hosting service: git push exited %s" %
e.returncode)
return CodeImportWorkerExitCode.FAILURE
return CodeImportWorkerExitCode.SUCCESS
def _getMirrorURL(self, db_branch_id):
"""Return the URL that `db_branch` is stored at."""
return urljoin(self.transport.base, '%08x' % db_branch_id)
