class TestWorkerMonitorIntegration(BzrTestCase):
layer = ZopelessAppServerLayer
run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=60)
def setUp(self):
BzrTestCase.setUp(self)
login('*****@*****.**')
self.factory = LaunchpadObjectFactory()
nuke_codeimport_sample_data()
self.repo_path = tempfile.mkdtemp()
self.disable_directory_isolation()
self.addCleanup(shutil.rmtree, self.repo_path)
self.foreign_commit_count = 0
def tearDown(self):
BzrTestCase.tearDown(self)
logout()
def makeCVSCodeImport(self):
"""Make a `CodeImport` that points to a real CVS repository."""
cvs_server = CVSServer(self.repo_path)
cvs_server.start_server()
self.addCleanup(cvs_server.stop_server)
cvs_server.makeModule('trunk', [('README', 'original\n')])
self.foreign_commit_count = 2
return self.factory.makeCodeImport(cvs_root=cvs_server.getRoot(),
cvs_module='trunk')
def makeSVNCodeImport(self):
"""Make a `CodeImport` that points to a real Subversion repository."""
self.subversion_server = SubversionServer(self.repo_path)
self.subversion_server.start_server()
self.addCleanup(self.subversion_server.stop_server)
url = self.subversion_server.makeBranch('trunk',
[('README', 'contents')])
self.foreign_commit_count = 2
return self.factory.makeCodeImport(svn_branch_url=url)
def makeBzrSvnCodeImport(self):
"""Make a `CodeImport` that points to a real Subversion repository."""
self.subversion_server = SubversionServer(self.repo_path,
use_svn_serve=True)
self.subversion_server.start_server()
self.addCleanup(self.subversion_server.stop_server)
url = self.subversion_server.makeBranch('trunk',
[('README', 'contents')])
self.foreign_commit_count = 2
return self.factory.makeCodeImport(
svn_branch_url=url, rcs_type=RevisionControlSystems.BZR_SVN)
def makeGitCodeImport(self):
"""Make a `CodeImport` that points to a real Git repository."""
self.git_server = GitServer(self.repo_path, use_server=False)
self.git_server.start_server()
self.addCleanup(self.git_server.stop_server)
self.git_server.makeRepo([('README', 'contents')])
self.foreign_commit_count = 1
return self.factory.makeCodeImport(
git_repo_url=self.git_server.get_url())
def makeBzrCodeImport(self):
"""Make a `CodeImport` that points to a real Bazaar branch."""
self.bzr_server = BzrServer(self.repo_path)
self.bzr_server.start_server()
self.addCleanup(self.bzr_server.stop_server)
self.bzr_server.makeRepo([('README', 'contents')])
self.foreign_commit_count = 1
return self.factory.makeCodeImport(
bzr_branch_url=self.bzr_server.get_url())
def getStartedJobForImport(self, code_import):
"""Get a started `CodeImportJob` for `code_import`.
This method approves the import, creates a job, marks it started and
returns the job. It also makes sure there are no branches or foreign
trees in the default stores to interfere with processing this job.
"""
source_details = CodeImportSourceDetails.fromCodeImport(code_import)
clean_up_default_stores_for_import(source_details)
self.addCleanup(clean_up_default_stores_for_import, source_details)
if code_import.review_status != CodeImportReviewStatus.REVIEWED:
code_import.updateFromData(
{'review_status': CodeImportReviewStatus.REVIEWED},
self.factory.makePerson())
job = getUtility(ICodeImportJobSet).getJobForMachine('machine', 10)
self.assertEqual(code_import, job.code_import)
return job
def assertCodeImportResultCreated(self, code_import):
"""Assert that a `CodeImportResult` was created for `code_import`."""
self.assertEqual(len(list(code_import.results)), 1)
result = list(code_import.results)[0]
self.assertEqual(result.status, CodeImportResultStatus.SUCCESS)
def assertBranchImportedOKForCodeImport(self, code_import):
"""Assert that a branch was pushed into the default branch store."""
url = get_default_bazaar_branch_store()._getMirrorURL(
code_import.branch.id)
branch = Branch.open(url)
self.assertEqual(self.foreign_commit_count, branch.revno())
def assertImported(self, ignored, code_import_id):
"""Assert that the `CodeImport` of the given id was imported."""
# In the in-memory tests, check that resetTimeout on the
# CodeImportWorkerMonitorProtocol was called at least once.
if self._protocol is not None:
self.assertPositive(self._protocol.reset_calls)
code_import = getUtility(ICodeImportSet).get(code_import_id)
self.assertCodeImportResultCreated(code_import)
self.assertBranchImportedOKForCodeImport(code_import)
def performImport(self, job_id):
"""Perform the import job with ID job_id.
Return a Deferred that fires when it the job is done.
This implementation does it in-process.
"""
logger = BufferLogger()
monitor = CIWorkerMonitorForTesting(
job_id, logger,
xmlrpc.Proxy(config.codeimportdispatcher.codeimportscheduler_url),
"anything")
deferred = monitor.run()
def save_protocol_object(result):
"""Save the process protocol object.
We do this in an addBoth so that it's called after the process
protocol is actually constructed but before we drop the last
reference to the monitor object.
"""
self._protocol = monitor._protocol
return result
return deferred.addBoth(save_protocol_object)
def test_import_cvs(self):
# Create a CVS CodeImport and import it.
job = self.getStartedJobForImport(self.makeCVSCodeImport())
code_import_id = job.code_import.id
job_id = job.id
self.layer.txn.commit()
result = self.performImport(job_id)
return result.addCallback(self.assertImported, code_import_id)
def test_import_subversion(self):
# Create a Subversion CodeImport and import it.
job = self.getStartedJobForImport(self.makeSVNCodeImport())
code_import_id = job.code_import.id
job_id = job.id
self.layer.txn.commit()
result = self.performImport(job_id)
return result.addCallback(self.assertImported, code_import_id)
def test_import_git(self):
# Create a Git CodeImport and import it.
job = self.getStartedJobForImport(self.makeGitCodeImport())
code_import_id = job.code_import.id
job_id = job.id
self.layer.txn.commit()
result = self.performImport(job_id)
return result.addCallback(self.assertImported, code_import_id)
def test_import_bzr(self):
# Create a Bazaar CodeImport and import it.
job = self.getStartedJobForImport(self.makeBzrCodeImport())
code_import_id = job.code_import.id
job_id = job.id
self.layer.txn.commit()
result = self.performImport(job_id)
return result.addCallback(self.assertImported, code_import_id)
def test_import_bzrsvn(self):
# Create a Subversion-via-bzr-svn CodeImport and import it.
job = self.getStartedJobForImport(self.makeBzrSvnCodeImport())
code_import_id = job.code_import.id
job_id = job.id
self.layer.txn.commit()
result = self.performImport(job_id)
return result.addCallback(self.assertImported, code_import_id)
class TestWorkerMonitorIntegration(BzrTestCase):
layer = ZopelessAppServerLayer
run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=60)
def setUp(self):
BzrTestCase.setUp(self)
login('*****@*****.**')
self.factory = LaunchpadObjectFactory()
nuke_codeimport_sample_data()
self.repo_path = tempfile.mkdtemp()
self.disable_directory_isolation()
self.addCleanup(shutil.rmtree, self.repo_path)
self.foreign_commit_count = 0
def tearDown(self):
BzrTestCase.tearDown(self)
logout()
def makeCVSCodeImport(self):
"""Make a `CodeImport` that points to a real CVS repository."""
cvs_server = CVSServer(self.repo_path)
cvs_server.start_server()
self.addCleanup(cvs_server.stop_server)
cvs_server.makeModule('trunk', [('README', 'original\n')])
self.foreign_commit_count = 2
return self.factory.makeCodeImport(
cvs_root=cvs_server.getRoot(), cvs_module='trunk')
def makeSVNCodeImport(self):
"""Make a `CodeImport` that points to a real Subversion repository."""
self.subversion_server = SubversionServer(self.repo_path)
self.subversion_server.start_server()
self.addCleanup(self.subversion_server.stop_server)
url = self.subversion_server.makeBranch(
'trunk', [('README', 'contents')])
self.foreign_commit_count = 2
return self.factory.makeCodeImport(svn_branch_url=url)
def makeBzrSvnCodeImport(self):
"""Make a `CodeImport` that points to a real Subversion repository."""
self.subversion_server = SubversionServer(
self.repo_path, use_svn_serve=True)
self.subversion_server.start_server()
self.addCleanup(self.subversion_server.stop_server)
url = self.subversion_server.makeBranch(
'trunk', [('README', 'contents')])
self.foreign_commit_count = 2
return self.factory.makeCodeImport(
svn_branch_url=url, rcs_type=RevisionControlSystems.BZR_SVN)
def makeGitCodeImport(self):
"""Make a `CodeImport` that points to a real Git repository."""
self.git_server = GitServer(self.repo_path, use_server=False)
self.git_server.start_server()
self.addCleanup(self.git_server.stop_server)
self.git_server.makeRepo([('README', 'contents')])
self.foreign_commit_count = 1
return self.factory.makeCodeImport(
git_repo_url=self.git_server.get_url())
def makeBzrCodeImport(self):
"""Make a `CodeImport` that points to a real Bazaar branch."""
self.bzr_server = BzrServer(self.repo_path)
self.bzr_server.start_server()
self.addCleanup(self.bzr_server.stop_server)
self.bzr_server.makeRepo([('README', 'contents')])
self.foreign_commit_count = 1
return self.factory.makeCodeImport(
bzr_branch_url=self.bzr_server.get_url())
def getStartedJobForImport(self, code_import):
"""Get a started `CodeImportJob` for `code_import`.
This method approves the import, creates a job, marks it started and
returns the job. It also makes sure there are no branches or foreign
trees in the default stores to interfere with processing this job.
"""
source_details = CodeImportSourceDetails.fromCodeImport(code_import)
clean_up_default_stores_for_import(source_details)
self.addCleanup(clean_up_default_stores_for_import, source_details)
if code_import.review_status != CodeImportReviewStatus.REVIEWED:
code_import.updateFromData(
{'review_status': CodeImportReviewStatus.REVIEWED},
self.factory.makePerson())
job = getUtility(ICodeImportJobSet).getJobForMachine('machine', 10)
self.assertEqual(code_import, job.code_import)
return job
def assertCodeImportResultCreated(self, code_import):
"""Assert that a `CodeImportResult` was created for `code_import`."""
self.assertEqual(len(list(code_import.results)), 1)
result = list(code_import.results)[0]
self.assertEqual(result.status, CodeImportResultStatus.SUCCESS)
def assertBranchImportedOKForCodeImport(self, code_import):
"""Assert that a branch was pushed into the default branch store."""
url = get_default_bazaar_branch_store()._getMirrorURL(
code_import.branch.id)
branch = Branch.open(url)
self.assertEqual(self.foreign_commit_count, branch.revno())
def assertImported(self, ignored, code_import_id):
"""Assert that the `CodeImport` of the given id was imported."""
# In the in-memory tests, check that resetTimeout on the
# CodeImportWorkerMonitorProtocol was called at least once.
if self._protocol is not None:
self.assertPositive(self._protocol.reset_calls)
code_import = getUtility(ICodeImportSet).get(code_import_id)
self.assertCodeImportResultCreated(code_import)
self.assertBranchImportedOKForCodeImport(code_import)
def performImport(self, job_id):
"""Perform the import job with ID job_id.
Return a Deferred that fires when it the job is done.
This implementation does it in-process.
"""
logger = BufferLogger()
monitor = CIWorkerMonitorForTesting(
job_id, logger,
xmlrpc.Proxy(config.codeimportdispatcher.codeimportscheduler_url),
"anything")
deferred = monitor.run()
def save_protocol_object(result):
"""Save the process protocol object.
We do this in an addBoth so that it's called after the process
protocol is actually constructed but before we drop the last
reference to the monitor object.
"""
self._protocol = monitor._protocol
return result
return deferred.addBoth(save_protocol_object)
def test_import_cvs(self):
# Create a CVS CodeImport and import it.
job = self.getStartedJobForImport(self.makeCVSCodeImport())
code_import_id = job.code_import.id
job_id = job.id
self.layer.txn.commit()
result = self.performImport(job_id)
return result.addCallback(self.assertImported, code_import_id)
def test_import_subversion(self):
# Create a Subversion CodeImport and import it.
job = self.getStartedJobForImport(self.makeSVNCodeImport())
code_import_id = job.code_import.id
job_id = job.id
self.layer.txn.commit()
result = self.performImport(job_id)
return result.addCallback(self.assertImported, code_import_id)
def test_import_git(self):
# Create a Git CodeImport and import it.
job = self.getStartedJobForImport(self.makeGitCodeImport())
code_import_id = job.code_import.id
job_id = job.id
self.layer.txn.commit()
result = self.performImport(job_id)
return result.addCallback(self.assertImported, code_import_id)
def test_import_bzr(self):
# Create a Bazaar CodeImport and import it.
job = self.getStartedJobForImport(self.makeBzrCodeImport())
code_import_id = job.code_import.id
job_id = job.id
self.layer.txn.commit()
result = self.performImport(job_id)
return result.addCallback(self.assertImported, code_import_id)
def test_import_bzrsvn(self):
# Create a Subversion-via-bzr-svn CodeImport and import it.
job = self.getStartedJobForImport(self.makeBzrSvnCodeImport())
code_import_id = job.code_import.id
job_id = job.id
self.layer.txn.commit()
result = self.performImport(job_id)
return result.addCallback(self.assertImported, code_import_id)
class TestWorkerMonitorIntegration(TestCaseInTempDir, TestCase):
layer = ZopelessAppServerLayer
run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=60)
def setUp(self):
super(TestWorkerMonitorIntegration, self).setUp()
login('*****@*****.**')
self.factory = LaunchpadObjectFactory()
nuke_codeimport_sample_data()
self.repo_path = tempfile.mkdtemp()
self.disable_directory_isolation()
self.addCleanup(shutil.rmtree, self.repo_path)
self.foreign_commit_count = 0
def tearDown(self):
super(TestWorkerMonitorIntegration, self).tearDown()
logout()
def makeCVSCodeImport(self):
"""Make a `CodeImport` that points to a real CVS repository."""
cvs_server = CVSServer(self.repo_path)
cvs_server.start_server()
self.addCleanup(cvs_server.stop_server)
cvs_server.makeModule('trunk', [('README', 'original\n')])
self.foreign_commit_count = 2
return self.factory.makeCodeImport(cvs_root=cvs_server.getRoot(),
cvs_module='trunk')
def makeSVNCodeImport(self):
"""Make a `CodeImport` that points to a real Subversion repository."""
self.subversion_server = SubversionServer(self.repo_path)
self.subversion_server.start_server()
self.addCleanup(self.subversion_server.stop_server)
url = self.subversion_server.makeBranch('trunk',
[('README', 'contents')])
self.foreign_commit_count = 2
return self.factory.makeCodeImport(svn_branch_url=url)
def makeBzrSvnCodeImport(self):
"""Make a `CodeImport` that points to a real Subversion repository."""
self.subversion_server = SubversionServer(self.repo_path,
use_svn_serve=True)
self.subversion_server.start_server()
self.addCleanup(self.subversion_server.stop_server)
url = self.subversion_server.makeBranch('trunk',
[('README', 'contents')])
self.foreign_commit_count = 2
return self.factory.makeCodeImport(
svn_branch_url=url, rcs_type=RevisionControlSystems.BZR_SVN)
def makeGitCodeImport(self, target_rcs_type=None):
"""Make a `CodeImport` that points to a real Git repository."""
self.git_server = GitServer(self.repo_path, use_server=False)
self.git_server.start_server()
self.addCleanup(self.git_server.stop_server)
self.git_server.makeRepo('source', [('README', 'contents')])
self.foreign_commit_count = 1
return self.factory.makeCodeImport(
git_repo_url=self.git_server.get_url('source'),
target_rcs_type=target_rcs_type)
def makeBzrCodeImport(self):
"""Make a `CodeImport` that points to a real Bazaar branch."""
self.bzr_server = BzrServer(self.repo_path)
self.bzr_server.start_server()
self.addCleanup(self.bzr_server.stop_server)
self.bzr_server.makeRepo([('README', 'contents')])
self.foreign_commit_count = 1
return self.factory.makeCodeImport(
bzr_branch_url=self.bzr_server.get_url())
def getStartedJobForImport(self, code_import):
"""Get a started `CodeImportJob` for `code_import`.
This method approves the import, creates a job, marks it started and
returns the job. It also makes sure there are no branches or foreign
trees in the default stores to interfere with processing this job.
"""
if code_import.review_status != CodeImportReviewStatus.REVIEWED:
code_import.updateFromData(
{'review_status': CodeImportReviewStatus.REVIEWED},
self.factory.makePerson())
job = getUtility(ICodeImportJobSet).getJobForMachine('machine', 10)
self.assertEqual(code_import, job.code_import)
source_details = CodeImportSourceDetails.fromArguments(
removeSecurityProxy(job.makeWorkerArguments()))
if IBranch.providedBy(code_import.target):
clean_up_default_stores_for_import(source_details)
self.addCleanup(clean_up_default_stores_for_import, source_details)
return job
def makeTargetGitServer(self):
"""Set up a target Git server that can receive imports."""
self.target_store = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, self.target_store)
self.target_git_server = GitServer(self.target_store, use_server=True)
self.target_git_server.start_server()
self.addCleanup(self.target_git_server.stop_server)
config_name = self.getUniqueString()
config_fixture = self.useFixture(
ConfigFixture(config_name,
self.layer.config_fixture.instance_name))
setting_lines = [
"[codehosting]",
"git_browse_root: %s" % self.target_git_server.get_url(""),
"",
"[launchpad]",
"internal_macaroon_secret_key: some-secret",
]
config_fixture.add_section("\n" + "\n".join(setting_lines))
self.useFixture(ConfigUseFixture(config_name))
self.useFixture(GitHostingFixture())
def assertCodeImportResultCreated(self, code_import):
"""Assert that a `CodeImportResult` was created for `code_import`."""
self.assertEqual(len(list(code_import.results)), 1)
result = list(code_import.results)[0]
self.assertEqual(result.status, CodeImportResultStatus.SUCCESS)
def assertBranchImportedOKForCodeImport(self, code_import):
"""Assert that a branch was pushed into the default branch store."""
if IBranch.providedBy(code_import.target):
url = get_default_bazaar_branch_store()._getMirrorURL(
code_import.branch.id)
branch = Branch.open(url)
commit_count = branch.revno()
else:
repo_path = os.path.join(self.target_store,
code_import.target.unique_name)
commit_count = int(
subprocess.check_output(["git", "rev-list", "--count", "HEAD"],
cwd=repo_path,
universal_newlines=True))
self.assertEqual(self.foreign_commit_count, commit_count)
def assertImported(self, code_import_id):
"""Assert that the `CodeImport` of the given id was imported."""
# In the in-memory tests, check that resetTimeout on the
# CodeImportWorkerMonitorProtocol was called at least once.
if self._protocol is not None:
self.assertPositive(self._protocol.reset_calls)
code_import = getUtility(ICodeImportSet).get(code_import_id)
self.assertCodeImportResultCreated(code_import)
self.assertBranchImportedOKForCodeImport(code_import)
def performImport(self, job_id):
"""Perform the import job with ID job_id.
Return a Deferred that fires when it the job is done.
This implementation does it in-process.
"""
logger = BufferLogger()
monitor = CIWorkerMonitorForTesting(
job_id, logger,
xmlrpc.Proxy(config.codeimportdispatcher.codeimportscheduler_url),
"anything")
deferred = monitor.run()
def save_protocol_object(result):
"""Save the process protocol object.
We do this in an addBoth so that it's called after the process
protocol is actually constructed but before we drop the last
reference to the monitor object.
"""
self._protocol = monitor._protocol
return result
return deferred.addBoth(save_protocol_object)
@defer.inlineCallbacks
def test_import_cvs(self):
# Create a CVS CodeImport and import it.
job = self.getStartedJobForImport(self.makeCVSCodeImport())
code_import_id = job.code_import.id
job_id = job.id
self.layer.txn.commit()
yield self.performImport(job_id)
self.assertImported(code_import_id)
@defer.inlineCallbacks
def test_import_subversion(self):
# Create a Subversion CodeImport and import it.
job = self.getStartedJobForImport(self.makeSVNCodeImport())
code_import_id = job.code_import.id
job_id = job.id
self.layer.txn.commit()
yield self.performImport(job_id)
self.assertImported(code_import_id)
@defer.inlineCallbacks
def test_import_git(self):
# Create a Git CodeImport and import it.
job = self.getStartedJobForImport(self.makeGitCodeImport())
code_import_id = job.code_import.id
job_id = job.id
self.layer.txn.commit()
yield self.performImport(job_id)
self.assertImported(code_import_id)
@defer.inlineCallbacks
def test_import_git_to_git(self):
# Create a Git-to-Git CodeImport and import it.
self.makeTargetGitServer()
job = self.getStartedJobForImport(
self.makeGitCodeImport(
target_rcs_type=TargetRevisionControlSystems.GIT))
code_import_id = job.code_import.id
job_id = job.id
self.layer.txn.commit()
target_repo_path = os.path.join(self.target_store,
job.code_import.target.unique_name)
os.makedirs(target_repo_path)
self.target_git_server.createRepository(target_repo_path, bare=True)
yield self.performImport(job_id)
self.assertImported(code_import_id)
target_repo = GitRepo(target_repo_path)
self.assertContentEqual(["heads/master"],
target_repo.refs.keys(base="refs"))
self.assertEqual("ref: refs/heads/master",
target_repo.refs.read_ref("HEAD"))
@defer.inlineCallbacks
def test_import_git_to_git_refs_changed(self):
# Create a Git-to-Git CodeImport and import it incrementally with
# ref and HEAD changes.
self.makeTargetGitServer()
job = self.getStartedJobForImport(
self.makeGitCodeImport(
target_rcs_type=TargetRevisionControlSystems.GIT))
code_import_id = job.code_import.id
job_id = job.id
self.layer.txn.commit()
source_repo = GitRepo(os.path.join(self.repo_path, "source"))
commit = source_repo.refs["refs/heads/master"]
source_repo.refs["refs/heads/one"] = commit
source_repo.refs["refs/heads/two"] = commit
source_repo.refs.set_symbolic_ref("HEAD", "refs/heads/one")
del source_repo.refs["refs/heads/master"]
target_repo_path = os.path.join(self.target_store,
job.code_import.target.unique_name)
self.target_git_server.makeRepo(job.code_import.target.unique_name,
[("NEWS", "contents")])
yield self.performImport(job_id)
self.assertImported(code_import_id)
target_repo = GitRepo(target_repo_path)
self.assertContentEqual(["heads/one", "heads/two"],
target_repo.refs.keys(base="refs"))
self.assertEqual("ref: refs/heads/one",
GitRepo(target_repo_path).refs.read_ref("HEAD"))
@defer.inlineCallbacks
def test_import_bzr(self):
# Create a Bazaar CodeImport and import it.
job = self.getStartedJobForImport(self.makeBzrCodeImport())
code_import_id = job.code_import.id
job_id = job.id
self.layer.txn.commit()
yield self.performImport(job_id)
self.assertImported(code_import_id)
@defer.inlineCallbacks
def test_import_bzrsvn(self):
# Create a Subversion-via-bzr-svn CodeImport and import it.
job = self.getStartedJobForImport(self.makeBzrSvnCodeImport())
code_import_id = job.code_import.id
job_id = job.id
self.layer.txn.commit()
yield self.performImport(job_id)
self.assertImported(code_import_id)