def test_init(self, getLogger):
bad = [
"git://user@@server/foo.git#bab0c73900241ef5c465d7e873e9d8b34c948e67",
"git://*****:*****@server/foo.git#bab0c73900241ef5c465d7e873e9d8b34c948e67",
"git://server/foo.git;params=not_allowed",
"git://server#asdasd", # no path
"git://server/foo.git", # no fragment
"http://localhost/foo.html",
"git://@localhost/foo/?a=bar/",
"http://localhost/foo.html?a=foo/",
"foo-1.1-1.src.rpm",
"git://",
"https://server/foo-1.1-1.src.rpm",
]
for url in bad:
with self.assertRaises(koji.GenericError):
scm = SCM(url)
url = "git://user@server/foo.git#bab0c73900241ef5c465d7e873e9d8b34c948e67"
scm = SCM(url)
self.assertEqual(scm.scheme, 'git://')
self.assertEqual(scm.user, 'user')
self.assertEqual(scm.host, 'server')
self.assertEqual(scm.repository, '/foo.git')
self.assertEqual(scm.module, '')
self.assertEqual(scm.revision,
'bab0c73900241ef5c465d7e873e9d8b34c948e67')
self.assertEqual(scm.use_common, True)
self.assertEqual(scm.source_cmd, ['make', 'sources'])
self.assertEqual(scm.scmtype, 'GIT')
def test_badrule(self, getLogger):
config = '''
bogus-entry-should-be-ignored
goodserver:*:no
!badserver:*
'''
url = "git://goodserver/path1#1234"
scm = SCM(url)
scm.assert_allowed(config)
def test_badrule(self, getLogger):
config = '''
bogus-entry-should-be-ignored
goodserver:*:no
!badserver:*
'''
url = "git://goodserver/path1#1234"
scm = SCM(url)
scm.assert_allowed(config)
def test_get_source_git(self, popen):
popen.return_value.wait.return_value = 0
popen.return_value.communicate = mock.MagicMock()
popen.return_value.communicate.return_value = (six.b('hash '),
six.b('any'))
url = "git://default/koji.git#asdasd"
scm = SCM(url)
scm.assert_allowed(self.config)
scm.checkout(self.tempdir,
session=self.session,
uploadpath=self.uploadpath,
logfile=self.logfile)
source = scm.get_source()
self.assertEqual(source, {
'url': url,
'source': 'git://default/koji.git#hash'
})
popen.return_value.wait.return_value = 1
with self.assertRaises(koji.GenericError) as cm:
source = scm.get_source()
self.assertEqual(cm.exception.args[0],
'Error getting commit hash for git')
def test_allowed_by_policy(self):
good = [
"git://goodserver/path1#1234",
"git+ssh://maybeserver/path1#1234",
]
bad = [
"cvs://badserver/projects/42#ref",
"svn://badserver/projects/42#ref",
"git://maybeserver/badpath/project#1234",
"git://maybeserver//badpath/project#1234",
"git://maybeserver////badpath/project#1234",
"git://maybeserver/./badpath/project#1234",
"git://maybeserver//.//badpath/project#1234",
"git://maybeserver/goodpath/../badpath/project#1234",
"git://maybeserver/goodpath/..//badpath/project#1234",
"git://maybeserver/..//badpath/project#1234",
]
session = mock.MagicMock()
session.evalPolicy.side_effect = FakePolicy(policy['one']).evalPolicy
for url in good:
scm = SCM(url)
scm.assert_allowed(session=session, by_config=False, by_policy=True)
for url in bad:
scm = SCM(url)
with self.assertRaises(koji.BuildError) as cm:
scm.assert_allowed(session=session, by_config=False, by_policy=True)
self.assertRegex(str(cm.exception), '^SCM: .* is not allowed, reason: None$')
def test_allowed(self, getLogger):
config = '''
goodserver:*:no
!badserver:*
!maybeserver:/badpath/*
maybeserver:*:no
'''
good = [
"git://goodserver/path1#1234",
"git+ssh://maybeserver/path1#1234",
]
bad = [
"cvs://badserver/projects/42#ref",
"svn://badserver/projects/42#ref",
"git://maybeserver/badpath/project#1234",
"git://maybeserver//badpath/project#1234",
"git://maybeserver////badpath/project#1234",
"git://maybeserver/./badpath/project#1234",
"git://maybeserver//.//badpath/project#1234",
"git://maybeserver/goodpath/../badpath/project#1234",
"git://maybeserver/goodpath/..//badpath/project#1234",
"git://maybeserver/..//badpath/project#1234",
]
for url in good:
scm = SCM(url)
scm.assert_allowed(config)
for url in bad:
scm = SCM(url)
try:
scm.assert_allowed(config)
except koji.BuildError:
pass
else:
raise AssertionError("allowed bad url: %s" % url)
def test_checkout_cvs_ssh(self):
url = "cvs+ssh://user@nocommon/cvsisdead?rpms/foo/EL3#sometag"
scm = SCM(url)
scm.assert_allowed(self.config)
scm.checkout(self.tempdir, session=self.session,
uploadpath=self.uploadpath, logfile=self.logfile)
self.assertEqual(scm.use_common, False)
self.symlink.assert_not_called()
# expected commands
cmd = ['cvs', '-d', ':ext:user@nocommon:/cvsisdead', 'checkout', '-r',
'sometag', 'rpms/foo/EL3']
call1 = mock.call(self.session, cmd[0], cmd, self.logfile,
self.uploadpath, cwd=self.tempdir, logerror=1,
append=False, env={'CVS_RSH': 'ssh'})
self.log_output.assert_has_calls([call1])
def test_checkout_svn_ssh(self):
url = "svn+ssh://user@nocommon/dist?rpms/foo/EL3#revision"
scm = SCM(url)
scm.assert_allowed(self.config)
scm.checkout(self.tempdir, session=self.session,
uploadpath=self.uploadpath, logfile=self.logfile)
self.assertEqual(scm.use_common, False)
self.symlink.assert_not_called()
# expected commands
cmd = ['svn', 'checkout', '-r', 'revision',
'svn+ssh://user@nocommon/dist/rpms/foo/EL3', 'rpms/foo/EL3']
call1 = mock.call(self.session, cmd[0], cmd, self.logfile,
self.uploadpath, cwd=self.tempdir, logerror=1,
append=False, env=None)
self.log_output.assert_has_calls([call1])
def fetchDockerfile(self, src, build_tag):
"""
Gets Dockerfile. Roughly corresponds to getSRPM method of build task
"""
scm = SCM(src)
scm.assert_allowed(self.options.allowed_scms)
scmdir = os.path.join(self.workdir, 'sources')
koji.ensuredir(scmdir)
logfile = os.path.join(self.workdir, 'checkout-for-labels.log')
uploadpath = self.getUploadDir()
koji.ensuredir(uploadpath)
self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag,
scratch=self.opts.get('scratch', False))
# Check out sources from the SCM
sourcedir = scm.checkout(scmdir, self.session, uploadpath, logfile)
self.run_callbacks("postSCMCheckout", scminfo=scm.get_info(), build_tag=build_tag,
scratch=self.opts.get('scratch', False), srcdir=sourcedir)
fn = os.path.join(sourcedir, 'Dockerfile')
if not os.path.exists(fn):
raise koji.BuildError("Dockerfile file missing: %s" % fn)
return fn
def test_checkout_error_in_command(self):
url = "git://nocommon/koji.git#asdasd"
scm = SCM(url)
scm.assert_allowed(self.config)
self.log_output.return_value = 1
with self.assertRaises(koji.BuildError):
scm.checkout(self.tempdir, session=self.session,
uploadpath=self.uploadpath, logfile=self.logfile)
self.assertEqual(scm.use_common, False)
self.symlink.assert_not_called()
# expected commands
cmd = ['git', 'clone', '-n', 'git://nocommon/koji.git',
self.tempdir + '/koji']
call1 = mock.call(self.session, cmd[0], cmd, self.logfile,
self.uploadpath, cwd=self.tempdir, logerror=1,
append=False, env=None)
# should have errored after first command
self.log_output.assert_has_calls([call1])
def test_checkout_gitssh_nocommon(self):
url = "git+ssh://user@nocommon/koji.git#asdasd"
scm = SCM(url)
scm.assert_allowed(self.config)
scm.checkout(self.tempdir, session=self.session,
uploadpath=self.uploadpath, logfile=self.logfile)
self.assertEqual(scm.use_common, False)
self.symlink.assert_not_called()
# expected commands
cmd = ['git', 'clone', '-n', 'git+ssh://user@nocommon/koji.git',
self.tempdir + '/koji']
call1 = mock.call(self.session, cmd[0], cmd, self.logfile,
self.uploadpath, cwd=self.tempdir, logerror=1,
append=False, env=None)
cmd = ['git', 'reset', '--hard', 'asdasd']
call2 = mock.call(self.session, cmd[0], cmd, self.logfile,
self.uploadpath, cwd=self.tempdir + '/koji',
logerror=1, append=True, env=None)
self.log_output.assert_has_calls([call1, call2])
def test_urlcheck(self):
good = [
"git://server/foo.git#bab0c73900241ef5c465d7e873e9d8b34c948e67",
"git+ssh://server2/other/path#bab0c73900241ef5c465d7e873e9d8b34c948e67",
"svn://server/path/to/code#bab0c73900241ef5c465d7e873e9d8b34c948e67",
"svn+ssh://server/some/path#bab0c73900241ef5c465d7e873e9d8b34c948e67",
"cvs://server/some/path#bab0c73900241ef5c465d7e873e9d8b34c948e67",
"cvs+ssh://server/some/path#bab0c73900241ef5c465d7e873e9d8b34c948e67",
]
bad = [
"http://localhost/foo.html",
"foo-1.1-1.src.rpm",
"https://server/foo-1.1-1.src.rpm",
"git:foobar",
"https:foo/bar",
"https://",
]
for url in good:
self.assertTrue(SCM.is_scm_url(url))
for url in bad:
self.assertFalse(SCM.is_scm_url(url))
def test_urlcheck(self):
good = [
"git://server/foo.git#bab0c73900241ef5c465d7e873e9d8b34c948e67",
"git+ssh://server2/other/path#bab0c73900241ef5c465d7e873e9d8b34c948e67",
"svn://server/path/to/code#bab0c73900241ef5c465d7e873e9d8b34c948e67",
"svn+ssh://server/some/path#bab0c73900241ef5c465d7e873e9d8b34c948e67",
"cvs://server/some/path#bab0c73900241ef5c465d7e873e9d8b34c948e67",
"cvs+ssh://server/some/path#bab0c73900241ef5c465d7e873e9d8b34c948e67",
]
bad = [
"http://localhost/foo.html",
"foo-1.1-1.src.rpm",
"https://server/foo-1.1-1.src.rpm",
"git:foobar",
"https:foo/bar",
"https://",
]
for url in good:
self.assertTrue(SCM.is_scm_url(url))
for url in bad:
self.assertFalse(SCM.is_scm_url(url))
def test_checkout_cvs_common(self):
url = "cvs://default/cvsisdead?rpms/foo/EL3#sometag"
scm = SCM(url)
scm.assert_allowed(self.config)
scm.checkout(self.tempdir, session=self.session,
uploadpath=self.uploadpath, logfile=self.logfile)
self.assertEqual(scm.use_common, True)
self.symlink.assert_called_once()
# expected commands
cmd = ['cvs', '-d', ':pserver:anonymous@default:/cvsisdead', 'checkout',
'-r', 'sometag', 'rpms/foo/EL3']
call1 = mock.call(self.session, cmd[0], cmd, self.logfile,
self.uploadpath, cwd=self.tempdir, logerror=1,
append=False, env=None)
cmd = ['cvs', '-d', ':pserver:anonymous@default:/cvsisdead', 'checkout',
'common']
call2 = mock.call(self.session, cmd[0], cmd, self.logfile,
self.uploadpath, cwd=self.tempdir, logerror=1,
append=True, env=None)
self.log_output.assert_has_calls([call1, call2])
def fetchDockerfile(self, src, build_tag):
"""
Gets Dockerfile. Roughly corresponds to getSRPM method of build task
"""
scm = SCM(src)
scm.assert_allowed(self.options.allowed_scms)
scmdir = os.path.join(self.workdir, 'sources')
koji.ensuredir(scmdir)
logfile = os.path.join(self.workdir, 'checkout-for-labels.log')
uploadpath = self.getUploadDir()
koji.ensuredir(uploadpath)
self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag,
scratch=self.opts.get('scratch', False))
# Check out sources from the SCM
sourcedir = scm.checkout(scmdir, self.session, uploadpath, logfile)
self.run_callbacks("postSCMCheckout", scminfo=scm.get_info(), build_tag=build_tag,
scratch=self.opts.get('scratch', False), srcdir=sourcedir)
fn = os.path.join(sourcedir, 'Dockerfile')
if not os.path.exists(fn):
raise koji.BuildError("Dockerfile file missing: %s" % fn)
return fn
def test_allowed(self, getLogger):
config = '''
goodserver:*:no
!badserver:*
!maybeserver:/badpath/*
maybeserver:*:no
'''
good = [
"git://goodserver/path1#1234",
"git+ssh://maybeserver/path1#1234",
]
bad = [
"cvs://badserver/projects/42#ref",
"svn://badserver/projects/42#ref",
"git://maybeserver/badpath/project#1234",
"git://maybeserver//badpath/project#1234",
"git://maybeserver////badpath/project#1234",
"git://maybeserver/./badpath/project#1234",
"git://maybeserver//.//badpath/project#1234",
"git://maybeserver/goodpath/../badpath/project#1234",
"git://maybeserver/goodpath/..//badpath/project#1234",
"git://maybeserver/..//badpath/project#1234",
]
for url in good:
scm = SCM(url)
scm.assert_allowed(config)
for url in bad:
scm = SCM(url)
try:
scm.assert_allowed(config)
except koji.BuildError:
pass
else:
raise AssertionError("allowed bad url: %s" % url)
def test_checkout_svn_ssh(self):
url = "svn+ssh://user@nocommon/dist?rpms/foo/EL3#revision"
scm = SCM(url)
scm.assert_allowed(self.config)
scm.checkout(self.tempdir,
session=self.session,
uploadpath=self.uploadpath,
logfile=self.logfile)
self.assertEqual(scm.use_common, False)
self.symlink.assert_not_called()
# expected commands
cmd = [
'svn', 'checkout', '-r', 'revision',
'svn+ssh://user@nocommon/dist/rpms/foo/EL3', 'rpms/foo/EL3'
]
call1 = mock.call(self.session,
cmd[0],
cmd,
self.logfile,
self.uploadpath,
cwd=self.tempdir,
logerror=1,
append=False,
env=None)
self.log_output.assert_has_calls([call1])
def test_checkout_cvs_ssh(self):
url = "cvs+ssh://user@nocommon/cvsisdead?rpms/foo/EL3#sometag"
scm = SCM(url)
scm.assert_allowed(self.config)
scm.checkout(self.tempdir,
session=self.session,
uploadpath=self.uploadpath,
logfile=self.logfile)
self.assertEqual(scm.use_common, False)
self.symlink.assert_not_called()
# expected commands
cmd = [
'cvs', '-d', ':ext:user@nocommon:/cvsisdead', 'checkout', '-r',
'sometag', 'rpms/foo/EL3'
]
call1 = mock.call(self.session,
cmd[0],
cmd,
self.logfile,
self.uploadpath,
cwd=self.tempdir,
logerror=1,
append=False,
env={'CVS_RSH': 'ssh'})
self.log_output.assert_has_calls([call1])
def test_checkout_error_in_command(self):
url = "git://nocommon/koji.git#asdasd"
scm = SCM(url)
scm.assert_allowed(self.config)
self.log_output.return_value = 1
with self.assertRaises(koji.BuildError):
scm.checkout(self.tempdir,
session=self.session,
uploadpath=self.uploadpath,
logfile=self.logfile)
self.assertEqual(scm.use_common, False)
self.symlink.assert_not_called()
# expected commands
cmd = [
'git', 'clone', '-n', 'git://nocommon/koji.git',
self.tempdir + '/koji'
]
call1 = mock.call(self.session,
cmd[0],
cmd,
self.logfile,
self.uploadpath,
cwd=self.tempdir,
logerror=1,
append=False,
env=None)
# should have errored after first command
self.log_output.assert_has_calls([call1])
def fetchDockerfile(self, src):
"""
Gets Dockerfile. Roughly corresponds to getSRPM method of build task
"""
scm = SCM(src)
scm.assert_allowed(self.options.allowed_scms)
scmdir = os.path.join(self.workdir, 'sources')
koji.ensuredir(scmdir)
logfile = os.path.join(self.workdir, 'checkout-for-labels.log')
uploadpath = self.getUploadDir()
koji.ensuredir(uploadpath)
# Check out sources from the SCM
sourcedir = scm.checkout(scmdir, self.session, uploadpath, logfile)
fn = os.path.join(sourcedir, 'Dockerfile')
if not os.path.exists(fn):
raise koji.BuildError, "Dockerfile file missing: %s" % fn
return fn
def fetchDockerfile(self, src):
"""
Gets Dockerfile. Roughly corresponds to getSRPM method of build task
"""
scm = SCM(src)
scm.assert_allowed(self.options.allowed_scms)
scmdir = os.path.join(self.workdir, 'sources')
koji.ensuredir(scmdir)
logfile = os.path.join(self.workdir, 'checkout-for-labels.log')
uploadpath = self.getUploadDir()
koji.ensuredir(uploadpath)
# Check out sources from the SCM
sourcedir = scm.checkout(scmdir, self.session, uploadpath, logfile)
fn = os.path.join(sourcedir, 'Dockerfile')
if not os.path.exists(fn):
raise koji.BuildError, "Dockerfile file missing: %s" % fn
return fn
def test_assert_allowed_basic(self):
scm = SCM("git://scm.example.com/path1#1234")
# session must be passed
with self.assertRaises(koji.GenericError) as cm:
scm.assert_allowed(session=None, by_config=False, by_policy=True)
self.assertEqual(str(cm.exception),
'When allowed SCM assertion is by policy, session must be passed in.')
# allowed could not be None
scm.assert_allowed_by_config = mock.MagicMock()
scm.assert_allowed(allowed=None, by_config=True, by_policy=False)
scm.assert_allowed_by_config.assert_called_once_with('')
def test_get_source_other(self, popen):
popen.return_value.wait.return_value = 0
popen.return_value.communicate = mock.MagicMock()
popen.return_value.communicate.return_value = ('hash ', 'any')
url = "svn+ssh://user@nocommon/dist?rpms/foo/EL3#revision"
scm = SCM(url)
scm.assert_allowed(self.config)
scm.checkout(self.tempdir, session=self.session,
uploadpath=self.uploadpath, logfile=self.logfile)
source = scm.get_source()
self.assertEqual(source, {'url': url, 'source': url})
def test_get_source_other(self, popen):
popen.return_value.wait.return_value = 0
popen.return_value.communicate = mock.MagicMock()
popen.return_value.communicate.return_value = ('hash ', 'any')
url = "svn+ssh://user@nocommon/dist?rpms/foo/EL3#revision"
scm = SCM(url)
scm.assert_allowed(self.config)
scm.checkout(self.tempdir, session=self.session,
uploadpath=self.uploadpath, logfile=self.logfile)
source = scm.get_source()
self.assertEqual(source, {'url': url, 'source': url})
def test_get_source_git(self, popen):
popen.return_value.wait.return_value = 0
popen.return_value.communicate = mock.MagicMock()
popen.return_value.communicate.return_value = (six.b('hash '), six.b('any'))
url = "git://default/koji.git#asdasd"
scm = SCM(url)
scm.assert_allowed(self.config)
scm.checkout(self.tempdir, session=self.session,
uploadpath=self.uploadpath, logfile=self.logfile)
source = scm.get_source()
self.assertEqual(source, {'url': url,
'source': 'git://default/koji.git#hash'})
popen.return_value.wait.return_value = 1
with self.assertRaises(koji.GenericError) as cm:
source = scm.get_source()
self.assertEqual(cm.exception.args[0],
'Error getting commit hash for git')
def test_checkout_git_common(self):
url = "git://default/koji.git#asdasd"
scm = SCM(url)
scm.assert_allowed(self.config)
scm.checkout(self.tempdir,
session=self.session,
uploadpath=self.uploadpath,
logfile=self.logfile)
self.assertEqual(scm.use_common, True)
self.symlink.assert_called_once()
# expected commands
cmd = [
'git', 'clone', '-n', 'git://default/koji.git',
self.tempdir + '/koji'
]
call1 = mock.call(self.session,
cmd[0],
cmd,
self.logfile,
self.uploadpath,
cwd=self.tempdir,
logerror=1,
append=False,
env=None)
cmd = ['git', 'reset', '--hard', 'asdasd']
call2 = mock.call(self.session,
cmd[0],
cmd,
self.logfile,
self.uploadpath,
cwd=self.tempdir + '/koji',
logerror=1,
append=True,
env=None)
cmd = ['git', 'clone', 'git://default/common.git', 'common']
call3 = mock.call(self.session,
cmd[0],
cmd,
self.logfile,
self.uploadpath,
cwd=self.tempdir,
logerror=1,
append=True,
env=None)
self.log_output.assert_has_calls([call1, call2, call3])
def test_checkout_cvs_common(self):
url = "cvs://default/cvsisdead?rpms/foo/EL3#sometag"
scm = SCM(url)
scm.assert_allowed(self.config)
scm.checkout(self.tempdir, session=self.session,
uploadpath=self.uploadpath, logfile=self.logfile)
self.assertEqual(scm.use_common, True)
self.symlink.assert_called_once()
# expected commands
cmd = ['cvs', '-d', ':pserver:anonymous@default:/cvsisdead', 'checkout',
'-r', 'sometag', 'rpms/foo/EL3']
call1 = mock.call(self.session, cmd[0], cmd, self.logfile, self.uploadpath,
cwd=self.tempdir, logerror=1, append=False, env=None)
cmd = ['cvs', '-d', ':pserver:anonymous@default:/cvsisdead', 'checkout', 'common']
call2 = mock.call(self.session, cmd[0], cmd, self.logfile, self.uploadpath,
cwd=self.tempdir, logerror=1, append=True, env=None)
self.log_output.assert_has_calls([call1, call2])
def handler(self, src, target, opts=None):
jsonschema.validate([src, target, opts], self.PARAMS_SCHEMA)
self.opts = opts
component = None
if not opts.get('git_branch'):
raise koji.BuildError("Git branch must be specified")
if opts.get('scratch') and opts.get('isolated'):
raise koji.BuildError("Build cannot be both isolated and scratch")
self.event_id = self.session.getLastEvent()['id']
target_info = self.session.getBuildTarget(target, event=self.event_id)
if not target_info:
raise koji.BuildError("Target `%s` not found" % target)
build_tag = target_info['build_tag']
archlist = self.getArchList(build_tag)
flatpak = opts.get('flatpak', False)
if flatpak:
if not osbs_flatpak_support:
raise koji.BuildError("osbs-client on koji builder doesn't have Flatpak support")
release_overwrite = None
else:
label_overwrites = {}
release_overwrite = opts.get('release')
if release_overwrite:
label_overwrites = {LABEL_NAME_MAP['RELEASE'][0]: release_overwrite}
component, expected_nvr = self.checkLabels(src, label_overwrites=label_overwrites,
build_tag=build_tag)
# scratch builds do not get imported, and consequently not tagged
if not self.opts.get('scratch') and not flatpak:
self.check_whitelist(component, target_info)
if flatpak:
expected_nvr = None
if not SCM.is_scm_url(src):
raise koji.BuildError('Invalid source specification: %s' % src)
# don't check build nvr for autorebuild (has triggered_after_koji_task)
# as they might be using add_timestamp_to_release
# and don't check it for skipped build, which might be enabling/disabling
# autorebuilds which use add_timestamp_to_release
triggered_after_koji_task = opts.get('triggered_after_koji_task', None)
skip_build = opts.get('skip_build', False)
if triggered_after_koji_task or skip_build:
expected_nvr = None
# Scratch and auto release builds shouldn't be checked for nvr
if not self.opts.get('scratch') and expected_nvr:
try:
build = self.session.getBuild(expected_nvr)
build_id = build['id']
except Exception:
self.logger.info("No build for %s found", expected_nvr, exc_info=True)
else:
if build['state'] in (koji.BUILD_STATES['FAILED'], koji.BUILD_STATES['CANCELED']):
self.logger.info("Build for %s found, but with reusable state %s",
expected_nvr, build['state'], exc_info=True)
else:
raise koji.BuildError("Build for %s already exists, id %s" %
(expected_nvr, build_id))
self.logger.debug("Spawning jobs for arches: %r", archlist)
kwargs = dict(
src=src,
target_info=target_info,
scratch=opts.get('scratch', False),
isolated=opts.get('isolated', False),
dependency_replacements=opts.get('dependency_replacements', None),
yum_repourls=opts.get('yum_repourls', None),
branch=opts.get('git_branch', None),
push_url=opts.get('push_url', None),
arches=archlist,
koji_parent_build=opts.get('koji_parent_build'),
release=release_overwrite,
flatpak=flatpak,
signing_intent=opts.get('signing_intent', None),
compose_ids=opts.get('compose_ids', None),
skip_build=skip_build,
triggered_after_koji_task=triggered_after_koji_task,
)
results = []
semi_results = self.createContainer(**kwargs)
if semi_results is not None:
results = [semi_results]
self.logger.debug("Results: %r", results)
all_repositories = []
all_koji_builds = []
if not results:
return {
'repositories': all_repositories,
'koji_builds': all_koji_builds,
'build': 'skipped',
}
for result in results:
try:
repository = result.get('repositories')
all_repositories.extend(repository)
except Exception as error:
self.logger.error("Failed to merge list of repositories "
"%r. Reason (%s): %s", repository,
type(error), error)
koji_build_id = result.get('koji_build_id')
if koji_build_id:
all_koji_builds.append(koji_build_id)
return {
'repositories': all_repositories,
'koji_builds': all_koji_builds,
}
def test_opts_by_policy(self):
session = mock.MagicMock()
session.evalPolicy.side_effect = FakePolicy(policy['two']).evalPolicy
url = "git://default/koji.git#1234"
scm = SCM(url)
scm.assert_allowed_by_policy(session=session)
self.assertEqual(scm.use_common, False)
self.assertEqual(scm.source_cmd, ['make', 'sources'])
url = "git://nocommon/koji.git#1234"
scm = SCM(url)
scm.assert_allowed_by_policy(session=session)
self.assertEqual(scm.use_common, False)
self.assertEqual(scm.source_cmd, ['make', 'sources'])
url = "git://common/koji.git#1234"
scm = SCM(url)
scm.assert_allowed_by_policy(session=session)
self.assertEqual(scm.use_common, True)
self.assertEqual(scm.source_cmd, ['make', 'sources'])
url = "git://srccmd/koji.git#1234"
scm = SCM(url)
scm.assert_allowed_by_policy(session=session)
self.assertEqual(scm.use_common, False)
self.assertEqual(scm.source_cmd, ['fedpkg', 'sources'])
url = "git://nosrc/koji.git#1234"
scm = SCM(url)
scm.assert_allowed_by_policy(session=session)
self.assertEqual(scm.use_common, False)
self.assertEqual(scm.source_cmd, None)
url = "git://mixed/foo/koji.git#1234"
scm = SCM(url)
scm.assert_allowed_by_policy(session=session)
self.assertEqual(scm.use_common, False)
self.assertEqual(scm.source_cmd, ['make', 'sources'])
url = "git://mixed/bar/koji.git#1234"
scm = SCM(url)
scm.assert_allowed_by_policy(session=session)
self.assertEqual(scm.use_common, True)
self.assertEqual(scm.source_cmd, ['make', 'sources'])
url = "git://mixed/baz/koji.git#1234"
scm = SCM(url)
scm.assert_allowed_by_policy(session=session)
self.assertEqual(scm.use_common, False)
self.assertEqual(scm.source_cmd, ['fedpkg', 'sources'])
url = "git://mixed/koji.git#1234"
scm = SCM(url)
with self.assertRaises(koji.BuildError):
scm.assert_allowed_by_policy(session=session)
url = "git://mixed/foo/koji.git#1234"
scm = SCM(url)
scm.assert_allowed_by_policy(session=session)
self.assertEqual(scm.use_common, False)
self.assertEqual(scm.source_cmd, ['make', 'sources'])
url = "git://mixed/bar/koji.git#1234"
scm = SCM(url)
scm.assert_allowed_by_policy(session=session)
self.assertEqual(scm.use_common, True)
self.assertEqual(scm.source_cmd, ['make', 'sources'])
url = "git://mixed/baz/koji.git#1234"
scm = SCM(url)
scm.assert_allowed_by_policy(session=session)
self.assertEqual(scm.use_common, False)
self.assertEqual(scm.source_cmd, ['fedpkg', 'sources'])
url = "git://mixed/foobar/koji.git#1234"
scm = SCM(url)
scm.assert_allowed_by_policy(session=session)
self.assertEqual(scm.use_common, True)
self.assertEqual(scm.source_cmd, ['fedpkg', 'sources'])
url = "git://mixed/foobaz/koji.git#1234"
scm = SCM(url)
scm.assert_allowed_by_policy(session=session)
self.assertEqual(scm.use_common, True)
self.assertIsNone(scm.source_cmd)
url = "git://nomatch/koji.git#1234"
scm = SCM(url)
with self.assertRaises(koji.BuildError):
scm.assert_allowed_by_policy(session=session)
def test_assert_allowed_by_both(self):
config = '''
default:*:no:
mixed:/foo/*:yes
mixed:/bar/*:no
mixed:/baz/*:no:centpkg,sources
mixed:/foobar/*:no:
mixed:/foobaz/*:no:centpkg,sources
'''
session = mock.MagicMock()
session.evalPolicy.side_effect = FakePolicy(policy['two']).evalPolicy
url = "git://default/koji.git#1234"
scm = SCM(url)
# match scm_host default :: allow
scm.assert_allowed(allowed=config, session=session, by_config=True, by_policy=True)
self.assertEqual(scm.use_common, False)
self.assertIsNone(scm.source_cmd)
url = "git://mixed/foo/koji.git#1234"
scm = SCM(url)
# match scm_host mixed && match scm_repository /foo/* :: allow
scm.assert_allowed(allowed=config, session=session, by_config=True, by_policy=True)
self.assertEqual(scm.use_common, False)
self.assertEqual(scm.source_cmd, ['make', 'sources'])
url = "git://mixed/bar/koji.git#1234"
scm = SCM(url)
# match scm_host mixed && match scm_repository /bar/* :: allow use_common
scm.assert_allowed(allowed=config, session=session, by_config=True, by_policy=True)
self.assertEqual(scm.use_common, True)
self.assertEqual(scm.source_cmd, ['make', 'sources'])
url = "git://mixed/baz/koji.git#1234"
scm = SCM(url)
# match scm_host mixed && match scm_repository /baz/* :: allow fedpkg sources
scm.assert_allowed(allowed=config, session=session, by_config=True, by_policy=True)
self.assertEqual(scm.use_common, False)
self.assertEqual(scm.source_cmd, ['fedpkg', 'sources'])
url = "git://mixed/foobar/koji.git#1234"
scm = SCM(url)
# match scm_host mixed && match scm_repository /foobar/* :: allow use_common fedpkg sources
scm.assert_allowed(allowed=config, session=session, by_config=True, by_policy=True)
self.assertEqual(scm.use_common, True)
self.assertEqual(scm.source_cmd, ['fedpkg', 'sources'])
url = "git://mixed/foobaz/koji.git#1234"
scm = SCM(url)
# match scm_host mixed && match scm_repository /foobaz/* :: allow use_common none
scm.assert_allowed(allowed=config, session=session, by_config=True, by_policy=True)
self.assertEqual(scm.use_common, True)
self.assertIsNone(scm.source_cmd)
def test_opts(self, getLogger):
config = '''
default:*
nocommon:*:no
srccmd:*:no:fedpkg,sources
nosrc:*:no:
mixed:/foo/*:no
mixed:/bar/*:yes
mixed:/baz/*:no:fedpkg,sources
'''
url = "git://default/koji.git#1234"
scm = SCM(url)
scm.assert_allowed(config)
self.assertEqual(scm.use_common, True)
self.assertEqual(scm.source_cmd, ['make', 'sources'])
url = "git://nocommon/koji.git#1234"
scm = SCM(url)
scm.assert_allowed(config)
self.assertEqual(scm.use_common, False)
self.assertEqual(scm.source_cmd, ['make', 'sources'])
url = "git://srccmd/koji.git#1234"
scm = SCM(url)
scm.assert_allowed(config)
self.assertEqual(scm.use_common, False)
self.assertEqual(scm.source_cmd, ['fedpkg', 'sources'])
url = "git://nosrc/koji.git#1234"
scm = SCM(url)
scm.assert_allowed(config)
self.assertEqual(scm.use_common, False)
self.assertEqual(scm.source_cmd, None)
url = "git://mixed/foo/koji.git#1234"
scm = SCM(url)
scm.assert_allowed(config)
self.assertEqual(scm.use_common, False)
self.assertEqual(scm.source_cmd, ['make', 'sources'])
url = "git://mixed/bar/koji.git#1234"
scm = SCM(url)
scm.assert_allowed(config)
self.assertEqual(scm.use_common, True)
self.assertEqual(scm.source_cmd, ['make', 'sources'])
url = "git://mixed/baz/koji.git#1234"
scm = SCM(url)
scm.assert_allowed(config)
self.assertEqual(scm.use_common, False)
self.assertEqual(scm.source_cmd, ['fedpkg', 'sources'])
url = "git://mixed/koji.git#1234"
scm = SCM(url)
with self.assertRaises(koji.BuildError):
scm.assert_allowed(config)
url = "git://mixed/foo/koji.git#1234"
scm = SCM(url)
scm.assert_allowed(config)
self.assertEqual(scm.use_common, False)
self.assertEqual(scm.source_cmd, ['make', 'sources'])
url = "git://mixed/bar/koji.git#1234"
scm = SCM(url)
scm.assert_allowed(config)
self.assertEqual(scm.use_common, True)
self.assertEqual(scm.source_cmd, ['make', 'sources'])
url = "git://mixed/baz/koji.git#1234"
scm = SCM(url)
scm.assert_allowed(config)
self.assertEqual(scm.use_common, False)
self.assertEqual(scm.source_cmd, ['fedpkg', 'sources'])
url = "git://mixed/koji.git#1234"
scm = SCM(url)
with self.assertRaises(koji.BuildError):
scm.assert_allowed(config)
def handler(self, src, target, opts=None):
if not opts:
opts = {}
self.opts = opts
data = {}
self.event_id = self.session.getLastEvent()['id']
target_info = self.session.getBuildTarget(target, event=self.event_id)
if not target_info:
raise koji.BuildError("Target `%s` not found" % target)
build_tag = target_info['build_tag']
archlist = self.getArchList(build_tag)
flatpak = opts.get('flatpak', False)
if flatpak:
if not osbs_flatpak_support:
raise koji.BuildError("osbs-client on koji builder doesn't have Flatpak support")
release_overwrite = None
else:
label_overwrites = {}
release_overwrite = opts.get('release')
if release_overwrite:
label_overwrites = {LABEL_DATA_MAP['RELEASE']: release_overwrite}
data, expected_nvr = self.checkLabels(src, label_overwrites=label_overwrites,
build_tag=build_tag)
admin_opts = self._get_admin_opts(opts)
data.update(admin_opts)
# scratch builds do not get imported, and consequently not tagged
if not self.opts.get('scratch') and not flatpak:
self.check_whitelist(data[LABEL_DATA_MAP['COMPONENT']], target_info)
try:
# Flatpak builds append .<N> to the release generated from module version
if flatpak:
auto_release = True
else:
auto_release = (data[LABEL_DATA_MAP['RELEASE']] ==
LABEL_DEFAULT_VALUES['RELEASE'])
if auto_release:
# Do not expose default release value
del data[LABEL_DATA_MAP['RELEASE']]
self.extra_information = {"src": src, "data": data,
"target": target}
if not SCM.is_scm_url(src):
raise koji.BuildError('Invalid source specification: %s' % src)
# Scratch and auto release builds shouldn't be checked for nvr
if not self.opts.get('scratch') and not auto_release:
try:
build_id = self.session.getBuild(expected_nvr)['id']
except:
self.logger.info("No build for %s found", expected_nvr, exc_info=True)
else:
raise koji.BuildError(
"Build for %s already exists, id %s" % (expected_nvr, build_id))
results = self.runBuilds(src, target_info, archlist,
scratch=opts.get('scratch', False),
isolated=opts.get('isolated', False),
yum_repourls=opts.get('yum_repourls', None),
branch=opts.get('git_branch', None),
push_url=opts.get('push_url', None),
koji_parent_build=opts.get('koji_parent_build'),
release=release_overwrite,
flatpak=flatpak,
compose_ids=opts.get('compose_ids', None),
signing_intent=opts.get('signing_intent', None),
)
all_repositories = []
all_koji_builds = []
for result in results:
try:
repository = result.get('repositories')
all_repositories.extend(repository)
except Exception as error:
self.logger.error("Failed to merge list of repositories "
"%r. Reason (%s): %s", repository,
type(error), error)
koji_build_id = result.get('koji_build_id')
if koji_build_id:
all_koji_builds.append(koji_build_id)
except (SystemExit, ServerExit, KeyboardInterrupt):
# we do not trap these
raise
except:
# reraise the exception
raise
return {
'repositories': all_repositories,
'koji_builds': all_koji_builds,
}
def test_opts(self, getLogger):
config = '''
default:*
nocommon:*:no
srccmd:*:no:fedpkg,sources
nosrc:*:no:
mixed:/foo/*:no
mixed:/bar/*:yes
mixed:/baz/*:no:fedpkg,sources
'''
url = "git://default/koji.git#1234"
scm = SCM(url)
scm.assert_allowed(config)
self.assertEqual(scm.use_common, True)
self.assertEqual(scm.source_cmd, ['make', 'sources'])
url = "git://nocommon/koji.git#1234"
scm = SCM(url)
scm.assert_allowed(config)
self.assertEqual(scm.use_common, False)
self.assertEqual(scm.source_cmd, ['make', 'sources'])
url = "git://srccmd/koji.git#1234"
scm = SCM(url)
scm.assert_allowed(config)
self.assertEqual(scm.use_common, False)
self.assertEqual(scm.source_cmd, ['fedpkg', 'sources'])
url = "git://nosrc/koji.git#1234"
scm = SCM(url)
scm.assert_allowed(config)
self.assertEqual(scm.use_common, False)
self.assertEqual(scm.source_cmd, None)
url = "git://mixed/foo/koji.git#1234"
scm = SCM(url)
scm.assert_allowed(config)
self.assertEqual(scm.use_common, False)
self.assertEqual(scm.source_cmd, ['make', 'sources'])
url = "git://mixed/bar/koji.git#1234"
scm = SCM(url)
scm.assert_allowed(config)
self.assertEqual(scm.use_common, True)
self.assertEqual(scm.source_cmd, ['make', 'sources'])
url = "git://mixed/baz/koji.git#1234"
scm = SCM(url)
scm.assert_allowed(config)
self.assertEqual(scm.use_common, False)
self.assertEqual(scm.source_cmd, ['fedpkg', 'sources'])
url = "git://mixed/koji.git#1234"
scm = SCM(url)
with self.assertRaises(koji.BuildError):
scm.assert_allowed(config)
url = "git://mixed/foo/koji.git#1234"
scm = SCM(url)
scm.assert_allowed(config)
self.assertEqual(scm.use_common, False)
self.assertEqual(scm.source_cmd, ['make', 'sources'])
url = "git://mixed/bar/koji.git#1234"
scm = SCM(url)
scm.assert_allowed(config)
self.assertEqual(scm.use_common, True)
self.assertEqual(scm.source_cmd, ['make', 'sources'])
url = "git://mixed/baz/koji.git#1234"
scm = SCM(url)
scm.assert_allowed(config)
self.assertEqual(scm.use_common, False)
self.assertEqual(scm.source_cmd, ['fedpkg', 'sources'])
url = "git://mixed/koji.git#1234"
scm = SCM(url)
with self.assertRaises(koji.BuildError):
scm.assert_allowed(config)
def handler(self, src, target, opts=None):
if not opts:
opts = {}
self.opts = opts
data = {}
self.event_id = self.session.getLastEvent()['id']
target_info = self.session.getBuildTarget(target, event=self.event_id)
build_tag = target_info['build_tag']
archlist = self.getArchList(build_tag)
flatpak = opts.get('flatpak', False)
if flatpak:
if not osbs_flatpak_support:
raise koji.BuildError(
"osbs-client on koji builder doesn't have Flatpak support")
module = opts.get('module', None)
if not module:
raise koji.BuildError(
"Module must be specified for a Flatpak build")
module_name, module_stream, module_version = split_module_spec(
module)
data = {
'name': module_name,
'version': module_stream,
}
if module_version is not None:
data['release'] = module_version
release_overwrite = None
else:
label_overwrites = {}
release_overwrite = opts.get('release')
if release_overwrite:
label_overwrites = {
LABEL_DATA_MAP['RELEASE']: release_overwrite
}
data, expected_nvr = self.checkLabels(
src, label_overwrites=label_overwrites, build_tag=build_tag)
admin_opts = self._get_admin_opts(opts)
data.update(admin_opts)
# scratch builds do not get imported, and consequently not tagged
if not self.opts.get('scratch'):
self.check_whitelist(data[LABEL_DATA_MAP['COMPONENT']],
target_info)
try:
# Flatpak builds append .<N> to the release generated from module version
if flatpak:
auto_release = True
else:
auto_release = (data[LABEL_DATA_MAP['RELEASE']] ==
LABEL_DEFAULT_VALUES['RELEASE'])
if auto_release:
# Do not expose default release value
del data[LABEL_DATA_MAP['RELEASE']]
self.extra_information = {
"src": src,
"data": data,
"target": target
}
if not SCM.is_scm_url(src):
raise koji.BuildError('Invalid source specification: %s' % src)
# Scratch and auto release builds shouldn't be checked for nvr
if not self.opts.get('scratch') and not auto_release:
try:
build_id = self.session.getBuild(expected_nvr)['id']
except:
self.logger.info("No build for %s found",
expected_nvr,
exc_info=True)
else:
raise koji.BuildError(
"Build for %s already exists, id %s" %
(expected_nvr, build_id))
results = self.runBuilds(
src,
target_info,
archlist,
scratch=opts.get('scratch', False),
isolated=opts.get('isolated', False),
yum_repourls=opts.get('yum_repourls', None),
branch=opts.get('git_branch', None),
push_url=opts.get('push_url', None),
koji_parent_build=opts.get('koji_parent_build'),
release=release_overwrite,
flatpak=flatpak,
module=opts.get('module', None),
compose_ids=opts.get('compose_ids', None),
signing_intent=opts.get('signing_intent', None),
)
all_repositories = []
all_koji_builds = []
for result in results:
try:
repository = result.get('repositories')
all_repositories.extend(repository)
except Exception, error:
self.logger.error(
"Failed to merge list of repositories "
"%r. Reason (%s): %s", repository, type(error), error)
koji_build_id = result.get('koji_build_id')
if koji_build_id:
all_koji_builds.append(koji_build_id)
except (SystemExit, ServerExit, KeyboardInterrupt):
# we do not trap these
raise
except:
# reraise the exception
raise
return {
'repositories': all_repositories,
'koji_builds': all_koji_builds,
}
