def setUp(self):
self.setUpTestReactor()
# ignore config error if txrequests is not installed
self.patch(config, '_errors', Mock())
self.master = fakemaster.make_master(self,
wantData=True,
wantDb=True,
wantMq=True)
yield self.master.startService()
self._http = yield fakehttpclientservice.HTTPClientService.getFakeService(
self.master,
self,
HOSTED_BASE_URL,
headers={'PRIVATE-TOKEN': 'XXYYZZ'},
debug=None,
verify=None)
self.sp = sp = GitLabStatusPush(Interpolate('XXYYZZ'))
sp.sessionFactory = Mock(return_value=Mock())
yield sp.setServiceParent(self.master)
def reconfigService(self,
base_url,
user,
password,
key=None,
statusName=None,
startDescription=None,
endDescription=None,
verbose=False,
**kwargs):
yield http.HttpStatusPushBase.reconfigService(self,
wantProperties=True,
**kwargs)
self.key = key or Interpolate('%(prop:buildername)s')
self.statusName = statusName
self.endDescription = endDescription or 'Build done.'
self.startDescription = startDescription or 'Build started.'
self.verbose = verbose
self._http = yield httpclientservice.HTTPClientService.getService(
self.master, base_url, auth=(user, password))
def reconfigService(self, token,
startDescription=None, endDescription=None,
context=None, baseURL=None, verbose=False, wantProperties=True, **kwargs):
token = yield self.renderSecrets(token)
yield super().reconfigService(wantProperties=wantProperties, **kwargs)
self.context = context or Interpolate('buildbot/%(prop:buildername)s')
self.startDescription = startDescription or 'Build started.'
self.endDescription = endDescription or 'Build done.'
if baseURL is None:
baseURL = HOSTED_BASE_URL
if baseURL.endswith('/'):
baseURL = baseURL[:-1]
self.baseURL = baseURL
self._http = yield httpclientservice.HTTPClientService.getService(
self.master, baseURL, headers={'PRIVATE-TOKEN': token},
debug=self.debug, verify=self.verify)
self.verbose = verbose
self.project_ids = {}
def reconfigService(self,
token,
startDescription=None,
endDescription=None,
context=None,
baseURL=None,
verbose=False,
**kwargs):
yield http.HttpStatusPushBase.reconfigService(self, **kwargs)
self.context = context or Interpolate('buildbot/%(prop:buildername)s')
self.startDescription = startDescription or 'Build started.'
self.endDescription = endDescription or 'Build done.'
if baseURL is None:
baseURL = HOSTED_BASE_URL
if baseURL.endswith('/'):
baseURL = baseURL[:-1]
self.baseURL = baseURL
self.session.headers.update({'Authorization': 'token ' + token})
self.verbose = verbose
def setUp(self):
self.setUpTestReactor()
self.setup_reporter_test()
# repository must be in the form http://gitlab/<owner>/<project>
self.reporter_test_repo = 'http://gitlab/buildbot/buildbot'
# ignore config error if txrequests is not installed
self.patch(config, '_errors', Mock())
self.master = fakemaster.make_master(self, wantData=True, wantDb=True,
wantMq=True)
yield self.master.startService()
self._http = yield fakehttpclientservice.HTTPClientService.getService(
self.master, self,
HOSTED_BASE_URL, headers={'PRIVATE-TOKEN': 'XXYYZZ'},
debug=None, verify=None)
self.sp = GitLabStatusPush(Interpolate('XXYYZZ'))
self.sp.sessionFactory = Mock(return_value=Mock())
yield self.sp.setServiceParent(self.master)
def __init__(self, token, repoOwner, repoName, sha=None,
startDescription=None, endDescription=None,
baseURL=None):
"""
Token for GitHub API.
"""
if not GitHubAPI:
config.error('GitHubStatus requires txgithub package installed')
StatusReceiverMultiService.__init__(self)
self._sha = sha or Interpolate("%(src::revision)s")
self._repoOwner = repoOwner
self._repoName = repoName
self._startDescription = startDescription or "Build started."
self._endDescription = endDescription or "Build done."
self._github = GitHubAPI(oauth2_token=token, baseURL=baseURL)
self._status = None
def makeOmnibusFactory(distribution):
factory = getFlockerFactory(python="python2.7")
factory.addSteps(installDependencies())
factory.addSteps(check_version())
factory.addStep(
ShellCommand(name='build-sdist',
description=["building", "sdist"],
descriptionDone=["build", "sdist"],
command=[
virtualenvBinary('python'),
"setup.py",
"sdist",
],
haltOnFailure=True))
factory.addStep(
ShellCommand(command=[
virtualenvBinary('python'),
'admin/build-package',
'--destination-path',
'repo',
'--distribution',
distribution,
Interpolate('/flocker/dist/Flocker-%(prop:version)s.tar.gz'),
],
name='build-package',
description=['building', 'package'],
descriptionDone=['build', 'package'],
haltOnFailure=True))
repository_path = resultPath('omnibus', discriminator=distribution)
factory.addStep(
DirectoryUpload(
'repo',
repository_path,
url=resultURL('omnibus', discriminator=distribution),
name="upload-repo",
))
factory.addSteps(createRepository(distribution, repository_path))
return factory
def setUp(self):
self.setUpTestReactor()
# ignore config error if txrequests is not installed
self.patch(config, '_errors', Mock())
self.master = fakemaster.make_master(testcase=self,
wantData=True,
wantDb=True,
wantMq=True)
yield self.master.startService()
self._http = yield fakehttpclientservice.HTTPClientService.getFakeService(
self.master,
self,
"http://gitea",
headers={'Authorization': 'token XXYYZZ'},
debug=None,
verify=None)
self.sp = sp = GiteaStatusPush("http://gitea/", Interpolate('XXYYZZ'))
sp.sessionFactory = Mock(return_value=Mock())
yield sp.setServiceParent(self.master)
def masterConfig():
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import schedulers, steps
c['schedulers'] = [
schedulers.ForceScheduler(
name="force",
builderNames=["testy"])]
c['secretsProviders'] = [FakeSecretStorage(
secretdict={"foo": "bar", "something": "more"})]
f = BuildFactory()
f.addStep(steps.ShellCommand(command=Interpolate('echo %(secrets:foo)s')))
c['builders'] = [
BuilderConfig(name="testy",
workernames=["local1"],
factory=f)]
return c
def setUp(self):
self.setUpTestReactor()
self.setup_reporter_test()
# repository must be in the form http://gitea/<owner>/<project>
self.reporter_test_repo = u'http://gitea/buildbot/buildbot'
# ignore config error if txrequests is not installed
self.patch(config, '_errors', Mock())
self.master = fakemaster.make_master(testcase=self,
wantData=True, wantDb=True, wantMq=True)
yield self.master.startService()
self._http = yield fakehttpclientservice.HTTPClientService.getService(
self.master, self,
"http://gitea", headers={'Authorization': 'token XXYYZZ'},
debug=None, verify=None)
self.sp = GiteaStatusPush("http://gitea/", Interpolate('XXYYZZ'))
yield self.sp.setServiceParent(self.master)
def test_constr_args(self):
# test that the args to IRC(..) make it all the way down to
# the IrcStatusBot class
s = self.makeIRC(host='host',
nick='nick',
channels=['channels'],
pm_to_nicks=['pm', 'to', 'nicks'],
noticeOnChannel=True,
port=1234,
tags=['tags'],
password=Interpolate('pass'),
notify_events={
'successToFailure': 1,
},
showBlameList=False,
useRevisions=True,
useSSL=False,
useSASL=False,
lostDelay=10,
failedDelay=20,
useColors=False)
yield s.startService()
# patch it up
factory = self.factory
proto_obj = mock.Mock(name='proto_obj')
factory.protocol = mock.Mock(name='protocol', return_value=proto_obj)
# run it
p = factory.buildProtocol('address')
self.assertIdentical(p, proto_obj)
factory.protocol.assert_called_with('nick',
'pass', ['channels'],
['pm', 'to', 'nicks'],
True, {}, ['tags'],
{'successToFailure': 1},
useColors=False,
useSASL=False,
useRevisions=True,
showBlameList=False)
def test_create_artifact_directory_with_customArtifactPath_as_Interpolate(
self):
customArtifactPath = Interpolate("Art-%(src:fmod:revision)s")
sourcestamp = StubSourceStamp()
self.setupStep(artifact.CreateArtifactDirectory(
artifactDirectory="mydir",
artifactServer='*****@*****.**',
artifactServerDir='/home/srv/web/dir',
artifactServerPort=222,
customArtifactPath=customArtifactPath),
sources={'fmod': sourcestamp})
self.expectCommands(
ExpectShell(
workdir='wkdir',
usePTY='slave-config',
command=[
'ssh', '*****@*****.**', '-p 222', 'cd /home/srv/web/dir;',
'mkdir -p ', 'Art-abcde/mydir'
]) + ExpectShell.log('stdio', stdout='') + 0)
self.expectOutcome(result=SUCCESS,
status_text=['Remote artifact directory created.'])
return self.runStep()
def reconfigService(self,
token,
startDescription=None,
endDescription=None,
context=None,
baseURL=None,
verbose=False,
**kwargs):
yield http.HttpStatusPushBase.reconfigService(self, **kwargs)
self.context = context or Interpolate('buildbot/%(prop:buildername)s')
self.startDescription = startDescription or 'Build started.'
self.endDescription = endDescription or 'Build done.'
if baseURL is None:
baseURL = HOSTED_BASE_URL
if baseURL.endswith('/'):
baseURL = baseURL[:-1]
self.baseURL = baseURL
self._http = yield httpclientservice.HTTPClientService.getService(
self.master, baseURL, headers={'PRIVATE-TOKEN': token})
self.verbose = verbose
self.project_ids = {}
def test_addBuildsetForChanges_properties_with_virtual_builders(self):
sched = self.makeScheduler(name='n', builderNames=['c'], properties={
'virtual_builder_name': Interpolate("myproject-%(src::branch)s")
})
self.db.insertTestData([
fakedb.SourceStamp(id=234, branch='dev1', project="linux"),
fakedb.Change(changeid=14, sourcestampid=234, branch="dev1"),
])
bsid, brids = yield sched.addBuildsetForChanges(reason='downstream',
waited_for=False, changeids=[14])
self.assertEqual((bsid, brids), self.exp_bsid_brids)
self.master.data.updates.addBuildset.assert_called_with(
waited_for=False,
builderids=[1],
external_idstring=None,
properties={
'virtual_builder_name': ("myproject-dev1", "Scheduler"),
'scheduler': ('n', 'Scheduler'),
},
reason='downstream',
scheduler='n',
sourcestamps=[234])
def reconfigService(self, token,
startDescription=None, endDescription=None,
context=None, baseURL=None, verbose=False,
destRepoOwner=None, destRepoName=None, **kwargs):
yield http.HttpStatusPushBase.reconfigService(self, **kwargs)
self.context = context or Interpolate('buildbot/%(prop:buildername)s')
self.startDescription = startDescription or 'Build started.'
self.endDescription = endDescription or 'Build done.'
self.destRepoOwner = destRepoOwner
self.destRepoName = destRepoName
if baseURL is None:
baseURL = HOSTED_BASE_URL
if baseURL.endswith('/'):
baseURL = baseURL[:-1]
self._http = yield httpclientservice.HTTPClientService.getService(
self.master, baseURL, headers={
'Authorization': 'token ' + token,
'User-Agent': 'Buildbot'
})
self.verbose = verbose
def get_clone_step():
from buildbot.steps.source.git import Git
from buildbot.steps.master import SetProperty
from buildbot.process.properties import Interpolate
return [
Git(description=['cloning'],
descriptionDone=['clone'],
hideStepIf=skipped_or_success,
repourl=Interpolate('%(prop:repository)s'),
mode='full',
shallow=16,
method='clobber',
getDescription={'tags': True},
retry=(1, 120),
progress=True,
env={
'GIT_CURL_VERBOSE': '1',
'GIT_TRACE': '1'
},
logEnviron=False)
]
def setUp(self):
self.setUpTestReactor()
self.setup_reporter_test()
# repository must be in the form http://gitlab/<owner>/<project>
self.reporter_test_repo = 'http://gitlab/buildbot/buildbot'
self.master = fakemaster.make_master(self,
wantData=True,
wantDb=True,
wantMq=True)
yield self.master.startService()
self._http = yield fakehttpclientservice.HTTPClientService.getService(
self.master,
self,
HOSTED_BASE_URL,
headers={'PRIVATE-TOKEN': 'XXYYZZ'},
debug=None,
verify=None)
self.sp = GitLabStatusPush(Interpolate('XXYYZZ'))
yield self.sp.setServiceParent(self.master)
def _make_factory_step_generator(project_name, project_git_uri, make_command=None, workdir="/srv/buildbot"):
make_factory_steps = [
Git(
name = "Executing %s content fetch" % project_name,
repourl=project_git_uri,
mode='incremental'
),
ShellCommand(
name = "Executing %s: 'make %s'" % ( project_name, make_command ),
command = [
"make",
make_command
]
),
DirectoryUpload(
slavesrc="build",
masterdest=Interpolate(
"/srv/output/%(kw:project_name)s/%(src::branch)s",
)
)
]
return make_factory_steps
def masterConfig():
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import schedulers
c['schedulers'] = [
schedulers.ForceScheduler(name="force", builderNames=["testy"])
]
c['secretsProviders'] = [
HashiCorpVaultSecretProvider(vaultToken='my_vaulttoken',
vaultServer="http://localhost:8200")
]
f = BuildFactory()
f.addStep(ShellCommand(command=[Interpolate('echo %(secret:key)s')]))
c['builders'] = [
BuilderConfig(name="testy", workernames=["local1"], factory=f)
]
return c
def reconfigService(self,
baseURL,
auth,
verification_name=None,
abstain=False,
category=None,
reporter=None,
verbose=False,
debug=None,
verify=None,
generators=None,
**kwargs):
auth = yield self.renderSecrets(auth)
self.debug = debug
self.verify = verify
self.verbose = verbose
if generators is None:
generators = self._create_default_generators()
yield super().reconfigService(generators=generators, **kwargs)
if baseURL.endswith('/'):
baseURL = baseURL[:-1]
self._http = yield httpclientservice.HTTPClientService.getService(
self.master,
baseURL,
auth=auth,
debug=self.debug,
verify=self.verify)
self._verification_name = verification_name or Interpolate(
'%(prop:buildername)s')
self._reporter = reporter or "buildbot"
self._abstain = abstain
self._category = category
self._verbose = verbose
def reconfigService(self,
base_url,
user,
password,
key=None,
statusName=None,
startDescription=None,
endDescription=None,
verbose=False,
**kwargs):
user, password = yield self.renderSecrets(user, password)
yield super().reconfigService(wantProperties=True, **kwargs)
self.key = key or Interpolate('%(prop:buildername)s')
self.context = statusName
self.endDescription = endDescription or 'Build done.'
self.startDescription = startDescription or 'Build started.'
self.verbose = verbose
self._http = yield httpclientservice.HTTPClientService.getService(
self.master,
base_url,
auth=(user, password),
debug=self.debug,
verify=self.verify)
def mkdocsfactory():
f = factory.BuildFactory()
f.addSteps([
gitStep,
FileDownload(mastersrc="virtualenv.py", slavedest="virtualenv.py", flunkOnFailure=True),
# run docs tools in their own virtualenv, otherwise we end up documenting
# the version of Buildbot running the metabuildbot!
VirtualenvSetup(name='virtualenv setup',
no_site_packages=True,
virtualenv_packages=['sphinx==1.2.2', 'Pygments==2.0.1', '--editable=master[docs]', '--editable=slave'],
virtualenv_dir='sandbox',
haltOnFailure=True),
# manual
ShellCommand(command=Interpolate(textwrap.dedent("""\
export VERSION=latest &&
. sandbox/bin/activate &&
gmake docs
""")), name="create docs"),
])
return f
def __init__(self, token, repoOwner, repoName, sha=None,
startDescription=None, endDescription=None):
"""
Token for GitHub API.
"""
StatusReceiverMultiService.__init__(self)
if not sha:
sha = Interpolate("%(src::revision)s")
if not startDescription:
startDescription = "Build started."
self._startDescription = startDescription
if not endDescription:
endDescription = "Build done."
self._endDescription = endDescription
self._token = token
self._sha = sha
self._repoOwner = repoOwner
self._repoName = repoName
self._github = GitHubAPI(oauth2_token=self._token)
def createReporter(self,
auth=("username", "passwd"),
headers=None,
**kwargs):
self._http = yield fakehttpclientservice.HTTPClientService.getService(
self.master,
self,
"serv",
auth=auth,
headers=headers,
debug=None,
verify=None)
interpolated_auth = None
if auth is not None:
username, passwd = auth
passwd = Interpolate(passwd)
interpolated_auth = (username, passwd)
self.sp = HttpStatusPush("serv",
auth=interpolated_auth,
headers=headers,
**kwargs)
yield self.sp.setServiceParent(self.master)
def StandardBuilderWorker(name, **kwargs):
password = ''.join(random.SystemRandom().choice(string.ascii_uppercase +
string.digits)
for _ in range(32))
tmpfs = docker.types.Mount('/tmp', None, type='tmpfs')
return DockerLatentWorker(
name,
password,
docker_host=config.docker_socket,
image=Interpolate('workers/%(prop:workerimage)s'),
masterFQDN=buildbot_ip,
volumes=[
'{0}/ccache:/data/ccache'.format(config.buildbot_data_dir),
'{0}/src:/data/src:ro'.format(config.buildbot_data_dir),
'{0}/builds:/data/builds'.format(config.buildbot_data_dir, name),
'{0}/packages:/data/packages'.format(config.buildbot_data_dir),
'{0}/bshomes:/data/bshomes'.format(config.buildbot_data_dir),
],
hostconfig={
'network_mode': 'workers-net',
'read_only': True,
'mounts': [tmpfs],
},
**kwargs)
def test_custom_name(self):
yield self.createGerritStatus(
verification_name=Interpolate("builder %(prop:buildername)s"))
build = yield self.setupBuildResults(SUCCESS)
# we make sure proper calls to txrequests have been made
self._http.expect(
method='post',
ep='/a/changes/12/revisions/2/verify-status~verifications',
json={
'comment': 'Build started.',
'abstain': False,
'name': u'builder Builder0',
'reporter': 'buildbot',
'url': 'http://localhost:8080/#builders/79/builds/0',
'value': 0,
'duration': 'pending'
})
self._http.expect(
method='post',
ep='/a/changes/12/revisions/2/verify-status~verifications',
json={
'comment': 'Build done.',
'abstain': False,
'name': u'builder Builder0',
'reporter': 'buildbot',
'url': 'http://localhost:8080/#builders/79/builds/0',
'value': 1,
'duration': '2h 1m 4s'
})
build['complete'] = False
build['complete_at'] = None
self.sp.buildStarted(("build", 20, "started"), build)
build['complete'] = True
build['complete_at'] = build['started_at'] + datetime.timedelta(
hours=2, minutes=1, seconds=4)
self.sp.buildFinished(("build", 20, "finished"), build)
def test_resultdir_renderable(self):
resultdir_text = "RESULT"
self.setup_step(
mock.Mock(root='TESTROOT',
resultdir=Interpolate('%(kw:resultdir)s',
resultdir=resultdir_text)))
self.expect_commands(
ExpectRmdir(dir=[
'build/RESULT/build.log', 'build/RESULT/root.log',
'build/RESULT/state.log'
],
log_environ=False).exit(0),
ExpectShell(workdir='wkdir',
command=[
'mock', '--root', 'TESTROOT', '--resultdir',
'RESULT'
],
logfiles={
'build.log': 'RESULT/build.log',
'root.log': 'RESULT/root.log',
'state.log': 'RESULT/state.log'
}).exit(0))
self.expect_outcome(result=SUCCESS, state_string="'mock --root ...'")
return self.run_step()
def test_architecture_renderable(self):
self.setup_step(
pbuilder.DebPbuilder(architecture=Interpolate('amd64')))
self.expect_commands(
ExpectStat(
file='/var/cache/pbuilder/stable-amd64-buildbot.tgz').exit(1),
ExpectShell(workdir='wkdir',
command=[
'sudo', '/usr/sbin/pbuilder', '--create',
'--basetgz',
'/var/cache/pbuilder/stable-amd64-buildbot.tgz',
'--distribution', 'stable', '--mirror',
'http://cdn.debian.net/debian/', '--architecture',
'amd64'
]).exit(0),
ExpectShell(workdir='wkdir',
command=[
'pdebuild', '--buildresult', '.', '--pbuilder',
'/usr/sbin/pbuilder', '--architecture', 'amd64',
'--', '--buildresult', '.', '--basetgz',
'/var/cache/pbuilder/stable-amd64-buildbot.tgz'
]).exit(0))
self.expect_outcome(result=SUCCESS)
return self.run_step()
def __init__(self, *args, **kwargs):
util.BuildFactory.__init__(self, *args, **kwargs)
title = 'Liri OS'
releasever = '30'
self.addSteps([
ImagePropertiesStep(name='set properties'),
steps.ShellCommand(
name='update container',
haltOnFailure=True,
command=['dnf', 'update', '-y'],
),
steps.ShellCommand(
name='install tools',
haltOnFailure=True,
command=[
'dnf', 'install', '-y', 'git', 'spin-kickstarts',
'pykickstart', 'livecd-tools'
],
),
steps.Git(
name='checkout sources',
codebase=util.Property('codebase'),
repourl=util.Property('repository'),
branch=util.Property('branch'),
mode='incremental',
submodules=True,
shallow=True,
),
steps.ShellCommand(
name='ksflatten',
haltOnFailure=True,
command=[
'ksflatten',
Interpolate('--config=%(prop:product)s-livecd.ks'), '-o',
'livecd.ks'
],
),
steps.RemoveDirectory(
name='clean cache',
dir='/build/cache',
doStepIf=IsCacheDisabled,
),
steps.ShellCommand(
name='build image',
haltOnFailure=True,
timeout=60 * 60,
command=[
'livecd-creator', '--releasever=' + releasever,
'--config=livecd.ks',
Interpolate('--fslabel=%(prop:imgname)s'), '--title',
title,
Interpolate('--product=%(prop:product)s'),
'--cache=/build/cache'
],
),
steps.ShellCommand(
name='checksum',
haltOnFailure=True,
command=[
'bash', '-c',
Interpolate(
'sha256sum -b --tag %(prop:isofilename)s > /repo/images/nightly/%(prop:checksumfilename)s'
)
],
),
steps.ShellCommand(
name='move file',
command=[
'mv',
Interpolate('%(prop:isofilename)s'),
'/repo/images/nightly/'
],
),
steps.ShellCommand(
name='remove old images',
command=[
'bash', '-c',
'find /repo/images/nightly -type f -mtime +7 -exec rm {} \;'
],
)
])
def get_whl_filename(abi):
"Returns the name of a .whl file for uploading, for the given Python ABI."
return Interpolate("panda3d-%s-%s-%s.whl", whl_version, abi,
platform_under)
