def setupProperties(self):
props = interfaces.IProperties(self)
# give the properties a reference back to this build
props.build = self
# start with global properties from the configuration
master = self.builder.botmaster.master
props.updateFromProperties(master.config.properties)
# from the SourceStamps, which have properties via Change
for change in self.allChanges():
props.updateFromProperties(change.properties)
# and finally, get any properties from requests (this is the path
# through which schedulers will send us properties)
for rq in self.requests:
props.updateFromProperties(rq.properties)
# now set some properties of our own, corresponding to the
# build itself
props.setProperty("buildnumber", self.build_status.number, "Build")
if self.sources and len(self.sources) == 1:
# old interface for backwards compatibility
source = self.sources[0]
props.setProperty("branch", source.branch, "Build")
props.setProperty("revision", source.revision, "Build")
props.setProperty("repository", source.repository, "Build")
props.setProperty("codebase", source.codebase, "Build")
props.setProperty("project", source.project, "Build")
self.builder.setupProperties(props)
def _flushProperties(self, results):
# `results` is just passed on to the next callback
props = interfaces.IProperties(self)
properties = props.getProperties().asList()
for name, value, source in properties:
yield self.master.data.updates.setBuildProperty(
self.buildid, name, value, source)
defer.returnValue(results)
def setupProperties(self):
props = interfaces.IProperties(self)
# give the properties a reference back to this build
props.build = self
# start with global properties from the configuration
props.updateFromProperties(self.master.config.properties)
# from the SourceStamps, which have properties via Change
for change in self.allChanges():
props.updateFromProperties(change.properties)
# and finally, get any properties from requests (this is the path
# through which schedulers will send us properties)
for rq in self.requests:
props.updateFromProperties(rq.properties)
self.builder.setupProperties(props)
def getSummaryStatistic(self, name, summary_fn, initial_value=_sentinel):
step_stats_list = [
st.getStatistic(name)
for st in self.executedSteps
if st.hasStatistic(name)]
if initial_value is self._sentinel:
return reduce(summary_fn, step_stats_list)
return reduce(summary_fn, step_stats_list, initial_value)
@defer.inlineCallbacks
def getUrl(self):
builder_id = yield self.builder.getBuilderId()
defer.returnValue(getURLForBuild(self.master, builder_id, self.number))
def waitUntilFinished(self):
return self.master.mq.waitUntilEvent(
('builds', str(self.buildid), 'finished'),
lambda: self.finished)
# IBuildControl
def getStatus(self):
return self.build_status
# stopBuild is defined earlier
components.registerAdapter(
lambda build: interfaces.IProperties(build.properties),
Build, interfaces.IProperties)
def setupStep(self, step, worker_version=None, worker_env=None,
buildFiles=None, wantDefaultWorkdir=True, wantData=True,
wantDb=False, wantMq=False):
"""
Set up C{step} for testing. This begins by using C{step} as a factory
to create a I{new} step instance, thereby testing that the factory
arguments are handled correctly. It then creates a comfortable
environment for the worker to run in, replete with a fake build and a
fake worker.
As a convenience, it can set the step's workdir with C{'wkdir'}.
@param worker_version: worker version to present, as a dictionary mapping
command name to version. A command name of '*' will apply for all
commands.
@param worker_env: environment from the worker at worker startup
@param wantData(bool): Set to True to add data API connector to master.
Default value: True.
@param wantDb(bool): Set to True to add database connector to master.
Default value: False.
@param wantMq(bool): Set to True to add mq connector to master.
Default value: False.
"""
if worker_version is None:
worker_version = {
'*': '99.99'
}
if worker_env is None:
worker_env = dict()
if buildFiles is None:
buildFiles = list()
factory = interfaces.IBuildStepFactory(step)
step = self.step = factory.buildStep()
self.master = fakemaster.make_master(wantData=wantData, wantDb=wantDb,
wantMq=wantMq, testcase=self)
# mock out the reactor for updateSummary's debouncing
self.debounceClock = task.Clock()
self.master.reactor = self.debounceClock
# set defaults
if wantDefaultWorkdir:
step.workdir = step._workdir or 'wkdir'
# step.build
b = self.build = fakebuild.FakeBuild(master=self.master)
b.allFiles = lambda: buildFiles
b.master = self.master
def getWorkerVersion(cmd, oldversion):
if cmd in worker_version:
return worker_version[cmd]
if '*' in worker_version:
return worker_version['*']
return oldversion
b.getWorkerCommandVersion = getWorkerVersion
b.workerEnvironment = worker_env.copy()
step.setBuild(b)
# watch for properties being set
self.properties = interfaces.IProperties(b)
# step.progress
step.progress = mock.Mock(name="progress")
# step.worker
self.worker = step.worker = worker.FakeWorker(self.master)
# step overrides
def addLog(name, type='s', logEncoding=None):
_log = logfile.FakeLogFile(name, step)
self.step.logs[name] = _log
return defer.succeed(_log)
step.addLog = addLog
step.addLog_newStyle = addLog
def addHTMLLog(name, html):
_log = logfile.FakeLogFile(name, step)
html = bytes2NativeString(html)
_log.addStdout(html)
return defer.succeed(None)
step.addHTMLLog = addHTMLLog
def addCompleteLog(name, text):
_log = logfile.FakeLogFile(name, step)
self.step.logs[name] = _log
_log.addStdout(text)
return defer.succeed(None)
step.addCompleteLog = addCompleteLog
step.logobservers = self.logobservers = {}
def addLogObserver(logname, observer):
self.logobservers.setdefault(logname, []).append(observer)
observer.step = step
step.addLogObserver = addLogObserver
# add any observers defined in the constructor, before this
# monkey-patch
for n, o in step._pendingLogObservers:
addLogObserver(n, o)
# expectations
self.exp_result = None
self.exp_state_string = None
self.exp_properties = {}
self.exp_missing_properties = []
self.exp_logfiles = {}
self.exp_hidden = False
self.exp_exception = None
# check that the step's name is not None
self.assertNotEqual(step.name, None)
return step
def runCommand(self, c):
c.buildslave = self.buildslave
d = c.run(self, self.remote)
return d
@staticmethod
def _maybeEvaluate(value, *args, **kwargs):
if callable(value):
value = value(*args, **kwargs)
return value
components.registerAdapter(BuildStep._getStepFactory, BuildStep,
interfaces.IBuildStepFactory)
components.registerAdapter(lambda step: interfaces.IProperties(step.build),
BuildStep, interfaces.IProperties)
class OutputProgressObserver(LogObserver):
length = 0
def __init__(self, name):
self.name = name
def logChunk(self, build, step, log, channel, text):
self.length += len(text)
self.step.setProgress(self.name, self.length)
class LoggingBuildStep(BuildStep):
def describe(self, done=False):
# old-style steps expect this function to exist
assert not self.isNewStyle()
desc = self._describe(done)
if not desc:
return []
if self.descriptionSuffix:
desc += self.descriptionSuffix
return desc
components.registerAdapter(
BuildStep._getStepFactory,
BuildStep, interfaces.IBuildStepFactory)
components.registerAdapter(
lambda step: interfaces.IProperties(step.build),
BuildStep, interfaces.IProperties)
class LoggingBuildStep(BuildStep):
progressMetrics = ('output',)
logfiles = {}
parms = BuildStep.parms + ['logfiles', 'lazylogfiles', 'log_eval_func']
cmd = None
renderables = ['logfiles', 'lazylogfiles']
def __init__(self, logfiles=None, lazylogfiles=False, log_eval_func=None,
*args, **kwargs):
def setupStep(self, step, slave_version={'*': "99.99"}, slave_env={}):
"""
Set up C{step} for testing. This begins by using C{step} as a factory
to create a I{new} step instance, thereby testing that the the factory
arguments are handled correctly. It then creates a comfortable
environment for the slave to run in, repleate with a fake build and a
fake slave.
As a convenience, it calls the step's setDefaultWorkdir method with
C{'wkdir'}.
@param slave_version: slave version to present, as a dictionary mapping
command name to version. A command name of '*' will apply for all
commands.
@param slave_env: environment from the slave at slave startup
"""
factory = interfaces.IBuildStepFactory(step)
step = self.step = factory.buildStep()
self.master = fakemaster.make_master(testcase=self)
# step.build
b = self.build = fakebuild.FakeBuild()
b.master = self.master
def getSlaveVersion(cmd, oldversion):
if cmd in slave_version:
return slave_version[cmd]
if '*' in slave_version:
return slave_version['*']
return oldversion
b.getSlaveCommandVersion = getSlaveVersion
b.slaveEnvironment = slave_env.copy()
step.setBuild(b)
# watch for properties being set
self.properties = interfaces.IProperties(b)
# step.progress
step.progress = mock.Mock(name="progress")
# step.buildslave
self.master = fakemaster.make_master(testcase=self)
self.buildslave = step.buildslave = slave.FakeSlave(self.master)
# step.step_status
ss = self.step_status = mock.Mock(name="step_status")
ss.status_text = None
ss.logs = {}
def ss_setText(strings):
ss.status_text = strings
ss.setText = ss_setText
ss.getLogs = lambda: ss.logs.values()
self.step_statistics = {}
ss.setStatistic = self.step_statistics.__setitem__
ss.getStatistic = self.step_statistics.get
ss.hasStatistic = self.step_statistics.__contains__
self.step.setStepStatus(ss)
# step overrides
def addLog(name):
l = remotecommand.FakeLogFile(name, step)
ss.logs[name] = l
return l
step.addLog = addLog
def addHTMLLog(name, html):
l = remotecommand.FakeLogFile(name, step)
l.addStdout(html)
ss.logs[name] = l
return l
step.addHTMLLog = addHTMLLog
def addCompleteLog(name, text):
l = remotecommand.FakeLogFile(name, step)
l.addStdout(text)
ss.logs[name] = l
return l
step.addCompleteLog = addCompleteLog
step.logobservers = self.logobservers = {}
def addLogObserver(logname, observer):
self.logobservers.setdefault(logname, []).append(observer)
observer.step = step
step.addLogObserver = addLogObserver
# add any observers defined in the constructor, before this monkey-patch
for n, o in step._pendingLogObservers:
addLogObserver(n, o)
# set defaults
step.setDefaultWorkdir('wkdir')
# expectations
self.exp_outcome = None
self.exp_properties = {}
self.exp_missing_properties = []
self.exp_logfiles = {}
self.exp_hidden = False
# check that the step's name is not None
self.assertNotEqual(step.name, None)
return step
def createEmail(self,
msgdict,
builderName,
title,
results,
builds=None,
patches=None,
logs=None):
text = msgdict['body'].encode(ENCODING)
type = msgdict['type']
if 'subject' in msgdict:
subject = msgdict['subject'].encode(ENCODING)
else:
subject = self.subject % {
'result': Results[results],
'projectName': title,
'title': title,
'builder': builderName,
}
assert '\n' not in subject, \
"Subject cannot contain newlines"
assert type in ('plain', 'html'), \
"'%s' message type must be 'plain' or 'html'." % type
if patches or logs:
m = MIMEMultipart()
m.attach(MIMEText(text, type, ENCODING))
else:
m = Message()
m.set_payload(text, ENCODING)
m.set_type("text/%s" % type)
m['Date'] = formatdate(localtime=True)
m['Subject'] = subject
m['From'] = self.fromaddr
# m['To'] is added later
if patches:
for (i, patch) in enumerate(patches):
a = self.patch_to_attachment(patch, i)
m.attach(a)
if logs:
for log in logs:
name = "%s.%s" % (log.getStep().getName(), log.getName())
if (self._shouldAttachLog(log.getName())
or self._shouldAttachLog(name)):
text = log.getText()
if not isinstance(text, unicode):
text = text.decode(LOG_ENCODING)
a = MIMEText(text.encode(ENCODING), _charset=ENCODING)
a.add_header('Content-Disposition',
"attachment",
filename=name)
m.attach(a)
#@todo: is there a better way to do this?
# Add any extra headers that were requested, doing WithProperties
# interpolation if only one build was given
if self.extraHeaders:
for k, v in self.extraHeaders.items():
if len(builds) == 1:
k = interfaces.IProperties(builds[0]).render(k)
if k in m:
twlog.msg("Warning: Got header " + k +
" in self.extraHeaders "
"but it already exists in the Message - "
"not adding it.")
if len(builds) == 1:
m[k] = interfaces.IProperties(builds[0]).render(v)
else:
m[k] = v
return m
self.build_status.buildFinished()
if self.progress and results == SUCCESS:
# XXX: also test a 'timing consistent' flag?
log.msg(" setting expectations for next time")
self.builder.setExpectations(self.progress)
eventually(self.releaseLocks)
self.deferred.callback(self)
self.deferred = None
def releaseLocks(self):
if self.locks:
log.msg("releaseLocks(%s): %s" % (self, self.locks))
for lock, access in self.locks:
if lock.isOwner(self, access):
lock.release(self, access)
else:
# This should only happen if we've been interrupted
assert self.stopped
# IBuildControl
def getStatus(self):
return self.build_status
# stopBuild is defined earlier
components.registerAdapter(
lambda build: interfaces.IProperties(build.build_status), Build,
interfaces.IProperties)
def setupStep(self,
step,
worker_version=None,
worker_env=None,
buildFiles=None,
wantDefaultWorkdir=True):
"""
Set up C{step} for testing. This begins by using C{step} as a factory
to create a I{new} step instance, thereby testing that the factory
arguments are handled correctly. It then creates a comfortable
environment for the worker to run in, replete with a fake build and a
fake worker.
As a convenience, it can set the step's workdir with C{'wkdir'}.
@param worker_version: worker version to present, as a dictionary mapping
command name to version. A command name of '*' will apply for all
commands.
@param worker_env: environment from the worker at worker startup
"""
if worker_version is None:
worker_version = {'*': '99.99'}
if worker_env is None:
worker_env = dict()
if buildFiles is None:
buildFiles = list()
factory = interfaces.IBuildStepFactory(step)
step = self.step = factory.buildStep()
# set defaults
if wantDefaultWorkdir:
step.workdir = step._workdir or 'wkdir'
# step.build
b = self.build = fakebuild.FakeBuild(master=self.master)
b.allFiles = lambda: buildFiles
b.master = self.master
def getWorkerVersion(cmd, oldversion):
if cmd in worker_version:
return worker_version[cmd]
if '*' in worker_version:
return worker_version['*']
return oldversion
b.getWorkerCommandVersion = getWorkerVersion
b.workerEnvironment = worker_env.copy()
step.setBuild(b)
# watch for properties being set
self.properties = interfaces.IProperties(b)
# step.progress
step.progress = mock.Mock(name="progress")
# step.worker
self.worker = step.worker = worker.FakeWorker(self.master)
self.worker.attached(None)
# step overrides
def addLog(name, type='s', logEncoding=None):
_log = logfile.FakeLogFile(name, step)
self.step.logs[name] = _log
return defer.succeed(_log)
step.addLog = addLog
step.addLog_newStyle = addLog
def addHTMLLog(name, html):
_log = logfile.FakeLogFile(name, step)
html = bytes2unicode(html)
_log.addStdout(html)
return defer.succeed(None)
step.addHTMLLog = addHTMLLog
def addCompleteLog(name, text):
_log = logfile.FakeLogFile(name, step)
self.step.logs[name] = _log
_log.addStdout(text)
return defer.succeed(None)
step.addCompleteLog = addCompleteLog
step.logobservers = self.logobservers = {}
def addLogObserver(logname, observer):
self.logobservers.setdefault(logname, []).append(observer)
observer.step = step
step.addLogObserver = addLogObserver
# add any observers defined in the constructor, before this
# monkey-patch
for n, o in step._pendingLogObservers:
addLogObserver(n, o)
self._got_test_result_sets = []
self._next_test_result_set_id = 1000
def add_test_result_set(description, category, value_unit):
self._got_test_result_sets.append(
(description, category, value_unit))
setid = self._next_test_result_set_id
self._next_test_result_set_id += 1
return defer.succeed(setid)
step.addTestResultSet = add_test_result_set
self._got_test_results = []
def add_test_result(setid,
value,
test_name=None,
test_code_path=None,
line=None,
duration_ns=None):
self._got_test_results.append(
(setid, value, test_name, test_code_path, line, duration_ns))
step.addTestResult = add_test_result
# expectations
self.exp_result = None
self.exp_state_string = None
self.exp_properties = {}
self.exp_missing_properties = []
self.exp_logfiles = {}
self.exp_hidden = False
self.exp_exception = None
self._exp_test_result_sets = []
self._exp_test_results = []
# check that the step's name is not None
self.assertNotEqual(step.name, None)
return step
def setupStep(self,
step,
slave_version={'*': "99.99"},
slave_env={},
buildFiles=[]):
"""
Set up C{step} for testing. This begins by using C{step} as a factory
to create a I{new} step instance, thereby testing that the the factory
arguments are handled correctly. It then creates a comfortable
environment for the slave to run in, replete with a fake build and a
fake slave.
As a convenience, it calls the step's setDefaultWorkdir method with
C{'wkdir'}.
@param slave_version: slave version to present, as a dictionary mapping
command name to version. A command name of '*' will apply for all
commands.
@param slave_env: environment from the slave at slave startup
"""
factory = interfaces.IBuildStepFactory(step)
step = self.step = factory.buildStep()
self.master = fakemaster.make_master(wantData=True, testcase=self)
# step.build
b = self.build = fakebuild.FakeBuild(master=self.master)
b.allFiles = lambda: buildFiles
b.master = self.master
def getSlaveVersion(cmd, oldversion):
if cmd in slave_version:
return slave_version[cmd]
if '*' in slave_version:
return slave_version['*']
return oldversion
b.getSlaveCommandVersion = getSlaveVersion
b.slaveEnvironment = slave_env.copy()
step.setBuild(b)
# watch for properties being set
self.properties = interfaces.IProperties(b)
# step.progress
step.progress = mock.Mock(name="progress")
# step.buildslave
self.buildslave = step.buildslave = slave.FakeSlave(self.master)
# step overrides
def addLog(name, type='s', logEncoding=None):
l = logfile.FakeLogFile(name, step)
self.step.logs[name] = l
return defer.succeed(l)
step.addLog = addLog
step.addLog_newStyle = addLog
def addHTMLLog(name, html):
l = logfile.FakeLogFile(name, step)
l.addStdout(html)
return defer.succeed(None)
step.addHTMLLog = addHTMLLog
def addCompleteLog(name, text):
l = logfile.FakeLogFile(name, step)
self.step.logs[name] = l
l.addStdout(text)
return defer.succeed(None)
step.addCompleteLog = addCompleteLog
step.logobservers = self.logobservers = {}
def addLogObserver(logname, observer):
self.logobservers.setdefault(logname, []).append(observer)
observer.step = step
step.addLogObserver = addLogObserver
# add any observers defined in the constructor, before this
# monkey-patch
for n, o in step._pendingLogObservers:
addLogObserver(n, o)
# set defaults
step.setDefaultWorkdir('wkdir')
# expectations
self.exp_result = None
self.exp_state_strings = None
self.exp_properties = {}
self.exp_missing_properties = []
self.exp_logfiles = {}
self.exp_hidden = False
# check that the step's name is not None
self.assertNotEqual(step.name, None)
# mock out the reactor for updateSummary's debouncing
self.debounceClock = task.Clock()
step.updateSummary._reactor = self.debounceClock
return step
def setupStep(self, step, slave_version="99.99", slave_env={}):
"""
Set up C{step} for testing. This begins by using C{step} as a factory
to create a I{new} step instance, thereby testing that the the factory
arguments are handled correctly. It then creates a comfortable
environment for the slave to run in, repleate with a fake build and a
fake slave.
As a convenience, it calls the step's setDefaultWorkdir method with
C{'wkdir'}.
@param slave_version: slave version to present; defaults to "99.99"
@param slave_env: environment from the slave at slave startup
"""
# yes, Virginia, "factory" refers both to the tuple and its first
# element TODO: fix that up
factory, args = step.getStepFactory()
step = self.step = factory(**args)
# step.build
b = self.build = fakebuild.FakeBuild()
b.getSlaveCommandVersion = lambda command, oldversion: slave_version
b.slaveEnvironment = slave_env.copy()
step.setBuild(b)
# watch for properties being set
self.properties = interfaces.IProperties(b)
# step.progress
step.progress = mock.Mock(name="progress")
# step.buildslave
step.buildslave = mock.Mock(name="buildslave")
# step.step_status
ss = self.step_status = mock.Mock(name="step_status")
ss.status_text = None
ss.logs = {}
def ss_setText(strings):
ss.status_text = strings
ss.setText = ss_setText
ss.getLogs = lambda: ss.logs.values()
self.step_statistics = {}
ss.setStatistic = self.step_statistics.__setitem__
ss.getStatistic = self.step_statistics.get
ss.hasStatistic = self.step_statistics.__contains__
self.step.setStepStatus(ss)
# step overrides
def addLog(name):
l = remotecommand.FakeLogFile(name)
ss.logs[name] = l
return l
step.addLog = addLog
def addHTMLLog(name, html):
l = remotecommand.FakeLogFile(name)
l.addStdout(html)
ss.logs[name] = l
return l
step.addHTMLLog = addHTMLLog
def addCompleteLog(name, text):
l = remotecommand.FakeLogFile(name)
l.addStdout(text)
ss.logs[name] = l
return l
step.addCompleteLog = addCompleteLog
# set defaults
step.setDefaultWorkdir('wkdir')
# expectations
self.exp_outcome = None
self.exp_properties = {}
self.exp_missing_properties = []
self.exp_logfiles = {}
return step
