def getLastCompletedBuild(self, builderid):
d = self.master.data.get(
('builds', ),
filters=[
resultspec.Filter('builderid', 'eq', [builderid]),
resultspec.Filter('complete', 'eq', [True])
],
order=['-number'],
limit=1)
@d.addCallback
def listAsOneOrNone(res):
if res:
return res[0]
return None
return d
def test_get_contains_two_tags_one_unknown(self):
resultSpec = resultspec.ResultSpec(
filters=[resultspec.Filter('tags', 'contains', ["tagA", "tagC"])])
builders = yield self.callGet(('builders', ))
builders = resultSpec.apply(builders)
[self.validateData(b) for b in builders]
self.assertEqual(sorted([b['builderid'] for b in builders]), [3, 5])
def test_endpoint_returns_total_without_applying_filters(self):
data = base.ListResult(mklist('x', *range(10, 20)))
data.total = 99
# apply doesn't want to get a total with filters still outstanding
f = resultspec.Filter(field='x', op='gt', values=[23])
self.assertRaises(
AssertionError,
lambda: resultspec.ResultSpec(filters=[f]).apply(data))
def _getUnclaimedBrs(self, builderid):
# Retrieve the list of Brs for all unclaimed builds
unclaim_brs = yield self.master.data.get(
('builders', builderid, 'buildrequests'),
[resultspec.Filter('claimed', 'eq', [False])])
# sort by submitted_at, so the first is the oldest
unclaim_brs.sort(key=lambda brd: brd['submitted_at'])
defer.returnValue(unclaim_brs)
def getBuilderNames(self):
d = self.master.data.get(('buildrequests', ),
filters=[resultspec.Filter('buildsetid', 'eq', [self.id])])
def get_names(brdicts):
return sorted([brd['buildername'] for brd in brdicts])
d.addCallback(get_names)
return d
def test_get_incomplete(self):
f = resultspec.Filter('complete', 'eq', [False])
buildsets = yield self.callGet(
('buildsets', ), resultSpec=resultspec.ResultSpec(filters=[f]))
self.assertEqual(len(buildsets), 1)
self.validateData(buildsets[0])
self.assertEqual(buildsets[0]['bsid'], 14)
def test_get_resultSpec_args(self):
ep = self.patchFooListPattern()
f = resultspec.Filter('val', 'gt', [909])
gotten = yield self.data.get(('foo',), filters=[f], fields=['val'],
order=['-val'], limit=2)
self.assertEqual(gotten, base.ListResult(
[{'val': 919}, {'val': 918}], total=10, limit=2))
ep.get.assert_called_once_with(mock.ANY, {})
def test_get_buildrequest_via_filter_with_string(self):
resultSpec = resultspec.OptimisedResultSpec(
filters=[resultspec.Filter('buildrequestid', 'eq', ['82'])])
builds = yield self.callGet(('builds', ), resultSpec=resultSpec)
for build in builds:
self.validateData(build)
self.assertEqual(sorted([b['number'] for b in builds]), [3, 4])
def testGetFilters(self):
getBuildRequestsMock = mock.Mock(return_value={})
self.patch(
self.master.db.buildrequests, 'getBuildRequests', getBuildRequestsMock)
f1 = resultspec.Filter('complete', 'eq', [False])
f2 = resultspec.Filter('claimed', 'eq', [True])
f3 = resultspec.Filter('buildsetid', 'eq', [55])
f4 = resultspec.Filter('branch', 'eq', ['mybranch'])
f5 = resultspec.Filter('repository', 'eq', ['myrepo'])
yield self.callGet(
('buildrequests',),
resultSpec=resultspec.ResultSpec(filters=[f1, f2, f3, f4, f5]))
getBuildRequestsMock.assert_called_with(
builderid=None,
bsid=55,
complete=False,
claimed=True,
resultSpec=resultspec.ResultSpec(filters=[f4, f5]))
def test_get_filter_ne(self):
resultSpec = resultspec.OptimisedResultSpec(
filters=[resultspec.Filter('builderid', 'ne', [78, 79])])
builds = yield self.callGet(('builds', ), resultSpec=resultSpec)
for b in builds:
self.validateData(b)
self.assertEqual(sorted([b['number'] for b in builds]), [3, 4])
def test_get_complete_at(self):
resultSpec = resultspec.OptimisedResultSpec(
filters=[resultspec.Filter('complete_at', 'eq', [None])])
builds = yield self.callGet(('builds', ), resultSpec=resultSpec)
for build in builds:
self.validateData(build)
self.assertEqual(sorted([b['number'] for b in builds]), [3, 4])
def _masterDeactivatedHousekeeping(self, masterid, name):
log.msg("doing housekeeping for master {} {}".format(masterid, name))
# common code for deactivating a master
yield self.master.data.rtypes.worker._masterDeactivated(
masterid=masterid)
yield self.master.data.rtypes.builder._masterDeactivated(
masterid=masterid)
yield self.master.data.rtypes.scheduler._masterDeactivated(
masterid=masterid)
yield self.master.data.rtypes.changesource._masterDeactivated(
masterid=masterid)
# for each build running on that instance..
builds = yield self.master.data.get(
('builds', ),
filters=[
resultspec.Filter('masterid', 'eq', [masterid]),
resultspec.Filter('complete', 'eq', [False])
])
for build in builds:
# stop any running steps..
steps = yield self.master.data.get(
('builds', build['buildid'], 'steps'),
filters=[resultspec.Filter('results', 'eq', [None])])
for step in steps:
# finish remaining logs for those steps..
logs = yield self.master.data.get(
('steps', step['stepid'], 'logs'),
filters=[resultspec.Filter('complete', 'eq', [False])])
for _log in logs:
yield self.master.data.updates.finishLog(
logid=_log['logid'])
yield self.master.data.updates.finishStep(
stepid=step['stepid'], results=RETRY, hidden=False)
# then stop the build itself
yield self.master.data.updates.finishBuild(
buildid=build['buildid'], results=RETRY)
# unclaim all of the build requests owned by the deactivated instance
buildrequests = yield self.master.db.buildrequests.getBuildRequests(
complete=False, claimed=masterid)
yield self.master.db.buildrequests.unclaimBuildRequests(
brids=[br['buildrequestid'] for br in buildrequests])
def test_getBuilds_resultSpecFilterContainsOneValue(self):
rs = resultspec.ResultSpec(
filters=[resultspec.Filter('state_string', 'contains', ['7'])])
rs.fieldMapping = {'state_string': 'builds.state_string'}
yield self.insertTestData(self.backgroundData + self.threeBuilds)
bdicts = yield self.db.builds.getBuilds(resultSpec=rs)
for bdict in bdicts:
validation.verifyDbDict(self, 'dbbuilddict', bdict)
self.assertEqual(sorted(bdicts, key=lambda bd: bd['id']),
[self.threeBdicts[52]])
def test_setWorkerStateFindByPaused(self):
yield self.master.data.updates.setWorkerState(2, True, False)
resultSpec = resultspec.OptimisedResultSpec(
filters=[resultspec.Filter('paused', 'eq', [True])])
workers = yield self.callGet(('workers',), resultSpec=resultSpec)
self.assertEqual(len(workers), 1)
worker = workers[0]
self.validateData(worker)
self.assertEqual(worker['paused'], True)
def test_getBuilds_resultSpecFilter(self):
rs = resultspec.ResultSpec(
filters=[resultspec.Filter('complete_at', 'ne', [None])])
rs.fieldMapping = {'complete_at': 'builds.complete_at'}
yield self.insertTestData(self.backgroundData + self.threeBuilds)
bdicts = yield self.db.builds.getBuilds(resultSpec=rs)
for bdict in bdicts:
validation.verifyDbDict(self, 'dbbuilddict', bdict)
self.assertEqual(sorted(bdicts, key=lambda bd: bd['id']),
[self.threeBdicts[52]])
def test_getBuilds_resultSpecFilterEqTwoValues(self):
rs = resultspec.ResultSpec(
filters=[resultspec.Filter('number', 'eq', [6, 7])])
rs.fieldMapping = {'number': 'builds.number'}
yield self.insertTestData(self.backgroundData + self.threeBuilds)
bdicts = yield self.db.builds.getBuilds(resultSpec=rs)
for bdict in bdicts:
validation.verifyDbDict(self, 'dbbuilddict', bdict)
self.assertEqual(sorted(bdicts, key=lambda bd: bd['id']),
[self.threeBdicts[51], self.threeBdicts[52]])
def test_get_filters(self):
ep = self.patchFooListPattern()
d = self.data.get(('foo',),
filters=[resultspec.Filter('val', 'lt', [902])])
@d.addCallback
def check(gotten):
self.assertEqual(gotten, base.ListResult(
[{'val': 900}, {'val': 901}], total=2))
ep.get.assert_called_once_with(mock.ANY, {})
return d
def test_get_incomplete(self):
f = resultspec.Filter('complete', 'eq', [False])
d = self.callGet(('buildsets',),
resultSpec=resultspec.ResultSpec(filters=[f]))
@d.addCallback
def check(buildsets):
self.assertEqual(len(buildsets), 1)
self.validateData(buildsets[0])
self.assertEqual(buildsets[0]['bsid'], 14)
return d
def _on_build_finished(self, key, build):
if build['results'] != FAILURE:
return
buildrequest = yield self.master.data.get(('buildrequests', build['buildrequestid']))
builder = yield self.master.data.get(("builders", build['builderid']))
buildset = yield self.master.data.get(('buildsets', buildrequest['buildsetid']))
sourcestamps = buildset['sourcestamps']
builders_to_cancel = set()
for ss in sourcestamps:
configs = self.filters.get_all_matched(builder['name'], ss)
for c in configs:
if builders_to_cancel is not None:
if c.builders_to_cancel is None:
builders_to_cancel = None
else:
builders_to_cancel.update(c.builders_to_cancel)
all_bs_buildrequests = yield self.master.data.get(
('buildrequests',),
filters=[resultspec.Filter('buildsetid', 'eq', [buildset['bsid']]),
resultspec.Filter('complete', 'eq', [False])])
all_bs_buildrequests = [br for br in all_bs_buildrequests
if br['buildrequestid'] != buildrequest['buildrequestid']]
for br in all_bs_buildrequests:
brid = br['buildrequestid']
if brid == buildrequest['buildrequestid']:
continue # this one has just failed
br_builder = yield self.master.data.get(("builders", br['builderid']))
if builders_to_cancel is not None and br_builder['name'] not in builders_to_cancel:
continue
reason = 'Build has been cancelled because another build in the same buildset failed'
self.master.data.control('cancel', {'reason': reason}, ('buildrequests', str(brid)))
def test_get_complete(self):
# override resultSpec implementation to be noop
class MyResultSpec(resultspec.ResultSpec):
def apply(self, data):
return data
resultSpec = MyResultSpec(
filters=[resultspec.Filter('complete', 'eq', [False])])
builds = yield self.callGet(('builds', ), resultSpec=resultSpec)
[self.validateData(build) for build in builds]
self.assertEqual(sorted([b['number'] for b in builds]), [3, 4])
def getOldestRequestTime(self):
"""Returns the submitted_at of the oldest unclaimed build request for
this builder, or None if there are no build requests.
@returns: datetime instance or None, via Deferred
"""
bldrid = yield self.getBuilderId()
unclaimed = yield self.master.data.get(
('builders', bldrid, 'buildrequests'),
[resultspec.Filter('claimed', 'eq', [False])])
if unclaimed:
unclaimed = sorted([brd['submitted_at'] for brd in unclaimed])
defer.returnValue(unclaimed[0])
def test_get_contains_two_tags(self):
resultSpec = resultspec.ResultSpec(
filters=[resultspec.Filter('tags', 'contains', ["tagA", "tagB"])])
d = self.callGet(('builders', ))
@d.addCallback
def check(builders):
builders = resultSpec.apply(builders)
[self.validateData(b) for b in builders]
self.assertEqual(sorted([b['builderid'] for b in builders]),
[3, 4, 5])
return d
def getBuilderNamesAndBuildRequests(self):
# returns a Deferred; undocumented method that may be removed
# without warning
d = self.master.data.get(('buildrequests', ),
filters=[resultspec.Filter('buildsetid', 'eq', [self.id])])
@d.addCallback
def get_objects(brdicts):
return dict([
(brd['buildername'], BuildRequestStatus(brd['buildername'],
brd['brid'], self.status))
for brd in brdicts])
return d
def getOldestRequestTime(self):
"""Returns the submitted_at of the oldest unclaimed build request for
this builder, or None if there are no build requests.
@returns: datetime instance or None, via Deferred
"""
bldrid = yield self.getBuilderId()
unclaimed = yield self.master.data.get(
('builders', bldrid, 'buildrequests'),
[resultspec.Filter('claimed', 'eq', [False])],
order=['submitted_at'], limit=1)
if unclaimed:
return unclaimed[0]['submitted_at']
return None
def doForceBuild(self,
wantSteps=False,
wantProperties=False,
wantLogs=False,
useChange=False):
# force a build, and wait until it is finished
d = defer.Deferred()
# in order to allow trigger based integration tests
# we wait until the first started build is finished
self.firstBuildRequestId = None
def newCallback(_, data):
if self.firstBuildRequestId is None:
self.firstBuildRequestId = data['buildrequestid']
newConsumer.stopConsuming()
def finishedCallback(_, data):
if self.firstBuildRequestId == data['buildrequestid']:
d.callback(data)
newConsumer = yield self.master.mq.startConsuming(
newCallback, ('buildrequests', None, 'new'))
finishedConsumer = yield self.master.mq.startConsuming(
finishedCallback, ('buildrequests', None, 'complete'))
if useChange is False:
# use data api to force a build
yield self.master.data.control("force", {},
("forceschedulers", "force"))
else:
# use data api to force a build, via a new change
yield self.master.data.updates.addChange(**useChange)
# wait until we receive the build finished event
buildrequest = yield d
builds = yield self.master.data.get(
('builds', ),
filters=[
resultspec.Filter('buildrequestid', 'eq',
[buildrequest['buildrequestid']])
])
# if the build has been retried, there will be several matching builds.
# We return the last build
build = builds[-1]
finishedConsumer.stopConsuming()
yield self.enrichBuild(build, wantSteps, wantProperties, wantLogs)
defer.returnValue(build)
def _fetchUnclaimedBrdicts(self):
# Sets up a cache of all the unclaimed brdicts. The cache is
# saved at self.unclaimedBrdicts cache. If the cache already
# exists, this function does nothing. If a refetch is desired, set
# the self.unclaimedBrdicts to None before calling."""
if self.unclaimedBrdicts is None:
# TODO: use order of the DATA API
brdicts = yield self.master.data.get(
('builders',
(yield self.bldr.getBuilderId()), 'buildrequests'),
[resultspec.Filter('claimed', 'eq', [False])])
# sort by submitted_at, so the first is the oldest
brdicts.sort(key=lambda brd: brd['submitted_at'])
self.unclaimedBrdicts = brdicts
defer.returnValue(self.unclaimedBrdicts)
def getNewestCompleteTime(self):
"""Returns the complete_at of the latest completed build request for
this builder, or None if there are no such build requests.
@returns: datetime instance or None, via Deferred
"""
bldrid = yield self.getBuilderId()
completed = yield self.master.data.get(
('builders', bldrid, 'buildrequests'),
[resultspec.Filter('complete', 'eq', [False])],
order=['-complete_at'], limit=1)
if completed:
defer.returnValue(completed[0]['complete_at'])
else:
defer.returnValue(None)
def getBuilder(self, buildername=None, builderid=None):
if buildername:
bdicts = yield self.master.data.get(('builders',),
filters=[resultspec.Filter('name', 'eq', [buildername])])
if bdicts:
bdict = bdicts[0] # Could there be more than one? One is enough.
else:
bdict = None
elif builderid:
bdict = yield self.master.data.get(('builders', builderid))
else:
raise UsageError("no builder specified")
if bdict is None:
if buildername:
which = '%s' % buildername
else:
which = 'number %s' % builderid
raise UsageError("no such builder '%s'" % which)
defer.returnValue(bdict)
def getBuilder(self, buildername=None, builderid=None):
if buildername:
bdicts = yield self.master.data.get(('builders',),
filters=[resultspec.Filter('name', 'eq', [buildername])])
if bdicts:
# Could there be more than one? One is enough.
bdict = bdicts[0]
else:
bdict = None
elif builderid:
bdict = yield self.master.data.get(('builders', builderid))
else:
raise UsageError("no builder specified")
if bdict is None:
if buildername:
which = buildername
else:
which = 'number {}'.format(builderid)
raise UsageError("no such builder '{}'".format(which))
return bdict
def getDetailsForBuildset(master,
bsid,
want_properties=False,
want_steps=False,
want_previous_build=False,
want_logs=False,
want_logs_content=False):
# Here we will do a bunch of data api calls on behalf of the reporters
# We do try to make *some* calls in parallel with the help of gatherResults, but don't commit
# to much in that. The idea is to do parallelism while keeping the code readable
# and maintainable.
# first, just get the buildset and all build requests for our buildset id
dl = [
master.data.get(("buildsets", bsid)),
master.data.get(
('buildrequests', ),
filters=[resultspec.Filter('buildsetid', 'eq', [bsid])])
]
(buildset, breqs) = yield defer.gatherResults(dl)
# next, get the bdictlist for each build request
dl = [
master.data.get(("buildrequests", breq['buildrequestid'], 'builds'))
for breq in breqs
]
builds = yield defer.gatherResults(dl)
builds = flatten(builds, types=(list, UserList))
if builds:
yield getDetailsForBuilds(master,
buildset,
builds,
want_properties=want_properties,
want_steps=want_steps,
want_previous_build=want_previous_build,
want_logs=want_logs,
want_logs_content=want_logs_content)
return dict(buildset=buildset, builds=builds)
