def setUp(self):
self.MASTER_ID = fakedb.FakeBuildRequestsComponent.MASTER_ID
self.OTHER_MASTER_ID = self.MASTER_ID + 1111
d = self.setUpConnectorComponent(table_names=[
'patches', 'changes', 'sourcestamp_changes', 'buildsets',
'buildset_properties', 'buildrequests', 'objects',
'buildrequest_claims', 'sourcestamps', 'sourcestampsets'
])
def finish_setup(_):
self.db.buildrequests = \
buildrequests.BuildRequestsConnectorComponent(self.db)
self.db.master.getObjectId = lambda: defer.succeed(self.MASTER_ID)
d.addCallback(finish_setup)
# set up a sourcestamp and buildset for use below
d.addCallback(lambda _: self.insertTestData([
fakedb.SourceStampSet(id=234),
fakedb.SourceStamp(id=234, sourcestampsetid=234),
fakedb.Object(id=self.MASTER_ID,
name="fake master",
class_name="BuildMaster"),
fakedb.Object(id=self.OTHER_MASTER_ID,
name="other master",
class_name="BuildMaster"),
fakedb.Buildset(id=self.BSID, sourcestampsetid=234),
]))
return d
class TestMastersConfigConnectorComponent(
connector_component.ConnectorComponentMixin,
unittest.TestCase):
def setUp(self):
d = self.setUpConnectorComponent(
table_names=['mastersconfig', 'objects',
'buildrequest_claims', 'buildrequests'])
def finish_setup(_):
self.db.mastersconfig = mastersconfig.MastersConfigConnectorComponent(self.db)
d.addCallback(finish_setup)
return d
def tearDown(self):
return self.tearDownConnectorComponent()
# common sample data
background_data = [
fakedb.Object(id=1, name='katana/buildmaster-01', class_name='buildbot.master.BuildMaster'),
fakedb.Object(id=2, name='katana/buildmaster-02', class_name='buildbot.master.BuildMaster'),
fakedb.MasterConfig(id=1, buildbotURL='http://localhost:8001/', objectid=1),
fakedb.MasterConfig(id=2, buildbotURL='http://localhost:8002/', objectid=2),
fakedb.BuildRequest(id=1, buildsetid=1, buildername='b1'),
fakedb.BuildRequest(id=2, buildsetid=2, buildername='b1'),
fakedb.BuildRequestClaim(brid=1, objectid=1, claimed_at='1416383733'),
fakedb.BuildRequestClaim(brid=2, objectid=2, claimed_at='1416383733'),
]
def test_setupMaster(self):
d = self.db.mastersconfig.setupMaster('http://localhost:8001/', 1)
def check(_):
def thd(conn):
r = conn.execute(self.db.model.mastersconfig.select())
rows = [(row.id, row.buildbotURL, row.objectid)
for row in r.fetchall()]
self.assertEqual(rows,
[(1, 'http://localhost:8001/', 1)])
return self.db.pool.do(thd)
d.addCallback(check)
return d
def test_getMasterURL(self):
d = self.insertTestData(self.background_data)
def check(row, expected_master={}):
self.assertEqual(row, expected_master)
d.addCallback(lambda _ : self.db.mastersconfig.getMasterURL(1))
d.addCallback(check, expected_master={'buildbotURL': 'http://localhost:8001/',
'id': 1, 'objectid': 1})
d.addCallback(lambda _ : self.db.mastersconfig.getMasterURL(2))
d.addCallback(check, expected_master={'buildbotURL': 'http://localhost:8002/',
'id': 2, 'objectid': 2})
return d
def test_savedProperties(self):
sched = self.makeScheduler(name='test', builderNames=['test'],
minute=[5], codebases={'cb':{'repository':'annoying'}})
self.db.insertTestData([
fakedb.Object(id=self.SCHEDULERID, name='test', class_name='NightlyTriggerable'),
fakedb.ObjectState(objectid=self.SCHEDULERID, name='lastTrigger',
value_json='[ {"cb": {"project": "p", "repository": "r", "branch": "br", "revision": "myrev"}} , {"testprop": ["test", "TEST"]} ]'),
])
sched.startService()
self.clock.advance(60*60) # Run for 1h
self.db.buildsets.assertBuildset('?',
dict(external_idstring=None,
properties=[
('scheduler', ('test', 'Scheduler')),
('testprop', ('test', 'TEST')),
],
reason="The NightlyTriggerable scheduler named 'test' triggered this build",
sourcestampsetid=100),
{'cb':
dict(branch='br', project='p', repository='r', codebase='cb',
revision='myrev', sourcestampsetid=100)
})
def test_triggerProperties(self):
sched = self.makeScheduler(name='test', builderNames=['test'],
minute=[5], codebases={'cb':{'repository':'annoying'}})
self.db.insertTestData([
fakedb.Object(id=self.SCHEDULERID, name='test', class_name='NightlyTriggerable'),
])
sched.startService()
sched.trigger({'cb':
dict(revision='myrev',
branch='br', project='p', repository='r'),
}, properties.Properties(testprop='test'))
self.db.state.assertState(self.SCHEDULERID, lastTrigger=[{'cb':
dict(revision='myrev',
branch='br', project='p', repository='r'),
}, {'testprop': ['test', 'TEST']}])
self.clock.advance(60*60) # Run for 1h
self.db.buildsets.assertBuildset('?',
dict(external_idstring=None,
properties=[
('scheduler', ('test', 'Scheduler')),
('testprop', ('test', 'TEST')),
],
reason="The NightlyTriggerable scheduler named 'test' triggered this build",
sourcestampsetid=100),
{'cb':
dict(branch='br', project='p', repository='r', codebase='cb',
revision='myrev', sourcestampsetid=100)
})
def test_preStartConsumingChanges_existing(self):
sched = self.makeFullScheduler(name='test',
builderNames=['test'],
treeStableTimer=None,
branch='master',
codebases=self.codebases,
createAbsoluteSourceStamps=True)
self.db.insertTestData([
fakedb.Object(id=self.OBJECTID,
name='test',
class_name='SingleBranchScheduler'),
fakedb.ObjectState(
objectid=self.OBJECTID,
name='lastCodebases',
value_json='{"a": {"branch": "master", "repository": "A", '
'"revision": "1234:abc", "lastChange": 13}}')
])
yield sched.preStartConsumingChanges()
self.assertEqual(
sched._lastCodebases, {
'a': {
'branch': 'master',
'lastChange': 13,
'repository': 'A',
'revision': '1234:abc'
}
})
def test_savedProperties(self):
sched = self.makeScheduler(
name='test',
builderNames=['test'],
minute=[5],
codebases={'cb': {
'repository': 'annoying'
}})
self.db.insertTestData([
fakedb.Object(id=self.SCHEDULERID,
name='test',
class_name='NightlyTriggerable'),
fakedb.ObjectState(
objectid=self.SCHEDULERID,
name='lastTrigger',
value_json=
'[ [ {"codebase": "cb", "project": "p", "repository": "r", "branch": "br", "revision": "myrev"} ], {"testprop": ["test", "TEST"]}, null, null ]'
),
])
sched.activate()
self.clock.advance(60 * 60) # Run for 1h
self.assertBuildsetAdded(properties={'testprop': (u'test', u'TEST')},
sourcestamps=[
dict(codebase='cb',
branch='br',
project='p',
repository='r',
revision='myrev'),
])
def test_triggerProperties(self):
sched = self.makeScheduler(name='test', builderNames=['test'],
minute=[5], codebases={'cb': {'repository': 'annoying'}})
self.db.insertTestData([
fakedb.Object(id=self.SCHEDULERID, name='test', class_name='NightlyTriggerable'),
])
sched.activate()
sched.trigger(False, [
dict(codebase='cb', revision='myrev', branch='br', project='p',
repository='r'),
], properties.Properties(testprop='test'))
self.db.state.assertState(self.SCHEDULERID, lastTrigger=[[
dict(codebase='cb', revision='myrev',
branch='br', project='p', repository='r'),
], {'testprop': ['test', 'TEST']}, None, None])
self.clock.advance(60 * 60) # Run for 1h
self.assertBuildsetAdded(
properties=dict(testprop=('test', 'TEST')),
sourcestamps=[
dict(codebase='cb', branch='br', project='p', repository='r',
revision='myrev'),
])
def setUpTests(self):
# set up a sourcestamp and buildset for use below
self.MASTER_ID = fakedb.FakeBuildRequestsComponent.MASTER_ID
self.OTHER_MASTER_ID = self.MASTER_ID + 1111
return self.insertTestData([
fakedb.SourceStampSet(id=234),
fakedb.SourceStamp(id=234, sourcestampsetid=234),
fakedb.Object(id=self.MASTER_ID,
name="fake master",
class_name="BuildMaster"),
fakedb.Object(id=self.OTHER_MASTER_ID,
name="other master",
class_name="BuildMaster"),
fakedb.Buildset(id=self.BSID, sourcestampsetid=234),
])
def test_atomicCreateState_nojsonable(self):
yield self.insertTestData([
fakedb.Object(id=10, name='-', class_name='-'),
])
d = self.db.state.atomicCreateState(10, 'x', object)
yield self.assertFailure(d, TypeError)
def test_getState_badjson(self):
d = self.insertTestData([
fakedb.Object(id=10, name='x', class_name='y'),
fakedb.ObjectState(objectid=10, name='x', value_json='ff[1'),
])
d.addCallback(lambda _: self.db.state.getState(10, 'x'))
return self.assertFailure(d, TypeError)
def test_setState_conflict(self):
d = self.insertTestData([
fakedb.Object(id=10, name='-', class_name='-'),
])
def hook(conn):
conn.execute(self.db.model.object_state.insert(),
objectid=10,
name='x',
value_json='22')
self.db.state._test_timing_hook = hook
d.addCallback(lambda _: self.db.state.setState(10, 'x', [1, 2]))
def check(_):
def thd(conn):
q = self.db.model.object_state.select()
rows = conn.execute(q).fetchall()
self.assertEqual([(r.objectid, r.name, r.value_json)
for r in rows], [(10, 'x', '22')])
return self.db.pool.do(thd)
d.addCallback(check)
return d
def test_setState_badjson(self):
d = self.insertTestData([
fakedb.Object(id=10, name='x', class_name='y'),
])
d.addCallback(lambda _:
self.db.state.setState(10, 'x', self)) # self is not JSON-able..
return self.assertFailure(d, TypeError)
def test_saveTrigger_noTrigger(self):
sched = self.makeScheduler(
name='test',
builderNames=['test'],
minute=[5],
codebases={'cb': {
'repository': 'annoying'
}})
self.db.insertTestData([
fakedb.Object(id=self.SCHEDULERID,
name='test',
class_name='NightlyTriggerable'),
])
sched.activate()
(idsDeferre, d) = sched.trigger(False, [
dict(codebase='cb',
revision='myrev',
branch='br',
project='p',
repository='r'),
],
set_props=None)
self.clock.advance(60 * 60) # Run for 1h
@d.addCallback
def cb(_):
self.db.state.assertState(self.SCHEDULERID, lastTrigger=None)
return d
def test_getObjectId_existing(self):
yield self.insertTestData([
fakedb.Object(id=19, name='someobj',
class_name='someclass')])
objectid = yield self.db.state.getObjectId('someobj', 'someclass')
self.assertEqual(objectid, 19)
def test_atomicCreateState(self):
yield self.insertTestData([
fakedb.Object(id=10, name='-', class_name='-'),
])
res = yield self.db.state.atomicCreateState(10, 'x', lambda: [1, 2])
self.assertEqual(res, [1, 2])
res = yield self.db.state.getState(10, 'x')
self.assertEqual(res, [1, 2])
def test_getState_present(self):
yield self.insertTestData([
fakedb.Object(id=10, name='x', class_name='y'),
fakedb.ObjectState(objectid=10, name='x', value_json='[1,2]'),
])
val = yield self.db.state.getState(10, 'x')
self.assertEqual(val, [1, 2])
def test_lineReceived_patchset_created(self):
self.master.db.insertTestData([
fakedb.Object(id=self.OBJECTID,
name='GerritEventLogPoller:gerrit',
class_name='GerritEventLogPoller')
])
yield self.newChangeSource()
self.changesource.now = lambda: datetime.datetime.utcfromtimestamp(
self.NOW_TIMESTAMP)
self._http.expect(method='get',
ep='/plugins/events-log/events/',
params={'t1': self.NOW_FORMATTED},
content_json=dict(
type="patchset-created",
change=dict(branch="br",
project="pr",
number="4321",
owner=dict(
name="Dustin",
email="*****@*****.**"),
url="http://buildbot.net",
subject="fix 1234"),
eventCreatedOn=self.EVENT_TIMESTAMP,
patchSet=dict(revision="abcdef", number="12")))
yield self.startChangeSource()
yield self.changesource.poll()
self.assertEqual(len(self.master.data.updates.changesAdded), 1)
c = self.master.data.updates.changesAdded[0]
for k, v in iteritems(c):
self.assertEqual(TestGerritChangeSource.expected_change[k], v)
self.master.db.state.assertState(self.OBJECTID,
last_event_ts=self.EVENT_TIMESTAMP)
# do a second poll, it should ask for the next events
self._http.expect(method='get',
ep='/plugins/events-log/events/',
params={'t1': self.EVENT_FORMATTED},
content_json=dict(
type="patchset-created",
change=dict(branch="br",
project="pr",
number="4321",
owner=dict(
name="Dustin",
email="*****@*****.**"),
url="http://buildbot.net",
subject="fix 1234"),
eventCreatedOn=self.EVENT_TIMESTAMP + 1,
patchSet=dict(revision="abcdef", number="12")))
yield self.changesource.poll()
self.master.db.state.assertState(self.OBJECTID,
last_event_ts=self.EVENT_TIMESTAMP +
1)
def test_getObjectId_existing(self):
d = self.insertTestData([
fakedb.Object(id=19, name='someobj',
class_name='someclass')])
d.addCallback(lambda _:
self.db.state.getObjectId('someobj', 'someclass'))
def check(objectid):
self.assertEqual(objectid, 19)
d.addCallback(check)
return d
def test_getState_present(self):
d = self.insertTestData([
fakedb.Object(id=10, name='x', class_name='y'),
fakedb.ObjectState(objectid=10, name='x', value_json='[1,2]'),
])
d.addCallback(lambda _:
self.db.state.getState(10, 'x'))
def check(val):
self.assertEqual(val, [1, 2])
d.addCallback(check)
return d
def test_pollDatabaseChanges_empty(self):
self.db.insertTestData([
fakedb.Object(id=22, name=self.master_name,
class_name='buildbot.master.BuildMaster'),
])
d = self.master.pollDatabaseChanges()
def check(_):
self.assertEqual(self.gotten_changes, [])
self.assertEqual(self.gotten_buildset_additions, [])
self.assertEqual(self.gotten_buildset_completions, [])
self.db.state.assertState(22, last_processed_change=0)
d.addCallback(check)
return d
def test_setState(self):
yield self.insertTestData([
fakedb.Object(id=10, name='-', class_name='-'),
])
yield self.db.state.setState(10, 'x', [1, 2])
def thd(conn):
q = self.db.model.object_state.select()
rows = conn.execute(q).fetchall()
self.assertEqual([(r.objectid, r.name, r.value_json)
for r in rows], [(10, 'x', '[1, 2]')])
yield self.db.pool.do(thd)
def test_atomicCreateState_conflict(self):
yield self.insertTestData([
fakedb.Object(id=10, name='-', class_name='-'),
])
def hook(conn):
conn.execute(self.db.model.object_state.insert(),
objectid=10, name='x', value_json='22')
self.db.state._test_timing_hook = hook
res = yield self.db.state.atomicCreateState(10, 'x', lambda: [1, 2])
self.assertEqual(res, 22)
res = yield self.db.state.getState(10, 'x')
self.assertEqual(res, 22)
def test_pruneChanges(self):
d = self.insertTestData(
[
fakedb.Object(id=29),
fakedb.SourceStamp(id=234),
fakedb.Change(changeid=11),
fakedb.Change(changeid=12),
fakedb.SchedulerChange(objectid=29, changeid=12),
fakedb.SourceStampChange(sourcestampid=234, changeid=12),
] +
self.change13_rows +
[
fakedb.SchedulerChange(objectid=29, changeid=13),
] +
self.change14_rows +
[
fakedb.SchedulerChange(objectid=29, changeid=14),
fakedb.Change(changeid=15),
fakedb.SourceStampChange(sourcestampid=234, changeid=15),
]
)
# pruning with a horizon of 2 should delete changes 11, 12 and 13
d.addCallback(lambda _: self.db.changes.pruneChanges(2))
def check(_):
def thd(conn):
results = {}
for tbl_name in ('scheduler_changes', 'sourcestamp_changes',
'change_files', 'change_properties',
'changes'):
tbl = self.db.model.metadata.tables[tbl_name]
res = conn.execute(sa.select([tbl.c.changeid]))
results[tbl_name] = sorted([row[0] for row in res.fetchall()])
self.assertEqual(results, {
'scheduler_changes': [14],
'sourcestamp_changes': [15],
'change_files': [14],
'change_properties': [],
'changes': [14, 15],
})
return self.db.pool.do(thd)
d.addCallback(check)
return d
def test_setState_badjson(self):
d = self.insertTestData([
fakedb.Object(id=10, name='x', class_name='y'),
])
d.addCallback(lambda _: self.db.state.setState(10, 'x', self)
) # self is not JSON-able..
def cb(_):
self.fail("should not have succeeded!")
def eb(f):
f.trap(TypeError)
d.addCallbacks(cb, eb)
return d
def test_getState_badjson(self):
d = self.insertTestData([
fakedb.Object(id=10, name='x', class_name='y'),
fakedb.ObjectState(objectid=10, name='x', value_json='ff[1'),
])
d.addCallback(lambda _: self.db.state.getState(10, 'x'))
def cb(_):
self.fail("should not have succeeded!")
def eb(f):
f.trap(TypeError)
d.addCallbacks(cb, eb)
return d
def test_startService_createAbsoluteSourceStamps_loadCodebase(self):
# check codebase is loaded and used on startup.
sched = self.makeFullScheduler(name='test',
builderNames=['test'],
treeStableTimer=None,
branch='master',
codebases=self.codebases,
createAbsoluteSourceStamps=True)
self.db.insertTestData([
fakedb.Object(id=self.OBJECTID,
name='test',
class_name='SingleBranchScheduler'),
fakedb.ObjectState(
objectid=self.OBJECTID,
name='lastCodebases',
value_json=
'{"a": {"branch": "master", "repository": "A", "revision": "1234:abc", "lastChange": 13}}'
)
])
d = sched.startService(_returnDeferred=True)
d.addCallback(lambda _: sched.gotChange(
self.mkch(
codebase='b', revision='2345:bcd', repository='B', number=14),
True))
def check(xxx_todo_changeme):
(bsid, brids) = xxx_todo_changeme
self.db.buildsets.assertBuildset(
bsid=bsid,
expected_buildset=self.mkbs(brids=brids),
expected_sourcestamps={
'a':
self.mkss(codebase='a',
revision='1234:abc',
repository='A'),
'b':
self.mkss(codebase='b',
revision='2345:bcd',
repository='B',
changeids=set([14]))
})
d.addCallback(check)
d.addCallback(lambda _: sched.stopService())
return d
def test_pollDatabaseChanges_nothing_new(self):
self.db.insertTestData([
fakedb.Object(id=53, name='master',
class_name='buildbot.master.BuildMaster'),
fakedb.ObjectState(objectid=53, name='last_processed_change',
value_json='10'),
fakedb.Change(changeid=10),
])
d = self.master.pollDatabaseChanges()
def check(_):
self.assertEqual(self.gotten_changes, [])
self.assertEqual(self.gotten_buildset_additions, [])
self.assertEqual(self.gotten_buildset_completions, [])
self.db.state.assertState(53, last_processed_change=10)
d.addCallback(check)
return d
def test_pollDatabaseChanges_catchup(self):
# with no existing state, it should catch up to the most recent change,
# but not process anything
self.db.insertTestData([
fakedb.Object(id=22, name=self.master_name,
class_name='buildbot.master.BuildMaster'),
fakedb.Change(changeid=10),
fakedb.Change(changeid=11),
])
d = self.master.pollDatabaseChanges()
def check(_):
self.assertEqual(self.gotten_changes, [])
self.assertEqual(self.gotten_buildset_additions, [])
self.assertEqual(self.gotten_buildset_completions, [])
self.db.state.assertState(22, last_processed_change=11)
d.addCallback(check)
return d
def test_gotChange_createAbsoluteSourceStamps_older_change(self):
# check codebase is not stored if it's older than the most recent
sched = self.makeFullScheduler(name='test',
builderNames=['test'],
treeStableTimer=None,
branch='master',
codebases=self.codebases,
createAbsoluteSourceStamps=True)
self.db.insertTestData([
fakedb.Object(id=self.OBJECTID,
name='test',
class_name='SingleBranchScheduler'),
fakedb.ObjectState(
objectid=self.OBJECTID,
name='lastCodebases',
value_json='{"a": {"branch": "master", "repository": "A", '
'"revision": "5555:def", "lastChange": 20}}')
])
d = sched.activate()
d.addCallback(
lambda _:
# this change is not recorded, since it's older than
# change 20
sched.gotChange(
self.mkch(codebase='a',
revision='1234:abc',
repository='A',
number=10), True))
def check(_):
self.db.state.assertState(self.OBJECTID,
lastCodebases={
'a':
dict(branch='master',
repository='A',
revision=u'5555:def',
lastChange=20)
})
d.addCallback(check)
d.addCallback(lambda _: sched.deactivate())
return d
def test_preStartConsumingChanges_no_createAbsoluteSourceStamps(self):
sched = self.makeFullScheduler(name='test',
builderNames=['test'],
treeStableTimer=None,
branch='master',
codebases=self.codebases)
self.db.insertTestData([
fakedb.Object(id=self.OBJECTID,
name='test',
class_name='SingleBranchScheduler'),
fakedb.ObjectState(objectid=self.OBJECTID,
name='lastCodebases',
value_json='{"a": {"branch": "master"}}')
])
yield sched.preStartConsumingChanges()
self.assertEqual(sched._lastCodebases, {})
