def get_build_info(self, bid):
# brid, buildername, buildnum
build_obj = rpc.RpcProxy('software_dev.commit')
res = build_obj.read(bid, ['buildername', 'build_number'])
if res:
return (res['id'], res['buildername'], res['build_number'])
return (None, None, None)
def scheduler_retire_changes(self, schedulerid, changeids, t):
scha_obj = rpc.RpcProxy('software_dev.sched_change')
# one time for important ones
sids = scha_obj.search([('sched_id', '=', schedulerid),
('commit_id', 'in', changeids)])
res = scha_obj.unlink(sids)
def create_buildset(self,
ssid,
reason,
properties,
builderNames,
t,
external_idstring=None):
# this creates both the BuildSet and the associated BuildRequests
now = self._getCurrentTime()
bset_obj = rpc.RpcProxy('software_dev.commit')
vals = { # sourcestamp:
'submitted_at': time2str(now),
}
if external_idstring:
vals['external_idstring'] = external_idstring
if reason:
vals['reason'] = reason
bsid = ssid # buildset == sourcestamp == change
bset_obj.write(bsid, vals)
for propname, propvalue in properties.properties.items():
bset_obj.setProperties(
bsid, [(pn, json.dumps(pv))
for pn, pv in properties.properties.items()])
# TODO: respect builderNames
brids = []
brid = ssid # buildrequest == sourcestamp
brids.append(brid)
self.notify("add-buildset", bsid)
self.notify("add-buildrequest", *brids)
return bsid
def getSourceStampNumberedNow(self, ssid, t=None, old_res=None):
assert isinstance(ssid, (int, long))
ss = self._sourcestamp_cache.get(ssid)
if ss:
return ss
assert isinstance(ssid, (int, long))
sstamp_obj = rpc.RpcProxy('software_dev.commit')
if not old_res:
res = sstamp_obj.read(ssid)
else:
res = old_res
if not res:
return None
branch = None # res['branch_url']
revision = res.get('revno', False) or res.get('hash', '')
patch = None
#if patchid is not None:
# raise NotImplementedError
changes = None
changeid = ssid
changes = [
self.getChangeNumberedNow(changeid, t),
]
ss = SourceStamp(branch, revision, patch, changes)
# project=project, repository=repository)
ss.ssid = ssid
self._sourcestamp_cache.add(ssid, ss)
return ss
def get_buildnums_for_brid(self, brid):
build_obj = rpc.RpcProxy('software_dev.commit')
# remember: buildrequest == build in our schema
res = build_obj.read(brid, ['build_number'])
return [
res['build_number'],
]
def _get_change_num(self, trans, changeid):
change_obj = rpc.RpcProxy('software_dev.commit')
if isinstance(changeid, (list, tuple)):
cids = changeid
else:
cids = [
changeid,
]
if not cids:
return []
res = change_obj.getChanges(cids)
ret = []
props = self.get_properties_from_db(change_obj, cids)
for cdict in res:
c = OpenObjectChange(**cdict)
p = props.get(cdict['id'], Properties())
c.properties.updateFromProperties(p)
self._change_cache.add(cdict['id'], c)
ret.append(c)
if isinstance(changeid, (list, tuple)):
return ret
else:
return ret[0]
def saveStatResults(self, changes, file_stats):
""" Try to save file_stats inside the filechanges of commits
@param changes is the list of changes (as in allChanges() )
@param file_stats is a dict of { fname:, { lines_add:, lines_rem: }}
"""
# commit_obj = rpc.RpcProxy('software_dev.commit')
fchange_obj = rpc.RpcProxy('software_dev.filechange')
commit_ids = []
for chg in changes:
if not chg.number:
continue
commit_ids.append(chg.number)
while len(commit_ids) and len(file_stats):
cid = commit_ids.pop() # so, we attribute the stats to the
# last commit that matches their files
fc_ids = fchange_obj.search([('commit_id', '=', cid)])
fcres = fchange_obj.read(fc_ids, ['filename'])
# We read all the filenames that belong to the commit and
# then try to see if we have any stats for them.
if not fcres:
continue
for fcd in fcres:
fcstat = file_stats.pop(fcd['filename'], False)
if not fcstat:
continue
# now, we have a filechange.id and stats
fchange_obj.write(fcd['id'], fcstat)
def get_buildername_for_brid(self, brid):
breq_obj = rpc.RpcProxy('software_dev.commit')
res = breq_obj.read(brid, ['buildername'])
if not res:
return None
return res['buildername']
def _txn_build_started(self, t, brid, buildnumber):
now = self._getCurrentTime()
build_obj = rpc.RpcProxy('software_dev.commit')
vals = {'build_number': buildnumber, 'build_start_time': time2str(now)}
build_obj.write(brid, vals)
bid = brid # one table is used for everything
self.notify("add-build", bid)
return bid
def scheduler_classify_change(self, schedulerid, number, important, t):
scha_obj = rpc.RpcProxy('software_dev.sched_change')
# print "Classify change %s at %s as important=%s" %( number, schedulerid, important)
scha_obj.create({
'commit_id': number,
'sched_id': schedulerid,
'important': important
})
def removeChangeNow(self, changeid):
"""Thoroughly remove a change from the database, including all dependent
tables"""
change_obj = rpc.RpcProxy('software_dev.commit')
change_obj.unlink([
changeid,
])
return None
def saveTResults(self, build_id, name, build_result, t_results):
bld_obj = rpc.RpcProxy('software_dev.commit')
tsum_obj = rpc.RpcProxy('software_dev.test_result')
for seq, tr in enumerate(t_results):
vals = {
'build_id': build_id,
'name': name,
'substep': '.'.join(tr.name), # it is a tuple in TestResult
'state': res_state.get(tr.results, 'unknown'),
'sequence': seq,
'blame_log': tr.text,
}
tsum_obj.create(vals)
return
def get_pending_brids_for_builder(self, buildername):
breq_obj = rpc.RpcProxy('software_dev.commit')
bids = breq_obj.search([('buildername', '=', buildername),
('complete', '=', False),
('claimed_at', '=', False)])
return list(bids)
def getChangeIdsLessThanIdNow(self, new_changeid):
"""Return a list of all extant change id's less than the given value,
sorted by number."""
change_obj = rpc.RpcProxy('software_dev.commit')
cids = change_obj.search([('id', '<', new_changeid)])
t = Token()
changes = self.runInteractionNow(self._get_change_num, cids)
changes.sort(key=lambda c: c.number)
return changes
def get_buildset_info(self, bsid):
bset_obj = rpc.RpcProxy('software_dev.commit')
res = bset_obj.read(
bsid, ['external_idstring', 'reason', 'complete', 'results'])
if res:
external_idstring = res['external_idstring'] or None
reason = res['reason'] or None
complete = bool(res['complete'])
return (external_idstring, reason, bsid, complete, res['results'])
return None # shouldn't happen
def poll_config(self):
bbot_obj = rpc.RpcProxy('software_dev.buildbot')
try:
new_tstamp = bbot_obj.get_conf_timestamp([
self.bbot_id,
])
# print "Got conf timestamp:", self.bbot_tstamp
except Exception, e:
print "Could not get timestamp: %s" % e
return
def __init__(self, svc, proxy, locals=None, timeout=60 * 5, pre_prompt=''):
if 0:
self.proxy = rpc.RpcProxy()
self.svc = rpc.RpcService()
self.proxy = proxy
self.svc = svc
self.proxy.timeout = timeout
self.log_hdlr = None
self.loged = True
self.pre_prompt = pre_prompt
self.init(locals)
def saveCStats(self, cid, cstats):
""" Try to save commit stats of change(s)
@param cstats is list of tuples of ( change_id:, { lines_add:, lines_rem:, ... })
"""
commit_obj = rpc.RpcProxy('software_dev.commit')
try:
commit_obj.saveCStats(cid, cstats)
except rpc.RpcException, e:
log.err("Cannot save commit stats: %s" % e)
return False
def _txn_resubmit_buildreqs(self, t, brids):
# the interrupted build that gets resubmitted will still have the
# same submitted_at value, so it should be re-started first
breq_obj = rpc.RpcProxy('software_dev.commit')
# remember: buildrequest == build in our schema
vals = {
'claimed_at': False,
'claimed_by_name': False,
'claimed_by_incarnation': False
}
breq_obj.write(list(brids), vals)
self.notify("add-buildrequest", *brids)
def getChangesGreaterThan(self, last_changeid, t=None):
"""Return a Deferred that fires with a list of all Change instances
with numbers greater than the given value, sorted by number. This is
useful for catching up with everything that's happened since you last
called this function."""
assert last_changeid >= 0
change_obj = rpc.RpcProxy('software_dev.commit')
cids = change_obj.search([('id', '>', last_changeid)])
# FIXME: defer
changes = self.runInteractionNow(self._get_change_num, cids)
changes.sort(key=lambda c: c.number)
return changes
def addSchedulers(self, added):
sched_obj = rpc.RpcProxy('software_dev.buildscheduler')
change_obj = rpc.RpcProxy('software_dev.commit')
for scheduler in added:
name = scheduler.name
assert name
class_name = "%s.%s" % (scheduler.__class__.__module__,
scheduler.__class__.__name__)
sids = sched_obj.search([('name', '=', name),
('class_name', '=', class_name)])
if sids:
sid = sids[0]
else:
sid = None
if sid is None:
# create a new row, with the latest changeid (so it won't try
# to process all of the old changes) new Schedulers are
# supposed to ignore pre-existing Changes
max_ids = change_obj.search([], 0, 1, 'id desc')
# TODO: really all changes?
if max_ids:
max_changeid = max_ids[0]
else:
max_changeid = 0
state = scheduler.get_initial_state(max_changeid)
state_json = json.dumps(state)
sid = sched_obj.create({
'name': name,
'class_name': class_name,
'state_dic': state_json
})
log.msg("scheduler '%s' got id %d" % (scheduler.name, sid))
scheduler.schedulerid = sid
def claim_buildrequests(self,
now,
master_name,
master_incarnation,
brids,
t=None):
if not brids:
return
breq_obj = rpc.RpcProxy('software_dev.commit')
vals = {
'claimed_at': time2str(now),
'claimed_by_name': master_name,
'claimed_by_incarnation': master_incarnation,
}
breq_obj.write(list(brids), vals)
def _txn_addChangeToDatabase(self, t, change):
change_obj = rpc.RpcProxy('software_dev.commit')
cdict = change.asDict()
cleanupDict(cdict)
for f in cdict['files']:
cleanupDict(f)
try:
change.number = change_obj.submit_change(cdict)
prop_arr = []
for propname, propvalue in change.properties.properties.items():
prop_arr.append((propname, json.dumps(propvalue)))
if prop_arr:
change_obj.setProperties(change.number, prop_arr)
self.notify("add-change", change.number)
except Exception, e:
log.err("Cannot add change: %s" % e)
def getBuildRequestWithNumber(self, brid, t=None):
assert isinstance(brid, (int, long))
breq_obj = rpc.RpcProxy('software_dev.commit')
res = breq_obj.read(brid)
if not res:
return None
ssid = brid # short-wire
ss = self.getSourceStampNumberedNow(ssid, t, res)
properties = self.get_properties_from_db(breq_obj, brid, t)
bsid = brid
br = BuildRequest(res['reason'], ss, res['buildername'], properties)
br.submittedAt = str2time(res['submitted_at'])
br.priority = res['priority']
br.id = brid
br.bsid = bsid
return br
def scheduler_get_classified_changes(self, schedulerid, t):
scha_obj = rpc.RpcProxy('software_dev.sched_change')
# one time for important ones
sids = scha_obj.search([('sched_id', '=', schedulerid),
('important', '=', True)])
res = scha_obj.read(sids, ['commit_id'])
important = self._get_change_num(Token(),
[r['commit_id'][0] for r in res])
# And one more time for unimportant ones
sids = scha_obj.search([('sched_id', '=', schedulerid),
('important', '=', False)])
res = scha_obj.read(sids, ['commit_id'])
unimportant = self._get_change_num(Token(),
[r['commit_id'][0] for r in res])
return (important, unimportant)
def changeEventGenerator(self,
branches=[],
categories=[],
committers=[],
minTime=0):
change_obj = rpc.RpcProxy('software_dev.commit')
domain = []
if branches:
domain.append(('branch_id', 'in', branches))
# if categories: Not Implemented yet
# domain.append( ('category_id', 'in', categories) )
if committers:
domain.append(('comitter_id', 'in', committers))
rows = change_obj.search(domain, 0, 0, 'id desc')
changes = self.runInteractionNow(self._get_change_num, rows)
for chg in changes:
yield chg
def get_unclaimed_buildrequests(self,
buildername,
old,
master_name,
master_incarnation,
t,
limit=None):
breq_obj = rpc.RpcProxy('software_dev.commit')
print "Get unclaimed buildrequests for %s after %s" % (buildername,
time2str(old))
bids = breq_obj.search(
[('buildername', '=', buildername),
('complete', '=', False), '|', '|', ('claimed_at', '=', False),
('claimed_at', '<', time2str(old)), '&',
('claimed_by_name', '=', master_name),
('claimed_by_incarnation', '!=', master_incarnation)], 0, limit
or False, 'priority DESC, submitted_at')
print "Got %d unclaimed buildrequests" % len(bids)
requests = [self.getBuildRequestWithNumber(bid, t) for bid in bids]
return requests
def _txn_retire_buildreqs(self, t, brids, results):
now = self._getCurrentTime()
breq_obj = rpc.RpcProxy('software_dev.commit')
# remember: buildrequest == build in our schema
vals = {
'complete': 1,
'results': results,
'complete_at': time2str(now)
}
breq_obj.write(brids, vals)
if True:
# now, does this cause any buildsets to complete?
# - Yes, since buildset == buildrequests (still)
bsids = brids
t = None
for bsid in bsids:
self._check_buildset(t, bsid, now)
self.notify("retire-buildrequest", *brids)
self.notify("modify-buildset", *bsids)
def _txn_cancel_buildrequest(self, t, brids):
# TODO: we aren't entirely sure if it'd be safe to just delete the
# buildrequest: what else might be waiting on it that would then just
# hang forever?. _check_buildset() should handle it well (an empty
# buildset will appear complete and SUCCESS-ful). But we haven't
# thought it through enough to be sure. So for now, "cancel" means
# "mark as complete and FAILURE".
now = self._getCurrentTime()
breq_obj = rpc.RpcProxy('software_dev.commit')
vals = {
'complete': True,
'results': FAILURE,
'complete_at': time2str(now)
}
breq_obj.write(brids, vals)
# now, does this cause any buildsets to complete?
bsids = brids
for bsid in bsids:
self._check_buildset(t, bsid, now)
self.notify("cancel-buildrequest", *brids)
self.notify("modify-buildset", *bsids)
def __init__(self, db_props, bmconfig):
"""
@param db_props a dict with info how to connect to db
@param c the BuildmasterConfig dict
"""
log.msg("Keeper config")
self.bmconfig = bmconfig
self.poll_interval = 560.0 #seconds
self.in_reset = False
self.bbot_tstamp = None
c = bmconfig
# some necessary definitions in the dict:
c['projectName'] = "OpenERP-Test"
c['buildbotURL'] = "http://test.openobject.com/"
c['db_url'] = 'openerp://' # it prevents the db_schema from going SQL
c['slavePortnum'] = 'tcp:8999:interface=127.0.0.1'
c['slaves'] = []
c['schedulers'] = []
c['builders'] = []
c['change_source'] = []
c['status'] = []
r = rpc.session.login(db_props)
if r != 0:
raise Exception("Could not login!")
bbot_obj = rpc.RpcProxy('software_dev.buildbot')
bbot_id = bbot_obj.search([('tech_code', '=',
db_props.get('code', 'buildbot'))])
assert bbot_id, "No buildbot for %r exists!" % db_props.get(
'code', 'buildbot')
self.bbot_id = bbot_id[0]
self.loop = twisted.internet.task.LoopingCall(self.poll_config)
self.loop.start(self.poll_interval)
