def remoteUpdate(self, update):
log.msg("remoteUpdate called with keys: " + ", ".join(update.keys()))
result = None
try:
self.rc = update.pop('rc')
log.msg('Comparison of localizations completed')
except KeyError:
pass
try:
# get the Observer data from the slave
result = update.pop('result')
except KeyError:
pass
try:
# get the Observer data from the slave
stats = update.pop('stats')
log.msg('untranslated count: %d' %
sum(map(lambda d: sum(d.values()), stats.values())))
self.addStats(stats)
except KeyError:
pass
if len(update):
# there's more than just us
LoggedRemoteCommand.remoteUpdate(self, update)
pass
if not result:
return
rmsg = {}
summary = result['summary']
self.completion = summary['completion']
changed = summary['changed']
unchanged = summary['unchanged']
tbmsg = ''
if 'tree' in self.args:
tbmsg = self.args['tree'] + ': '
tbmsg += "%(tree)s %(locale)s" % self.args
if self.rc == FAILURE:
missing = sum([summary[k] \
for k in ['missing', 'missingInFiles'] \
if k in summary])
self.logs['stdio'].addEntry(5, json.dumps(result, indent=2))
self.addSummary(summary)
es = elasticsearch.Elasticsearch(hosts=settings.ES_COMPARE_HOST)
# create our ES document to index in ES
# details from result, and self.dbrun was created in addSummary above
body = {'run': self.dbrun.id, 'details': result['details']}
rv = es.index(index=settings.ES_COMPARE_INDEX,
body=body,
doc_type='comparison',
id=self.dbrun.id)
log.msg('es.index: ' + json.dumps(rv))
def startVC(self, branch, revision, patch):
slavever = self.slaveVersion("bzr")
if not slavever:
m = "slave is too old, does not know about bzr"
raise NotImplementedError(m)
if self.repourl:
# assert not branch # we need baseURL= to use branches
self.args['repourl'] = self.repourl
else:
self.args['repourl'] = self.baseURL + self.branch # self.baseURL + branch
if not self.alwaysUseLatest:
if not self.args['repourl'].endswith(branch):
log.err("Repo url %s != %s" % (self.args['repourl'], branch))
self.args['revision'] = revision
self.setProperty('branch_url', self.args['repourl'])
self.setProperty('revision_hash', revision) # FIXME
else:
self.args['revision'] = None
self.args['patch'] = patch
revstuff = []
self.description.extend(revstuff)
self.descriptionDone.extend(revstuff)
cmd = LoggedRemoteCommand("openobjectbzr", self.args)
self.startCommand(cmd)
def start(self):
modules=['base']
s = self.build.getSourceStamp()
for change in s.changes:
for f in change.files:
try:
module = f.split('/')[1]
if module not in modules:
modules.append(module)
except:
pass
self.args['modules'] = ','.join(modules)
commands = ['python','bin/openerp-server.py']
if self.args['addonsdir']:
commands.append("--addons-path=%s"%(self.args['addonsdir']))
if self.args['port']:
commands.append("--port=%s"%(self.args['port']))
if self.args['netport']:
self.args['netport'].append("--net_port=%s"%(self.args['netport']))
if self.args['dbname']:
commands.append("--database=%s"%(self.args['dbname']))
commands.append("--update=%s"%(self.args['modules']))
commands.append("--stop-after-init")
self.args['command'] = commands
cmd = LoggedRemoteCommand("OpenObjectShell",self.args)
self.startCommand(cmd)
def startVC(self, branch, revision, patch):
slavever = self.slaveVersion("hg")
if not slavever:
raise BuildSlaveTooOldError("slave is too old, does not know "
"about hg")
if self.repourl:
# we need baseURL= to use dirname branches
assert self.branchType == 'inrepo' or not branch
self.args['repourl'] = self.repourl
if branch:
self.args['branch'] = branch
else:
self.args['repourl'] = self.baseURL + (branch or '')
self.args['revision'] = revision
self.args['patch'] = patch
revstuff = []
if branch is not None and branch != self.branch:
revstuff.append("[branch]")
self.description.extend(revstuff)
self.descriptionDone.extend(revstuff)
cmd = LoggedRemoteCommand("hg", self.args)
self.startCommand(cmd)
def start(self):
log.msg('starting with compare')
args = {}
args.update(self.args)
for k, v in args.iteritems():
if isinstance(v, WithProperties):
args[k] = self.build.getProperties().render(v)
try:
args['tree'] = self.build.getProperty('tree')
except KeyError:
pass
buildername = self.build.getProperty('buildername')
buildnumber = self.build.getProperty('buildnumber')
args['srctime'] = self.build.getProperty('srctime')
try:
build = Build.objects.get(builder__master__name=self.master,
builder__name=buildername,
buildnumber=buildnumber)
args['build'] = build.id
except Build.DoesNotExist:
args['build'] = None
args['revs'] = []
for rev in self.build.getProperty('revisions'):
ident = self.build.getProperty('%s_revision' % rev)
args['revs'].append(ident)
self.descriptionDone = [args['locale'], args['tree']]
cmd = LoggedRemoteCommand(self.cmd_name, args)
self.startCommand(cmd, [])
def remoteUpdate(self, update):
log.msg("remoteUpdate called with keys: " + ", ".join(update.keys()))
result = None
try:
self.rc = update.pop("rc")
log.msg("Comparison of localizations completed")
except KeyError:
pass
try:
# get the Observer data from the slave
result = update.pop("result")
except KeyError:
pass
try:
# get the Observer data from the slave
stats = update.pop("stats")
log.msg("untranslated count: %d" % sum(map(lambda d: sum(d.values()), stats.values())))
self.addStats(stats)
except KeyError:
pass
if len(update):
# there's more than just us
LoggedRemoteCommand.remoteUpdate(self, update)
pass
if not result:
return
rmsg = {}
summary = result["summary"]
self.completion = summary["completion"]
changed = summary["changed"]
unchanged = summary["unchanged"]
tbmsg = ""
if "tree" in self.args:
tbmsg = self.args["tree"] + ": "
tbmsg += "%(tree)s %(locale)s" % self.args
if self.rc == FAILURE:
missing = sum([summary[k] for k in ["missing", "missingInFiles"] if k in summary])
self.logs["stdio"].addEntry(5, json.dumps(result, indent=2))
self.addSummary(summary)
es = elasticsearch.Elasticsearch(hosts=settings.ES_COMPARE_HOST)
# create our ES document to index in ES
# details from result, and self.dbrun was created in addSummary above
body = {"run": self.dbrun.id, "details": result["details"]}
rv = es.index(index=settings.ES_COMPARE_INDEX, body=body, doc_type="comparison", id=self.dbrun.id)
log.msg("es.index: " + json.dumps(rv))
def start(self):
slavever = self.slaveVersion('stat')
if not slavever:
raise BuildSlaveTooOldError("slave is too old, does not know "
"about stat")
cmd = LoggedRemoteCommand('stat', {'file': self.file})
d = self.runCommand(cmd)
d.addCallback(lambda res: self.commandComplete(cmd))
d.addErrback(self.failed)
def startVC(self, branch, revision, patch):
slavever = self.slaveVersion("p4")
assert slavever, "slave is too old, does not know about p4"
args = dict(self.args)
args['branch'] = branch or self.branch
args['revision'] = revision
args['patch'] = patch
cmd = LoggedRemoteCommand("p4", args)
self.startCommand(cmd)
def startVC(self, branch, revision, patch):
self.args['branch'] = branch
self.args['revision'] = revision
self.args['patch'] = patch
slavever = self.slaveVersion("git")
if not slavever:
raise BuildSlaveTooOldError("slave is too old, does not know "
"about git")
cmd = LoggedRemoteCommand("git", self.args)
self.startCommand(cmd)
def start(self):
slavever = self.slaveVersion('rmdir')
if not slavever:
raise BuildSlaveTooOldError("slave is too old, does not know "
"about rmdir")
properties = self.build.getProperties()
cmd = LoggedRemoteCommand('rmdir',
{'dir': properties.render(self.dir)})
d = self.runCommand(cmd)
d.addCallback(lambda res: self.commandComplete(cmd))
d.addErrback(self.failed)
def remoteUpdate(self, update):
log.msg("remoteUpdate called with keys: " + ", ".join(update.keys()))
result = None
try:
self.rc = update.pop("rc")
log.msg("Comparison of localizations completed")
except KeyError:
pass
try:
# get the Observer data from the slave
result = update.pop("result")
except KeyError:
pass
try:
# get the Observer data from the slave
stats = update.pop("stats")
log.msg("untranslated count: %d" % sum(map(lambda d: sum(d.values()), stats.values())))
self.addStats(stats)
except KeyError:
pass
if len(update):
# there's more than just us
LoggedRemoteCommand.remoteUpdate(self, update)
pass
if not result:
return
rmsg = {}
summary = result["summary"]
self.completion = summary["completion"]
changed = summary["changed"]
unchanged = summary["unchanged"]
tbmsg = ""
if "tree" in self.args:
tbmsg = self.args["tree"] + ": "
tbmsg += "%(tree)s %(locale)s" % self.args
if self.rc == FAILURE:
missing = sum([summary[k] for k in ["missing", "missingInFiles"] if k in summary])
self.logs["stdio"].addEntry(5, json.dumps(result, indent=2))
self.addSummary(summary)
def start(self):
s = self.build.getSourceStamp()
latest_rev_no = False
for change in s.changes:
latest_rev_no = change.revision
self.args['command']=["bzr","merge"]
if latest_rev_no:
self.args['command'] += ["-r", str(latest_rev_no)]
if self.args['branch']:
self.args['command'].append(self.args['branch'])
cmd = LoggedRemoteCommand("OpenObjectShell",self.args)
self.startCommand(cmd)
def __call__(self, step):
slavever = step.slaveVersion('stat')
if not slavever:
raise BuildSlaveTooOldError("slave is too old, does not know "
"about stat")
def commandComplete(cmd):
# False if any filesystem object with the given name exists.
return (cmd.rc != 0)
cmd = LoggedRemoteCommand('stat', {'file': self.filename})
d = step.runCommand(cmd)
d.addCallback(lambda res: commandComplete(cmd))
return d
def startVC(self, branch, revision, patch):
self.args['version'] = branch
self.args['revision'] = revision
self.args['patch'] = patch
warnings = self.checkSlaveVersion("bazaar", branch)
revstuff = []
if branch is not None and branch != self.branch:
revstuff.append("[branch]")
if revision is not None:
revstuff.append("patch%s" % revision)
self.description.extend(revstuff)
self.descriptionDone.extend(revstuff)
cmd = LoggedRemoteCommand("bazaar", self.args)
self.startCommand(cmd, warnings)
def startVC(self, branch, revision, patch):
if self.slaveVersionIsOlderThan("cvs", "1.39"):
# the slave doesn't know to avoid re-using the same sourcedir
# when the branch changes. We have no way of knowing which branch
# the last build used, so if we're using a non-default branch and
# either 'update' or 'copy' modes, it is safer to refuse to
# build, and tell the user they need to upgrade the buildslave.
if (branch != self.branch
and self.args['mode'] in ("update", "copy")):
m = ("This buildslave (%s) does not know about multiple "
"branches, and using mode=%s would probably build the "
"wrong tree. "
"Refusing to build. Please upgrade the buildslave to "
"buildbot-0.7.0 or newer." %
(self.build.slavename, self.args['mode']))
log.msg(m)
raise BuildSlaveTooOldError(m)
if branch is None:
branch = "HEAD"
self.args['branch'] = branch
self.args['revision'] = revision
self.args['patch'] = patch
if self.args['branch'] == "HEAD" and self.args['revision']:
# special case. 'cvs update -r HEAD -D today' gives no files
# TODO: figure out why, see if it applies to -r BRANCH
self.args['branch'] = None
# deal with old slaves
warnings = []
slavever = self.slaveVersion("cvs", "old")
if slavever == "old":
# 0.5.0
if self.args['mode'] == "export":
self.args['export'] = 1
elif self.args['mode'] == "clobber":
self.args['clobber'] = 1
elif self.args['mode'] == "copy":
self.args['copydir'] = "source"
self.args['tag'] = self.args['branch']
assert not self.args['patch'] # 0.5.0 slave can't do patch
cmd = LoggedRemoteCommand("cvs", self.args)
self.startCommand(cmd, warnings)
def startVC(self, branch, revision, patch):
slavever = self.slaveVersion("darcs")
if not slavever:
m = "slave is too old, does not know about darcs"
raise BuildSlaveTooOldError(m)
if self.slaveVersionIsOlderThan("darcs", "1.39"):
if revision:
# TODO: revisit this once we implement computeSourceRevision
m = "0.6.6 slaves can't handle args['revision']"
raise BuildSlaveTooOldError(m)
# the slave doesn't know to avoid re-using the same sourcedir
# when the branch changes. We have no way of knowing which branch
# the last build used, so if we're using a non-default branch and
# either 'update' or 'copy' modes, it is safer to refuse to
# build, and tell the user they need to upgrade the buildslave.
if (branch != self.branch
and self.args['mode'] in ("update", "copy")):
m = ("This buildslave (%s) does not know about multiple "
"branches, and using mode=%s would probably build the "
"wrong tree. "
"Refusing to build. Please upgrade the buildslave to "
"buildbot-0.7.0 or newer." %
(self.build.slavename, self.args['mode']))
raise BuildSlaveTooOldError(m)
if self.repourl:
assert not branch # we need baseURL= to use branches
self.args['repourl'] = self.repourl
else:
self.args['repourl'] = self.baseURL + branch
self.args['revision'] = revision
self.args['patch'] = patch
revstuff = []
if branch is not None and branch != self.branch:
revstuff.append("[branch]")
self.description.extend(revstuff)
self.descriptionDone.extend(revstuff)
cmd = LoggedRemoteCommand("darcs", self.args)
self.startCommand(cmd)
def startVC(self, branch, revision, patch):
slavever = self.slaveVersion("bzr")
if not slavever:
m = "slave is too old, does not know about bzr"
raise BuildSlaveTooOldError(m)
if self.repourl:
assert not branch # we need baseURL= to use branches
self.args['repourl'] = self.repourl
else:
self.args['repourl'] = self.baseURL + branch
self.args['revision'] = revision
self.args['patch'] = patch
revstuff = []
if branch is not None and branch != self.branch:
revstuff.append("[branch]")
self.description.extend(revstuff)
self.descriptionDone.extend(revstuff)
cmd = LoggedRemoteCommand("bzr", self.args)
self.startCommand(cmd)
def __call__(self, step):
slavever = step.slaveVersion('stat')
if not slavever:
raise BuildSlaveTooOldError("slave is too old, does not know "
"about stat")
def commandComplete(cmd):
if cmd.rc != 0:
return False
s = cmd.updates["stat"][-1]
filemode = s[stat.ST_MODE]
if stat.S_ISREG(filemode) or stat.S_ISLNK(filemode):
# True only if this is a file or a link and not any other file
# system object.
return True
else:
return False
cmd = LoggedRemoteCommand('stat', {'file': self.filename})
d = step.runCommand(cmd)
d.addCallback(lambda res: commandComplete(cmd))
return d
def start(self):
self.args['command']=["bzr","revert"]
cmd = LoggedRemoteCommand("OpenObjectShell",self.args)
self.startCommand(cmd)
# put them in -m so that both install-module and check-quality use them.
self.args['command'] += [ '-m', str(mc) ]
# Here goes the test sequence, TODO make custom
self.args['command'] += ['--', '-drop-db']
self.args['command'] += ['--', 'create-db']
if len(mods_changed) or all_modules:
self.args['command'] += ['--', 'install-module'] #+ [ modules...]
if self.args['test_mode'] == 'check-quality':
self.args['command'] += ['--', 'check-quality' ] # + [modules]
elif self.args['test_mode'] == 'check-fvg':
self.args['command'] += ['--', 'fields-view-get' ]
self.args['command'] += ['--', '+drop-db']
cmd = LoggedRemoteCommand("OpenObjectShell",self.args)
self.startCommand(cmd)
def createSummary(self, plog):
global log
logs = self.cmd.logs
bqi_num_modules = None
# buildbotURL = self.build.builder.botmaster.parent.buildbotURL
bqi_re = re.compile(r'([^\>\|]+)(\|[^\>]+)?\> (.*)$')
qlog_re = re.compile(r'Module: "(.+)", score: (.*)$')
def bq2tr(bqi_name):
return tuple(bqi_name.split('.'))
logkeys = logs.keys()
def startVC(self, branch, revision, patch):
slavever = self.slaveVersion("p4sync")
assert slavever, "slave is too old, does not know about p4"
cmd = LoggedRemoteCommand("p4sync", self.args)
self.startCommand(cmd)
def remoteComplete(self, maybeFailure):
log.msg('end with compare, rc: %s, maybeFailure: %s' %
(self.rc, maybeFailure))
LoggedRemoteCommand.remoteComplete(self, maybeFailure)
return maybeFailure
def start(self):
args = {'objdir': self.objdir, 'timeout': 60}
cmd = LoggedRemoteCommand("setMozillaBuildProperties", args)
self.startCommand(cmd)
def startVC(self, branch, revision, patch):
# handle old slaves
warnings = []
slavever = self.slaveVersion("svn", "old")
if not slavever:
m = "slave does not have the 'svn' command"
raise BuildSlaveTooOldError(m)
if self.slaveVersionIsOlderThan("svn", "1.39"):
# the slave doesn't know to avoid re-using the same sourcedir
# when the branch changes. We have no way of knowing which branch
# the last build used, so if we're using a non-default branch and
# either 'update' or 'copy' modes, it is safer to refuse to
# build, and tell the user they need to upgrade the buildslave.
if (branch != self.branch
and self.args['mode'] in ("update", "copy")):
m = ("This buildslave (%s) does not know about multiple "
"branches, and using mode=%s would probably build the "
"wrong tree. "
"Refusing to build. Please upgrade the buildslave to "
"buildbot-0.7.0 or newer." %
(self.build.slavename, self.args['mode']))
raise BuildSlaveTooOldError(m)
if slavever == "old":
# 0.5.0 compatibility
if self.args['mode'] in ("clobber", "copy"):
# TODO: use some shell commands to make up for the
# deficiency, by blowing away the old directory first (thus
# forcing a full checkout)
warnings.append("WARNING: this slave can only do SVN updates"
", not mode=%s\n" % self.args['mode'])
log.msg("WARNING: this slave only does mode=update")
if self.args['mode'] == "export":
raise BuildSlaveTooOldError("old slave does not have "
"mode=export")
self.args['directory'] = self.args['workdir']
if revision is not None:
# 0.5.0 can only do HEAD. We have no way of knowing whether
# the requested revision is HEAD or not, and for
# slowly-changing trees this will probably do the right
# thing, so let it pass with a warning
m = ("WARNING: old slave can only update to HEAD, not "
"revision=%s" % revision)
log.msg(m)
warnings.append(m + "\n")
revision = "HEAD" # interprets this key differently
if patch:
raise BuildSlaveTooOldError("old slave can't do patch")
if self.svnurl:
assert not branch # we need baseURL= to use branches
self.args['svnurl'] = self.svnurl
else:
self.args['svnurl'] = self.baseURL + branch
self.args['revision'] = revision
self.args['patch'] = patch
revstuff = []
if branch is not None and branch != self.branch:
revstuff.append("[branch]")
if revision is not None:
revstuff.append("r%s" % revision)
if patch is not None:
revstuff.append("[patch]")
self.description.extend(revstuff)
self.descriptionDone.extend(revstuff)
cmd = LoggedRemoteCommand("svn", self.args)
self.startCommand(cmd, warnings)
def startVC(self):
slavever = self.slaveVersion("monotone")
assert slavever, "slave is too old, does not know about monotone"
cmd = LoggedRemoteCommand("monotone", self.args)
self.startCommand(cmd)
def remoteComplete(self, maybeFailure):
log.msg("end with compare, rc: %s, maybeFailure: %s" % (self.rc, maybeFailure))
LoggedRemoteCommand.remoteComplete(self, maybeFailure)
return maybeFailure
def __init__(self, name, args):
LoggedRemoteCommand.__init__(self, name, args)
self.dbrun = None
def start(self):
args = {'handle': (self.handle, self.build.reason)}
cmd = LoggedRemoteCommand("dummy.wait", args)
self.startCommand(cmd)
def start(self):
args = {'timeout': self.timeout}
cmd = LoggedRemoteCommand("dummy", args)
self.startCommand(cmd)
def __init__(self, name, args):
LoggedRemoteCommand.__init__(self, name, args)
self.dbrun = None
