def SubmitJob(self, parsed_job, changeids):
if not parsed_job['bot']:
raise BadJobfile(
'incoming Try job did not specify any allowed builder names')
# Verify the try job patch is not more than 20MB.
if parsed_job.get('patch'):
patchsize = len(parsed_job['patch'])
if patchsize > 20 * 1024 * 1024: # 20MB
raise BadJobfile('incoming Try job patch is %s bytes, '
'must be less than 20MB' % (patchsize))
d = self.master.db.sourcestamps.addSourceStamp(
branch=parsed_job['branch'],
revision=parsed_job['revision'],
patch_body=parsed_job['patch'],
patch_level=parsed_job['patchlevel'],
patch_subdir=parsed_job['root'],
project=parsed_job['project'],
repository=parsed_job['repository'] or '',
changeids=changeids)
d.addCallback(self.create_buildset, parsed_job)
d.addErrback(log.err, "Failed to queue a try job!")
return d
def parse_options(options, builders, pools):
"""Converts try job settings into a dict.
The dict is received as dict(str, list(str)).
"""
try:
# Flush empty values.
options = dict((k, v) for k, v in options.iteritems() if v)
flatten(options, 'name', 'Unnamed')
flatten(options, 'user', 'John Doe')
comma_separated(options, 'email')
for email in options['email']:
# Access to a protected member XXX of a client class
# pylint: disable=W0212
if not TryJobBase._EMAIL_VALIDATOR.match(email):
raise BadJobfile("'%s' is an invalid email address!" % email)
flatten(options, 'patch', None)
flatten(options, 'root', None)
try_int(options, 'patchlevel', 0)
flatten(options, 'branch', None)
flatten(options, 'revision', None)
flatten(options, 'reason',
'%s: %s' % (options['user'], options['name']))
try_bool(options, 'clobber', False)
flatten(options, 'project', pools.default_pool_name if pools else None)
flatten(options, 'repository', None)
# Code review info. Enforce numbers.
try_int(options, 'patchset', None)
try_int(options, 'issue', None)
# Manages bot selection and test filtering. It is preferable to use
# multiple bot=unit_tests:Gtest.Filter lines. DEFAULT_TESTS is a marker to
# specify that the default tests should be run for this builder.
options['bot'] = dict_comma(options.get('bot', []), builders,
DEFAULT_TESTS)
if not options['bot'] and pools:
options['bot'] = pools.Select(None, options['project'])
if not options['bot']:
raise BadJobfile('No builder selected')
comma_separated(options, 'testfilter')
if options['testfilter']:
for k in options['bot']:
options['bot'][k].update(options['testfilter'])
# Convert the set back to list.
for bot in options['bot']:
options['bot'][bot] = list(options['bot'][bot])
log.msg('Chose %s for job %s' %
(','.join(options['bot']), options['reason']))
return options
except (TypeError, ValueError), e:
lines = [(i[0].rsplit('/', 1)[-1], i[1])
for i in traceback.extract_tb(sys.exc_info()[2])]
raise BadJobfile('Failed to parse the metadata: %r' % options, e,
lines)
def dict_comma(values, valid_keys, default):
"""Splits comma separated strings with : to create a dict."""
out = {}
# We accept two formats:
# 1. Builder(s) only, will run default tests. Eg. win,mac
# 2. Builder(s) with test(s): Eg. win,mac:test1:filter1,test2:filter2,test3
# In this example, win will run default tests, and mac will be ran with
# test1:filter1, test2:filter2, and test3.
# In no cases are builders allowed to be specified after tests are specified.
for value in values:
sections_match = re.match(r'^([^\:]+):?(.*)$', value)
if sections_match:
builders_raw, tests_raw = sections_match.groups()
builders = builders_raw.split(',')
tests = [test for test in tests_raw.split(',') if test]
for builder in builders:
if not builder:
# Empty builder means trailing comma.
raise BadJobfile('Bad input - trailing comma: %s' % value)
if builder not in valid_keys:
# Drop unrecognized builders.
log.msg('%s not in builders, dropping.' % builder)
continue
if not tests or builder != builders[-1]:
# We can only add filters to the last builder.
out.setdefault(builder, set()).add(default)
else:
for test in tests:
if len(test.split(':')) > 2:
# Don't support test:filter:foo.
raise BadJobfile('Failed to process test %s' %
value)
out.setdefault(builder, set()).add(test)
else:
raise BadJobfile('Failed to process value %s' % value)
return out
def SubmitJobs(self, jobs):
"""Submit pending try jobs to slaves for building.
Args:
jobs: a list of jobs. Each job is a dictionary of properties describing
what to build.
"""
log.msg('TryJobRietveld.SubmitJobs: %s' % json.dumps(jobs, indent=2))
for job in jobs:
try:
# Gate the try job on the user that requested the job, not the one that
# authored the CL.
if not self._valid_users.contains(job['requester']):
raise BadJobfile('TryJobRietveld rejecting job from %s' %
job['requester'])
if job['email'] != job['requester']:
# Note the fact the try job was requested by someone else in the
# 'reason'.
job['reason'] = job.get('reason') or ''
if job['reason']:
job['reason'] += '; '
job['reason'] += "This CL was triggered by %s" % job[
'requester']
options = {
'bot': {
job['builder']: job['tests']
},
'email': [job['email']],
'project': [self._project],
'try_job_key': job['key'],
}
# Transform some properties as is expected by parse_options().
for key in ('name', 'user', 'root', 'reason', 'clobber',
'patchset', 'issue', 'requester', 'revision'):
options[key] = [job[key]]
# Now cleanup the job dictionary and submit it.
cleaned_job = self.parse_options(options)
yield self.get_lkgr(cleaned_job)
c = yield self.master.addChange(
author=','.join(cleaned_job['email']),
# TODO(maruel): Get patchset properties to get the list of files.
# files=[],
revision=cleaned_job['revision'],
comments='')
changeids = [c.number]
cleaned_job['patch_storage'] = 'rietveld'
yield self.SubmitJob(cleaned_job, changeids)
except BadJobfile, e:
# We need to mark it as failed otherwise it'll stay in the pending
# state. Simulate a buildFinished event on the build.
log.err('Got "%s" for issue %s' % (e, job.get('issue')))
for service in self.master.services:
if service.__class__.__name__ == 'TryServerHttpStatusPush':
build = {
'properties': [
('buildername', job.get('builder'), None),
('buildnumber', -1, None),
('issue', job['issue'], None),
('patchset', job['patchset'], None),
('project', self._project, None),
('revision', '', None),
('slavename', '', None),
('try_job_key', job['key'], None),
],
'reason':
job.get('reason', ''),
# Use EXCEPTION until SKIPPED results in a non-green try job
# results on Rietveld.
'results':
EXCEPTION,
}
service.push('buildFinished', build=build)
log.err('Rietveld updated')
break
else:
log.err(
'Rietveld not updated: no corresponding service found.'
)
def dict_comma(values, valid_keys, default):
"""Splits comma separated strings with : to create a dict."""
out = {}
# See the unit test to understand all the supported formats. It's insane
# for now.
for value in values:
# Slowly process the line as a state machine.
# The inputs are:
# A. The next symbol, a string and the one after.
# B. The separator after: (',', ':', None).
# C. The last processed bot name.
# D. The last processed test name.
# There is 4 states:
# 1. Next item must be a bot: value='builder'. Next state can be 1 or 2.
# 2. Next item must be a test: value='builder:test' when the index is at
# test. Next state can be 3 or 4.
# 3. Next item must be a test filter: builder:test:filter when the index is
# at filter. Next state is 4.
# 4. Next item can be a bot or is a test: value='builder:test,not_sure' or
# value='builder:test:filter,not_sure' when the index is at not_sure. It
# depends on not_sure being in valid_keys or not. Next state is 2 or 4.
items = re.split(r'([\,\:])', value)
if not items or ((len(items) - 1) % 2) != 0:
raise BadJobfile('Failed to process value %s', value)
last_key = None # A bot
last_value = None # A test
state = 1 # The initial item is a last_key (bot).
while items:
# Eat two items at a time:
item = items.pop(0)
separator = items.pop(
0) if items else None # Technically, next_separator
# Refuse "foo,:bar"
if item in (',', ':') or separator not in (',', ':', None):
raise BadJobfile('Failed to process value %s', value)
if state == 1:
assert last_key is None
assert last_value is None
if item not in valid_keys:
raise BadJobfile('Failed to process value %s', value)
if separator == ':':
# Don't save it yet.
last_key = item
state = 2
else:
out.setdefault(item, set()).add(default)
state = 1
elif state == 2:
assert last_key is not None
assert last_value is None
if separator == ':':
last_value = item
state = 3
else:
out.setdefault(last_key, set()).add(item)
last_value = None
state = 4
elif state == 3:
assert last_key is not None
assert last_value is not None
if separator == ':':
raise BadJobfile('Failed to process value %s', value)
out.setdefault(last_key,
set()).add('%s:%s' % (last_value, item))
last_value = None
state = 4
elif state == 4:
assert last_key is not None
assert last_value is None
if separator == ':':
if item not in valid_keys:
last_value = item
state = 3
else:
last_key = item
state = 2
else:
if item not in valid_keys:
# A value.
out.setdefault(last_key, set()).add(item)
else:
# A key.
out.setdefault(item, set()).add(default)
return out
def TryJobRietveldSubmitJobs(self, jobs):
""" Override of master.try_job_rietveld.TryJobRietveld.SubmitJobs:
http://src.chromium.org/viewvc/chrome/trunk/tools/build/scripts/master/try_job_rietveld.py?view=markup
We modify it to include "baseurl" as a build property.
"""
log.msg('TryJobRietveld.SubmitJobs: %s' % json.dumps(jobs, indent=2))
for job in jobs:
try:
# Gate the try job on the user that requested the job, not the one that
# authored the CL.
# pylint: disable=W0212
########################## Added by rmistry ##########################
if (job.get('requester')
and not job['requester'].endswith('@google.com')
and not job['requester'].endswith('@chromium.org')
and not job['requester'] in TRYBOTS_REQUESTER_WHITELIST):
# Reject the job only if the requester has an email not ending in
# google.com or chromium.org
raise BadJobfile('TryJobRietveld rejecting job from %s' %
job['requester'])
######################################################################
########################## Added by borenet ##########################
if not (job.get('baseurl')
and config_private.Master.Skia.project_name.lower()
in job['baseurl']):
raise BadJobfile(
'TryJobRietveld rejecting job with unknown baseurl: %s' %
job.get('baseurl'))
######################################################################
if job['email'] != job['requester']:
# Note the fact the try job was requested by someone else in the
# 'reason'.
job['reason'] = job.get('reason') or ''
if job['reason']:
job['reason'] += '; '
job['reason'] += "This CL was triggered by %s" % job[
'requester']
options = {
'bot': {
job['builder']: job['tests']
},
'email': [job['email']],
'project': [self._project],
'try_job_key': job['key'],
}
# Transform some properties as is expected by parse_options().
for key in (
########################## Added by borenet ##########################
'baseurl',
######################################################################
'name',
'user',
'root',
'reason',
'clobber',
'patchset',
'issue',
'requester',
'revision'):
options[key] = [job[key]]
# Now cleanup the job dictionary and submit it.
cleaned_job = self.parse_options(options)
wfd = defer.waitForDeferred(self.get_lkgr(cleaned_job))
yield wfd
wfd.getResult()
wfd = defer.waitForDeferred(
self.master.addChange(
author=','.join(cleaned_job['email']),
# TODO(maruel): Get patchset properties to get the list of files.
# files=[],
revision=cleaned_job['revision'],
comments=''))
yield wfd
changeids = [wfd.getResult().number]
wfd = defer.waitForDeferred(self.SubmitJob(cleaned_job, changeids))
yield wfd
wfd.getResult()
except BadJobfile, e:
# We need to mark it as failed otherwise it'll stay in the pending
# state. Simulate a buildFinished event on the build.
if not job.get('key'):
log.err(
'Got %s for issue %s but not key, not updating Rietveld' %
(e, job.get('issue')))
continue
log.err('Got %s for issue %s, updating Rietveld' %
(e, job.get('issue')))
for service in self.master.services:
if service.__class__.__name__ == 'TryServerHttpStatusPush':
# pylint: disable=W0212,W0612
build = {
'properties': [
('buildername', job.get('builder'), None),
('buildnumber', -1, None),
('issue', job['issue'], None),
('patchset', job['patchset'], None),
('project', self._project, None),
('revision', '', None),
('slavename', '', None),
('try_job_key', job['key'], None),
],
'reason':
job.get('reason', ''),
# Use EXCEPTION until SKIPPED results in a non-green try job
# results on Rietveld.
'results':
EXCEPTION,
}
########################## Added by rmistry #########################
# Do not update Rietveld to mark the try job request as failed.
# See https://code.google.com/p/chromium/issues/detail?id=224014 for
# more context.
# service.push('buildFinished', build=build)
#####################################################################
break