def _GenTSBotSpec(checkouts, change, changed_files, options):
bot_spec = []
# Get try slaves from PRESUBMIT.py files if not specified.
# Even if the diff comes from options.url, use the local checkout for bot
# selection.
try:
import presubmit_support
root_presubmit = checkouts[0].ReadRootFile('PRESUBMIT.py')
if not change:
if not changed_files:
changed_files = checkouts[0].file_tuples
change = presubmit_support.Change(options.name, '',
checkouts[0].checkout_root,
changed_files, options.issue,
options.patchset, options.email)
trybots = presubmit_support.DoGetTrySlaves(
change, checkouts[0].GetFileNames(), checkouts[0].checkout_root,
root_presubmit, options.project, options.verbose, sys.stdout)
if trybots:
old_style = filter(lambda x: isinstance(x, basestring), trybots)
new_style = filter(lambda x: isinstance(x, tuple), trybots)
# _ParseBotList's testfilter is set to None otherwise it will complain.
bot_spec = _ApplyTestFilter(options.testfilter,
_ParseBotList(old_style, None))
bot_spec.extend(_ApplyTestFilter(options.testfilter, new_style))
except ImportError:
pass
return bot_spec
def TryChange(argv, file_list, swallow_exception, prog=None):
"""
Args:
argv: Arguments and options.
file_list: Default value to pass to --file.
swallow_exception: Whether we raise or swallow exceptions.
"""
default_settings = GetTryServerSettings()
transport_functions = {'http': _SendChangeHTTP, 'svn': _SendChangeSVN}
default_transport = transport_functions.get(
default_settings.get('default_transport'))
# Parse argv
parser = optparse.OptionParser(usage=USAGE, version=__version__, prog=prog)
group = optparse.OptionGroup(parser, "Result and status")
group.add_option("-u",
"--user",
default=getpass.getuser(),
help="Owner user name [default: %default]")
group.add_option(
"-e",
"--email",
default=os.environ.get('TRYBOT_RESULTS_EMAIL_ADDRESS',
os.environ.get('EMAIL_ADDRESS')),
help="Email address where to send the results. Use either "
"the TRYBOT_RESULTS_EMAIL_ADDRESS environment "
"variable or EMAIL_ADDRESS to set the email address "
"the try bots report results to [default: %default]")
group.add_option("-n", "--name", help="Descriptive name of the try job")
group.add_option("--issue",
type='int',
help="Update rietveld issue try job status")
group.add_option("--patchset",
type='int',
help="Update rietveld issue try job status")
parser.add_option_group(group)
group = optparse.OptionGroup(parser, "Try job options")
group.add_option(
"-b",
"--bot",
action="append",
help="Only use specifics build slaves, ex: '--bot win' to "
"run the try job only on the 'win' slave; see the try "
"server waterfall for the slave's name")
group.add_option("-r",
"--revision",
help="Revision to use for the try job; default: the "
"revision will be determined by the try server; see "
"its waterfall for more info")
group.add_option("-c",
"--clobber",
action="store_true",
help="Force a clobber before building; e.g. don't do an "
"incremental build")
# TODO(maruel): help="Select a specific configuration, usually 'debug' or "
# "'release'"
group.add_option("--target", help=optparse.SUPPRESS_HELP)
# TODO(bradnelson): help="Override which project to use"
group.add_option("--project",
help=optparse.SUPPRESS_HELP,
default=default_settings['default_project'])
# Override the list of tests to run, use multiple times to list many tests
# (or comma separated)
group.add_option("-t",
"--tests",
action="append",
help=optparse.SUPPRESS_HELP)
parser.add_option_group(group)
group = optparse.OptionGroup(parser, "Patch to run")
group.add_option("-f",
"--file",
default=file_list,
dest="files",
metavar="FILE",
action="append",
help="Use many times to list the files to include in the "
"try, relative to the repository root")
group.add_option("--diff", help="File containing the diff to try")
group.add_option("--url", help="Url where to grab a patch")
group.add_option("--root",
help="Root to use for the patch; base subdirectory for "
"patch created in a subdirectory",
default=default_settings["default_root"])
group.add_option("--patchlevel",
type='int',
metavar="LEVEL",
help="Used as -pN parameter to patch",
default=default_settings["default_patchlevel"])
parser.add_option_group(group)
group = optparse.OptionGroup(parser, "Access the try server by HTTP")
group.add_option("--use_http",
action="store_const",
const=_SendChangeHTTP,
dest="send_patch",
default=default_transport,
help="Use HTTP to talk to the try server [default]")
group.add_option("--host",
default=default_settings['http_host'],
help="Host address")
group.add_option("--port",
default=default_settings['http_port'],
help="HTTP port")
group.add_option("--proxy", help="HTTP proxy")
parser.add_option_group(group)
group = optparse.OptionGroup(parser, "Access the try server with SVN")
group.add_option("--use_svn",
action="store_const",
const=_SendChangeSVN,
dest="send_patch",
help="Use SVN to talk to the try server")
group.add_option(
"--svn_repo",
metavar="SVN_URL",
default=default_settings['svn_repo'],
help="SVN url to use to write the changes in; --use_svn is "
"implied when using --svn_repo")
parser.add_option_group(group)
options, args = parser.parse_args(argv)
# Switch the default accordingly if there was no default send_patch.
if not options.send_patch:
if options.port and options.host:
options.send_patch = _SendChangeHTTP
elif options.svn_repo:
options.send_patch = _SendChangeSVN
else:
parser.error('Please specify an access method.')
if len(args) == 1 and args[0] == 'help':
parser.print_help()
if (not options.files and (not options.issue and options.patchset)
and not options.diff and not options.url):
# TODO(maruel): It should just try the modified files showing up in a
# svn status.
parser.error('Nothing to try, changelist is empty.')
try:
# Convert options.diff into the content of the diff.
if options.url:
options.diff = urllib.urlopen(options.url).read()
elif options.diff:
options.diff = gcl.ReadFile(options.diff)
# Process the VCS in any case at least to retrieve the email address.
try:
options.scm = GuessVCS(options)
options.scm.ProcessOptions()
except NoTryServerAccess, e:
# If we got the diff, we don't care.
if not options.diff:
# TODO(maruel): Raise what?
raise
# Get try slaves from PRESUBMIT.py files if not specified.
if not options.bot:
if options.url:
parser.error('You need to specify which bots to use.')
root_presubmit = gcl.GetCachedFile('PRESUBMIT.py', use_root=True)
options.bot = presubmit_support.DoGetTrySlaves(
options.scm.GetFileNames(), options.scm.GetLocalRoot(),
root_presubmit, False, sys.stdout)
if options.name is None:
if options.issue:
options.name = 'Issue %s' % options.issue
else:
options.name = 'Unnamed'
print('Note: use --name NAME to change the try job name.')
if not options.email:
print(
'Warning: TRYBOT_RESULTS_EMAIL_ADDRESS is not set. Try server '
'results might\ngo to: %[email protected].\n' % options.user)
else:
print('Results will be emailed to: ' + options.email)
# Send the patch.
options.send_patch(options)
print 'Patch \'%s\' sent to try server: %s' % (options.name, ', '.join(
options.bot))
def TryChange(argv, change, swallow_exception, prog=None, extra_epilog=None):
"""
Args:
argv: Arguments and options.
change: Change instance corresponding to the CL.
swallow_exception: Whether we raise or swallow exceptions.
"""
parser = gen_parser(prog)
epilog = EPILOG % {'prog': prog}
if extra_epilog:
epilog += extra_epilog
parser.epilog = epilog
# Remove epilog formatting
parser.format_epilog = lambda x: parser.epilog
options, args = parser.parse_args(argv)
# If they've asked for help, give it to them
if len(args) == 1 and args[0] == 'help':
parser.print_help()
return 0
# If they've said something confusing, don't spawn a try job until you
# understand what they want.
if args:
parser.error('Extra argument(s) "%s" not understood' % ' '.join(args))
if options.dry_run:
options.verbose += 1
LOG_FORMAT = '%(levelname)s %(filename)s(%(lineno)d): %(message)s'
if not swallow_exception:
if options.verbose == 0:
logging.basicConfig(level=logging.WARNING, format=LOG_FORMAT)
elif options.verbose == 1:
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
elif options.verbose > 1:
logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)
logging.debug(argv)
if (options.patchlevel is not None
and (options.patchlevel < 0 or options.patchlevel > 10)):
parser.error(
'Have you tried --port instead? You probably confused -p and -P.')
# Strip off any @ in the user, otherwise svn gets confused.
options.user = options.user.split('@', 1)[0]
if options.rietveld_url:
# Try to extract the review number if possible and fix the protocol.
if not '://' in options.rietveld_url:
options.rietveld_url = 'http://' + options.rietveld_url
match = re.match(r'^(.*)/(\d+)/?$', options.rietveld_url)
if match:
if options.issue or options.patchset:
parser.error(
'Cannot use both --issue and use a review number url')
options.issue = int(match.group(2))
options.rietveld_url = match.group(1)
try:
changed_files = None
# Always include os.getcwd() in the checkout settings.
checkouts = []
path = os.getcwd()
file_list = []
if options.files:
file_list = options.files
elif change:
file_list = [f.LocalPath() for f in change.AffectedFiles()]
if options.upstream_branch:
path += '@' + options.upstream_branch
# Clear file list so that the correct list will be retrieved from the
# upstream branch.
file_list = []
checkouts.append(GuessVCS(options, path, file_list))
checkouts[0].AutomagicalSettings()
for item in options.sub_rep:
# Pass file_list=None because we don't know the sub repo's file list.
checkout = GuessVCS(options,
os.path.join(checkouts[0].checkout_root, item),
None)
if checkout.checkout_root in [c.checkout_root for c in checkouts]:
parser.error('Specified the root %s two times.' %
checkout.checkout_root)
checkouts.append(checkout)
can_http = options.port and options.host
can_svn = options.svn_repo
# If there was no transport selected yet, now we must have enough data to
# select one.
if not options.send_patch and not (can_http or can_svn):
parser.error('Please specify an access method.')
# Convert options.diff into the content of the diff.
if options.url:
if options.files:
parser.error(
'You cannot specify files and --url at the same time.')
options.diff = urllib.urlopen(options.url).read()
elif options.diff:
if options.files:
parser.error(
'You cannot specify files and --diff at the same time.')
options.diff = gclient_utils.FileRead(options.diff, 'rb')
elif options.issue and options.patchset is None:
# Retrieve the patch from rietveld when the diff is not specified.
# When patchset is specified, it's because it's done by gcl/git-try.
api_url = '%s/api/%d' % (options.rietveld_url, options.issue)
logging.debug(api_url)
contents = json.loads(urllib.urlopen(api_url).read())
options.patchset = contents['patchsets'][-1]
diff_url = (
'%s/download/issue%d_%d.diff' %
(options.rietveld_url, options.issue, options.patchset))
diff = GetMungedDiff('', urllib.urlopen(diff_url).readlines())
options.diff = ''.join(diff[0])
changed_files = diff[1]
else:
# Use this as the base.
root = checkouts[0].checkout_root
diffs = []
for checkout in checkouts:
raw_diff = checkout.GenerateDiff()
if not raw_diff:
logging.error('Empty or non-existant diff, exiting.')
return 1
diff = raw_diff.splitlines(True)
path_diff = gclient_utils.PathDifference(
root, checkout.checkout_root)
# Munge it.
diffs.extend(GetMungedDiff(path_diff, diff)[0])
options.diff = ''.join(diffs)
if not options.name:
if options.issue:
options.name = 'Issue %s' % options.issue
else:
options.name = 'Unnamed'
print('Note: use --name NAME to change the try job name.')
if not options.email:
parser.error(
'Using an anonymous checkout. Please use --email or set '
'the TRYBOT_RESULTS_EMAIL_ADDRESS environment variable.')
print('Results will be emailed to: ' + options.email)
if not options.bot:
# Get try slaves from PRESUBMIT.py files if not specified.
# Even if the diff comes from options.url, use the local checkout for bot
# selection.
try:
import presubmit_support
root_presubmit = checkouts[0].ReadRootFile('PRESUBMIT.py')
if not change:
if not changed_files:
changed_files = checkouts[0].file_tuples
change = presubmit_support.Change(
options.name, '', checkouts[0].checkout_root,
changed_files, options.issue, options.patchset,
options.email)
options.bot = presubmit_support.DoGetTrySlaves(
change, checkouts[0].GetFileNames(),
checkouts[0].checkout_root, root_presubmit,
options.project, options.verbose, sys.stdout)
except ImportError:
pass
if options.testfilter:
bots = set()
for bot in options.bot:
assert ',' not in bot
if bot.endswith(':compile'):
# Skip over compile-only builders for now.
continue
bots.add(bot.split(':', 1)[0])
options.bot = list(bots)
# If no bot is specified, either the default pool will be selected or the
# try server will refuse the job. Either case we don't need to interfere.
if any('triggered' in b.split(':', 1)[0] for b in options.bot):
print >> sys.stderr, (
'ERROR You are trying to send a job to a triggered bot. This type of'
' bot requires an\ninitial job from a parent (usually a builder). '
'Instead send your job to the parent.\nBot list: %s' %
options.bot)
return 1
if options.print_bots:
print 'Bots which would be used:'
for bot in options.bot:
print ' %s' % bot
return 0
# Send the patch.
if options.send_patch:
# If forced.
options.send_patch(options)
PrintSuccess(options)
return 0
try:
if can_http:
_SendChangeHTTP(options)
PrintSuccess(options)
return 0
except NoTryServerAccess:
if not can_svn:
raise
_SendChangeSVN(options)
PrintSuccess(options)
return 0
except (InvalidScript, NoTryServerAccess), e:
if swallow_exception:
return 1
print >> sys.stderr, e
return 1
def TryChange(argv,
file_list,
swallow_exception,
prog=None,
extra_epilog=None):
"""
Args:
argv: Arguments and options.
file_list: Default value to pass to --file.
swallow_exception: Whether we raise or swallow exceptions.
"""
# Parse argv
parser = optparse.OptionParser(usage=USAGE, version=__version__, prog=prog)
epilog = EPILOG % {'prog': prog}
if extra_epilog:
epilog += extra_epilog
parser.epilog = epilog
# Remove epilog formatting
parser.format_epilog = lambda x: parser.epilog
parser.add_option("-v",
"--verbose",
action="count",
default=0,
help="Prints debugging infos")
group = optparse.OptionGroup(parser, "Result and status")
group.add_option("-u",
"--user",
default=getpass.getuser(),
help="Owner user name [default: %default]")
group.add_option(
"-e",
"--email",
default=os.environ.get('TRYBOT_RESULTS_EMAIL_ADDRESS',
os.environ.get('EMAIL_ADDRESS')),
help="Email address where to send the results. Use either "
"the TRYBOT_RESULTS_EMAIL_ADDRESS environment "
"variable or EMAIL_ADDRESS to set the email address "
"the try bots report results to [default: %default]")
group.add_option("-n", "--name", help="Descriptive name of the try job")
group.add_option("--issue",
type='int',
help="Update rietveld issue try job status")
group.add_option("--patchset",
type='int',
help="Update rietveld issue try job status. This is "
"optional if --issue is used, In that case, the "
"latest patchset will be used.")
group.add_option("--dry_run",
action='store_true',
help="Don't send the try job. This implies --verbose, so "
"it will print the diff.")
parser.add_option_group(group)
group = optparse.OptionGroup(parser, "Try job options")
group.add_option("-b",
"--bot",
action="append",
help="Only use specifics build slaves, ex: "
"'--bot win,layout_mac'; see the try "
"server waterfall for the slave's name")
group.add_option("-r",
"--revision",
help="Revision to use for the try job; default: the "
"revision will be determined by the try server; see "
"its waterfall for more info")
group.add_option("-c",
"--clobber",
action="store_true",
help="Force a clobber before building; e.g. don't do an "
"incremental build")
# TODO(maruel): help="Select a specific configuration, usually 'debug' or "
# "'release'"
group.add_option("--target", help=optparse.SUPPRESS_HELP)
group.add_option(
"--project",
help="Override which project to use. Projects are defined "
"server-side to define what default bot set to use")
group.add_option(
"-t",
"--testfilter",
action="append",
help="Add a gtest_filter to a test. Use multiple times to "
"specify filters for different tests. (i.e. "
"--testfilter base_unittests:ThreadTest.* "
"--testfilter ui_tests) If you specify any testfilters "
"the test results will not be reported in rietveld and "
"only tests with filters will run.")
parser.add_option_group(group)
group = optparse.OptionGroup(parser, "Patch to run")
group.add_option("-f",
"--file",
default=file_list,
dest="files",
metavar="FILE",
action="append",
help="Use many times to list the files to include in the "
"try, relative to the repository root")
group.add_option("--diff", help="File containing the diff to try")
group.add_option("--url",
help="Url where to grab a patch, e.g. "
"http://example.com/x.diff")
group.add_option("-R",
"--rietveld_url",
default="codereview.appspot.com",
metavar="URL",
help="Has 2 usages, both refer to the rietveld instance: "
"Specify which code review patch to use as the try job "
"or rietveld instance to update the try job results "
"Default:%default")
group.add_option("--root",
help="Root to use for the patch; base subdirectory for "
"patch created in a subdirectory")
group.add_option("-p",
"--patchlevel",
type='int',
metavar="LEVEL",
help="Used as -pN parameter to patch")
group.add_option("-s",
"--sub_rep",
action="append",
default=[],
help="Subcheckout to use in addition. This is mainly "
"useful for gclient-style checkouts. In git, checkout "
"the branch with changes first. Use @rev or "
"@branch to specify the "
"revision/branch to diff against. If no @branch is "
"given the diff will be against the upstream branch. "
"If @branch then the diff is branch..HEAD. "
"All edits must be checked in.")
group.add_option("--no_gclient",
action="store_true",
help="Disable automatic search for gclient checkout.")
group.add_option(
"-E",
"--exclude",
action="append",
default=['ChangeLog'],
metavar='REGEXP',
help="Regexp patterns to exclude files. Default: %default")
group.add_option("--upstream_branch",
action="store",
help="Specify the upstream branch to diff against in the "
"main checkout")
parser.add_option_group(group)
group = optparse.OptionGroup(parser, "Access the try server by HTTP")
group.add_option("--use_http",
action="store_const",
const=_SendChangeHTTP,
dest="send_patch",
help="Use HTTP to talk to the try server [default]")
group.add_option("-H", "--host", help="Host address")
group.add_option("-P", "--port", help="HTTP port")
group.add_option("--proxy", help="HTTP proxy")
parser.add_option_group(group)
group = optparse.OptionGroup(parser, "Access the try server with SVN")
group.add_option("--use_svn",
action="store_const",
const=_SendChangeSVN,
dest="send_patch",
help="Use SVN to talk to the try server")
group.add_option(
"-S",
"--svn_repo",
metavar="SVN_URL",
help="SVN url to use to write the changes in; --use_svn is "
"implied when using --svn_repo")
parser.add_option_group(group)
options, args = parser.parse_args(argv)
# If they've asked for help, give it to them
if len(args) == 1 and args[0] == 'help':
parser.print_help()
return 0
# If they've said something confusing, don't spawn a try job until you
# understand what they want.
if args:
parser.error('Extra argument(s) "%s" not understood' % ' '.join(args))
if options.dry_run:
options.verbose += 1
LOG_FORMAT = '%(levelname)s %(filename)s(%(lineno)d): %(message)s'
if not swallow_exception:
if options.verbose == 0:
logging.basicConfig(level=logging.WARNING, format=LOG_FORMAT)
elif options.verbose == 1:
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
elif options.verbose > 1:
logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)
logging.debug(argv)
# Strip off any @ in the user, otherwise svn gets confused.
options.user = options.user.split('@', 1)[0]
if options.rietveld_url:
# Try to extract the review number if possible and fix the protocol.
if not '://' in options.rietveld_url:
options.rietveld_url = 'http://' + options.rietveld_url
match = re.match(r'^(.*)/(\d+)/?$', options.rietveld_url)
if match:
if options.issue or options.patchset:
parser.error(
'Cannot use both --issue and use a review number url')
options.issue = int(match.group(2))
options.rietveld_url = match.group(1)
try:
# Always include os.getcwd() in the checkout settings.
checkouts = []
path = os.getcwd()
if options.upstream_branch:
path += '@' + options.upstream_branch
checkouts.append(GuessVCS(options, path))
checkouts[0].AutomagicalSettings()
for item in options.sub_rep:
checkout = GuessVCS(options,
os.path.join(checkouts[0].checkout_root, item))
if checkout.checkout_root in [c.checkout_root for c in checkouts]:
parser.error('Specified the root %s two times.' %
checkout.checkout_root)
checkouts.append(checkout)
can_http = options.port and options.host
can_svn = options.svn_repo
# If there was no transport selected yet, now we must have enough data to
# select one.
if not options.send_patch and not (can_http or can_svn):
parser.error('Please specify an access method.')
# Convert options.diff into the content of the diff.
if options.url:
if options.files:
parser.error(
'You cannot specify files and --url at the same time.')
options.diff = urllib.urlopen(options.url).read()
elif options.diff:
if options.files:
parser.error(
'You cannot specify files and --diff at the same time.')
options.diff = gclient_utils.FileRead(options.diff, 'rb')
elif options.issue and options.patchset is None:
# Retrieve the patch from rietveld when the diff is not specified.
# When patchset is specified, it's because it's done by gcl/git-try.
if json is None:
parser.error(
'json or simplejson library is missing, please install.')
api_url = '%s/api/%d' % (options.rietveld_url, options.issue)
logging.debug(api_url)
contents = json.loads(urllib.urlopen(api_url).read())
options.patchset = contents['patchsets'][-1]
diff_url = (
'%s/download/issue%d_%d.diff' %
(options.rietveld_url, options.issue, options.patchset))
diff = GetMungedDiff('', urllib.urlopen(diff_url).readlines())
options.diff = ''.join(diff)
else:
# Use this as the base.
root = checkouts[0].checkout_root
diffs = []
for checkout in checkouts:
diff = checkout.GenerateDiff().splitlines(True)
path_diff = gclient_utils.PathDifference(
root, checkout.checkout_root)
# Munge it.
diffs.extend(GetMungedDiff(path_diff, diff))
options.diff = ''.join(diffs)
if not options.bot:
# Get try slaves from PRESUBMIT.py files if not specified.
# Even if the diff comes from options.url, use the local checkout for bot
# selection.
try:
import presubmit_support
root_presubmit = checkouts[0].ReadRootFile('PRESUBMIT.py')
options.bot = presubmit_support.DoGetTrySlaves(
checkouts[0].GetFileNames(), checkouts[0].checkout_root,
root_presubmit, options.project, False, sys.stdout)
except ImportError:
pass
# If no bot is specified, either the default pool will be selected or the
# try server will refuse the job. Either case we don't need to interfere.
if options.name is None:
if options.issue:
options.name = 'Issue %s' % options.issue
else:
options.name = 'Unnamed'
print('Note: use --name NAME to change the try job name.')
if not options.email:
parser.error(
'Using an anonymous checkout. Please use --email or set '
'the TRYBOT_RESULTS_EMAIL_ADDRESS environment variable.')
else:
print('Results will be emailed to: ' + options.email)
# Prevent rietveld updates if we aren't running all the tests.
if options.testfilter is not None:
options.issue = None
options.patchset = None
# Send the patch.
if options.send_patch:
# If forced.
options.send_patch(options)
PrintSuccess(options)
return 0
try:
if can_http:
_SendChangeHTTP(options)
PrintSuccess(options)
return 0
except NoTryServerAccess:
if not can_svn:
raise
_SendChangeSVN(options)
PrintSuccess(options)
return 0
except (InvalidScript, NoTryServerAccess), e:
if swallow_exception:
return 1
print >> sys.stderr, e
return 1