def main(args):
tools.disable_buffering()
parser = tools.OptionParserWithLogging(usage="%prog <options>", version=__version__, log_file=RUN_ISOLATED_LOG_FILE)
data_group = optparse.OptionGroup(parser, "Data source")
data_group.add_option("-s", "--isolated", metavar="FILE", help="File/url describing what to map or run")
data_group.add_option("-H", "--hash", help="Hash of the .isolated to grab from the hash table")
isolateserver.add_isolate_server_options(data_group, True)
parser.add_option_group(data_group)
cache_group = optparse.OptionGroup(parser, "Cache management")
cache_group.add_option("--cache", default="cache", metavar="DIR", help="Cache directory, default=%default")
cache_group.add_option(
"--max-cache-size",
type="int",
metavar="NNN",
default=20 * 1024 * 1024 * 1024,
help="Trim if the cache gets larger than this value, default=%default",
)
cache_group.add_option(
"--min-free-space",
type="int",
metavar="NNN",
default=2 * 1024 * 1024 * 1024,
help="Trim if disk free space becomes lower than this value, " "default=%default",
)
cache_group.add_option(
"--max-items",
type="int",
metavar="NNN",
default=100000,
help="Trim if more than this number of items are in the cache " "default=%default",
)
parser.add_option_group(cache_group)
auth.add_auth_options(parser)
options, args = parser.parse_args(args)
auth.process_auth_options(parser, options)
isolateserver.process_isolate_server_options(data_group, options)
if bool(options.isolated) == bool(options.hash):
logging.debug("One and only one of --isolated or --hash is required.")
parser.error("One and only one of --isolated or --hash is required.")
options.cache = os.path.abspath(options.cache)
policies = CachePolicies(options.max_cache_size, options.min_free_space, options.max_items)
try:
# |options.cache| path may not exist until DiskCache() instance is created.
cache = DiskCache(options.cache, policies, isolateserver.get_hash_algo(options.namespace))
remote = options.isolate_server or options.indir
with isolateserver.get_storage(remote, options.namespace) as storage:
# Hashing schemes used by |storage| and |cache| MUST match.
assert storage.hash_algo == cache.hash_algo
return run_tha_test(options.isolated or options.hash, storage, cache, args)
except Exception as e:
# Make sure any exception is logged.
tools.report_error(e)
logging.exception(e)
return 1
def __init__(self, **kwargs):
logging_utils.OptionParserWithLogging.__init__(self, **kwargs)
self.server_group = optparse.OptionGroup(self, 'Server')
self.server_group.add_option('-S',
'--swarming',
metavar='URL',
default=os.environ.get(
'SWARMING_SERVER', ''),
help='Swarming server to use')
isolateserver.add_isolate_server_options(self.server_group)
self.add_option_group(self.server_group)
auth.add_auth_options(self)
self.add_option('-d',
'--dimension',
default=[],
action='append',
nargs=2,
dest='dimensions',
metavar='FOO bar',
help='dimension to filter on')
self.add_option(
'--priority',
type='int',
help='The lower value, the more important the task is. It may be '
'important to specify a higher priority since the default value '
'will make the task to be triggered only when the bots are idle.')
self.add_option(
'--deadline',
type='int',
default=6 * 60 * 60,
help=
'Seconds to allow the task to be pending for a bot to run before '
'this task request expires.')
def CMDarchive(parser, args):
"""Creates a .isolated file and uploads the tree to an isolate server.
All the files listed in the .isolated file are put in the isolate server
cache via isolateserver.py.
"""
add_isolate_options(parser)
add_subdir_option(parser)
isolateserver.add_isolate_server_options(parser)
auth.add_auth_options(parser)
options, args = parser.parse_args(args)
if args:
parser.error('Unsupported argument: %s' % args)
process_isolate_options(parser, options)
auth.process_auth_options(parser, options)
isolateserver.process_isolate_server_options(parser, options, True, True)
server_ref = isolate_storage.ServerRef(
options.isolate_server, options.namespace)
result = isolate_and_archive([(options, unicode(os.getcwd()))], server_ref)
if result is None:
return EXIT_CODE_UPLOAD_ERROR
assert len(result) == 1, result
if result.values()[0] is None:
return EXIT_CODE_ISOLATE_ERROR
return 0
def add_trigger_options(parser):
"""Adds all options to trigger a task on Swarming."""
isolateserver.add_isolate_server_options(parser, True)
add_filter_options(parser)
parser.task_group = tools.optparse.OptionGroup(parser, 'Task properties')
parser.task_group.add_option(
'-e', '--env', default=[], action='append', nargs=2, metavar='FOO bar',
help='Environment variables to set')
parser.task_group.add_option(
'--priority', type='int', default=100,
help='The lower value, the more important the task is')
parser.task_group.add_option(
'-T', '--task-name',
help='Display name of the task. It uniquely identifies the task. '
'Defaults to <base_name>/<dimensions>/<isolated hash>/<timestamp> '
'if an isolated file is provided, if a hash is provided, it '
'defaults to <user>/<dimensions>/<isolated hash>/<timestamp>')
parser.task_group.add_option(
'--deadline', type='int', default=6*60*60,
help='Seconds to allow the task to be pending for a bot to run before '
'this task request expires.')
parser.add_option_group(parser.task_group)
# TODO(maruel): This is currently written in a chromium-specific way.
parser.group_logging.add_option(
'--profile', action='store_true',
default=bool(os.environ.get('ISOLATE_DEBUG')),
help='Have run_isolated.py print profiling info')
def CMDarchive(parser, args):
"""Creates a .isolated file and uploads the tree to an isolate server.
All the files listed in the .isolated file are put in the isolate server
cache via isolateserver.py.
"""
add_isolate_options(parser)
add_subdir_option(parser)
isolateserver.add_isolate_server_options(parser)
auth.add_auth_options(parser)
options, args = parser.parse_args(args)
if args:
parser.error('Unsupported argument: %s' % args)
process_isolate_options(parser, options)
auth.process_auth_options(parser, options)
isolateserver.process_isolate_server_options(parser, options, True)
result = isolate_and_archive(
[(options, unicode(os.getcwd()))],
options.isolate_server,
options.namespace)
if result is None:
return EXIT_CODE_UPLOAD_ERROR
assert len(result) == 1, result
if result.values()[0] is None:
return EXIT_CODE_ISOLATE_ERROR
return 0
def main(args):
parser = logging_utils.OptionParserWithLogging(
usage='%prog <options>',
version=__version__,
log_file=RUN_ISOLATED_LOG_FILE)
parser.add_option(
'--json',
help=
'dump output metadata to json file. When used, run_isolated returns '
'non-zero only on internal failure')
parser.add_option('--hard-timeout',
type='float',
help='Enforce hard timeout in execution')
parser.add_option('--grace-period',
type='float',
help='Grace period between SIGTERM and SIGKILL')
data_group = optparse.OptionGroup(parser, 'Data source')
data_group.add_option(
'-s',
'--isolated',
help='Hash of the .isolated to grab from the isolate server')
isolateserver.add_isolate_server_options(data_group)
parser.add_option_group(data_group)
isolateserver.add_cache_options(parser)
parser.set_defaults(cache='cache')
debug_group = optparse.OptionGroup(parser, 'Debugging')
debug_group.add_option(
'--leak-temp-dir',
action='store_true',
help='Deliberately leak isolate\'s temp dir for later examination '
'[default: %default]')
debug_group.add_option('--root-dir',
help='Use a directory instead of a random one')
parser.add_option_group(debug_group)
auth.add_auth_options(parser)
options, args = parser.parse_args(args)
if not options.isolated:
parser.error('--isolated is required.')
auth.process_auth_options(parser, options)
isolateserver.process_isolate_server_options(parser, options, True)
cache = isolateserver.process_cache_options(options)
if options.root_dir:
options.root_dir = unicode(os.path.abspath(options.root_dir))
if options.json:
options.json = unicode(os.path.abspath(options.json))
with isolateserver.get_storage(options.isolate_server,
options.namespace) as storage:
# Hashing schemes used by |storage| and |cache| MUST match.
assert storage.hash_algo == cache.hash_algo
return run_tha_test(options.isolated, storage, cache,
options.leak_temp_dir, options.json,
options.root_dir, options.hard_timeout,
options.grace_period, args)
def main(args):
tools.disable_buffering()
parser = logging_utils.OptionParserWithLogging(
usage='%prog <options>',
version=__version__,
log_file=RUN_ISOLATED_LOG_FILE)
parser.add_option(
'--json',
help='dump output metadata to json file. When used, run_isolated returns '
'non-zero only on internal failure')
parser.add_option(
'--hard-timeout', type='int', help='Enforce hard timeout in execution')
parser.add_option(
'--grace-period', type='int',
help='Grace period between SIGTERM and SIGKILL')
data_group = optparse.OptionGroup(parser, 'Data source')
data_group.add_option(
'-s', '--isolated',
help='Hash of the .isolated to grab from the isolate server')
isolateserver.add_isolate_server_options(data_group)
parser.add_option_group(data_group)
isolateserver.add_cache_options(parser)
parser.set_defaults(cache='cache')
debug_group = optparse.OptionGroup(parser, 'Debugging')
debug_group.add_option(
'--leak-temp-dir',
action='store_true',
help='Deliberately leak isolate\'s temp dir for later examination '
'[default: %default]')
debug_group.add_option(
'--root-dir', help='Use a directory instead of a random one')
parser.add_option_group(debug_group)
auth.add_auth_options(parser)
options, args = parser.parse_args(args)
if not options.isolated:
parser.error('--isolated is required.')
auth.process_auth_options(parser, options)
isolateserver.process_isolate_server_options(parser, options, True)
cache = isolateserver.process_cache_options(options)
if options.root_dir:
options.root_dir = unicode(os.path.abspath(options.root_dir))
if options.json:
options.json = unicode(os.path.abspath(options.json))
with isolateserver.get_storage(
options.isolate_server, options.namespace) as storage:
# Hashing schemes used by |storage| and |cache| MUST match.
assert storage.hash_algo == cache.hash_algo
return run_tha_test(
options.isolated, storage, cache, options.leak_temp_dir, options.json,
options.root_dir, options.hard_timeout, options.grace_period, args)
def test_indir(self):
data = [
(['-I', 'http://foo.com/'], ('http://foo.com', None)),
(['--indir', ROOT_DIR], ('', ROOT_DIR)),
]
for value, (expected_isolate_server, expected_indir) in data:
parser = isolateserver.OptionParserIsolateServer()
isolateserver.add_isolate_server_options(parser, True)
options, _ = parser.parse_args(value)
isolateserver.process_isolate_server_options(parser, options)
self.assertEqual(expected_isolate_server, options.isolate_server)
self.assertEqual(expected_indir, options.indir)
def test_indir(self):
data = [
(['-I', 'http://foo.com/'], ('http://foo.com', None)),
(['--indir', ROOT_DIR], ('', ROOT_DIR)),
]
for value, (expected_isolate_server, expected_indir) in data:
parser = isolateserver.OptionParserIsolateServer()
isolateserver.add_isolate_server_options(parser, True)
options, _ = parser.parse_args(value)
isolateserver.process_isolate_server_options(parser, options)
self.assertEqual(expected_isolate_server, options.isolate_server)
self.assertEqual(expected_indir, options.indir)
def add_trigger_options(parser):
"""Adds all options to trigger a task on Swarming."""
isolateserver.add_isolate_server_options(parser)
add_filter_options(parser)
parser.task_group = optparse.OptionGroup(parser, "Task properties")
parser.task_group.add_option("-s", "--isolated", help="Hash of the .isolated to grab from the isolate server")
parser.task_group.add_option(
"-e", "--env", default=[], action="append", nargs=2, metavar="FOO bar", help="Environment variables to set"
)
parser.task_group.add_option(
"--priority", type="int", default=100, help="The lower value, the more important the task is"
)
parser.task_group.add_option(
"-T",
"--task-name",
help="Display name of the task. Defaults to "
"<base_name>/<dimensions>/<isolated hash>/<timestamp> if an "
"isolated file is provided, if a hash is provided, it defaults to "
"<user>/<dimensions>/<isolated hash>/<timestamp>",
)
parser.task_group.add_option("--tags", action="append", default=[], help="Tags to assign to the task.")
parser.task_group.add_option(
"--user", default="", help="User associated with the task. Defaults to authenticated user on " "the server."
)
parser.task_group.add_option(
"--idempotent",
action="store_true",
default=False,
help="When set, the server will actively try to find a previous task "
"with the same parameter and return this result instead if possible",
)
parser.task_group.add_option(
"--expiration",
type="int",
default=6 * 60 * 60,
help="Seconds to allow the task to be pending for a bot to run before " "this task request expires.",
)
parser.task_group.add_option("--deadline", type="int", dest="expiration", help=optparse.SUPPRESS_HELP)
parser.task_group.add_option(
"--hard-timeout", type="int", default=60 * 60, help="Seconds to allow the task to complete."
)
parser.task_group.add_option(
"--io-timeout", type="int", default=20 * 60, help="Seconds to allow the task to be silent."
)
parser.task_group.add_option(
"--raw-cmd",
action="store_true",
default=False,
help="When set, the command after -- is used as-is without run_isolated. "
"In this case, no .isolated file is expected.",
)
parser.add_option_group(parser.task_group)
def add_trigger_options(parser):
"""Adds all options to trigger a task on Swarming."""
isolateserver.add_isolate_server_options(parser)
add_filter_options(parser)
parser.task_group = optparse.OptionGroup(parser, 'Task properties')
parser.task_group.add_option(
'-s', '--isolated',
help='Hash of the .isolated to grab from the isolate server')
parser.task_group.add_option(
'-e', '--env', default=[], action='append', nargs=2, metavar='FOO bar',
help='Environment variables to set')
parser.task_group.add_option(
'--priority', type='int', default=100,
help='The lower value, the more important the task is')
parser.task_group.add_option(
'-T', '--task-name',
help='Display name of the task. Defaults to '
'<base_name>/<dimensions>/<isolated hash>/<timestamp> if an '
'isolated file is provided, if a hash is provided, it defaults to '
'<user>/<dimensions>/<isolated hash>/<timestamp>')
parser.task_group.add_option(
'--tags', action='append', default=[],
help='Tags to assign to the task.')
parser.task_group.add_option(
'--user', default='',
help='User associated with the task. Defaults to authenticated user on '
'the server.')
parser.task_group.add_option(
'--idempotent', action='store_true', default=False,
help='When set, the server will actively try to find a previous task '
'with the same parameter and return this result instead if possible')
parser.task_group.add_option(
'--expiration', type='int', default=6*60*60,
help='Seconds to allow the task to be pending for a bot to run before '
'this task request expires.')
parser.task_group.add_option(
'--deadline', type='int', dest='expiration',
help=optparse.SUPPRESS_HELP)
parser.task_group.add_option(
'--hard-timeout', type='int', default=60*60,
help='Seconds to allow the task to complete.')
parser.task_group.add_option(
'--io-timeout', type='int', default=20*60,
help='Seconds to allow the task to be silent.')
parser.task_group.add_option(
'--raw-cmd', action='store_true', default=False,
help='When set, the command after -- is used as-is without run_isolated. '
'In this case, no .isolated file is expected.')
parser.add_option_group(parser.task_group)
def add_trigger_options(parser):
"""Adds all options to trigger a task on Swarming."""
isolateserver.add_isolate_server_options(parser, True)
add_filter_options(parser)
parser.task_group = tools.optparse.OptionGroup(parser, 'Task properties')
parser.task_group.add_option(
'-w',
'--working-dir',
default='swarm_tests',
help='Working directory on the swarming slave side. default: %default.'
)
parser.task_group.add_option('--working_dir',
help=tools.optparse.SUPPRESS_HELP)
parser.task_group.add_option('-e',
'--env',
default=[],
action='append',
nargs=2,
metavar='FOO bar',
help='environment variables to set')
parser.task_group.add_option(
'--priority',
type='int',
default=100,
help='The lower value, the more important the task is')
parser.task_group.add_option('--shards',
type='int',
default=1,
help='number of shards to use')
parser.task_group.add_option(
'-T',
'--task-name',
help='Display name of the task. It uniquely identifies the task. '
'Defaults to <base_name>/<dimensions>/<isolated hash>/<timestamp> '
'if an isolated file is provided, if a hash is provided, it '
'defaults to <user>/<dimensions>/<isolated hash>/<timestamp>')
parser.task_group.add_option(
'--deadline',
type='int',
default=6 * 60 * 60,
help='Seconds to allow the task to be pending for a bot to run before '
'this task request expires.')
parser.add_option_group(parser.task_group)
# TODO(maruel): This is currently written in a chromium-specific way.
parser.group_logging.add_option(
'--profile',
action='store_true',
default=bool(os.environ.get('ISOLATE_DEBUG')),
help='Have run_isolated.py print profiling info')
def test_isolate_server(self):
data = [
(['-I', 'http://foo.com/'], 'http://foo.com'),
(['-I', 'https://foo.com/'], 'https://foo.com'),
(['-I', 'https://foo.com'], 'https://foo.com'),
(['-I', 'https://foo.com/a'], 'https://foo.com/a'),
(['-I', 'https://foo.com/a/'], 'https://foo.com/a'),
(['-I', 'https://foo.com:8080/a/'], 'https://foo.com:8080/a'),
(['-I', 'foo.com'], 'https://foo.com'),
(['-I', 'foo.com:8080'], 'https://foo.com:8080'),
(['-I', 'foo.com/'], 'https://foo.com'),
(['-I', 'foo.com/a/'], 'https://foo.com/a'),
]
for value, expected in data:
parser = isolateserver.OptionParserIsolateServer()
isolateserver.add_isolate_server_options(parser, False)
options, _ = parser.parse_args(value)
isolateserver.process_isolate_server_options(parser, options)
self.assertEqual(expected, options.isolate_server)
def test_isolate_server(self):
data = [
(['-I', 'http://foo.com/'], 'http://foo.com'),
(['-I', 'https://foo.com/'], 'https://foo.com'),
(['-I', 'https://foo.com'], 'https://foo.com'),
(['-I', 'https://foo.com/a'], 'https://foo.com/a'),
(['-I', 'https://foo.com/a/'], 'https://foo.com/a'),
(['-I', 'https://foo.com:8080/a/'], 'https://foo.com:8080/a'),
(['-I', 'foo.com'], 'https://foo.com'),
(['-I', 'foo.com:8080'], 'https://foo.com:8080'),
(['-I', 'foo.com/'], 'https://foo.com'),
(['-I', 'foo.com/a/'], 'https://foo.com/a'),
]
for value, expected in data:
parser = isolateserver.OptionParserIsolateServer()
isolateserver.add_isolate_server_options(parser, False)
options, _ = parser.parse_args(value)
isolateserver.process_isolate_server_options(parser, options)
self.assertEqual(expected, options.isolate_server)
def add_trigger_options(parser):
"""Adds all options to trigger a task on Swarming."""
isolateserver.add_isolate_server_options(parser, True)
add_filter_options(parser)
parser.task_group = tools.optparse.OptionGroup(parser, "Task properties")
parser.task_group.add_option(
"-w",
"--working-dir",
default="swarm_tests",
help="Working directory on the swarming slave side. default: %default.",
)
parser.task_group.add_option("--working_dir", help=tools.optparse.SUPPRESS_HELP)
parser.task_group.add_option(
"-e", "--env", default=[], action="append", nargs=2, metavar="FOO bar", help="environment variables to set"
)
parser.task_group.add_option(
"--priority", type="int", default=100, help="The lower value, the more important the task is"
)
parser.task_group.add_option("--shards", type="int", default=1, help="number of shards to use")
parser.task_group.add_option(
"-T",
"--task-name",
help="Display name of the task. It uniquely identifies the task. "
"Defaults to <base_name>/<dimensions>/<isolated hash>/<timestamp> "
"if an isolated file is provided, if a hash is provided, it "
"defaults to <user>/<dimensions>/<isolated hash>/<timestamp>",
)
parser.task_group.add_option(
"--deadline",
type="int",
default=6 * 60 * 60,
help="Seconds to allow the task to be pending for a bot to run before " "this task request expires.",
)
parser.add_option_group(parser.task_group)
# TODO(maruel): This is currently written in a chromium-specific way.
parser.group_logging.add_option(
"--profile",
action="store_true",
default=bool(os.environ.get("ISOLATE_DEBUG")),
help="Have run_isolated.py print profiling info",
)
def main(args):
tools.disable_buffering()
parser = logging_utils.OptionParserWithLogging(
usage='%prog <options>',
version=__version__,
log_file=RUN_ISOLATED_LOG_FILE)
data_group = optparse.OptionGroup(parser, 'Data source')
data_group.add_option(
'-s', '--isolated',
help='Hash of the .isolated to grab from the isolate server')
data_group.add_option(
'-H', dest='isolated', help=optparse.SUPPRESS_HELP)
isolateserver.add_isolate_server_options(data_group)
parser.add_option_group(data_group)
isolateserver.add_cache_options(parser)
parser.set_defaults(cache='cache')
debug_group = optparse.OptionGroup(parser, 'Debugging')
debug_group.add_option(
'--leak-temp-dir',
action='store_true',
help='Deliberately leak isolate\'s temp dir for later examination '
'[default: %default]')
parser.add_option_group(debug_group)
auth.add_auth_options(parser)
options, args = parser.parse_args(args)
if not options.isolated:
parser.error('--isolated is required.')
auth.process_auth_options(parser, options)
isolateserver.process_isolate_server_options(parser, options, True)
cache = isolateserver.process_cache_options(options)
with isolateserver.get_storage(
options.isolate_server, options.namespace) as storage:
# Hashing schemes used by |storage| and |cache| MUST match.
assert storage.hash_algo == cache.hash_algo
return run_tha_test(
options.isolated, storage, cache, options.leak_temp_dir, args)
def main(args):
tools.disable_buffering()
parser = tools.OptionParserWithLogging(
usage='%prog <options>',
version=__version__,
log_file=RUN_ISOLATED_LOG_FILE)
data_group = optparse.OptionGroup(parser, 'Data source')
data_group.add_option(
'-s', '--isolated',
help='Hash of the .isolated to grab from the isolate server')
data_group.add_option(
'-H', dest='isolated', help=optparse.SUPPRESS_HELP)
isolateserver.add_isolate_server_options(data_group)
parser.add_option_group(data_group)
isolateserver.add_cache_options(parser)
parser.set_defaults(cache='cache')
debug_group = optparse.OptionGroup(parser, 'Debugging')
debug_group.add_option(
'--leak-temp-dir',
action='store_true',
help='Deliberately leak isolate\'s temp dir for later examination '
'[default: %default]')
parser.add_option_group(debug_group)
auth.add_auth_options(parser)
options, args = parser.parse_args(args)
if not options.isolated:
parser.error('--isolated is required.')
auth.process_auth_options(parser, options)
isolateserver.process_isolate_server_options(parser, options, True)
cache = isolateserver.process_cache_options(options)
with isolateserver.get_storage(
options.isolate_server, options.namespace) as storage:
# Hashing schemes used by |storage| and |cache| MUST match.
assert storage.hash_algo == cache.hash_algo
return run_tha_test(
options.isolated, storage, cache, options.leak_temp_dir, args)
def main(args):
tools.disable_buffering()
parser = logging_utils.OptionParserWithLogging(
usage="%prog <options>", version=__version__, log_file=RUN_ISOLATED_LOG_FILE
)
parser.add_option(
"--json",
help="dump output metadata to json file. When used, run_isolated returns " "non-zero only on internal failure",
)
data_group = optparse.OptionGroup(parser, "Data source")
data_group.add_option("-s", "--isolated", help="Hash of the .isolated to grab from the isolate server")
isolateserver.add_isolate_server_options(data_group)
parser.add_option_group(data_group)
isolateserver.add_cache_options(parser)
parser.set_defaults(cache="cache")
debug_group = optparse.OptionGroup(parser, "Debugging")
debug_group.add_option(
"--leak-temp-dir",
action="store_true",
help="Deliberately leak isolate's temp dir for later examination " "[default: %default]",
)
debug_group.add_option("--root-dir", help="Use a directory instead of a random one")
parser.add_option_group(debug_group)
auth.add_auth_options(parser)
options, args = parser.parse_args(args)
if not options.isolated:
parser.error("--isolated is required.")
auth.process_auth_options(parser, options)
isolateserver.process_isolate_server_options(parser, options, True)
cache = isolateserver.process_cache_options(options)
with isolateserver.get_storage(options.isolate_server, options.namespace) as storage:
# Hashing schemes used by |storage| and |cache| MUST match.
assert storage.hash_algo == cache.hash_algo
return run_tha_test(
options.isolated, storage, cache, options.leak_temp_dir, options.json, options.root_dir, args
)
def __init__(self, **kwargs):
tools.OptionParserWithLogging.__init__(self, **kwargs)
self.server_group = tools.optparse.OptionGroup(self, 'Server')
self.server_group.add_option(
'-S', '--swarming',
metavar='URL', default=os.environ.get('SWARMING_SERVER', ''),
help='Swarming server to use')
isolateserver.add_isolate_server_options(self.server_group)
self.add_option_group(self.server_group)
auth.add_auth_options(self)
self.add_option(
'-d', '--dimension', default=[], action='append', nargs=2,
dest='dimensions', metavar='FOO bar',
help='dimension to filter on')
self.add_option(
'--priority', type='int',
help='The lower value, the more important the task is. It may be '
'important to specify a higher priority since the default value '
'will make the task to be triggered only when the bots are idle.')
self.add_option(
'--deadline', type='int', default=6*60*60,
help='Seconds to allow the task to be pending for a bot to run before '
'this task request expires.')
def add_trigger_options(parser):
"""Adds all options to trigger a task on Swarming."""
isolateserver.add_isolate_server_options(parser, True)
parser.filter_group = tools.optparse.OptionGroup(parser, 'Filtering slaves')
parser.filter_group.add_option(
'-d', '--dimension', default=[], action='append', nargs=2,
dest='dimensions', metavar='FOO bar',
help='dimension to filter on')
parser.add_option_group(parser.filter_group)
parser.task_group = tools.optparse.OptionGroup(parser, 'Task properties')
parser.task_group.add_option(
'-w', '--working-dir', default='swarm_tests',
help='Working directory on the swarming slave side. default: %default.')
parser.task_group.add_option(
'--working_dir', help=tools.optparse.SUPPRESS_HELP)
parser.task_group.add_option(
'-e', '--env', default=[], action='append', nargs=2, metavar='FOO bar',
help='environment variables to set')
parser.task_group.add_option(
'--priority', type='int', default=100,
help='The lower value, the more important the task is')
parser.task_group.add_option(
'--shards', type='int', default=1, help='number of shards to use')
parser.task_group.add_option(
'-T', '--task-name',
help='Display name of the task. It uniquely identifies the task. '
'Defaults to <base_name>/<dimensions>/<isolated hash> if an '
'isolated file is provided, if a hash is provided, it defaults to '
'<user>/<dimensions>/<isolated hash>')
parser.add_option_group(parser.task_group)
# TODO(maruel): This is currently written in a chromium-specific way.
parser.group_logging.add_option(
'--profile', action='store_true',
default=bool(os.environ.get('ISOLATE_DEBUG')),
help='Have run_isolated.py print profiling info')
def main(args):
tools.disable_buffering()
parser = tools.OptionParserWithLogging(
usage='%prog <options>',
version=__version__,
log_file=RUN_ISOLATED_LOG_FILE)
data_group = optparse.OptionGroup(parser, 'Data source')
data_group.add_option(
'-s', '--isolated',
metavar='FILE',
help='File/url describing what to map or run')
data_group.add_option(
'-H', '--hash',
help='Hash of the .isolated to grab from the hash table')
isolateserver.add_isolate_server_options(data_group, True)
parser.add_option_group(data_group)
cache_group = optparse.OptionGroup(parser, 'Cache management')
cache_group.add_option(
'--cache',
default='cache',
metavar='DIR',
help='Cache directory, default=%default')
cache_group.add_option(
'--max-cache-size',
type='int',
metavar='NNN',
default=20*1024*1024*1024,
help='Trim if the cache gets larger than this value, default=%default')
cache_group.add_option(
'--min-free-space',
type='int',
metavar='NNN',
default=2*1024*1024*1024,
help='Trim if disk free space becomes lower than this value, '
'default=%default')
cache_group.add_option(
'--max-items',
type='int',
metavar='NNN',
default=100000,
help='Trim if more than this number of items are in the cache '
'default=%default')
parser.add_option_group(cache_group)
auth.add_auth_options(parser)
options, args = parser.parse_args(args)
auth.process_auth_options(parser, options)
isolateserver.process_isolate_server_options(data_group, options)
if bool(options.isolated) == bool(options.hash):
logging.debug('One and only one of --isolated or --hash is required.')
parser.error('One and only one of --isolated or --hash is required.')
options.cache = os.path.abspath(options.cache)
policies = CachePolicies(
options.max_cache_size, options.min_free_space, options.max_items)
algo = isolateserver.get_hash_algo(options.namespace)
try:
# |options.cache| may not exist until DiskCache() instance is created.
cache = DiskCache(options.cache, policies, algo)
remote = options.isolate_server or options.indir
with isolateserver.get_storage(remote, options.namespace) as storage:
return run_tha_test(
options.isolated or options.hash, storage, cache, algo, args)
except Exception as e:
# Make sure any exception is logged.
tools.report_error(e)
logging.exception(e)
return 1
def CMDbatcharchive(parser, args):
"""Archives multiple isolated trees at once.
Using single command instead of multiple sequential invocations allows to cut
redundant work when isolated trees share common files (e.g. file hashes are
checked only once, their presence on the server is checked only once, and
so on).
Takes a list of paths to *.isolated.gen.json files that describe what trees to
isolate. Format of files is:
{
"version": 1,
"dir": <absolute path to a directory all other paths are relative to>,
"args": [list of command line arguments for single 'archive' command]
}
"""
isolateserver.add_isolate_server_options(parser)
isolateserver.add_archive_options(parser)
auth.add_auth_options(parser)
parser.add_option(
"--dump-json", metavar="FILE", help="Write isolated hashes of archived trees to this file as JSON"
)
options, args = parser.parse_args(args)
auth.process_auth_options(parser, options)
isolateserver.process_isolate_server_options(parser, options, True)
# Validate all incoming options, prepare what needs to be archived as a list
# of tuples (archival options, working directory).
work_units = []
for gen_json_path in args:
# Validate JSON format of a *.isolated.gen.json file.
data = tools.read_json(gen_json_path)
if data.get("version") != ISOLATED_GEN_JSON_VERSION:
parser.error("Invalid version in %s" % gen_json_path)
cwd = data.get("dir")
if not isinstance(cwd, unicode) or not os.path.isdir(cwd):
parser.error("Invalid dir in %s" % gen_json_path)
args = data.get("args")
if not isinstance(args, list) or not all(isinstance(x, unicode) for x in args):
parser.error("Invalid args in %s" % gen_json_path)
# Convert command line (embedded in JSON) to Options object.
work_units.append((parse_archive_command_line(args, cwd), cwd))
# Perform the archival, all at once.
isolated_hashes = isolate_and_archive(work_units, options.isolate_server, options.namespace)
# TODO(vadimsh): isolate_and_archive returns None on upload failure, there's
# no way currently to figure out what *.isolated file from a batch were
# successfully uploaded, so consider them all failed (and emit empty dict
# as JSON result).
if options.dump_json:
tools.write_json(options.dump_json, isolated_hashes or {}, False)
if isolated_hashes is None:
return EXIT_CODE_UPLOAD_ERROR
# isolated_hashes[x] is None if 'x.isolate' contains a error.
if not all(isolated_hashes.itervalues()):
return EXIT_CODE_ISOLATE_ERROR
return 0
def create_option_parser():
parser = logging_utils.OptionParserWithLogging(
usage='%prog <options> [command to run or extra args]',
version=__version__,
log_file=RUN_ISOLATED_LOG_FILE)
parser.add_option(
'--clean',
action='store_true',
help=
'Cleans the cache, trimming it necessary and remove corrupted items '
'and returns without executing anything; use with -v to know what '
'was done')
parser.add_option(
'--no-clean',
action='store_true',
help=
'Do not clean the cache automatically on startup. This is meant for '
'bots where a separate execution with --clean was done earlier so '
'doing it again is redundant')
parser.add_option('--use-symlinks',
action='store_true',
help='Use symlinks instead of hardlinks')
parser.add_option(
'--json',
help=
'dump output metadata to json file. When used, run_isolated returns '
'non-zero only on internal failure')
parser.add_option('--hard-timeout',
type='float',
help='Enforce hard timeout in execution')
parser.add_option('--grace-period',
type='float',
help='Grace period between SIGTERM and SIGKILL')
parser.add_option(
'--bot-file',
help='Path to a file describing the state of the host. The content is '
'defined by on_before_task() in bot_config.')
parser.add_option(
'--output',
action='append',
help='Specifies an output to return. If no outputs are specified, all '
'files located in $(ISOLATED_OUTDIR) will be returned; '
'otherwise, outputs in both $(ISOLATED_OUTDIR) and those '
'specified by --output option (there can be multiple) will be '
'returned. Note that if a file in OUT_DIR has the same path '
'as an --output option, the --output version will be returned.')
parser.add_option(
'-a',
'--argsfile',
# This is actually handled in parse_args; it's included here purely so it
# can make it into the help text.
help='Specify a file containing a JSON array of arguments to this '
'script. If --argsfile is provided, no other argument may be '
'provided on the command line.')
data_group = optparse.OptionGroup(parser, 'Data source')
data_group.add_option(
'-s',
'--isolated',
help='Hash of the .isolated to grab from the isolate server.')
isolateserver.add_isolate_server_options(data_group)
parser.add_option_group(data_group)
isolateserver.add_cache_options(parser)
cipd.add_cipd_options(parser)
named_cache.add_named_cache_options(parser)
debug_group = optparse.OptionGroup(parser, 'Debugging')
debug_group.add_option(
'--leak-temp-dir',
action='store_true',
help='Deliberately leak isolate\'s temp dir for later examination. '
'Default: %default')
debug_group.add_option('--root-dir',
help='Use a directory instead of a random one')
parser.add_option_group(debug_group)
auth.add_auth_options(parser)
parser.set_defaults(cache='cache',
cipd_cache='cipd_cache',
named_cache_root='named_caches')
return parser
def main(args):
tools.disable_buffering()
parser = tools.OptionParserWithLogging(
usage='%prog <options>',
version=__version__,
log_file=RUN_ISOLATED_LOG_FILE)
data_group = optparse.OptionGroup(parser, 'Data source')
data_group.add_option(
'-s', '--isolated',
metavar='FILE',
help='File/url describing what to map or run')
data_group.add_option(
'-H', '--hash',
help='Hash of the .isolated to grab from the hash table')
isolateserver.add_isolate_server_options(data_group, True)
parser.add_option_group(data_group)
cache_group = optparse.OptionGroup(parser, 'Cache management')
cache_group.add_option(
'--cache',
default='cache',
metavar='DIR',
help='Cache directory, default=%default')
cache_group.add_option(
'--max-cache-size',
type='int',
metavar='NNN',
default=20*1024*1024*1024,
help='Trim if the cache gets larger than this value, default=%default')
cache_group.add_option(
'--min-free-space',
type='int',
metavar='NNN',
default=2*1024*1024*1024,
help='Trim if disk free space becomes lower than this value, '
'default=%default')
cache_group.add_option(
'--max-items',
type='int',
metavar='NNN',
default=100000,
help='Trim if more than this number of items are in the cache '
'default=%default')
parser.add_option_group(cache_group)
auth.add_auth_options(parser)
options, args = parser.parse_args(args)
auth.process_auth_options(parser, options)
isolateserver.process_isolate_server_options(data_group, options)
if bool(options.isolated) == bool(options.hash):
logging.debug('One and only one of --isolated or --hash is required.')
parser.error('One and only one of --isolated or --hash is required.')
options.cache = os.path.abspath(options.cache)
policies = CachePolicies(
options.max_cache_size, options.min_free_space, options.max_items)
try:
# |options.cache| path may not exist until DiskCache() instance is created.
cache = DiskCache(
options.cache, policies, isolateserver.get_hash_algo(options.namespace))
remote = options.isolate_server or options.indir
with isolateserver.get_storage(remote, options.namespace) as storage:
# Hashing schemes used by |storage| and |cache| MUST match.
assert storage.hash_algo == cache.hash_algo
return run_tha_test(
options.isolated or options.hash, storage, cache, args)
except Exception as e:
# Make sure any exception is logged.
tools.report_error(e)
logging.exception(e)
return 1
def CMDbatcharchive(parser, args):
"""Archives multiple isolated trees at once.
Using single command instead of multiple sequential invocations allows to cut
redundant work when isolated trees share common files (e.g. file hashes are
checked only once, their presence on the server is checked only once, and
so on).
Takes a list of paths to *.isolated.gen.json files that describe what trees to
isolate. Format of files is:
{
"version": 1,
"dir": <absolute path to a directory all other paths are relative to>,
"args": [list of command line arguments for single 'archive' command]
}
"""
isolateserver.add_isolate_server_options(parser)
isolateserver.add_archive_options(parser)
auth.add_auth_options(parser)
parser.add_option(
'--dump-json',
metavar='FILE',
help='Write isolated hashes of archived trees to this file as JSON')
options, args = parser.parse_args(args)
auth.process_auth_options(parser, options)
isolateserver.process_isolate_server_options(parser, options, True, True)
# Validate all incoming options, prepare what needs to be archived as a list
# of tuples (archival options, working directory).
work_units = []
for gen_json_path in args:
# Validate JSON format of a *.isolated.gen.json file.
try:
data = tools.read_json(gen_json_path)
except IOError as e:
parser.error('Failed to open %s: %s' % (gen_json_path, e))
if data.get('version') != ISOLATED_GEN_JSON_VERSION:
parser.error('Invalid version in %s' % gen_json_path)
cwd = data.get('dir')
if not isinstance(cwd, unicode) or not fs.isdir(cwd):
parser.error('Invalid dir in %s' % gen_json_path)
args = data.get('args')
if (not isinstance(args, list)
or not all(isinstance(x, unicode) for x in args)):
parser.error('Invalid args in %s' % gen_json_path)
# Convert command line (embedded in JSON) to Options object.
work_units.append((parse_archive_command_line(args, cwd), cwd))
# Perform the archival, all at once.
isolated_hashes = isolate_and_archive(work_units, options.isolate_server,
options.namespace)
# TODO(vadimsh): isolate_and_archive returns None on upload failure, there's
# no way currently to figure out what *.isolated file from a batch were
# successfully uploaded, so consider them all failed (and emit empty dict
# as JSON result).
if options.dump_json:
tools.write_json(options.dump_json, isolated_hashes or {}, False)
if isolated_hashes is None:
return EXIT_CODE_UPLOAD_ERROR
# isolated_hashes[x] is None if 'x.isolate' contains a error.
if not all(isolated_hashes.itervalues()):
return EXIT_CODE_ISOLATE_ERROR
return 0
def create_option_parser():
parser = logging_utils.OptionParserWithLogging(
usage='%prog <options> [command to run or extra args]',
version=__version__,
log_file=RUN_ISOLATED_LOG_FILE)
parser.add_option(
'--clean', action='store_true',
help='Cleans the cache, trimming it necessary and remove corrupted items '
'and returns without executing anything; use with -v to know what '
'was done')
parser.add_option(
'--no-clean', action='store_true',
help='Do not clean the cache automatically on startup. This is meant for '
'bots where a separate execution with --clean was done earlier so '
'doing it again is redundant')
parser.add_option(
'--use-symlinks', action='store_true',
help='Use symlinks instead of hardlinks')
parser.add_option(
'--json',
help='dump output metadata to json file. When used, run_isolated returns '
'non-zero only on internal failure')
parser.add_option(
'--hard-timeout', type='float', help='Enforce hard timeout in execution')
parser.add_option(
'--grace-period', type='float',
help='Grace period between SIGTERM and SIGKILL')
parser.add_option(
'--raw-cmd', action='store_true',
help='Ignore the isolated command, use the one supplied at the command '
'line')
parser.add_option(
'--relative-cwd',
help='Ignore the isolated \'relative_cwd\' and use this one instead; '
'requires --raw-cmd')
parser.add_option(
'--env', default=[], action='append',
help='Environment variables to set for the child process')
parser.add_option(
'--env-prefix', default=[], action='append',
help='Specify a VAR=./path/fragment to put in the environment variable '
'before executing the command. The path fragment must be relative '
'to the isolated run directory, and must not contain a `..` token. '
'The path will be made absolute and prepended to the indicated '
'$VAR using the OS\'s path separator. Multiple items for the same '
'$VAR will be prepended in order.')
parser.add_option(
'--bot-file',
help='Path to a file describing the state of the host. The content is '
'defined by on_before_task() in bot_config.')
parser.add_option(
'--switch-to-account',
help='If given, switches LUCI_CONTEXT to given logical service account '
'(e.g. "task" or "system") before launching the isolated process.')
parser.add_option(
'--output', action='append',
help='Specifies an output to return. If no outputs are specified, all '
'files located in $(ISOLATED_OUTDIR) will be returned; '
'otherwise, outputs in both $(ISOLATED_OUTDIR) and those '
'specified by --output option (there can be multiple) will be '
'returned. Note that if a file in OUT_DIR has the same path '
'as an --output option, the --output version will be returned.')
parser.add_option(
'-a', '--argsfile',
# This is actually handled in parse_args; it's included here purely so it
# can make it into the help text.
help='Specify a file containing a JSON array of arguments to this '
'script. If --argsfile is provided, no other argument may be '
'provided on the command line.')
data_group = optparse.OptionGroup(parser, 'Data source')
data_group.add_option(
'-s', '--isolated',
help='Hash of the .isolated to grab from the isolate server.')
isolateserver.add_isolate_server_options(data_group)
parser.add_option_group(data_group)
isolateserver.add_cache_options(parser)
cipd.add_cipd_options(parser)
group = optparse.OptionGroup(parser, 'Named caches')
group.add_option(
'--named-cache',
dest='named_caches',
action='append',
nargs=2,
default=[],
help='A named cache to request. Accepts two arguments, name and path. '
'name identifies the cache, must match regex [a-z0-9_]{1,4096}. '
'path is a path relative to the run dir where the cache directory '
'must be put to. '
'This option can be specified more than once.')
group.add_option(
'--named-cache-root', default='named_caches',
help='Cache root directory. Default=%default')
parser.add_option_group(group)
debug_group = optparse.OptionGroup(parser, 'Debugging')
debug_group.add_option(
'--leak-temp-dir',
action='store_true',
help='Deliberately leak isolate\'s temp dir for later examination. '
'Default: %default')
debug_group.add_option(
'--root-dir', help='Use a directory instead of a random one')
parser.add_option_group(debug_group)
auth.add_auth_options(parser)
parser.set_defaults(cache='cache', cipd_cache='cipd_cache')
return parser
def create_option_parser():
parser = logging_utils.OptionParserWithLogging(
usage='%prog <options> [command to run or extra args]',
version=__version__,
log_file=RUN_ISOLATED_LOG_FILE)
parser.add_option(
'--clean',
action='store_true',
help=
'Cleans the cache, trimming it necessary and remove corrupted items '
'and returns without executing anything; use with -v to know what '
'was done')
parser.add_option(
'--no-clean',
action='store_true',
help=
'Do not clean the cache automatically on startup. This is meant for '
'bots where a separate execution with --clean was done earlier so '
'doing it again is redundant')
parser.add_option('--use-symlinks',
action='store_true',
help='Use symlinks instead of hardlinks')
parser.add_option(
'--json',
help=
'dump output metadata to json file. When used, run_isolated returns '
'non-zero only on internal failure')
parser.add_option('--hard-timeout',
type='float',
help='Enforce hard timeout in execution')
parser.add_option('--grace-period',
type='float',
help='Grace period between SIGTERM and SIGKILL')
parser.add_option(
'--bot-file',
help='Path to a file describing the state of the host. The content is '
'defined by on_before_task() in bot_config.')
data_group = optparse.OptionGroup(parser, 'Data source')
data_group.add_option(
'-s',
'--isolated',
help='Hash of the .isolated to grab from the isolate server.')
isolateserver.add_isolate_server_options(data_group)
parser.add_option_group(data_group)
isolateserver.add_cache_options(parser)
cipd.add_cipd_options(parser)
debug_group = optparse.OptionGroup(parser, 'Debugging')
debug_group.add_option(
'--leak-temp-dir',
action='store_true',
help='Deliberately leak isolate\'s temp dir for later examination. '
'Default: %default')
debug_group.add_option('--root-dir',
help='Use a directory instead of a random one')
parser.add_option_group(debug_group)
auth.add_auth_options(parser)
parser.set_defaults(cache='cache', cipd_cache='cipd_cache')
return parser
