def setUp(self):
super(DiskCacheTest, self).setUp()
# If this fails on Windows, please rerun this tests as an elevated user with
# administrator access right.
self.assertEqual(True, file_path.enable_symlink())
self._algo = isolated_format.get_hash_algo('default-gzip')
self._free_disk = 1000
# Max: 100 bytes, 2 items
# Min free disk: 1000 bytes.
self._policies = isolateserver.CachePolicies(100, 1000, 2)
def get_free_space(p):
self.assertEqual(p, self.tempdir)
return self._free_disk
self.mock(file_path, 'get_free_space', get_free_space)
def setUp(self):
super(DiskContentAddressedCacheTest, self).setUp()
# If this fails on Windows, please rerun this tests as an elevated user with
# administrator access right.
self.assertEqual(True, file_path.enable_symlink())
self._algo = hashlib.sha256
self._free_disk = 1000
# Max: 100 bytes, 2 items
# Min free disk: 1000 bytes.
self._policies = local_caching.CachePolicies(max_cache_size=100,
min_free_space=1000,
max_items=2,
max_age_secs=0)
def get_free_space(p):
self.assertEqual(p, os.path.join(self.tempdir, 'cache'))
return self._free_disk
self.mock(file_path, 'get_free_space', get_free_space)
def setUp(self):
super(DiskContentAddressedCacheTest, self).setUp()
# If this fails on Windows, please rerun this tests as an elevated user with
# administrator access right.
self.assertEqual(True, file_path.enable_symlink())
storage = isolateserver.get_storage(options.isolate_server,
options.namespace)
with storage:
# Hashing schemes used by |storage| and |isolate_cache| MUST match.
assert storage.hash_algo == isolate_cache.hash_algo
return run_tha_test(args, options.isolated, storage,
isolate_cache, options.output,
init_named_caches, options.leak_temp_dir,
options.json, options.root_dir,
options.hard_timeout, options.grace_period,
options.bot_file, install_packages_fn,
options.use_symlinks)
return run_tha_test(args, options.isolated, None, isolate_cache,
options.output, init_named_caches,
options.leak_temp_dir, options.json,
options.root_dir, options.hard_timeout,
options.grace_period, options.bot_file,
install_packages_fn, options.use_symlinks)
except (cipd.Error, named_cache.Error) as ex:
print >> sys.stderr, ex.message
return 1
if __name__ == '__main__':
subprocess42.inhibit_os_error_reporting()
# Ensure that we are always running with the correct encoding.
fix_encoding.fix_encoding()
file_path.enable_symlink()
sys.exit(main(sys.argv[1:]))
def setUpClass(cls):
if not file_path.enable_symlink():
raise Exception('Failed to enable symlink support')
def main(args):
# Warning: when --argsfile is used, the strings are unicode instances, when
# parsed normally, the strings are str instances.
(parser, options, args) = parse_args(args)
if not file_path.enable_symlink():
logging.error('Symlink support is not enabled')
isolate_cache = isolateserver.process_cache_options(options, trim=False)
named_cache_manager = named_cache.process_named_cache_options(parser, options)
if options.clean:
if options.isolated:
parser.error('Can\'t use --isolated with --clean.')
if options.isolate_server:
parser.error('Can\'t use --isolate-server with --clean.')
if options.json:
parser.error('Can\'t use --json with --clean.')
if options.named_caches:
parser.error('Can\t use --named-cache with --clean.')
clean_caches(options, isolate_cache, named_cache_manager)
return 0
if not options.no_clean:
clean_caches(options, isolate_cache, named_cache_manager)
if not options.isolated and not args:
parser.error('--isolated or command to run is required.')
auth.process_auth_options(parser, options)
isolateserver.process_isolate_server_options(
parser, options, True, False)
if not options.isolate_server:
if options.isolated:
parser.error('--isolated requires --isolate-server')
if ISOLATED_OUTDIR_PARAMETER in args:
parser.error(
'%s in args requires --isolate-server' % ISOLATED_OUTDIR_PARAMETER)
if options.root_dir:
options.root_dir = unicode(os.path.abspath(options.root_dir))
if options.json:
options.json = unicode(os.path.abspath(options.json))
if any('=' not in i for i in options.env):
parser.error(
'--env required key=value form. value can be skipped to delete '
'the variable')
options.env = dict(i.split('=', 1) for i in options.env)
prefixes = {}
cwd = os.path.realpath(os.getcwd())
for item in options.env_prefix:
if '=' not in item:
parser.error(
'--env-prefix %r is malformed, must be in the form `VAR=./path`'
% item)
key, opath = item.split('=', 1)
if os.path.isabs(opath):
parser.error('--env-prefix %r path is bad, must be relative.' % opath)
opath = os.path.normpath(opath)
if not os.path.realpath(os.path.join(cwd, opath)).startswith(cwd):
parser.error(
'--env-prefix %r path is bad, must be relative and not contain `..`.'
% opath)
prefixes.setdefault(key, []).append(opath)
options.env_prefix = prefixes
cipd.validate_cipd_options(parser, options)
install_packages_fn = noop_install_packages
if options.cipd_enabled:
install_packages_fn = lambda run_dir: install_client_and_packages(
run_dir, cipd.parse_package_args(options.cipd_packages),
options.cipd_server, options.cipd_client_package,
options.cipd_client_version, cache_dir=options.cipd_cache)
@contextlib.contextmanager
def install_named_caches(run_dir):
# WARNING: this function depends on "options" variable defined in the outer
# function.
caches = [
(os.path.join(run_dir, unicode(relpath)), name)
for name, relpath in options.named_caches
]
with named_cache_manager.open():
for path, name in caches:
named_cache_manager.install(path, name)
try:
yield
finally:
# Uninstall each named cache, returning it to the cache pool. If an
# uninstall fails for a given cache, it will remain in the task's
# temporary space, get cleaned up by the Swarming bot, and be lost.
#
# If the Swarming bot cannot clean up the cache, it will handle it like
# any other bot file that could not be removed.
with named_cache_manager.open():
for path, name in caches:
try:
named_cache_manager.uninstall(path, name)
except named_cache.Error:
logging.exception('Error while removing named cache %r at %r. '
'The cache will be lost.', path, name)
extra_args = []
command = []
if options.raw_cmd:
command = args
if options.relative_cwd:
a = os.path.normpath(os.path.abspath(options.relative_cwd))
if not a.startswith(os.getcwd()):
parser.error(
'--relative-cwd must not try to escape the working directory')
else:
if options.relative_cwd:
parser.error('--relative-cwd requires --raw-cmd')
extra_args = args
data = TaskData(
command=command,
relative_cwd=options.relative_cwd,
extra_args=extra_args,
isolated_hash=options.isolated,
storage=None,
isolate_cache=isolate_cache,
outputs=options.output,
install_named_caches=install_named_caches,
leak_temp_dir=options.leak_temp_dir,
root_dir=_to_unicode(options.root_dir),
hard_timeout=options.hard_timeout,
grace_period=options.grace_period,
bot_file=options.bot_file,
switch_to_account=options.switch_to_account,
install_packages_fn=install_packages_fn,
use_symlinks=options.use_symlinks,
env=options.env,
env_prefix=options.env_prefix)
try:
if options.isolate_server:
storage = isolateserver.get_storage(
options.isolate_server, options.namespace)
with storage:
data = data._replace(storage=storage)
# Hashing schemes used by |storage| and |isolate_cache| MUST match.
assert storage.hash_algo == isolate_cache.hash_algo
return run_tha_test(data, options.json)
return run_tha_test(data, options.json)
except (cipd.Error, named_cache.Error) as ex:
print >> sys.stderr, ex.message
return 1