def isolated_to_hash(isolate_server, namespace, arg, algo, verbose):
"""Archives a .isolated file if needed.
Returns the file hash to trigger and a bool specifying if it was a file (True)
or a hash (False).
"""
if arg.endswith('.isolated'):
file_hash = isolated_archive(isolate_server, namespace, arg, algo, verbose)
if not file_hash:
on_error.report('Archival failure %s' % arg)
return None, True
return file_hash, True
elif isolated_format.is_valid_hash(arg, algo):
return arg, False
else:
on_error.report('Invalid hash %s' % arg)
return None, False
def isolated_to_hash(arg, algo):
"""Archives a .isolated file if needed.
Returns the file hash to trigger and a bool specifying if it was a file (True)
or a hash (False).
"""
if arg.endswith(".isolated"):
file_hash = isolated_format.hash_file(arg, algo)
if not file_hash:
on_error.report("Archival failure %s" % arg)
return None, True
return file_hash, True
elif isolated_format.is_valid_hash(arg, algo):
return arg, False
else:
on_error.report("Invalid hash %s" % arg)
return None, False
def get_client(service_url,
package_template,
version,
cache_dir,
timeout=None):
"""Returns a context manager that yields a CipdClient. A blocking call.
Upon exit from the context manager, the client binary may be deleted
(if the internal cache is full).
Args:
service_url (str): URL of the CIPD backend.
package_template (str): package name template of the CIPD client.
version (str): version of CIPD client package.
cache_dir: directory to store instance cache, version cache
and a hardlink to the client binary.
timeout (int): if not None, timeout in seconds for this function.
Yields:
CipdClient.
Raises:
Error if CIPD client version cannot be resolved or client cannot be fetched.
"""
timeoutfn = tools.sliding_timeout(timeout)
# Package names are always lower case.
# TODO(maruel): Assert instead?
package_name = package_template.lower().replace('${platform}',
get_platform())
# Resolve version to instance id.
# Is it an instance id already? They look like HEX SHA1.
if isolated_format.is_valid_hash(version, hashlib.sha1):
instance_id = version
elif ':' in version: # it's an immutable tag, cache the resolved version
# version_cache is {hash(package_name, tag) -> instance id} mapping.
# It does not take a lot of disk space.
version_cache = isolateserver.DiskCache(
unicode(os.path.join(cache_dir, 'versions')),
isolateserver.CachePolicies(0, 0, 300),
hashlib.sha1,
trim=True)
with version_cache:
version_cache.cleanup()
# Convert (package_name, version) to a string that may be used as a
# filename in disk cache by hashing it.
version_digest = hashlib.sha1('%s\n%s' %
(package_name, version)).hexdigest()
try:
with version_cache.getfileobj(version_digest) as f:
instance_id = f.read()
except isolateserver.CacheMiss:
instance_id = resolve_version(service_url,
package_name,
version,
timeout=timeoutfn())
version_cache.write(version_digest, instance_id)
else: # it's a ref, hit the backend
instance_id = resolve_version(service_url,
package_name,
version,
timeout=timeoutfn())
# instance_cache is {instance_id -> client binary} mapping.
# It is bounded by 5 client versions.
instance_cache = isolateserver.DiskCache(
unicode(os.path.join(cache_dir, 'clients')),
isolateserver.CachePolicies(0, 0, 5),
hashlib.sha1,
trim=True)
with instance_cache:
instance_cache.cleanup()
if instance_id not in instance_cache:
logging.info('Fetching CIPD client %s:%s', package_name,
instance_id)
fetch_url = get_client_fetch_url(service_url,
package_name,
instance_id,
timeout=timeoutfn())
_fetch_cipd_client(instance_cache, instance_id, fetch_url,
timeoutfn)
# A single host can run multiple swarming bots, but ATM they do not share
# same root bot directory. Thus, it is safe to use the same name for the
# binary.
cipd_bin_dir = unicode(os.path.join(cache_dir, 'bin'))
binary_path = os.path.join(cipd_bin_dir, 'cipd' + EXECUTABLE_SUFFIX)
if fs.isfile(binary_path):
file_path.remove(binary_path)
else:
file_path.ensure_tree(cipd_bin_dir)
with instance_cache.getfileobj(instance_id) as f:
isolateserver.putfile(f, binary_path, 0511) # -r-x--x--x
_ensure_batfile(binary_path)
yield CipdClient(binary_path,
package_name=package_name,
instance_id=instance_id,
service_url=service_url)
def get_client(service_url,
package_template,
version,
cache_dir,
timeout=None):
"""Returns a context manager that yields a CipdClient. A blocking call.
Upon exit from the context manager, the client binary may be deleted
(if the internal cache is full).
Args:
service_url (str): URL of the CIPD backend.
package_template (str): package name template of the CIPD client.
version (str): version of CIPD client package.
cache_dir: directory to store instance cache, version cache
and a hardlink to the client binary.
timeout (int): if not None, timeout in seconds for this function.
Yields:
CipdClient.
Raises:
Error if CIPD client version cannot be resolved or client cannot be fetched.
"""
timeoutfn = tools.sliding_timeout(timeout)
# Package names are always lower case.
# TODO(maruel): Assert instead?
package_name = package_template.lower().replace('${platform}',
get_platform())
# Resolve version to instance id.
# Is it an instance id already? They look like HEX SHA1.
if isolated_format.is_valid_hash(version, hashlib.sha1):
instance_id = version
elif ':' in version: # it's an immutable tag, cache the resolved version
# version_cache is {hash(package_name, tag) -> instance id} mapping.
# It does not take a lot of disk space.
version_cache = local_caching.DiskContentAddressedCache(
six.text_type(os.path.join(cache_dir, 'versions')),
local_caching.CachePolicies(
# 1GiB.
max_cache_size=1024 * 1024 * 1024,
min_free_space=0,
max_items=300,
# 3 weeks.
max_age_secs=21 * 24 * 60 * 60),
trim=True)
# Convert (package_name, version) to a string that may be used as a
# filename in disk cache by hashing it.
version_digest = hashlib.sha1('%s\n%s' %
(package_name, version)).hexdigest()
try:
with version_cache.getfileobj(version_digest) as f:
instance_id = f.read()
except local_caching.CacheMiss:
instance_id = resolve_version(service_url,
package_name,
version,
timeout=timeoutfn())
version_cache.write(version_digest, instance_id)
version_cache.trim()
else: # it's a ref, hit the backend
instance_id = resolve_version(service_url,
package_name,
version,
timeout=timeoutfn())
# instance_cache is {instance_id -> client binary} mapping.
# It is bounded by 5 client versions.
instance_cache = local_caching.DiskContentAddressedCache(
six.text_type(os.path.join(cache_dir, 'clients')),
local_caching.CachePolicies(
# 1GiB.
max_cache_size=1024 * 1024 * 1024,
min_free_space=0,
max_items=10,
# 3 weeks.
max_age_secs=21 * 24 * 60 * 60),
trim=True)
if instance_id not in instance_cache:
logging.info('Fetching CIPD client %s:%s', package_name, instance_id)
fetch_url = get_client_fetch_url(service_url,
package_name,
instance_id,
timeout=timeoutfn())
_fetch_cipd_client(instance_cache, instance_id, fetch_url, timeoutfn)
# A single host can run multiple swarming bots, but they cannot share same
# root bot directory. Thus, it is safe to use the same name for the binary.
cipd_bin_dir = six.text_type(os.path.join(cache_dir, 'bin'))
binary_path = os.path.join(cipd_bin_dir, 'cipd' + EXECUTABLE_SUFFIX)
if fs.isfile(binary_path):
# TODO(maruel): Do not unconditionally remove the binary.
try:
file_path.remove(binary_path)
except WindowsError: # pylint: disable=undefined-variable
# See whether cipd.exe is running for crbug.com/1028781
ret = subprocess42.call(['tasklist.exe'])
if ret:
logging.error('tasklist returns non-zero: %d', ret)
raise
else:
file_path.ensure_tree(cipd_bin_dir)
with instance_cache.getfileobj(instance_id) as f:
isolateserver.putfile(f, binary_path, 0o511) # -r-x--x--x
_ensure_batfile(binary_path)
yield CipdClient(binary_path,
package_name=package_name,
instance_id=instance_id,
service_url=service_url)
instance_cache.trim()