def get_client(service_url,
package_template,
version,
cache_dir,
timeout=None):
"""Returns a context manager that yields a CipdClient. A blocking call.
Upon exit from the context manager, the client binary may be deleted
(if the internal cache is full).
Args:
service_url (str): URL of the CIPD backend.
package_template (str): package name template of the CIPD client.
version (str): version of CIPD client package.
cache_dir: directory to store instance cache, version cache
and a hardlink to the client binary.
timeout (int): if not None, timeout in seconds for this function.
Yields:
CipdClient.
Raises:
Error if CIPD client version cannot be resolved or client cannot be fetched.
"""
timeoutfn = tools.sliding_timeout(timeout)
# Package names are always lower case.
# TODO(maruel): Assert instead?
package_name = package_template.lower().replace('${platform}',
get_platform())
# Resolve version to instance id.
# Is it an instance id already? They look like HEX SHA1.
if isolated_format.is_valid_hash(version, hashlib.sha1):
instance_id = version
elif ':' in version: # it's an immutable tag, cache the resolved version
# version_cache is {hash(package_name, tag) -> instance id} mapping.
# It does not take a lot of disk space.
version_cache = isolateserver.DiskCache(
unicode(os.path.join(cache_dir, 'versions')),
isolateserver.CachePolicies(0, 0, 300),
hashlib.sha1,
trim=True)
with version_cache:
version_cache.cleanup()
# Convert (package_name, version) to a string that may be used as a
# filename in disk cache by hashing it.
version_digest = hashlib.sha1('%s\n%s' %
(package_name, version)).hexdigest()
try:
with version_cache.getfileobj(version_digest) as f:
instance_id = f.read()
except isolateserver.CacheMiss:
instance_id = resolve_version(service_url,
package_name,
version,
timeout=timeoutfn())
version_cache.write(version_digest, instance_id)
else: # it's a ref, hit the backend
instance_id = resolve_version(service_url,
package_name,
version,
timeout=timeoutfn())
# instance_cache is {instance_id -> client binary} mapping.
# It is bounded by 5 client versions.
instance_cache = isolateserver.DiskCache(
unicode(os.path.join(cache_dir, 'clients')),
isolateserver.CachePolicies(0, 0, 5),
hashlib.sha1,
trim=True)
with instance_cache:
instance_cache.cleanup()
if instance_id not in instance_cache:
logging.info('Fetching CIPD client %s:%s', package_name,
instance_id)
fetch_url = get_client_fetch_url(service_url,
package_name,
instance_id,
timeout=timeoutfn())
_fetch_cipd_client(instance_cache, instance_id, fetch_url,
timeoutfn)
# A single host can run multiple swarming bots, but ATM they do not share
# same root bot directory. Thus, it is safe to use the same name for the
# binary.
cipd_bin_dir = unicode(os.path.join(cache_dir, 'bin'))
binary_path = os.path.join(cipd_bin_dir, 'cipd' + EXECUTABLE_SUFFIX)
if fs.isfile(binary_path):
file_path.remove(binary_path)
else:
file_path.ensure_tree(cipd_bin_dir)
with instance_cache.getfileobj(instance_id) as f:
isolateserver.putfile(f, binary_path, 0511) # -r-x--x--x
_ensure_batfile(binary_path)
yield CipdClient(binary_path,
package_name=package_name,
instance_id=instance_id,
service_url=service_url)
def ensure(self,
site_root,
packages,
cache_dir=None,
tmp_dir=None,
timeout=None):
"""Ensures that packages installed in |site_root| equals |packages| set.
Blocking call.
Args:
site_root (str): where to install packages.
packages: dict of subdir -> list of (package_template, version) tuples.
cache_dir (str): if set, cache dir for cipd binary own cache.
Typically contains packages and tags.
tmp_dir (str): if not None, dir for temp files.
timeout (int): if not None, timeout in seconds for this function to run.
Returns:
Pinned packages in the form of {subdir: [(package_name, package_id)]},
which correspond 1:1 with the input packages argument.
Raises:
Error if could not install packages or timed out.
"""
timeoutfn = tools.sliding_timeout(timeout)
logging.info('Installing packages %r into %s', packages, site_root)
ensure_file_handle, ensure_file_path = tempfile.mkstemp(
dir=tmp_dir, prefix=u'cipd-ensure-file-', suffix='.txt')
json_out_file_handle, json_file_path = tempfile.mkstemp(
dir=tmp_dir, prefix=u'cipd-ensure-result-', suffix='.json')
os.close(json_out_file_handle)
try:
try:
for subdir, pkgs in sorted(packages.iteritems()):
if '\n' in subdir:
raise Error(
'Could not install packages; subdir %r contains newline'
% subdir)
os.write(ensure_file_handle, '@Subdir %s\n' % (subdir, ))
for pkg, version in pkgs:
os.write(ensure_file_handle,
'%s %s\n' % (pkg, version))
finally:
os.close(ensure_file_handle)
cmd = [
self.binary_path,
'ensure',
'-root',
site_root,
'-ensure-file',
ensure_file_path,
'-verbose', # this is safe because cipd-ensure does not print a lot
'-json-output',
json_file_path,
]
if cache_dir:
cmd += ['-cache-dir', cache_dir]
if self.service_url:
cmd += ['-service-url', self.service_url]
logging.debug('Running %r', cmd)
process = subprocess42.Popen(cmd,
stdout=subprocess42.PIPE,
stderr=subprocess42.PIPE)
output = []
for pipe_name, line in process.yield_any_line(timeout=0.1):
to = timeoutfn()
if to is not None and to <= 0:
raise Error(
'Could not install packages; took more than %d seconds'
% timeout)
if not pipe_name:
# stdout or stderr was closed, but yield_any_line still may have
# something to yield.
continue
output.append(line)
if pipe_name == 'stderr':
logging.debug('cipd client: %s', line)
else:
logging.info('cipd client: %s', line)
exit_code = process.wait(timeout=timeoutfn())
if exit_code != 0:
raise Error(
'Could not install packages; exit code %d\noutput:%s' %
(exit_code, '\n'.join(output)))
with open(json_file_path) as jfile:
result_json = json.load(jfile)
return {
subdir: [(x['package'], x['instance_id']) for x in pins]
for subdir, pins in result_json['result'].iteritems()
}
finally:
fs.remove(ensure_file_path)
fs.remove(json_file_path)
def install_client_and_packages(run_dir,
packages,
service_url,
client_package_name,
client_version,
cache_dir,
timeout=None):
"""Bootstraps CIPD client and installs CIPD packages.
Yields CipdClient, stats, client info and pins (as single CipdInfo object).
Pins and the CIPD client info are in the form of:
[
{
"path": path, "package_name": package_name, "version": version,
},
...
]
(the CIPD client info is a single dictionary instead of a list)
such that they correspond 1:1 to all input package arguments from the command
line. These dictionaries make their all the way back to swarming, where they
become the arguments of CipdPackage.
If 'packages' list is empty, will bootstrap CIPD client, but won't install
any packages.
The bootstrapped client (regardless whether 'packages' list is empty or not),
will be made available to the task via $PATH.
Args:
run_dir (str): root of installation.
packages: packages to install, list [(path, package_name, version), ...].
service_url (str): CIPD server url, e.g.
"https://chrome-infra-packages.appspot.com."
client_package_name (str): CIPD package name of CIPD client.
client_version (str): Version of CIPD client.
cache_dir (str): where to keep cache of cipd clients, packages and tags.
timeout: max duration in seconds that this function can take.
"""
assert cache_dir
timeoutfn = tools.sliding_timeout(timeout)
start = time.time()
cache_dir = os.path.abspath(cache_dir)
cipd_cache_dir = os.path.join(cache_dir,
'cache') # tag and instance caches
run_dir = os.path.abspath(run_dir)
packages = packages or []
get_client_start = time.time()
client_manager = cipd.get_client(service_url,
client_package_name,
client_version,
cache_dir,
timeout=timeoutfn())
with client_manager as client:
get_client_duration = time.time() - get_client_start
package_pins = []
if packages:
package_pins = _install_packages(run_dir, cipd_cache_dir, client,
packages, timeoutfn())
file_path.make_tree_files_read_only(run_dir)
total_duration = time.time() - start
logging.info('Installing CIPD client and packages took %d seconds',
total_duration)
yield CipdInfo(client=client,
cache_dir=cipd_cache_dir,
stats={
'duration': total_duration,
'get_client_duration': get_client_duration,
},
pins={
'client_package': {
'package_name': client.package_name,
'version': client.instance_id,
},
'packages': package_pins,
})
def get_client(service_url,
package_template,
version,
cache_dir,
timeout=None):
"""Returns a context manager that yields a CipdClient. A blocking call.
Upon exit from the context manager, the client binary may be deleted
(if the internal cache is full).
Args:
service_url (str): URL of the CIPD backend.
package_template (str): package name template of the CIPD client.
version (str): version of CIPD client package.
cache_dir: directory to store instance cache, version cache
and a hardlink to the client binary.
timeout (int): if not None, timeout in seconds for this function.
Yields:
CipdClient.
Raises:
Error if CIPD client version cannot be resolved or client cannot be fetched.
"""
timeoutfn = tools.sliding_timeout(timeout)
# Package names are always lower case.
# TODO(maruel): Assert instead?
package_name = package_template.lower().replace('${platform}',
get_platform())
# Resolve version to instance id.
# Is it an instance id already? They look like HEX SHA1.
if isolated_format.is_valid_hash(version, hashlib.sha1):
instance_id = version
elif ':' in version: # it's an immutable tag, cache the resolved version
# version_cache is {hash(package_name, tag) -> instance id} mapping.
# It does not take a lot of disk space.
version_cache = local_caching.DiskContentAddressedCache(
six.text_type(os.path.join(cache_dir, 'versions')),
local_caching.CachePolicies(
# 1GiB.
max_cache_size=1024 * 1024 * 1024,
min_free_space=0,
max_items=300,
# 3 weeks.
max_age_secs=21 * 24 * 60 * 60),
trim=True)
# Convert (package_name, version) to a string that may be used as a
# filename in disk cache by hashing it.
version_digest = hashlib.sha1('%s\n%s' %
(package_name, version)).hexdigest()
try:
with version_cache.getfileobj(version_digest) as f:
instance_id = f.read()
except local_caching.CacheMiss:
instance_id = resolve_version(service_url,
package_name,
version,
timeout=timeoutfn())
version_cache.write(version_digest, instance_id)
version_cache.trim()
else: # it's a ref, hit the backend
instance_id = resolve_version(service_url,
package_name,
version,
timeout=timeoutfn())
# instance_cache is {instance_id -> client binary} mapping.
# It is bounded by 5 client versions.
instance_cache = local_caching.DiskContentAddressedCache(
six.text_type(os.path.join(cache_dir, 'clients')),
local_caching.CachePolicies(
# 1GiB.
max_cache_size=1024 * 1024 * 1024,
min_free_space=0,
max_items=10,
# 3 weeks.
max_age_secs=21 * 24 * 60 * 60),
trim=True)
if instance_id not in instance_cache:
logging.info('Fetching CIPD client %s:%s', package_name, instance_id)
fetch_url = get_client_fetch_url(service_url,
package_name,
instance_id,
timeout=timeoutfn())
_fetch_cipd_client(instance_cache, instance_id, fetch_url, timeoutfn)
# A single host can run multiple swarming bots, but they cannot share same
# root bot directory. Thus, it is safe to use the same name for the binary.
cipd_bin_dir = six.text_type(os.path.join(cache_dir, 'bin'))
binary_path = os.path.join(cipd_bin_dir, 'cipd' + EXECUTABLE_SUFFIX)
if fs.isfile(binary_path):
# TODO(maruel): Do not unconditionally remove the binary.
try:
file_path.remove(binary_path)
except WindowsError: # pylint: disable=undefined-variable
# See whether cipd.exe is running for crbug.com/1028781
ret = subprocess42.call(['tasklist.exe'])
if ret:
logging.error('tasklist returns non-zero: %d', ret)
raise
else:
file_path.ensure_tree(cipd_bin_dir)
with instance_cache.getfileobj(instance_id) as f:
isolateserver.putfile(f, binary_path, 0o511) # -r-x--x--x
_ensure_batfile(binary_path)
yield CipdClient(binary_path,
package_name=package_name,
instance_id=instance_id,
service_url=service_url)
instance_cache.trim()
def install_packages(run_dir,
packages,
service_url,
client_package_name,
client_version,
cache_dir=None,
timeout=None):
"""Installs packages. Returns stats, cipd client info and pins.
pins and the cipd client info are in the form of:
[
{
"path": path, "package_name": package_name, "version": version,
},
...
]
(the cipd client info is a single dictionary instead of a list)
such that they correspond 1:1 to all input package arguments from the command
line. These dictionaries make their all the way back to swarming, where they
become the arguments of CipdPackage.
Args:
run_dir (str): root of installation.
packages: packages to install, list [(path, package_name, version), ...]
service_url (str): CIPD server url, e.g.
"https://chrome-infra-packages.appspot.com."
client_package_name (str): CIPD package name of CIPD client.
client_version (str): Version of CIPD client.
cache_dir (str): where to keep cache of cipd clients, packages and tags.
timeout: max duration in seconds that this function can take.
"""
assert cache_dir
if not packages:
return None
timeoutfn = tools.sliding_timeout(timeout)
start = time.time()
cache_dir = os.path.abspath(cache_dir)
run_dir = os.path.abspath(run_dir)
package_pins = [None] * len(packages)
def insert_pin(path, name, version, idx):
path = path.replace(os.path.sep, '/')
package_pins[idx] = {
'package_name': name,
'path': path,
'version': version,
}
get_client_start = time.time()
client_manager = cipd.get_client(service_url,
client_package_name,
client_version,
cache_dir,
timeout=timeoutfn())
by_path = collections.defaultdict(list)
for i, (path, name, version) in enumerate(packages):
path = path.replace('/', os.path.sep)
by_path[path].append((name, version, i))
with client_manager as client:
client_package = {
'package_name': client.package_name,
'version': client.instance_id,
}
get_client_duration = time.time() - get_client_start
for path, pkgs in sorted(by_path.iteritems()):
site_root = os.path.abspath(os.path.join(run_dir, path))
if not site_root.startswith(run_dir):
raise cipd.Error('Invalid CIPD package path "%s"' % path)
# Do not clean site_root before installation because it may contain other
# site roots.
file_path.ensure_tree(site_root, 0770)
pins = client.ensure(site_root,
[(name, vers) for name, vers, _ in pkgs],
cache_dir=os.path.join(
cache_dir, 'cipd_internal'),
timeout=timeoutfn())
for i, pin in enumerate(pins):
insert_pin(path, pin[0], pin[1], pkgs[i][2])
file_path.make_tree_files_read_only(site_root)
total_duration = time.time() - start
logging.info('Installing CIPD client and packages took %d seconds',
total_duration)
assert None not in package_pins
return {
'stats': {
'duration': total_duration,
'get_client_duration': get_client_duration,
},
'cipd_pins': {
'client_package': client_package,
'packages': package_pins,
}
}
def ensure(self,
site_root,
packages,
cache_dir=None,
tmp_dir=None,
timeout=None):
"""Ensures that packages installed in |site_root| equals |packages| set.
Blocking call.
Args:
site_root (str): where to install packages.
packages: list of (package_template, version) tuples.
cache_dir (str): if set, cache dir for cipd binary own cache.
Typically contains packages and tags.
tmp_dir (str): if not None, dir for temp files.
timeout (int): if not None, timeout in seconds for this function to run.
Returns:
Pinned packages in the form of [(package_name, package_id)], which
correspond 1:1 with the input packages argument.
Raises:
Error if could not install packages or timed out.
"""
timeoutfn = tools.sliding_timeout(timeout)
logging.info('Installing packages %r into %s', packages, site_root)
list_file_handle, list_file_path = tempfile.mkstemp(
dir=tmp_dir, prefix=u'cipd-ensure-list-', suffix='.txt')
json_out_file_handle, json_file_path = tempfile.mkstemp(
dir=tmp_dir, prefix=u'cipd-ensure-result-', suffix='.json')
os.close(json_out_file_handle)
try:
try:
for pkg, version in packages:
pkg = render_package_name_template(pkg)
os.write(list_file_handle, '%s %s\n' % (pkg, version))
finally:
os.close(list_file_handle)
cmd = [
self.binary_path,
'ensure',
'-root',
site_root,
'-list',
list_file_path,
'-verbose', # this is safe because cipd-ensure does not print a lot
'-json-output',
json_file_path,
]
if cache_dir:
cmd += ['-cache-dir', cache_dir]
if self.service_url:
cmd += ['-service-url', self.service_url]
logging.debug('Running %r', cmd)
process = subprocess42.Popen(cmd,
stdout=subprocess42.PIPE,
stderr=subprocess42.PIPE)
output = []
for pipe_name, line in process.yield_any_line(timeout=0.1):
to = timeoutfn()
if to is not None and to <= 0:
raise Error(
'Could not install packages; took more than %d seconds'
% timeout)
if not pipe_name:
# stdout or stderr was closed, but yield_any_line still may have
# something to yield.
continue
output.append(line)
if pipe_name == 'stderr':
logging.debug('cipd client: %s', line)
else:
logging.info('cipd client: %s', line)
exit_code = process.wait(timeout=timeoutfn())
if exit_code != 0:
raise Error(
'Could not install packages; exit code %d\noutput:%s' %
(exit_code, '\n'.join(output)))
with open(json_file_path) as jfile:
result_json = json.load(jfile)
# TEMPORARY(iannucci): this code handles cipd <1.4 and cipd >=1.5
# formatted ensure result formats. Cipd 1.5 added support for subdirs, and
# as part of the transition, the result of the ensure command needed to
# change. To ease the transition, we always return data as-if we're using
# the new format. Once cipd 1.5+ is deployed everywhere, this type switch
# can be removed.
if isinstance(result_json['result'], dict):
# cipd 1.5
return {
subdir: [(x['package'], x['instance_id']) for x in pins]
for subdir, pins in result_json['result'].iteritems()
}
else:
# cipd 1.4
return {
"": [(x['package'], x['instance_id'])
for x in result_json['result']],
}
finally:
fs.remove(list_file_path)
fs.remove(json_file_path)
