def change_tree_read_only(rootdir, read_only):
"""Changes the tree read-only bits according to the read_only specification.
The flag can be 0, 1 or 2, which will affect the possibility to modify files
and create or delete files.
"""
if read_only == 2:
# Files and directories (except on Windows) are marked read only. This
# inhibits modifying, creating or deleting files in the test directory,
# except on Windows where creating and deleting files is still possible.
file_path.make_tree_read_only(rootdir)
elif read_only == 1:
# Files are marked read only but not the directories. This inhibits
# modifying files but creating or deleting files is still possible.
file_path.make_tree_files_read_only(rootdir)
elif read_only in (0, None):
# Anything can be modified.
# TODO(maruel): This is currently dangerous as long as DiskCache.touch()
# is not yet changed to verify the hash of the content of the files it is
# looking at, so that if a test modifies an input file, the file must be
# deleted.
file_path.make_tree_writeable(rootdir)
else:
raise ValueError(
'change_tree_read_only(%s, %s): Unknown flag %s' %
(rootdir, read_only, read_only))
def change_tree_read_only(rootdir, read_only):
"""Changes the tree read-only bits according to the read_only specification.
The flag can be 0, 1 or 2, which will affect the possibility to modify files
and create or delete files.
"""
if read_only == 2:
# Files and directories (except on Windows) are marked read only. This
# inhibits modifying, creating or deleting files in the test directory,
# except on Windows where creating and deleting files is still possible.
file_path.make_tree_read_only(rootdir)
elif read_only == 1:
# Files are marked read only but not the directories. This inhibits
# modifying files but creating or deleting files is still possible.
file_path.make_tree_files_read_only(rootdir)
elif read_only in (0, None):
# Anything can be modified.
# TODO(maruel): This is currently dangerous as long as DiskCache.touch()
# is not yet changed to verify the hash of the content of the files it is
# looking at, so that if a test modifies an input file, the file must be
# deleted.
file_path.make_tree_writeable(rootdir)
else:
raise ValueError('change_tree_read_only(%s, %s): Unknown flag %s' %
(rootdir, read_only, read_only))
def install_client_and_packages(run_dir,
packages,
service_url,
client_package_name,
client_version,
cache_dir,
timeout=None):
"""Bootstraps CIPD client and installs CIPD packages.
Yields CipdClient, stats, client info and pins (as single CipdInfo object).
Pins and the CIPD client info are in the form of:
[
{
"path": path, "package_name": package_name, "version": version,
},
...
]
(the CIPD client info is a single dictionary instead of a list)
such that they correspond 1:1 to all input package arguments from the command
line. These dictionaries make their all the way back to swarming, where they
become the arguments of CipdPackage.
If 'packages' list is empty, will bootstrap CIPD client, but won't install
any packages.
The bootstrapped client (regardless whether 'packages' list is empty or not),
will be made available to the task via $PATH.
Args:
run_dir (str): root of installation.
packages: packages to install, list [(path, package_name, version), ...].
service_url (str): CIPD server url, e.g.
"https://chrome-infra-packages.appspot.com."
client_package_name (str): CIPD package name of CIPD client.
client_version (str): Version of CIPD client.
cache_dir (str): where to keep cache of cipd clients, packages and tags.
timeout: max duration in seconds that this function can take.
"""
assert cache_dir
timeoutfn = tools.sliding_timeout(timeout)
start = time.time()
cache_dir = os.path.abspath(cache_dir)
cipd_cache_dir = os.path.join(cache_dir,
'cache') # tag and instance caches
run_dir = os.path.abspath(run_dir)
packages = packages or []
get_client_start = time.time()
client_manager = cipd.get_client(service_url,
client_package_name,
client_version,
cache_dir,
timeout=timeoutfn())
with client_manager as client:
get_client_duration = time.time() - get_client_start
package_pins = []
if packages:
package_pins = _install_packages(run_dir, cipd_cache_dir, client,
packages, timeoutfn())
file_path.make_tree_files_read_only(run_dir)
total_duration = time.time() - start
logging.info('Installing CIPD client and packages took %d seconds',
total_duration)
yield CipdInfo(client=client,
cache_dir=cipd_cache_dir,
stats={
'duration': total_duration,
'get_client_duration': get_client_duration,
},
pins={
'client_package': {
'package_name': client.package_name,
'version': client.instance_id,
},
'packages': package_pins,
})
def install_packages(run_dir,
packages,
service_url,
client_package_name,
client_version,
cache_dir=None,
timeout=None):
"""Installs packages. Returns stats, cipd client info and pins.
pins and the cipd client info are in the form of:
[
{
"path": path, "package_name": package_name, "version": version,
},
...
]
(the cipd client info is a single dictionary instead of a list)
such that they correspond 1:1 to all input package arguments from the command
line. These dictionaries make their all the way back to swarming, where they
become the arguments of CipdPackage.
Args:
run_dir (str): root of installation.
packages: packages to install, list [(path, package_name, version), ...]
service_url (str): CIPD server url, e.g.
"https://chrome-infra-packages.appspot.com."
client_package_name (str): CIPD package name of CIPD client.
client_version (str): Version of CIPD client.
cache_dir (str): where to keep cache of cipd clients, packages and tags.
timeout: max duration in seconds that this function can take.
"""
assert cache_dir
if not packages:
return None
timeoutfn = tools.sliding_timeout(timeout)
start = time.time()
cache_dir = os.path.abspath(cache_dir)
run_dir = os.path.abspath(run_dir)
package_pins = [None] * len(packages)
def insert_pin(path, name, version, idx):
path = path.replace(os.path.sep, '/')
package_pins[idx] = {
'package_name': name,
'path': path,
'version': version,
}
get_client_start = time.time()
client_manager = cipd.get_client(service_url,
client_package_name,
client_version,
cache_dir,
timeout=timeoutfn())
by_path = collections.defaultdict(list)
for i, (path, name, version) in enumerate(packages):
path = path.replace('/', os.path.sep)
by_path[path].append((name, version, i))
with client_manager as client:
client_package = {
'package_name': client.package_name,
'version': client.instance_id,
}
get_client_duration = time.time() - get_client_start
for path, pkgs in sorted(by_path.iteritems()):
site_root = os.path.abspath(os.path.join(run_dir, path))
if not site_root.startswith(run_dir):
raise cipd.Error('Invalid CIPD package path "%s"' % path)
# Do not clean site_root before installation because it may contain other
# site roots.
file_path.ensure_tree(site_root, 0770)
pins = client.ensure(site_root,
[(name, vers) for name, vers, _ in pkgs],
cache_dir=os.path.join(
cache_dir, 'cipd_internal'),
timeout=timeoutfn())
for i, pin in enumerate(pins):
insert_pin(path, pin[0], pin[1], pkgs[i][2])
file_path.make_tree_files_read_only(site_root)
total_duration = time.time() - start
logging.info('Installing CIPD client and packages took %d seconds',
total_duration)
assert None not in package_pins
return {
'stats': {
'duration': total_duration,
'get_client_duration': get_client_duration,
},
'cipd_pins': {
'client_package': client_package,
'packages': package_pins,
}
}
