def _copy_package_include_modules(src_package, dest_pkg_repo, overrides=None):
src_include_modules_path = \
os.path.join(src_package.base, IncludeModuleManager.include_modules_subpath)
if not os.path.exists(src_include_modules_path):
return
dest_package_name = overrides.get("name") or src_package.name
dest_package_version = overrides.get("version") or src_package.version
pkg_install_path = dest_pkg_repo.get_package_payload_path(
package_name=dest_package_name, package_version=dest_package_version)
dest_include_modules_path = \
os.path.join(pkg_install_path, IncludeModuleManager.include_modules_subpath)
last_dir = get_existing_path(
dest_include_modules_path,
topmost_path=os.path.dirname(pkg_install_path))
if last_dir:
ctxt = make_path_writable(last_dir)
else:
ctxt = with_noop()
with ctxt:
safe_makedirs(dest_include_modules_path)
additive_copytree(src_include_modules_path, dest_include_modules_path)
def _install_include_modules(self, install_path):
# install 'include' sourcefiles, used by funcs decorated with @include
if not self.package.includes:
return
install_path = install_path or self.package.config.local_packages_path
base_path = self.get_package_install_path(install_path)
path = os.path.join(base_path,
IncludeModuleManager.include_modules_subpath)
safe_makedirs(path)
definition_python_path = self.package.config.package_definition_python_path
for name in self.package.includes:
filepath = os.path.join(definition_python_path, name) + ".py"
with open(filepath, "rb") as f:
txt = f.read().strip()
uuid = sha1(txt).hexdigest()
dest_filepath = os.path.join(path, "%s.py" % name)
shutil.copy(filepath, dest_filepath) # overwrite if exists
sha1_filepath = os.path.join(path, "%s.sha1" % name)
with open(sha1_filepath, "w") as f: # overwrite if exists
f.write(uuid)
def _build_variant_base(self,
variant,
build_type,
install_path=None,
clean=False,
install=False,
**kwargs):
# create build/install paths
install_path = install_path or self.package.config.local_packages_path
variant_install_path = self.get_package_install_path(install_path)
variant_build_path = self.build_path
if variant.subpath:
variant_build_path = os.path.join(variant_build_path,
variant.subpath)
variant_install_path = os.path.join(variant_install_path,
variant.subpath)
# create directories (build, install)
if clean and os.path.exists(variant_build_path):
shutil.rmtree(variant_build_path)
safe_makedirs(variant_build_path)
if install:
# inform package repo that a variant is about to be built/installed
pkg_repo = package_repository_manager.get_repository(install_path)
pkg_repo.pre_variant_install(variant.resource)
if not os.path.exists(variant_install_path):
safe_makedirs(variant_install_path)
# create build environment
context, rxt_filepath = self.create_build_context(
variant=variant,
build_type=build_type,
build_path=variant_build_path)
# run build system
build_system_name = self.build_system.name()
self._print("\nInvoking %s build system...", build_system_name)
build_result = self.build_system.build(
context=context,
variant=variant,
build_path=variant_build_path,
install_path=variant_install_path,
install=install,
build_type=build_type)
if not build_result.get("success"):
raise BuildError("The %s build system failed." % build_system_name)
if install:
# install some files for debugging purposes
extra_files = build_result.get("extra_files", []) + [rxt_filepath]
for file_ in extra_files:
copy_or_replace(file_, variant_install_path)
return build_result
def _copy_package_include_modules(src_package, dest_pkg_repo, overrides=None):
src_include_modules_path = \
os.path.join(src_package.base, IncludeModuleManager.include_modules_subpath)
if not os.path.exists(src_include_modules_path):
return
dest_package_name = overrides.get("name") or src_package.name
dest_package_version = overrides.get("version") or src_package.version
pkg_install_path = dest_pkg_repo.get_package_payload_path(
package_name=dest_package_name,
package_version=dest_package_version
)
dest_include_modules_path = \
os.path.join(pkg_install_path, IncludeModuleManager.include_modules_subpath)
last_dir = get_existing_path(dest_include_modules_path,
topmost_path=os.path.dirname(pkg_install_path))
if last_dir:
ctxt = make_path_writable(last_dir)
else:
ctxt = with_noop()
with ctxt:
safe_makedirs(dest_include_modules_path)
additive_copytree(src_include_modules_path, dest_include_modules_path)
def _copy_variant_payload(src_variant,
dest_pkg_repo,
shallow=False,
overrides=None):
# Get payload path of source variant. For some types (eg from a "memory"
# type repo) there may not be a root.
#
variant_root = getattr(src_variant, "root", None)
if not variant_root:
raise PackageCopyError(
"Cannot copy source variant %s - it is a type of variant that "
"does not have a root.", src_variant.uri)
if not os.path.isdir(variant_root):
raise PackageCopyError(
"Cannot copy source variant %s - its root does not appear to "
"be present on disk (%s).", src_variant.uri, variant_root)
dest_variant_name = overrides.get("name") or src_variant.name
dest_variant_version = overrides.get("version") or src_variant.version
# determine variant installation path
variant_install_path = dest_pkg_repo.get_package_payload_path(
package_name=dest_variant_name, package_version=dest_variant_version)
if src_variant.subpath:
variant_install_path = os.path.join(variant_install_path,
src_variant.subpath)
# perform the copy/symlinking
if shallow:
maybe_symlink = replacing_symlink
else:
maybe_symlink = replacing_copy
if src_variant.subpath:
# symlink/copy the last install dir to the variant root
safe_makedirs(os.path.dirname(variant_install_path))
maybe_symlink(variant_root, variant_install_path)
else:
safe_makedirs(variant_install_path)
# copy all files, and symlink/copy all dirs within the variant
for name in os.listdir(variant_root):
src_path = os.path.join(variant_root, name)
dest_path = os.path.join(variant_install_path, name)
if os.path.isdir(src_path) and not os.path.islink(src_path):
maybe_symlink(src_path, dest_path)
else:
replacing_copy(src_path, dest_path)
def _copy_package_include_modules(src_package, dest_pkg_repo, overrides=None):
src_include_modules_path = \
os.path.join(src_package.base, IncludeModuleManager.include_modules_subpath)
if not os.path.exists(src_include_modules_path):
return
dest_package_name = overrides.get("name") or src_package.name
dest_package_version = overrides.get("version") or src_package.version
pkg_install_path = dest_pkg_repo.get_package_payload_path(
package_name=dest_package_name, package_version=dest_package_version)
dest_include_modules_path = \
os.path.join(pkg_install_path, IncludeModuleManager.include_modules_subpath)
safe_makedirs(dest_include_modules_path)
additive_copytree(src_include_modules_path, dest_include_modules_path)
def _install_include_modules(self, install_path):
# install 'include' sourcefiles, used by funcs decorated with @include
if not self.package.includes:
return
install_path = install_path or self.package.config.local_packages_path
base_path = self.get_package_install_path(install_path)
path = os.path.join(base_path, IncludeModuleManager.include_modules_subpath)
safe_makedirs(path)
definition_python_path = self.package.config.package_definition_python_path
for name in self.package.includes:
filepath = os.path.join(definition_python_path, name) + ".py"
with open(filepath) as f:
txt = f.read().strip()
uuid = sha1(txt).hexdigest()
dest_filepath = os.path.join(path, "%s-%s.py" % (name, uuid))
if not os.path.exists(dest_filepath):
shutil.copy(filepath, dest_filepath)
def __init__(self, path):
"""Create a package cache.
Args:
path (str): Path on disk, must exist.
"""
if not os.path.isdir(path):
raise PackageCacheError("Not a directory: %s" % path)
self.path = path
# make dirs for internal use
safe_makedirs(self._log_dir)
safe_makedirs(self._pending_dir)
safe_makedirs(self._remove_dir)
def _copy_variant_payload(src_variant,
dest_pkg_repo,
shallow=False,
follow_symlinks=False,
overrides=None,
verbose=False):
# Get payload path of source variant. For some types (eg from a "memory"
# type repo) there may not be a root.
#
variant_root = getattr(src_variant, "root", None)
if not variant_root:
raise PackageCopyError(
"Cannot copy source variant %s - it is a type of variant that "
"does not have a root.", src_variant.uri)
if not os.path.isdir(variant_root):
raise PackageCopyError(
"Cannot copy source variant %s - its root does not appear to "
"be present on disk (%s).", src_variant.uri, variant_root)
dest_variant_name = overrides.get("name") or src_variant.name
dest_variant_version = overrides.get("version") or src_variant.version
# determine variant installation path
dest_pkg_payload_path = dest_pkg_repo.get_package_payload_path(
package_name=dest_variant_name, package_version=dest_variant_version)
if src_variant.subpath:
variant_install_path = os.path.join(dest_pkg_payload_path,
src_variant.subpath)
else:
variant_install_path = dest_pkg_payload_path
# get ready for copy/symlinking
copy_func = partial(replacing_copy, follow_symlinks=follow_symlinks)
if shallow:
maybe_symlink = replacing_symlink
else:
maybe_symlink = copy_func
# possibly make install path temporarily writable
last_dir = get_existing_path(
variant_install_path,
topmost_path=os.path.dirname(dest_pkg_payload_path))
if last_dir:
ctxt = make_path_writable(last_dir)
else:
ctxt = with_noop()
# copy the variant payload
with ctxt:
safe_makedirs(variant_install_path)
# determine files not to copy
skip_files = []
if src_variant.subpath:
# Detect overlapped variants. This is the case where one variant subpath
# might be A, and another is A/B. We must ensure that A/B is not created
# as a symlink during shallow install of variant A - that would then
# cause A/B payload to be installed back into original package, possibly
# corrupting it.
#
# Here we detect this case, and create a list of dirs not to copy/link,
# because they are in fact a subpath dir for another variant.
#
skip_files.extend(_get_overlapped_variant_dirs(src_variant))
else:
# just skip package definition file
for name in config.plugins.package_repository.filesystem.package_filenames:
for fmt in (FileFormat.py, FileFormat.yaml):
filename = name + '.' + fmt.extension
skip_files.append(filename)
# copy/link all topmost files within the variant root
for name in os.listdir(variant_root):
if name in skip_files:
filepath = os.path.join(variant_root, name)
if verbose:
if src_variant.subpath:
msg = ("Did not copy %s - this is part of an "
"overlapping variant's root path.")
else:
msg = "Did not copy package definition file %s"
print_info(msg, filepath)
continue
src_path = os.path.join(variant_root, name)
dest_path = os.path.join(variant_install_path, name)
if os.path.islink(src_path):
copy_func(src_path, dest_path)
else:
maybe_symlink(src_path, dest_path)
# copy permissions of source variant dirs onto dest
src_package = src_variant.parent
src_pkg_repo = src_package.repository
src_pkg_payload_path = src_pkg_repo.get_package_payload_path(
package_name=src_package.name, package_version=src_package.version)
shutil.copystat(src_pkg_payload_path, dest_pkg_payload_path)
subpath = src_variant.subpath
while subpath:
src_path = os.path.join(src_pkg_payload_path, subpath)
dest_path = os.path.join(dest_pkg_payload_path, subpath)
shutil.copystat(src_path, dest_path)
subpath = os.path.dirname(subpath)
def _build_variant_base(self,
variant,
build_type,
install_path=None,
clean=False,
install=False,
**kwargs):
# create build/install paths
install_path = install_path or self.package.config.local_packages_path
package_install_path = self.get_package_install_path(install_path)
variant_build_path = self.build_path
if variant.index is None:
variant_install_path = package_install_path
else:
subpath = variant._non_shortlinked_subpath
variant_build_path = os.path.join(variant_build_path, subpath)
variant_install_path = os.path.join(package_install_path, subpath)
# create directories (build, install)
if clean and os.path.exists(variant_build_path):
self._rmtree(variant_build_path)
safe_makedirs(variant_build_path)
# find last dir of installation path that exists, and possibly make it
# writable during variant installation
#
last_dir = get_existing_path(variant_install_path,
topmost_path=install_path)
if last_dir:
ctxt = make_path_writable(last_dir)
else:
ctxt = with_noop()
with ctxt:
if install:
# inform package repo that a variant is about to be built/installed
pkg_repo = package_repository_manager.get_repository(
install_path)
pkg_repo.pre_variant_install(variant.resource)
if not os.path.exists(variant_install_path):
safe_makedirs(variant_install_path)
# if hashed variants are enabled, create the variant shortlink
if variant.parent.hashed_variants:
try:
# create the dir containing all shortlinks
base_shortlinks_path = os.path.join(
package_install_path,
variant.parent.config.variant_shortlinks_dirname)
safe_makedirs(base_shortlinks_path)
# create the shortlink
rel_variant_path = os.path.relpath(
variant_install_path, base_shortlinks_path)
create_unique_base26_symlink(base_shortlinks_path,
rel_variant_path)
except Exception as e:
# Treat any error as warning - lack of shortlink is not
# a breaking issue, it just means the variant root path
# will be long.
#
print_warning(
"Error creating variant shortlink for %s: %s: %s",
variant_install_path, e.__class__.__name__, e)
# Re-evaluate the variant, so that variables such as 'building' and
# 'build_variant_index' are set, and any early-bound package attribs
# are re-evaluated wrt these vars. This is done so that attribs such as
# 'requires' can change depending on whether a build is occurring or not.
#
# Note that this re-evaluated variant is ONLY used here, for the purposes
# of creating the build context. The variant that is actually installed
# is the one evaluated where 'building' is False.
#
re_evaluated_package = variant.parent.get_reevaluated({
"building":
True,
"build_variant_index":
variant.index or 0,
"build_variant_requires":
variant.variant_requires
})
re_evaluated_variant = re_evaluated_package.get_variant(
variant.index)
# create build environment (also creates build.rxt file)
context, rxt_filepath = self.create_build_context(
variant=re_evaluated_variant,
build_type=build_type,
build_path=variant_build_path)
# list of extra files (build.rxt etc) that are installed if an
# installation is taking place
#
extra_install_files = [rxt_filepath]
# create variant.json file. This identifies which variant this is.
# This is important for hashed variants, where it is not obvious
# which variant is in which root path. The file is there for
# debugging purposes only.
#
if variant.index is not None:
data = {
"index": variant.index,
"data": variant.parent.data["variants"][variant.index]
}
filepath = os.path.join(variant_build_path, "variant.json")
extra_install_files.append(filepath)
with open(filepath, 'w') as f:
json.dump(data, f, indent=2)
# run build system
build_system_name = self.build_system.name()
self._print("\nInvoking %s build system...", build_system_name)
build_result = self.build_system.build(
context=context,
variant=variant,
build_path=variant_build_path,
install_path=variant_install_path,
install=install,
build_type=build_type)
if not build_result.get("success"):
# delete the possibly partially installed variant payload
if install:
self._rmtree(variant_install_path)
raise BuildError("The %s build system failed." %
build_system_name)
if install:
# add some installation details to build result
build_result.update({
"package_install_path":
package_install_path,
"variant_install_path":
variant_install_path
})
# the build system can also specify extra files that need to
# be installed
filepaths = build_result.get("extra_files")
if filepaths:
extra_install_files.extend(filepaths)
# install extra files
for file_ in extra_install_files:
copy_or_replace(file_, variant_install_path)
# Install include modules. Note that this doesn't need to be done
# multiple times, but for subsequent variants it has no effect.
#
self._install_include_modules(install_path)
return build_result
def _build_variant_base(self, variant, build_type, install_path=None,
clean=False, install=False, **kwargs):
# create build/install paths
install_path = install_path or self.package.config.local_packages_path
package_install_path = self.get_package_install_path(install_path)
variant_build_path = self.build_path
if variant.index is None:
variant_install_path = package_install_path
else:
subpath = variant._non_shortlinked_subpath
variant_build_path = os.path.join(variant_build_path, subpath)
variant_install_path = os.path.join(package_install_path, subpath)
# create directories (build, install)
if clean and os.path.exists(variant_build_path):
shutil.rmtree(variant_build_path)
safe_makedirs(variant_build_path)
# find last dir of installation path that exists, and possibly make it
# writable during variant installation
#
last_dir = get_existing_path(variant_install_path,
topmost_path=install_path)
if last_dir:
ctxt = make_path_writable(last_dir)
else:
ctxt = with_noop()
with ctxt:
if install:
# inform package repo that a variant is about to be built/installed
pkg_repo = package_repository_manager.get_repository(install_path)
pkg_repo.pre_variant_install(variant.resource)
if not os.path.exists(variant_install_path):
safe_makedirs(variant_install_path)
# if hashed variants are enabled, create the variant shortlink
if variant.parent.hashed_variants:
try:
# create the dir containing all shortlinks
base_shortlinks_path = os.path.join(
package_install_path,
variant.parent.config.variant_shortlinks_dirname
)
safe_makedirs(base_shortlinks_path)
# create the shortlink
rel_variant_path = os.path.relpath(
variant_install_path, base_shortlinks_path)
create_unique_base26_symlink(
base_shortlinks_path, rel_variant_path)
except Exception as e:
# Treat any error as warning - lack of shortlink is not
# a breaking issue, it just means the variant root path
# will be long.
#
print_warning(
"Error creating variant shortlink for %s: %s: %s",
variant_install_path, e.__class__.__name__, e
)
# Re-evaluate the variant, so that variables such as 'building' and
# 'build_variant_index' are set, and any early-bound package attribs
# are re-evaluated wrt these vars. This is done so that attribs such as
# 'requires' can change depending on whether a build is occurring or not.
#
# Note that this re-evaluated variant is ONLY used here, for the purposes
# of creating the build context. The variant that is actually installed
# is the one evaluated where 'building' is False.
#
re_evaluated_package = variant.parent.get_reevaluated({
"building": True,
"build_variant_index": variant.index or 0,
"build_variant_requires": variant.variant_requires
})
re_evaluated_variant = re_evaluated_package.get_variant(variant.index)
# create build environment
context, rxt_filepath = self.create_build_context(
variant=re_evaluated_variant,
build_type=build_type,
build_path=variant_build_path)
# run build system
build_system_name = self.build_system.name()
self._print("\nInvoking %s build system...", build_system_name)
build_result = self.build_system.build(
context=context,
variant=variant,
build_path=variant_build_path,
install_path=variant_install_path,
install=install,
build_type=build_type)
if not build_result.get("success"):
raise BuildError("The %s build system failed." % build_system_name)
if install:
# Install the 'variant.json' file, which identifies which variant
# this is. This is important for hashed variants, where it is not
# obvious which variant is in which root path. The file is there
# for debugging purposes only.
#
if variant.index is not None:
data = {
"index": variant.index,
"data": variant.parent.data["variants"][variant.index]
}
filepath = os.path.join(variant_install_path, "variant.json")
with open(filepath, 'w') as f:
json.dump(data, f, indent=2)
# install some files for debugging purposes (incl build.rxt)
extra_files = build_result.get("extra_files", [])
if rxt_filepath:
extra_files = extra_files + [rxt_filepath]
for file_ in extra_files:
copy_or_replace(file_, variant_install_path)
# Install include modules. Note that this doesn't need to be done
# multiple times, but for subsequent variants it has no effect.
#
self._install_include_modules(install_path)
return build_result
def _copy_variant_payload(src_variant, dest_pkg_repo, shallow=False,
follow_symlinks=False, overrides=None, verbose=False):
# Get payload path of source variant. For some types (eg from a "memory"
# type repo) there may not be a root.
#
variant_root = getattr(src_variant, "root", None)
if not variant_root:
raise PackageCopyError(
"Cannot copy source variant %s - it is a type of variant that "
"does not have a root.", src_variant.uri
)
if not os.path.isdir(variant_root):
raise PackageCopyError(
"Cannot copy source variant %s - its root does not appear to "
"be present on disk (%s).", src_variant.uri, variant_root
)
dest_variant_name = overrides.get("name") or src_variant.name
dest_variant_version = overrides.get("version") or src_variant.version
# determine variant installation path
dest_pkg_payload_path = dest_pkg_repo.get_package_payload_path(
package_name=dest_variant_name,
package_version=dest_variant_version
)
if src_variant.subpath:
variant_install_path = os.path.join(dest_pkg_payload_path,
src_variant.subpath)
else:
variant_install_path = dest_pkg_payload_path
# get ready for copy/symlinking
copy_func = partial(replacing_copy,
follow_symlinks=follow_symlinks)
if shallow:
maybe_symlink = replacing_symlink
else:
maybe_symlink = copy_func
# possibly make install path temporarily writable
last_dir = get_existing_path(
variant_install_path,
topmost_path=os.path.dirname(dest_pkg_payload_path))
if last_dir:
ctxt = make_path_writable(last_dir)
else:
ctxt = with_noop()
# copy the variant payload
with ctxt:
safe_makedirs(variant_install_path)
# determine files not to copy
skip_files = []
if src_variant.subpath:
# Detect overlapped variants. This is the case where one variant subpath
# might be A, and another is A/B. We must ensure that A/B is not created
# as a symlink during shallow install of variant A - that would then
# cause A/B payload to be installed back into original package, possibly
# corrupting it.
#
# Here we detect this case, and create a list of dirs not to copy/link,
# because they are in fact a subpath dir for another variant.
#
skip_files.extend(_get_overlapped_variant_dirs(src_variant))
else:
# just skip package definition file
for name in config.plugins.package_repository.filesystem.package_filenames:
for fmt in (FileFormat.py, FileFormat.yaml):
filename = name + '.' + fmt.extension
skip_files.append(filename)
# copy/link all topmost files within the variant root
for name in os.listdir(variant_root):
if name in skip_files:
filepath = os.path.join(variant_root, name)
if verbose:
if src_variant.subpath:
msg = ("Did not copy %s - this is part of an "
"overlapping variant's root path.")
else:
msg = "Did not copy package definition file %s"
print_info(msg, filepath)
continue
src_path = os.path.join(variant_root, name)
dest_path = os.path.join(variant_install_path, name)
if os.path.islink(src_path):
copy_func(src_path, dest_path)
else:
maybe_symlink(src_path, dest_path)
# copy permissions of source variant dirs onto dest
src_package = src_variant.parent
src_pkg_repo = src_package.repository
src_pkg_payload_path = src_pkg_repo.get_package_payload_path(
package_name=src_package.name,
package_version=src_package.version
)
shutil.copystat(src_pkg_payload_path, dest_pkg_payload_path)
subpath = src_variant.subpath
while subpath:
src_path = os.path.join(src_pkg_payload_path, subpath)
dest_path = os.path.join(dest_pkg_payload_path, subpath)
shutil.copystat(src_path, dest_path)
subpath = os.path.dirname(subpath)
def _build_variant_base(self, variant, build_type, install_path=None,
clean=False, install=False, **kwargs):
# create build/install paths
install_path = install_path or self.package.config.local_packages_path
variant_install_path = self.get_package_install_path(install_path)
variant_build_path = self.build_path
if variant.subpath:
variant_build_path = os.path.join(variant_build_path, variant.subpath)
variant_install_path = os.path.join(variant_install_path, variant.subpath)
# create directories (build, install)
if clean and os.path.exists(variant_build_path):
shutil.rmtree(variant_build_path)
safe_makedirs(variant_build_path)
# find last dir of installation path that exists, and possibly make it
# writable during variant installation
#
last_dir = get_existing_path(variant_install_path,
topmost_path=install_path)
if last_dir:
ctxt = make_path_writable(last_dir)
else:
ctxt = with_noop()
with ctxt:
if install:
# inform package repo that a variant is about to be built/installed
pkg_repo = package_repository_manager.get_repository(install_path)
pkg_repo.pre_variant_install(variant.resource)
if not os.path.exists(variant_install_path):
safe_makedirs(variant_install_path)
# Re-evaluate the variant, so that variables such as 'building' and
# 'build_variant_index' are set, and any early-bound package attribs
# are re-evaluated wrt these vars. This is done so that attribs such as
# 'requires' can change depending on whether a build is occurring or not.
#
# Note that this re-evaluated variant is ONLY used here, for the purposes
# of creating the build context. The variant that is actually installed
# is the one evaluated where 'building' is False.
#
re_evaluated_package = variant.parent.get_reevaluated({
"building": True,
"build_variant_index": variant.index or 0,
"build_variant_requires": variant.variant_requires
})
re_evaluated_variant = re_evaluated_package.get_variant(variant.index)
# create build environment
context, rxt_filepath = self.create_build_context(
variant=re_evaluated_variant,
build_type=build_type,
build_path=variant_build_path)
# run build system
build_system_name = self.build_system.name()
self._print("\nInvoking %s build system...", build_system_name)
build_result = self.build_system.build(
context=context,
variant=variant,
build_path=variant_build_path,
install_path=variant_install_path,
install=install,
build_type=build_type)
if not build_result.get("success"):
raise BuildError("The %s build system failed." % build_system_name)
if install:
# install some files for debugging purposes
extra_files = build_result.get("extra_files", [])
if rxt_filepath:
extra_files = extra_files + [rxt_filepath]
for file_ in extra_files:
copy_or_replace(file_, variant_install_path)
# Install include modules. Note that this doesn't need to be done
# multiple times, but for subsequent variants it has no effect.
#
self._install_include_modules(install_path)
return build_result
def add_variant(self, variant, force=False):
"""Copy a variant's payload into the cache.
The following steps are taken to ensure muti-thread/proc safety, and to
guarantee that a partially-copied variant payload is never able to be
used:
1. The hash dir (eg '/<cache_dir>/foo/1.0.0/af8d') is created;
2. A file lock mutex ('/<cache_dir>/.lock') is acquired;
3. The file '/<cache_dir>/foo/1.0.0/af8d/.copying-a' (or -b, -c etc) is
created. This tells rez that this variant is being copied and cannot
be used yet;
4. The file '/<cache_dir>/foo/1.0.0/af8d/a.json' is created. Now
another proc/thread can't create the same local variant;
5. The file lock is released;
6. The variant payload is copied to '/<cache_dir>/foo/1.0.0/af8d/a';
7. The '.copying-a' file is removed.
Note that the variant will not be cached in the following circumstances,
unless `force` is True:
- The variant is not cachable as determined by `Variant.is_cachable`;
- The variant is from a local package, and 'config.package_cache_local'
is False;
- The variant is stored on the same disk device as this cache, and
config.package_cache_same_device' is False.
Args:
variant (`Variant`): The variant to copy into this cache
force (bool): Copy the variant regardless. Use at your own risk (there
is no guarantee the resulting variant payload will be functional).
Returns:
2-tuple:
- str: Path to cached payload
- int: One of:
- VARIANT_FOUND
- VARIANT_CREATED
- VARIANT_COPYING
- VARIANT_COPY_STALLED
"""
from rez.utils.base26 import get_next_base26
from rez.utils.filesystem import safe_makedirs
# do some sanity checking on variant to cache
package = variant.parent
variant_root = getattr(variant, "root", None)
if not variant_root:
raise PackageCacheError(
"Not cached - variant is a type that does not have a root: %s"
% variant.uri)
if not os.path.isdir(variant_root):
raise PackageCacheError(
"Not cached - variant %s root does not appear on disk: %s" %
(variant.uri, variant_root))
if not force:
# package is configured to not be cachable
if not package.is_cachable:
raise PackageCacheError(
"Not cached - package is not cachable: %s" % package.uri)
# package is local
if not config.package_cache_local and variant.is_local:
raise PackageCacheError("Not cached - package is local: %s" %
package.uri)
# Package is already on same disk device as package cache. Note that
# this check is skipped on Windows + Py<3.4, as os.stat does not
# support device identification.
#
dev_stat_not_supported = (platform.system() == "Windows"
and sys.version_info[:2] < (3, 4))
if not config.package_cache_same_device and not dev_stat_not_supported:
st_pkgcache = os.stat(self.path)
st_variant = os.stat(variant_root)
if st_pkgcache.st_dev == st_variant.st_dev:
raise PackageCacheError(
"Not cached - variant %s is on same device as cache: %s"
% (variant.uri, variant_root))
# Package belongs to a temp repo (this occurs when a package is
# tested on pre_build/pre_release - see
# https://github.com/nerdvegas/rez/wiki/Package-Definition-Guide#tests)
#
if package.repository.name() == "filesystem" and \
package.repository.location.startswith(config.tmpdir + os.sep):
raise PackageCacheError(
"Not cached - package is in temp repository %s" %
package.repository)
no_op_statuses = (self.VARIANT_FOUND, self.VARIANT_COPYING,
self.VARIANT_COPY_STALLED)
# variant already exists, or is being copied to cache by another thread/proc
status, rootpath = self._get_cached_root(variant)
if status in no_op_statuses:
return (rootpath, status)
# 1.
path = self._get_hash_path(variant)
safe_makedirs(path)
# construct data to store to json file
data = {"handle": variant.handle.to_dict()}
if variant.index is not None:
# just added for debugging purposes
data["data"] = package.data["variants"][variant.index]
# 2. + 5.
with self._lock():
# Check if variant exists again, another proc could have created it
# just before lock acquire
#
status, rootpath = self._get_cached_root(variant)
if status in no_op_statuses:
return (rootpath, status)
# determine next increment name ('a', 'b' etc)
names = os.listdir(path)
names = [x for x in names if x.endswith(".json")]
if names:
prev = os.path.splitext(max(names))[0]
else:
prev = None
incname = get_next_base26(prev)
# 3.
copying_filepath = os.path.join(path, ".copying-" + incname)
with open(copying_filepath, 'w'):
pass
# 4.
json_filepath = os.path.join(path, incname + ".json")
with open(json_filepath, 'w') as f:
f.write(json.dumps(data))
# 6.
#
# Here we continually update mtime on the .copying file, to indicate
# that the copy is active. This allows us to detect stalled/errored
# copies, and report them as VARIANT_COPY_STALLED status.
#
still_copying = True
def _while_copying():
while still_copying:
time.sleep(self._COPYING_TIME_INC)
try:
os.utime(copying_filepath, None)
except:
pass
rootpath = os.path.join(path, incname)
th = threading.Thread(target=_while_copying)
th.daemon = True
th.start()
try:
shutil.copytree(variant_root, rootpath)
finally:
still_copying = False
# 7.
th.join()
os.remove(copying_filepath)
return (rootpath, self.VARIANT_CREATED)
def _copy_variant_payload(src_variant, dest_pkg_repo, shallow=False,
follow_symlinks=False, overrides=None, verbose=False):
# Get payload path of source variant. For some types (eg from a "memory"
# type repo) there may not be a root.
#
variant_root = getattr(src_variant, "root", None)
if not variant_root:
raise PackageCopyError(
"Cannot copy source variant %s - it is a type of variant that "
"does not have a root." % src_variant.uri
)
if not os.path.isdir(variant_root):
raise PackageCopyError(
"Cannot copy source variant %s - its root does not appear to "
"be present on disk (%s)." % src_variant.uri, variant_root
)
dest_variant_name = overrides.get("name") or src_variant.name
dest_variant_version = overrides.get("version") or src_variant.version
# determine variant installation path
dest_pkg_payload_path = dest_pkg_repo.get_package_payload_path(
package_name=dest_variant_name,
package_version=dest_variant_version
)
is_varianted = (src_variant.index is not None)
src_variant_subpath = None
if is_varianted:
src_variant_subpath = src_variant._non_shortlinked_subpath
variant_install_path = os.path.join(
dest_pkg_payload_path, src_variant_subpath)
else:
variant_install_path = dest_pkg_payload_path
# get ready for copy/symlinking
copy_func = partial(replacing_copy,
follow_symlinks=follow_symlinks)
if shallow:
maybe_symlink = replacing_symlink
else:
maybe_symlink = copy_func
# possibly make install path temporarily writable
last_dir = get_existing_path(
variant_install_path,
topmost_path=os.path.dirname(dest_pkg_payload_path))
if last_dir and config.make_package_temporarily_writable:
ctxt = make_path_writable(last_dir)
else:
ctxt = with_noop()
# copy the variant payload
with ctxt:
safe_makedirs(variant_install_path)
# determine files not to copy
skip_files = []
if is_varianted and not src_variant.parent.hashed_variants:
# Detect overlapped variants. This is the case where one variant subpath
# might be A, and another is A/B. We must ensure that A/B is not created
# as a symlink during shallow install of variant A - that would then
# cause A/B payload to be installed back into original package, possibly
# corrupting it.
#
# Here we detect this case, and create a list of dirs not to copy/link,
# because they are in fact a subpath dir for another variant.
#
# Note that for hashed variants, we don't do this check because overlapped
# variants are not possible.
#
skip_files.extend(_get_overlapped_variant_dirs(src_variant))
else:
# just skip package definition file
for name in config.plugins.package_repository.filesystem.package_filenames:
for fmt in (FileFormat.py, FileFormat.yaml):
filename = name + '.' + fmt.extension
skip_files.append(filename)
# copy/link all topmost files within the variant root
for name in os.listdir(variant_root):
if name in skip_files:
filepath = os.path.join(variant_root, name)
if verbose and is_varianted:
print_info(
"Did not copy %s - this is part of an overlapping "
"variant's root path.", filepath
)
continue
src_path = os.path.join(variant_root, name)
dest_path = os.path.join(variant_install_path, name)
if os.path.islink(src_path):
copy_func(src_path, dest_path)
else:
maybe_symlink(src_path, dest_path)
# copy permissions of source variant dirs onto dest
src_package = src_variant.parent
src_pkg_repo = src_package.repository
src_pkg_payload_path = src_pkg_repo.get_package_payload_path(
package_name=src_package.name,
package_version=src_package.version
)
shutil.copystat(src_pkg_payload_path, dest_pkg_payload_path)
subpath = src_variant_subpath
while subpath:
src_path = os.path.join(src_pkg_payload_path, subpath)
dest_path = os.path.join(dest_pkg_payload_path, subpath)
shutil.copystat(src_path, dest_path)
subpath = os.path.dirname(subpath)
# create the variant shortlink
if src_variant.parent.hashed_variants:
try:
# base _v dir
base_shortlinks_path = os.path.join(
dest_pkg_payload_path,
src_package.config.variant_shortlinks_dirname
)
safe_makedirs(base_shortlinks_path)
# shortlink
rel_variant_path = os.path.relpath(
variant_install_path, base_shortlinks_path)
create_unique_base26_symlink(
base_shortlinks_path, rel_variant_path)
except Exception as e:
# Treat any error as warning - lack of shortlink is not
# a breaking issue, it just means the variant root path
# will be long.
#
print_warning(
"Error creating variant shortlink for %s: %s: %s",
variant_install_path, e.__class__.__name__, e
)
def _copy_variant_payload(src_variant,
dest_pkg_repo,
shallow=False,
follow_symlinks=False,
overrides=None):
# Get payload path of source variant. For some types (eg from a "memory"
# type repo) there may not be a root.
#
variant_root = getattr(src_variant, "root", None)
if not variant_root:
raise PackageCopyError(
"Cannot copy source variant %s - it is a type of variant that "
"does not have a root.", src_variant.uri)
if not os.path.isdir(variant_root):
raise PackageCopyError(
"Cannot copy source variant %s - its root does not appear to "
"be present on disk (%s).", src_variant.uri, variant_root)
dest_variant_name = overrides.get("name") or src_variant.name
dest_variant_version = overrides.get("version") or src_variant.version
# determine variant installation path
variant_install_path = dest_pkg_repo.get_package_payload_path(
package_name=dest_variant_name, package_version=dest_variant_version)
if src_variant.subpath:
variant_install_path = os.path.join(variant_install_path,
src_variant.subpath)
# perform the copy/symlinking
copy_func = partial(replacing_copy, follow_symlinks=follow_symlinks)
if shallow:
maybe_symlink = replacing_symlink
else:
maybe_symlink = copy_func
if src_variant.subpath:
# symlink/copy the last install dir to the variant root
safe_makedirs(os.path.dirname(variant_install_path))
maybe_symlink(variant_root, variant_install_path)
else:
safe_makedirs(variant_install_path)
# Symlink/copy all files and dirs within the null variant, except
# for the package definition itself.
#
for name in os.listdir(variant_root):
is_pkg_defn = False
# skip package definition file
name_ = os.path.splitext(name)[0]
if name_ in config.plugins.package_repository.filesystem.package_filenames:
for fmt in (FileFormat.py, FileFormat.yaml):
filename = name_ + '.' + fmt.extension
if name == filename:
is_pkg_defn = True
break
if is_pkg_defn:
continue
src_path = os.path.join(variant_root, name)
dest_path = os.path.join(variant_install_path, name)
if os.path.islink(src_path):
copy_func(src_path, dest_path)
else:
maybe_symlink(src_path, dest_path)
