def _copy_package_include_modules(src_package, dest_pkg_repo, overrides=None): src_include_modules_path = \ os.path.join(src_package.base, IncludeModuleManager.include_modules_subpath) if not os.path.exists(src_include_modules_path): return dest_package_name = overrides.get("name") or src_package.name dest_package_version = overrides.get("version") or src_package.version pkg_install_path = dest_pkg_repo.get_package_payload_path( package_name=dest_package_name, package_version=dest_package_version) dest_include_modules_path = \ os.path.join(pkg_install_path, IncludeModuleManager.include_modules_subpath) last_dir = get_existing_path( dest_include_modules_path, topmost_path=os.path.dirname(pkg_install_path)) if last_dir: ctxt = make_path_writable(last_dir) else: ctxt = with_noop() with ctxt: safe_makedirs(dest_include_modules_path) additive_copytree(src_include_modules_path, dest_include_modules_path)
def _install_include_modules(self, install_path): # install 'include' sourcefiles, used by funcs decorated with @include if not self.package.includes: return install_path = install_path or self.package.config.local_packages_path base_path = self.get_package_install_path(install_path) path = os.path.join(base_path, IncludeModuleManager.include_modules_subpath) safe_makedirs(path) definition_python_path = self.package.config.package_definition_python_path for name in self.package.includes: filepath = os.path.join(definition_python_path, name) + ".py" with open(filepath, "rb") as f: txt = f.read().strip() uuid = sha1(txt).hexdigest() dest_filepath = os.path.join(path, "%s.py" % name) shutil.copy(filepath, dest_filepath) # overwrite if exists sha1_filepath = os.path.join(path, "%s.sha1" % name) with open(sha1_filepath, "w") as f: # overwrite if exists f.write(uuid)
def _build_variant_base(self, variant, build_type, install_path=None, clean=False, install=False, **kwargs): # create build/install paths install_path = install_path or self.package.config.local_packages_path variant_install_path = self.get_package_install_path(install_path) variant_build_path = self.build_path if variant.subpath: variant_build_path = os.path.join(variant_build_path, variant.subpath) variant_install_path = os.path.join(variant_install_path, variant.subpath) # create directories (build, install) if clean and os.path.exists(variant_build_path): shutil.rmtree(variant_build_path) safe_makedirs(variant_build_path) if install: # inform package repo that a variant is about to be built/installed pkg_repo = package_repository_manager.get_repository(install_path) pkg_repo.pre_variant_install(variant.resource) if not os.path.exists(variant_install_path): safe_makedirs(variant_install_path) # create build environment context, rxt_filepath = self.create_build_context( variant=variant, build_type=build_type, build_path=variant_build_path) # run build system build_system_name = self.build_system.name() self._print("\nInvoking %s build system...", build_system_name) build_result = self.build_system.build( context=context, variant=variant, build_path=variant_build_path, install_path=variant_install_path, install=install, build_type=build_type) if not build_result.get("success"): raise BuildError("The %s build system failed." % build_system_name) if install: # install some files for debugging purposes extra_files = build_result.get("extra_files", []) + [rxt_filepath] for file_ in extra_files: copy_or_replace(file_, variant_install_path) return build_result
def _copy_package_include_modules(src_package, dest_pkg_repo, overrides=None): src_include_modules_path = \ os.path.join(src_package.base, IncludeModuleManager.include_modules_subpath) if not os.path.exists(src_include_modules_path): return dest_package_name = overrides.get("name") or src_package.name dest_package_version = overrides.get("version") or src_package.version pkg_install_path = dest_pkg_repo.get_package_payload_path( package_name=dest_package_name, package_version=dest_package_version ) dest_include_modules_path = \ os.path.join(pkg_install_path, IncludeModuleManager.include_modules_subpath) last_dir = get_existing_path(dest_include_modules_path, topmost_path=os.path.dirname(pkg_install_path)) if last_dir: ctxt = make_path_writable(last_dir) else: ctxt = with_noop() with ctxt: safe_makedirs(dest_include_modules_path) additive_copytree(src_include_modules_path, dest_include_modules_path)
def _copy_variant_payload(src_variant, dest_pkg_repo, shallow=False, overrides=None): # Get payload path of source variant. For some types (eg from a "memory" # type repo) there may not be a root. # variant_root = getattr(src_variant, "root", None) if not variant_root: raise PackageCopyError( "Cannot copy source variant %s - it is a type of variant that " "does not have a root.", src_variant.uri) if not os.path.isdir(variant_root): raise PackageCopyError( "Cannot copy source variant %s - its root does not appear to " "be present on disk (%s).", src_variant.uri, variant_root) dest_variant_name = overrides.get("name") or src_variant.name dest_variant_version = overrides.get("version") or src_variant.version # determine variant installation path variant_install_path = dest_pkg_repo.get_package_payload_path( package_name=dest_variant_name, package_version=dest_variant_version) if src_variant.subpath: variant_install_path = os.path.join(variant_install_path, src_variant.subpath) # perform the copy/symlinking if shallow: maybe_symlink = replacing_symlink else: maybe_symlink = replacing_copy if src_variant.subpath: # symlink/copy the last install dir to the variant root safe_makedirs(os.path.dirname(variant_install_path)) maybe_symlink(variant_root, variant_install_path) else: safe_makedirs(variant_install_path) # copy all files, and symlink/copy all dirs within the variant for name in os.listdir(variant_root): src_path = os.path.join(variant_root, name) dest_path = os.path.join(variant_install_path, name) if os.path.isdir(src_path) and not os.path.islink(src_path): maybe_symlink(src_path, dest_path) else: replacing_copy(src_path, dest_path)
def _copy_package_include_modules(src_package, dest_pkg_repo, overrides=None): src_include_modules_path = \ os.path.join(src_package.base, IncludeModuleManager.include_modules_subpath) if not os.path.exists(src_include_modules_path): return dest_package_name = overrides.get("name") or src_package.name dest_package_version = overrides.get("version") or src_package.version pkg_install_path = dest_pkg_repo.get_package_payload_path( package_name=dest_package_name, package_version=dest_package_version) dest_include_modules_path = \ os.path.join(pkg_install_path, IncludeModuleManager.include_modules_subpath) safe_makedirs(dest_include_modules_path) additive_copytree(src_include_modules_path, dest_include_modules_path)
def _install_include_modules(self, install_path): # install 'include' sourcefiles, used by funcs decorated with @include if not self.package.includes: return install_path = install_path or self.package.config.local_packages_path base_path = self.get_package_install_path(install_path) path = os.path.join(base_path, IncludeModuleManager.include_modules_subpath) safe_makedirs(path) definition_python_path = self.package.config.package_definition_python_path for name in self.package.includes: filepath = os.path.join(definition_python_path, name) + ".py" with open(filepath) as f: txt = f.read().strip() uuid = sha1(txt).hexdigest() dest_filepath = os.path.join(path, "%s-%s.py" % (name, uuid)) if not os.path.exists(dest_filepath): shutil.copy(filepath, dest_filepath)
def __init__(self, path): """Create a package cache. Args: path (str): Path on disk, must exist. """ if not os.path.isdir(path): raise PackageCacheError("Not a directory: %s" % path) self.path = path # make dirs for internal use safe_makedirs(self._log_dir) safe_makedirs(self._pending_dir) safe_makedirs(self._remove_dir)
def _copy_variant_payload(src_variant, dest_pkg_repo, shallow=False, follow_symlinks=False, overrides=None, verbose=False): # Get payload path of source variant. For some types (eg from a "memory" # type repo) there may not be a root. # variant_root = getattr(src_variant, "root", None) if not variant_root: raise PackageCopyError( "Cannot copy source variant %s - it is a type of variant that " "does not have a root.", src_variant.uri) if not os.path.isdir(variant_root): raise PackageCopyError( "Cannot copy source variant %s - its root does not appear to " "be present on disk (%s).", src_variant.uri, variant_root) dest_variant_name = overrides.get("name") or src_variant.name dest_variant_version = overrides.get("version") or src_variant.version # determine variant installation path dest_pkg_payload_path = dest_pkg_repo.get_package_payload_path( package_name=dest_variant_name, package_version=dest_variant_version) if src_variant.subpath: variant_install_path = os.path.join(dest_pkg_payload_path, src_variant.subpath) else: variant_install_path = dest_pkg_payload_path # get ready for copy/symlinking copy_func = partial(replacing_copy, follow_symlinks=follow_symlinks) if shallow: maybe_symlink = replacing_symlink else: maybe_symlink = copy_func # possibly make install path temporarily writable last_dir = get_existing_path( variant_install_path, topmost_path=os.path.dirname(dest_pkg_payload_path)) if last_dir: ctxt = make_path_writable(last_dir) else: ctxt = with_noop() # copy the variant payload with ctxt: safe_makedirs(variant_install_path) # determine files not to copy skip_files = [] if src_variant.subpath: # Detect overlapped variants. This is the case where one variant subpath # might be A, and another is A/B. We must ensure that A/B is not created # as a symlink during shallow install of variant A - that would then # cause A/B payload to be installed back into original package, possibly # corrupting it. # # Here we detect this case, and create a list of dirs not to copy/link, # because they are in fact a subpath dir for another variant. # skip_files.extend(_get_overlapped_variant_dirs(src_variant)) else: # just skip package definition file for name in config.plugins.package_repository.filesystem.package_filenames: for fmt in (FileFormat.py, FileFormat.yaml): filename = name + '.' + fmt.extension skip_files.append(filename) # copy/link all topmost files within the variant root for name in os.listdir(variant_root): if name in skip_files: filepath = os.path.join(variant_root, name) if verbose: if src_variant.subpath: msg = ("Did not copy %s - this is part of an " "overlapping variant's root path.") else: msg = "Did not copy package definition file %s" print_info(msg, filepath) continue src_path = os.path.join(variant_root, name) dest_path = os.path.join(variant_install_path, name) if os.path.islink(src_path): copy_func(src_path, dest_path) else: maybe_symlink(src_path, dest_path) # copy permissions of source variant dirs onto dest src_package = src_variant.parent src_pkg_repo = src_package.repository src_pkg_payload_path = src_pkg_repo.get_package_payload_path( package_name=src_package.name, package_version=src_package.version) shutil.copystat(src_pkg_payload_path, dest_pkg_payload_path) subpath = src_variant.subpath while subpath: src_path = os.path.join(src_pkg_payload_path, subpath) dest_path = os.path.join(dest_pkg_payload_path, subpath) shutil.copystat(src_path, dest_path) subpath = os.path.dirname(subpath)
def _build_variant_base(self, variant, build_type, install_path=None, clean=False, install=False, **kwargs): # create build/install paths install_path = install_path or self.package.config.local_packages_path package_install_path = self.get_package_install_path(install_path) variant_build_path = self.build_path if variant.index is None: variant_install_path = package_install_path else: subpath = variant._non_shortlinked_subpath variant_build_path = os.path.join(variant_build_path, subpath) variant_install_path = os.path.join(package_install_path, subpath) # create directories (build, install) if clean and os.path.exists(variant_build_path): self._rmtree(variant_build_path) safe_makedirs(variant_build_path) # find last dir of installation path that exists, and possibly make it # writable during variant installation # last_dir = get_existing_path(variant_install_path, topmost_path=install_path) if last_dir: ctxt = make_path_writable(last_dir) else: ctxt = with_noop() with ctxt: if install: # inform package repo that a variant is about to be built/installed pkg_repo = package_repository_manager.get_repository( install_path) pkg_repo.pre_variant_install(variant.resource) if not os.path.exists(variant_install_path): safe_makedirs(variant_install_path) # if hashed variants are enabled, create the variant shortlink if variant.parent.hashed_variants: try: # create the dir containing all shortlinks base_shortlinks_path = os.path.join( package_install_path, variant.parent.config.variant_shortlinks_dirname) safe_makedirs(base_shortlinks_path) # create the shortlink rel_variant_path = os.path.relpath( variant_install_path, base_shortlinks_path) create_unique_base26_symlink(base_shortlinks_path, rel_variant_path) except Exception as e: # Treat any error as warning - lack of shortlink is not # a breaking issue, it just means the variant root path # will be long. # print_warning( "Error creating variant shortlink for %s: %s: %s", variant_install_path, e.__class__.__name__, e) # Re-evaluate the variant, so that variables such as 'building' and # 'build_variant_index' are set, and any early-bound package attribs # are re-evaluated wrt these vars. This is done so that attribs such as # 'requires' can change depending on whether a build is occurring or not. # # Note that this re-evaluated variant is ONLY used here, for the purposes # of creating the build context. The variant that is actually installed # is the one evaluated where 'building' is False. # re_evaluated_package = variant.parent.get_reevaluated({ "building": True, "build_variant_index": variant.index or 0, "build_variant_requires": variant.variant_requires }) re_evaluated_variant = re_evaluated_package.get_variant( variant.index) # create build environment (also creates build.rxt file) context, rxt_filepath = self.create_build_context( variant=re_evaluated_variant, build_type=build_type, build_path=variant_build_path) # list of extra files (build.rxt etc) that are installed if an # installation is taking place # extra_install_files = [rxt_filepath] # create variant.json file. This identifies which variant this is. # This is important for hashed variants, where it is not obvious # which variant is in which root path. The file is there for # debugging purposes only. # if variant.index is not None: data = { "index": variant.index, "data": variant.parent.data["variants"][variant.index] } filepath = os.path.join(variant_build_path, "variant.json") extra_install_files.append(filepath) with open(filepath, 'w') as f: json.dump(data, f, indent=2) # run build system build_system_name = self.build_system.name() self._print("\nInvoking %s build system...", build_system_name) build_result = self.build_system.build( context=context, variant=variant, build_path=variant_build_path, install_path=variant_install_path, install=install, build_type=build_type) if not build_result.get("success"): # delete the possibly partially installed variant payload if install: self._rmtree(variant_install_path) raise BuildError("The %s build system failed." % build_system_name) if install: # add some installation details to build result build_result.update({ "package_install_path": package_install_path, "variant_install_path": variant_install_path }) # the build system can also specify extra files that need to # be installed filepaths = build_result.get("extra_files") if filepaths: extra_install_files.extend(filepaths) # install extra files for file_ in extra_install_files: copy_or_replace(file_, variant_install_path) # Install include modules. Note that this doesn't need to be done # multiple times, but for subsequent variants it has no effect. # self._install_include_modules(install_path) return build_result
def _build_variant_base(self, variant, build_type, install_path=None, clean=False, install=False, **kwargs): # create build/install paths install_path = install_path or self.package.config.local_packages_path package_install_path = self.get_package_install_path(install_path) variant_build_path = self.build_path if variant.index is None: variant_install_path = package_install_path else: subpath = variant._non_shortlinked_subpath variant_build_path = os.path.join(variant_build_path, subpath) variant_install_path = os.path.join(package_install_path, subpath) # create directories (build, install) if clean and os.path.exists(variant_build_path): shutil.rmtree(variant_build_path) safe_makedirs(variant_build_path) # find last dir of installation path that exists, and possibly make it # writable during variant installation # last_dir = get_existing_path(variant_install_path, topmost_path=install_path) if last_dir: ctxt = make_path_writable(last_dir) else: ctxt = with_noop() with ctxt: if install: # inform package repo that a variant is about to be built/installed pkg_repo = package_repository_manager.get_repository(install_path) pkg_repo.pre_variant_install(variant.resource) if not os.path.exists(variant_install_path): safe_makedirs(variant_install_path) # if hashed variants are enabled, create the variant shortlink if variant.parent.hashed_variants: try: # create the dir containing all shortlinks base_shortlinks_path = os.path.join( package_install_path, variant.parent.config.variant_shortlinks_dirname ) safe_makedirs(base_shortlinks_path) # create the shortlink rel_variant_path = os.path.relpath( variant_install_path, base_shortlinks_path) create_unique_base26_symlink( base_shortlinks_path, rel_variant_path) except Exception as e: # Treat any error as warning - lack of shortlink is not # a breaking issue, it just means the variant root path # will be long. # print_warning( "Error creating variant shortlink for %s: %s: %s", variant_install_path, e.__class__.__name__, e ) # Re-evaluate the variant, so that variables such as 'building' and # 'build_variant_index' are set, and any early-bound package attribs # are re-evaluated wrt these vars. This is done so that attribs such as # 'requires' can change depending on whether a build is occurring or not. # # Note that this re-evaluated variant is ONLY used here, for the purposes # of creating the build context. The variant that is actually installed # is the one evaluated where 'building' is False. # re_evaluated_package = variant.parent.get_reevaluated({ "building": True, "build_variant_index": variant.index or 0, "build_variant_requires": variant.variant_requires }) re_evaluated_variant = re_evaluated_package.get_variant(variant.index) # create build environment context, rxt_filepath = self.create_build_context( variant=re_evaluated_variant, build_type=build_type, build_path=variant_build_path) # run build system build_system_name = self.build_system.name() self._print("\nInvoking %s build system...", build_system_name) build_result = self.build_system.build( context=context, variant=variant, build_path=variant_build_path, install_path=variant_install_path, install=install, build_type=build_type) if not build_result.get("success"): raise BuildError("The %s build system failed." % build_system_name) if install: # Install the 'variant.json' file, which identifies which variant # this is. This is important for hashed variants, where it is not # obvious which variant is in which root path. The file is there # for debugging purposes only. # if variant.index is not None: data = { "index": variant.index, "data": variant.parent.data["variants"][variant.index] } filepath = os.path.join(variant_install_path, "variant.json") with open(filepath, 'w') as f: json.dump(data, f, indent=2) # install some files for debugging purposes (incl build.rxt) extra_files = build_result.get("extra_files", []) if rxt_filepath: extra_files = extra_files + [rxt_filepath] for file_ in extra_files: copy_or_replace(file_, variant_install_path) # Install include modules. Note that this doesn't need to be done # multiple times, but for subsequent variants it has no effect. # self._install_include_modules(install_path) return build_result
def _copy_variant_payload(src_variant, dest_pkg_repo, shallow=False, follow_symlinks=False, overrides=None, verbose=False): # Get payload path of source variant. For some types (eg from a "memory" # type repo) there may not be a root. # variant_root = getattr(src_variant, "root", None) if not variant_root: raise PackageCopyError( "Cannot copy source variant %s - it is a type of variant that " "does not have a root.", src_variant.uri ) if not os.path.isdir(variant_root): raise PackageCopyError( "Cannot copy source variant %s - its root does not appear to " "be present on disk (%s).", src_variant.uri, variant_root ) dest_variant_name = overrides.get("name") or src_variant.name dest_variant_version = overrides.get("version") or src_variant.version # determine variant installation path dest_pkg_payload_path = dest_pkg_repo.get_package_payload_path( package_name=dest_variant_name, package_version=dest_variant_version ) if src_variant.subpath: variant_install_path = os.path.join(dest_pkg_payload_path, src_variant.subpath) else: variant_install_path = dest_pkg_payload_path # get ready for copy/symlinking copy_func = partial(replacing_copy, follow_symlinks=follow_symlinks) if shallow: maybe_symlink = replacing_symlink else: maybe_symlink = copy_func # possibly make install path temporarily writable last_dir = get_existing_path( variant_install_path, topmost_path=os.path.dirname(dest_pkg_payload_path)) if last_dir: ctxt = make_path_writable(last_dir) else: ctxt = with_noop() # copy the variant payload with ctxt: safe_makedirs(variant_install_path) # determine files not to copy skip_files = [] if src_variant.subpath: # Detect overlapped variants. This is the case where one variant subpath # might be A, and another is A/B. We must ensure that A/B is not created # as a symlink during shallow install of variant A - that would then # cause A/B payload to be installed back into original package, possibly # corrupting it. # # Here we detect this case, and create a list of dirs not to copy/link, # because they are in fact a subpath dir for another variant. # skip_files.extend(_get_overlapped_variant_dirs(src_variant)) else: # just skip package definition file for name in config.plugins.package_repository.filesystem.package_filenames: for fmt in (FileFormat.py, FileFormat.yaml): filename = name + '.' + fmt.extension skip_files.append(filename) # copy/link all topmost files within the variant root for name in os.listdir(variant_root): if name in skip_files: filepath = os.path.join(variant_root, name) if verbose: if src_variant.subpath: msg = ("Did not copy %s - this is part of an " "overlapping variant's root path.") else: msg = "Did not copy package definition file %s" print_info(msg, filepath) continue src_path = os.path.join(variant_root, name) dest_path = os.path.join(variant_install_path, name) if os.path.islink(src_path): copy_func(src_path, dest_path) else: maybe_symlink(src_path, dest_path) # copy permissions of source variant dirs onto dest src_package = src_variant.parent src_pkg_repo = src_package.repository src_pkg_payload_path = src_pkg_repo.get_package_payload_path( package_name=src_package.name, package_version=src_package.version ) shutil.copystat(src_pkg_payload_path, dest_pkg_payload_path) subpath = src_variant.subpath while subpath: src_path = os.path.join(src_pkg_payload_path, subpath) dest_path = os.path.join(dest_pkg_payload_path, subpath) shutil.copystat(src_path, dest_path) subpath = os.path.dirname(subpath)
def _build_variant_base(self, variant, build_type, install_path=None, clean=False, install=False, **kwargs): # create build/install paths install_path = install_path or self.package.config.local_packages_path variant_install_path = self.get_package_install_path(install_path) variant_build_path = self.build_path if variant.subpath: variant_build_path = os.path.join(variant_build_path, variant.subpath) variant_install_path = os.path.join(variant_install_path, variant.subpath) # create directories (build, install) if clean and os.path.exists(variant_build_path): shutil.rmtree(variant_build_path) safe_makedirs(variant_build_path) # find last dir of installation path that exists, and possibly make it # writable during variant installation # last_dir = get_existing_path(variant_install_path, topmost_path=install_path) if last_dir: ctxt = make_path_writable(last_dir) else: ctxt = with_noop() with ctxt: if install: # inform package repo that a variant is about to be built/installed pkg_repo = package_repository_manager.get_repository(install_path) pkg_repo.pre_variant_install(variant.resource) if not os.path.exists(variant_install_path): safe_makedirs(variant_install_path) # Re-evaluate the variant, so that variables such as 'building' and # 'build_variant_index' are set, and any early-bound package attribs # are re-evaluated wrt these vars. This is done so that attribs such as # 'requires' can change depending on whether a build is occurring or not. # # Note that this re-evaluated variant is ONLY used here, for the purposes # of creating the build context. The variant that is actually installed # is the one evaluated where 'building' is False. # re_evaluated_package = variant.parent.get_reevaluated({ "building": True, "build_variant_index": variant.index or 0, "build_variant_requires": variant.variant_requires }) re_evaluated_variant = re_evaluated_package.get_variant(variant.index) # create build environment context, rxt_filepath = self.create_build_context( variant=re_evaluated_variant, build_type=build_type, build_path=variant_build_path) # run build system build_system_name = self.build_system.name() self._print("\nInvoking %s build system...", build_system_name) build_result = self.build_system.build( context=context, variant=variant, build_path=variant_build_path, install_path=variant_install_path, install=install, build_type=build_type) if not build_result.get("success"): raise BuildError("The %s build system failed." % build_system_name) if install: # install some files for debugging purposes extra_files = build_result.get("extra_files", []) if rxt_filepath: extra_files = extra_files + [rxt_filepath] for file_ in extra_files: copy_or_replace(file_, variant_install_path) # Install include modules. Note that this doesn't need to be done # multiple times, but for subsequent variants it has no effect. # self._install_include_modules(install_path) return build_result
def add_variant(self, variant, force=False): """Copy a variant's payload into the cache. The following steps are taken to ensure muti-thread/proc safety, and to guarantee that a partially-copied variant payload is never able to be used: 1. The hash dir (eg '/<cache_dir>/foo/1.0.0/af8d') is created; 2. A file lock mutex ('/<cache_dir>/.lock') is acquired; 3. The file '/<cache_dir>/foo/1.0.0/af8d/.copying-a' (or -b, -c etc) is created. This tells rez that this variant is being copied and cannot be used yet; 4. The file '/<cache_dir>/foo/1.0.0/af8d/a.json' is created. Now another proc/thread can't create the same local variant; 5. The file lock is released; 6. The variant payload is copied to '/<cache_dir>/foo/1.0.0/af8d/a'; 7. The '.copying-a' file is removed. Note that the variant will not be cached in the following circumstances, unless `force` is True: - The variant is not cachable as determined by `Variant.is_cachable`; - The variant is from a local package, and 'config.package_cache_local' is False; - The variant is stored on the same disk device as this cache, and config.package_cache_same_device' is False. Args: variant (`Variant`): The variant to copy into this cache force (bool): Copy the variant regardless. Use at your own risk (there is no guarantee the resulting variant payload will be functional). Returns: 2-tuple: - str: Path to cached payload - int: One of: - VARIANT_FOUND - VARIANT_CREATED - VARIANT_COPYING - VARIANT_COPY_STALLED """ from rez.utils.base26 import get_next_base26 from rez.utils.filesystem import safe_makedirs # do some sanity checking on variant to cache package = variant.parent variant_root = getattr(variant, "root", None) if not variant_root: raise PackageCacheError( "Not cached - variant is a type that does not have a root: %s" % variant.uri) if not os.path.isdir(variant_root): raise PackageCacheError( "Not cached - variant %s root does not appear on disk: %s" % (variant.uri, variant_root)) if not force: # package is configured to not be cachable if not package.is_cachable: raise PackageCacheError( "Not cached - package is not cachable: %s" % package.uri) # package is local if not config.package_cache_local and variant.is_local: raise PackageCacheError("Not cached - package is local: %s" % package.uri) # Package is already on same disk device as package cache. Note that # this check is skipped on Windows + Py<3.4, as os.stat does not # support device identification. # dev_stat_not_supported = (platform.system() == "Windows" and sys.version_info[:2] < (3, 4)) if not config.package_cache_same_device and not dev_stat_not_supported: st_pkgcache = os.stat(self.path) st_variant = os.stat(variant_root) if st_pkgcache.st_dev == st_variant.st_dev: raise PackageCacheError( "Not cached - variant %s is on same device as cache: %s" % (variant.uri, variant_root)) # Package belongs to a temp repo (this occurs when a package is # tested on pre_build/pre_release - see # https://github.com/nerdvegas/rez/wiki/Package-Definition-Guide#tests) # if package.repository.name() == "filesystem" and \ package.repository.location.startswith(config.tmpdir + os.sep): raise PackageCacheError( "Not cached - package is in temp repository %s" % package.repository) no_op_statuses = (self.VARIANT_FOUND, self.VARIANT_COPYING, self.VARIANT_COPY_STALLED) # variant already exists, or is being copied to cache by another thread/proc status, rootpath = self._get_cached_root(variant) if status in no_op_statuses: return (rootpath, status) # 1. path = self._get_hash_path(variant) safe_makedirs(path) # construct data to store to json file data = {"handle": variant.handle.to_dict()} if variant.index is not None: # just added for debugging purposes data["data"] = package.data["variants"][variant.index] # 2. + 5. with self._lock(): # Check if variant exists again, another proc could have created it # just before lock acquire # status, rootpath = self._get_cached_root(variant) if status in no_op_statuses: return (rootpath, status) # determine next increment name ('a', 'b' etc) names = os.listdir(path) names = [x for x in names if x.endswith(".json")] if names: prev = os.path.splitext(max(names))[0] else: prev = None incname = get_next_base26(prev) # 3. copying_filepath = os.path.join(path, ".copying-" + incname) with open(copying_filepath, 'w'): pass # 4. json_filepath = os.path.join(path, incname + ".json") with open(json_filepath, 'w') as f: f.write(json.dumps(data)) # 6. # # Here we continually update mtime on the .copying file, to indicate # that the copy is active. This allows us to detect stalled/errored # copies, and report them as VARIANT_COPY_STALLED status. # still_copying = True def _while_copying(): while still_copying: time.sleep(self._COPYING_TIME_INC) try: os.utime(copying_filepath, None) except: pass rootpath = os.path.join(path, incname) th = threading.Thread(target=_while_copying) th.daemon = True th.start() try: shutil.copytree(variant_root, rootpath) finally: still_copying = False # 7. th.join() os.remove(copying_filepath) return (rootpath, self.VARIANT_CREATED)
def _copy_variant_payload(src_variant, dest_pkg_repo, shallow=False, follow_symlinks=False, overrides=None, verbose=False): # Get payload path of source variant. For some types (eg from a "memory" # type repo) there may not be a root. # variant_root = getattr(src_variant, "root", None) if not variant_root: raise PackageCopyError( "Cannot copy source variant %s - it is a type of variant that " "does not have a root." % src_variant.uri ) if not os.path.isdir(variant_root): raise PackageCopyError( "Cannot copy source variant %s - its root does not appear to " "be present on disk (%s)." % src_variant.uri, variant_root ) dest_variant_name = overrides.get("name") or src_variant.name dest_variant_version = overrides.get("version") or src_variant.version # determine variant installation path dest_pkg_payload_path = dest_pkg_repo.get_package_payload_path( package_name=dest_variant_name, package_version=dest_variant_version ) is_varianted = (src_variant.index is not None) src_variant_subpath = None if is_varianted: src_variant_subpath = src_variant._non_shortlinked_subpath variant_install_path = os.path.join( dest_pkg_payload_path, src_variant_subpath) else: variant_install_path = dest_pkg_payload_path # get ready for copy/symlinking copy_func = partial(replacing_copy, follow_symlinks=follow_symlinks) if shallow: maybe_symlink = replacing_symlink else: maybe_symlink = copy_func # possibly make install path temporarily writable last_dir = get_existing_path( variant_install_path, topmost_path=os.path.dirname(dest_pkg_payload_path)) if last_dir and config.make_package_temporarily_writable: ctxt = make_path_writable(last_dir) else: ctxt = with_noop() # copy the variant payload with ctxt: safe_makedirs(variant_install_path) # determine files not to copy skip_files = [] if is_varianted and not src_variant.parent.hashed_variants: # Detect overlapped variants. This is the case where one variant subpath # might be A, and another is A/B. We must ensure that A/B is not created # as a symlink during shallow install of variant A - that would then # cause A/B payload to be installed back into original package, possibly # corrupting it. # # Here we detect this case, and create a list of dirs not to copy/link, # because they are in fact a subpath dir for another variant. # # Note that for hashed variants, we don't do this check because overlapped # variants are not possible. # skip_files.extend(_get_overlapped_variant_dirs(src_variant)) else: # just skip package definition file for name in config.plugins.package_repository.filesystem.package_filenames: for fmt in (FileFormat.py, FileFormat.yaml): filename = name + '.' + fmt.extension skip_files.append(filename) # copy/link all topmost files within the variant root for name in os.listdir(variant_root): if name in skip_files: filepath = os.path.join(variant_root, name) if verbose and is_varianted: print_info( "Did not copy %s - this is part of an overlapping " "variant's root path.", filepath ) continue src_path = os.path.join(variant_root, name) dest_path = os.path.join(variant_install_path, name) if os.path.islink(src_path): copy_func(src_path, dest_path) else: maybe_symlink(src_path, dest_path) # copy permissions of source variant dirs onto dest src_package = src_variant.parent src_pkg_repo = src_package.repository src_pkg_payload_path = src_pkg_repo.get_package_payload_path( package_name=src_package.name, package_version=src_package.version ) shutil.copystat(src_pkg_payload_path, dest_pkg_payload_path) subpath = src_variant_subpath while subpath: src_path = os.path.join(src_pkg_payload_path, subpath) dest_path = os.path.join(dest_pkg_payload_path, subpath) shutil.copystat(src_path, dest_path) subpath = os.path.dirname(subpath) # create the variant shortlink if src_variant.parent.hashed_variants: try: # base _v dir base_shortlinks_path = os.path.join( dest_pkg_payload_path, src_package.config.variant_shortlinks_dirname ) safe_makedirs(base_shortlinks_path) # shortlink rel_variant_path = os.path.relpath( variant_install_path, base_shortlinks_path) create_unique_base26_symlink( base_shortlinks_path, rel_variant_path) except Exception as e: # Treat any error as warning - lack of shortlink is not # a breaking issue, it just means the variant root path # will be long. # print_warning( "Error creating variant shortlink for %s: %s: %s", variant_install_path, e.__class__.__name__, e )
def _copy_variant_payload(src_variant, dest_pkg_repo, shallow=False, follow_symlinks=False, overrides=None): # Get payload path of source variant. For some types (eg from a "memory" # type repo) there may not be a root. # variant_root = getattr(src_variant, "root", None) if not variant_root: raise PackageCopyError( "Cannot copy source variant %s - it is a type of variant that " "does not have a root.", src_variant.uri) if not os.path.isdir(variant_root): raise PackageCopyError( "Cannot copy source variant %s - its root does not appear to " "be present on disk (%s).", src_variant.uri, variant_root) dest_variant_name = overrides.get("name") or src_variant.name dest_variant_version = overrides.get("version") or src_variant.version # determine variant installation path variant_install_path = dest_pkg_repo.get_package_payload_path( package_name=dest_variant_name, package_version=dest_variant_version) if src_variant.subpath: variant_install_path = os.path.join(variant_install_path, src_variant.subpath) # perform the copy/symlinking copy_func = partial(replacing_copy, follow_symlinks=follow_symlinks) if shallow: maybe_symlink = replacing_symlink else: maybe_symlink = copy_func if src_variant.subpath: # symlink/copy the last install dir to the variant root safe_makedirs(os.path.dirname(variant_install_path)) maybe_symlink(variant_root, variant_install_path) else: safe_makedirs(variant_install_path) # Symlink/copy all files and dirs within the null variant, except # for the package definition itself. # for name in os.listdir(variant_root): is_pkg_defn = False # skip package definition file name_ = os.path.splitext(name)[0] if name_ in config.plugins.package_repository.filesystem.package_filenames: for fmt in (FileFormat.py, FileFormat.yaml): filename = name_ + '.' + fmt.extension if name == filename: is_pkg_defn = True break if is_pkg_defn: continue src_path = os.path.join(variant_root, name) dest_path = os.path.join(variant_install_path, name) if os.path.islink(src_path): copy_func(src_path, dest_path) else: maybe_symlink(src_path, dest_path)