def _copy_package_include_modules(src_package, dest_pkg_repo, overrides=None): src_include_modules_path = \ os.path.join(src_package.base, IncludeModuleManager.include_modules_subpath) if not os.path.exists(src_include_modules_path): return dest_package_name = overrides.get("name") or src_package.name dest_package_version = overrides.get("version") or src_package.version pkg_install_path = dest_pkg_repo.get_package_payload_path( package_name=dest_package_name, package_version=dest_package_version) dest_include_modules_path = \ os.path.join(pkg_install_path, IncludeModuleManager.include_modules_subpath) last_dir = get_existing_path( dest_include_modules_path, topmost_path=os.path.dirname(pkg_install_path)) if last_dir: ctxt = make_path_writable(last_dir) else: ctxt = with_noop() with ctxt: safe_makedirs(dest_include_modules_path) additive_copytree(src_include_modules_path, dest_include_modules_path)
def _copy_package_include_modules(src_package, dest_pkg_repo, overrides=None): src_include_modules_path = \ os.path.join(src_package.base, IncludeModuleManager.include_modules_subpath) if not os.path.exists(src_include_modules_path): return dest_package_name = overrides.get("name") or src_package.name dest_package_version = overrides.get("version") or src_package.version pkg_install_path = dest_pkg_repo.get_package_payload_path( package_name=dest_package_name, package_version=dest_package_version ) dest_include_modules_path = \ os.path.join(pkg_install_path, IncludeModuleManager.include_modules_subpath) last_dir = get_existing_path(dest_include_modules_path, topmost_path=os.path.dirname(pkg_install_path)) if last_dir: ctxt = make_path_writable(last_dir) else: ctxt = with_noop() with ctxt: safe_makedirs(dest_include_modules_path) additive_copytree(src_include_modules_path, dest_include_modules_path)
def _remove_deprecated_packages(package, namespaces, deprecate): """Remove Rez package requirements from a Rez package, if needed. If the Python imports defined in `deprecate` are no longer present in `namespaces` then that means that `package` no longer depends on the stuff listed in `deprecate` and the requirement effectively doesn't matter anymore and can be removed. Args: package (:class:`rez.packges_.DeveloperPackage`): Some Rez package whose requirements may change as a result of this function getting ran. namespaces (iter[str]): The Python dot-separated namespaces that a Rez package uses. In short, these are all of the import statements that a Rez package has inside of it and can be thought of as its "dependencies". The function uses this list to figure out if `user_namespaces` is actually still being imported. deprecate (iter[tuple[:class:`rez.vendor.version.requirement.Requirement`, tuple[str]]]): E ach Rez package that is (assumed to already be) a dependency of `package` and the Python import namespaces that the package takes up. If any namespace in `namespaces` is still around, then that means `package` actually still depends on the Rez package so we can't safely remove it (because it's still a dependency). But if it isn't there, remove it. """ packages_to_remove = set() for package_, package_namespaces in deprecate: if _is_package_needed(package_namespaces, namespaces): continue packages_to_remove.add(package_.name) if not packages_to_remove: # Nothing to do so exit early. return with open(package.filepath, "r") as handler: code = handler.read() new_code = api.remove_from_attribute("requires", list(packages_to_remove), code) with filesystem.make_path_writable( os.path.dirname(os.path.dirname(package.filepath))): with serialise.open_file_for_write(package.filepath) as handler: handler.write(new_code)
def _bump(package, increment, new_dependencies): rez_bump_api.bump(package, **{increment: 1}) with open(package.filepath, "r") as handler: code = handler.read() new_code = api.add_to_attribute("requires", new_dependencies, code) with filesystem.make_path_writable( os.path.dirname(os.path.dirname(package.filepath))): with serialise.open_file_for_write(package.filepath) as handler: handler.write(new_code) root = finder.get_package_root(package) return finder.get_nearest_rez_package(root)
def patch_rpaths(elfpath, rpaths): """Replace an elf's rpath header with those provided. """ # this is a hack to get around https://github.com/nerdvegas/rez/issues/1074 # I actually hit a case where patchelf was installed as a rez suite tool, # causing '$ORIGIN' to be expanded early (to empty string). # TODO remove this hack when bug is fixed # env = os.environ.copy() env["ORIGIN"] = "$ORIGIN" with make_path_writable(elfpath): if rpaths: _run("patchelf", "--set-rpath", ':'.join(rpaths), elfpath, env=env) else: _run("patchelf", "--remove-rpath")
def _run_command_on_package(package): """Run the user-provided command on the given Rez package. Args: package (:class:`rez.developer_package.DeveloperPackage`): The Rez package that presumably is a package.yaml file that needs to be changed. Raises: :class:`.InvalidPackage: If `package` is not a package.yaml file. Returns: str: Any error message that occurred from this command, if any. """ if not package.filepath.endswith(".yaml"): raise exceptions.InvalidPackage( package, finder.get_package_root(package), 'Package "{package}" is not a YAML file.'.format( package=package), ) buffer_ = io.StringIO() package.print_info(format_=serialise.FileFormat.py, buf=buffer_) code = buffer_.getvalue() path = os.path.join(finder.get_package_root(package), "package.py") _LOGGER.info('Now creating "%s".', path) # Writing to DeveloperPackage objects is currently bugged. # So instead, we disable caching during write. # # Reference: https://github.com/nerdvegas/rez/issues/857 # with filesystem.make_path_writable( os.path.dirname(os.path.dirname(path))): with serialise.open_file_for_write(path) as handler: handler.write(code) _LOGGER.info('Now removing the old "%s".', package.filepath) os.remove(package.filepath) return ""
def _write_package_to_disk(package, version): """Update a Rez package on-disk with its new contents. Args: package (:class:`rez.packages_.DeveloperPackage`): Some package on-disk to write out. version (str): The new semantic version that will be written to-disk. """ with open(package, "r") as handler: code = handler.read() graph = parso.parse(code) try: assignment = parso_utility.find_assignment_nodes("version", graph)[-1] except IndexError: assignment = None prefix = " " if assignment: prefix = assignment.children[0].prefix.strip("\n") or prefix node = tree.String('"{version}"'.format(version=version), (0, 0), prefix=prefix) graph = convention.insert_or_append(node, graph, assignment, "version") # Writing to DeveloperPackage objects is currently bugged. # So instead, we disable caching during write. # # Reference: https://github.com/nerdvegas/rez/issues/857 # with filesystem.make_path_writable( os.path.dirname(os.path.dirname(package))): with serialise.open_file_for_write(package) as handler: handler.write(graph.get_code())
def _copy_variant_payload(src_variant, dest_pkg_repo, shallow=False, follow_symlinks=False, overrides=None, verbose=False): # Get payload path of source variant. For some types (eg from a "memory" # type repo) there may not be a root. # variant_root = getattr(src_variant, "root", None) if not variant_root: raise PackageCopyError( "Cannot copy source variant %s - it is a type of variant that " "does not have a root.", src_variant.uri) if not os.path.isdir(variant_root): raise PackageCopyError( "Cannot copy source variant %s - its root does not appear to " "be present on disk (%s).", src_variant.uri, variant_root) dest_variant_name = overrides.get("name") or src_variant.name dest_variant_version = overrides.get("version") or src_variant.version # determine variant installation path dest_pkg_payload_path = dest_pkg_repo.get_package_payload_path( package_name=dest_variant_name, package_version=dest_variant_version) if src_variant.subpath: variant_install_path = os.path.join(dest_pkg_payload_path, src_variant.subpath) else: variant_install_path = dest_pkg_payload_path # get ready for copy/symlinking copy_func = partial(replacing_copy, follow_symlinks=follow_symlinks) if shallow: maybe_symlink = replacing_symlink else: maybe_symlink = copy_func # possibly make install path temporarily writable last_dir = get_existing_path( variant_install_path, topmost_path=os.path.dirname(dest_pkg_payload_path)) if last_dir: ctxt = make_path_writable(last_dir) else: ctxt = with_noop() # copy the variant payload with ctxt: safe_makedirs(variant_install_path) # determine files not to copy skip_files = [] if src_variant.subpath: # Detect overlapped variants. This is the case where one variant subpath # might be A, and another is A/B. We must ensure that A/B is not created # as a symlink during shallow install of variant A - that would then # cause A/B payload to be installed back into original package, possibly # corrupting it. # # Here we detect this case, and create a list of dirs not to copy/link, # because they are in fact a subpath dir for another variant. # skip_files.extend(_get_overlapped_variant_dirs(src_variant)) else: # just skip package definition file for name in config.plugins.package_repository.filesystem.package_filenames: for fmt in (FileFormat.py, FileFormat.yaml): filename = name + '.' + fmt.extension skip_files.append(filename) # copy/link all topmost files within the variant root for name in os.listdir(variant_root): if name in skip_files: filepath = os.path.join(variant_root, name) if verbose: if src_variant.subpath: msg = ("Did not copy %s - this is part of an " "overlapping variant's root path.") else: msg = "Did not copy package definition file %s" print_info(msg, filepath) continue src_path = os.path.join(variant_root, name) dest_path = os.path.join(variant_install_path, name) if os.path.islink(src_path): copy_func(src_path, dest_path) else: maybe_symlink(src_path, dest_path) # copy permissions of source variant dirs onto dest src_package = src_variant.parent src_pkg_repo = src_package.repository src_pkg_payload_path = src_pkg_repo.get_package_payload_path( package_name=src_package.name, package_version=src_package.version) shutil.copystat(src_pkg_payload_path, dest_pkg_payload_path) subpath = src_variant.subpath while subpath: src_path = os.path.join(src_pkg_payload_path, subpath) dest_path = os.path.join(dest_pkg_payload_path, subpath) shutil.copystat(src_path, dest_path) subpath = os.path.dirname(subpath)
def _build_variant_base(self, variant, build_type, install_path=None, clean=False, install=False, **kwargs): # create build/install paths install_path = install_path or self.package.config.local_packages_path package_install_path = self.get_package_install_path(install_path) variant_build_path = self.build_path if variant.index is None: variant_install_path = package_install_path else: subpath = variant._non_shortlinked_subpath variant_build_path = os.path.join(variant_build_path, subpath) variant_install_path = os.path.join(package_install_path, subpath) # create directories (build, install) if clean and os.path.exists(variant_build_path): self._rmtree(variant_build_path) safe_makedirs(variant_build_path) # find last dir of installation path that exists, and possibly make it # writable during variant installation # last_dir = get_existing_path(variant_install_path, topmost_path=install_path) if last_dir: ctxt = make_path_writable(last_dir) else: ctxt = with_noop() with ctxt: if install: # inform package repo that a variant is about to be built/installed pkg_repo = package_repository_manager.get_repository( install_path) pkg_repo.pre_variant_install(variant.resource) if not os.path.exists(variant_install_path): safe_makedirs(variant_install_path) # if hashed variants are enabled, create the variant shortlink if variant.parent.hashed_variants: try: # create the dir containing all shortlinks base_shortlinks_path = os.path.join( package_install_path, variant.parent.config.variant_shortlinks_dirname) safe_makedirs(base_shortlinks_path) # create the shortlink rel_variant_path = os.path.relpath( variant_install_path, base_shortlinks_path) create_unique_base26_symlink(base_shortlinks_path, rel_variant_path) except Exception as e: # Treat any error as warning - lack of shortlink is not # a breaking issue, it just means the variant root path # will be long. # print_warning( "Error creating variant shortlink for %s: %s: %s", variant_install_path, e.__class__.__name__, e) # Re-evaluate the variant, so that variables such as 'building' and # 'build_variant_index' are set, and any early-bound package attribs # are re-evaluated wrt these vars. This is done so that attribs such as # 'requires' can change depending on whether a build is occurring or not. # # Note that this re-evaluated variant is ONLY used here, for the purposes # of creating the build context. The variant that is actually installed # is the one evaluated where 'building' is False. # re_evaluated_package = variant.parent.get_reevaluated({ "building": True, "build_variant_index": variant.index or 0, "build_variant_requires": variant.variant_requires }) re_evaluated_variant = re_evaluated_package.get_variant( variant.index) # create build environment (also creates build.rxt file) context, rxt_filepath = self.create_build_context( variant=re_evaluated_variant, build_type=build_type, build_path=variant_build_path) # list of extra files (build.rxt etc) that are installed if an # installation is taking place # extra_install_files = [rxt_filepath] # create variant.json file. This identifies which variant this is. # This is important for hashed variants, where it is not obvious # which variant is in which root path. The file is there for # debugging purposes only. # if variant.index is not None: data = { "index": variant.index, "data": variant.parent.data["variants"][variant.index] } filepath = os.path.join(variant_build_path, "variant.json") extra_install_files.append(filepath) with open(filepath, 'w') as f: json.dump(data, f, indent=2) # run build system build_system_name = self.build_system.name() self._print("\nInvoking %s build system...", build_system_name) build_result = self.build_system.build( context=context, variant=variant, build_path=variant_build_path, install_path=variant_install_path, install=install, build_type=build_type) if not build_result.get("success"): # delete the possibly partially installed variant payload if install: self._rmtree(variant_install_path) raise BuildError("The %s build system failed." % build_system_name) if install: # add some installation details to build result build_result.update({ "package_install_path": package_install_path, "variant_install_path": variant_install_path }) # the build system can also specify extra files that need to # be installed filepaths = build_result.get("extra_files") if filepaths: extra_install_files.extend(filepaths) # install extra files for file_ in extra_install_files: copy_or_replace(file_, variant_install_path) # Install include modules. Note that this doesn't need to be done # multiple times, but for subsequent variants it has no effect. # self._install_include_modules(install_path) return build_result
def _build_variant_base(self, variant, build_type, install_path=None, clean=False, install=False, **kwargs): # create build/install paths install_path = install_path or self.package.config.local_packages_path package_install_path = self.get_package_install_path(install_path) variant_build_path = self.build_path if variant.index is None: variant_install_path = package_install_path else: subpath = variant._non_shortlinked_subpath variant_build_path = os.path.join(variant_build_path, subpath) variant_install_path = os.path.join(package_install_path, subpath) # create directories (build, install) if clean and os.path.exists(variant_build_path): shutil.rmtree(variant_build_path) safe_makedirs(variant_build_path) # find last dir of installation path that exists, and possibly make it # writable during variant installation # last_dir = get_existing_path(variant_install_path, topmost_path=install_path) if last_dir: ctxt = make_path_writable(last_dir) else: ctxt = with_noop() with ctxt: if install: # inform package repo that a variant is about to be built/installed pkg_repo = package_repository_manager.get_repository(install_path) pkg_repo.pre_variant_install(variant.resource) if not os.path.exists(variant_install_path): safe_makedirs(variant_install_path) # if hashed variants are enabled, create the variant shortlink if variant.parent.hashed_variants: try: # create the dir containing all shortlinks base_shortlinks_path = os.path.join( package_install_path, variant.parent.config.variant_shortlinks_dirname ) safe_makedirs(base_shortlinks_path) # create the shortlink rel_variant_path = os.path.relpath( variant_install_path, base_shortlinks_path) create_unique_base26_symlink( base_shortlinks_path, rel_variant_path) except Exception as e: # Treat any error as warning - lack of shortlink is not # a breaking issue, it just means the variant root path # will be long. # print_warning( "Error creating variant shortlink for %s: %s: %s", variant_install_path, e.__class__.__name__, e ) # Re-evaluate the variant, so that variables such as 'building' and # 'build_variant_index' are set, and any early-bound package attribs # are re-evaluated wrt these vars. This is done so that attribs such as # 'requires' can change depending on whether a build is occurring or not. # # Note that this re-evaluated variant is ONLY used here, for the purposes # of creating the build context. The variant that is actually installed # is the one evaluated where 'building' is False. # re_evaluated_package = variant.parent.get_reevaluated({ "building": True, "build_variant_index": variant.index or 0, "build_variant_requires": variant.variant_requires }) re_evaluated_variant = re_evaluated_package.get_variant(variant.index) # create build environment context, rxt_filepath = self.create_build_context( variant=re_evaluated_variant, build_type=build_type, build_path=variant_build_path) # run build system build_system_name = self.build_system.name() self._print("\nInvoking %s build system...", build_system_name) build_result = self.build_system.build( context=context, variant=variant, build_path=variant_build_path, install_path=variant_install_path, install=install, build_type=build_type) if not build_result.get("success"): raise BuildError("The %s build system failed." % build_system_name) if install: # Install the 'variant.json' file, which identifies which variant # this is. This is important for hashed variants, where it is not # obvious which variant is in which root path. The file is there # for debugging purposes only. # if variant.index is not None: data = { "index": variant.index, "data": variant.parent.data["variants"][variant.index] } filepath = os.path.join(variant_install_path, "variant.json") with open(filepath, 'w') as f: json.dump(data, f, indent=2) # install some files for debugging purposes (incl build.rxt) extra_files = build_result.get("extra_files", []) if rxt_filepath: extra_files = extra_files + [rxt_filepath] for file_ in extra_files: copy_or_replace(file_, variant_install_path) # Install include modules. Note that this doesn't need to be done # multiple times, but for subsequent variants it has no effect. # self._install_include_modules(install_path) return build_result
def _create_variant(self, variant, dry_run=False, overrides=None): # special case overrides variant_name = overrides.get("name") or variant.name variant_version = overrides.get("version") or variant.version overrides = (overrides or {}).copy() overrides.pop("name", None) overrides.pop("version", None) # find or create the package family family = self.get_package_family(variant_name) if not family: family = self._create_family(variant_name) if isinstance(family, FileSystemCombinedPackageFamilyResource): raise NotImplementedError( "Cannot install variant into combined-style package file %r." % family.filepath) # find the package if it already exists existing_package = None for package in self.iter_packages(family): if package.version == variant_version: # during a build, the family/version dirs get created ahead of # time, which causes a 'missing package definition file' error. # This is fine, we can just ignore it and write the new file. try: package.validate_data() except PackageDefinitionFileMissing: break uuids = set([variant.uuid, package.uuid]) if len(uuids) > 1 and None not in uuids: raise ResourceError( "Cannot install variant %r into package %r - the " "packages are not the same (UUID mismatch)" % (variant, package)) existing_package = package if variant.index is None: if package.variants: raise ResourceError( "Attempting to install a package without variants " "(%r) into an existing package with variants (%r)" % (variant, package)) elif not package.variants: raise ResourceError( "Attempting to install a variant (%r) into an existing " "package without variants (%r)" % (variant, package)) existing_package_data = None release_data = {} # Need to treat 'config' as special case. In validated data, this is # converted to a Config object. We need it as the raw dict that you'd # see in a package.py. # def _get_package_data(pkg): data = pkg.validated_data() if hasattr(pkg, "_data"): raw_data = pkg._data else: raw_data = pkg.resource._data raw_config_data = raw_data.get('config') data.pop("config", None) if raw_config_data: data["config"] = raw_config_data return data def _remove_build_keys(obj): for key in package_build_only_keys: obj.pop(key, None) new_package_data = _get_package_data(variant.parent) new_package_data.pop("variants", None) new_package_data["name"] = variant_name if variant_version: new_package_data["version"] = variant_version package_changed = False _remove_build_keys(new_package_data) if existing_package: debug_print( "Found existing package for installation of variant %s: %s", variant.uri, existing_package.uri ) existing_package_data = _get_package_data(existing_package) _remove_build_keys(existing_package_data) # detect case where new variant introduces package changes outside of variant data_1 = existing_package_data.copy() data_2 = new_package_data.copy() for key in package_release_keys: data_2.pop(key, None) value = data_1.pop(key, None) if value is not None: release_data[key] = value for key in ("format_version", "base", "variants"): data_1.pop(key, None) data_2.pop(key, None) package_changed = (data_1 != data_2) if debug_print: if package_changed: from rez.utils.data_utils import get_dict_diff_str debug_print("Variant %s package data differs from package %s", variant.uri, existing_package.uri) txt = get_dict_diff_str(data_1, data_2, "Changes:") debug_print(txt) else: debug_print("Variant %s package data matches package %s", variant.uri, existing_package.uri) # check for existing installed variant existing_installed_variant = None installed_variant_index = None if existing_package: if variant.index is None: existing_installed_variant = \ self.iter_variants(existing_package).next() else: variant_requires = variant.variant_requires for variant_ in self.iter_variants(existing_package): variant_requires_ = existing_package.variants[variant_.index] if variant_requires_ == variant_requires: installed_variant_index = variant_.index existing_installed_variant = variant_ if existing_installed_variant: debug_print( "Variant %s already has installed equivalent: %s", variant.uri, existing_installed_variant.uri ) if dry_run: if not package_changed: return existing_installed_variant else: return None # construct package data for new installed package definition if existing_package: _, file_ = os.path.split(existing_package.filepath) package_filename, package_extension = os.path.splitext(file_) package_extension = package_extension[1:] package_format = FileFormat[package_extension] if package_changed: # graft together new package data, with existing package variants, # and other data that needs to stay unchanged (eg timestamp) package_data = new_package_data if variant.index is not None: package_data["variants"] = existing_package_data.get("variants", []) else: package_data = existing_package_data else: package_data = new_package_data package_filename = _settings.package_filenames[0] package_extension = "py" package_format = FileFormat.py # merge existing release data (if any) into the package. Note that when # this data becomes variant-specific, this step will no longer be needed package_data.update(release_data) # merge the new variant into the package if installed_variant_index is None and variant.index is not None: variant_requires = variant.variant_requires if not package_data.get("variants"): package_data["variants"] = [] package_data["variants"].append(variant_requires) installed_variant_index = len(package_data["variants"]) - 1 # a little data massaging is needed package_data.pop("base", None) # create version dir if it doesn't already exist family_path = os.path.join(self.location, variant_name) if variant_version: pkg_base_path = os.path.join(family_path, str(variant_version)) else: pkg_base_path = family_path if not os.path.exists(pkg_base_path): os.makedirs(pkg_base_path) # Apply overrides. # # If we're installing into an existing package, then existing attributes # in that package take precedence over `overrides`. If we're installing # to a new package, then `overrides` takes precedence always. # # This is done so that variants added to an existing package don't change # attributes such as 'timestamp' or release-related fields like 'revision'. # for key, value in overrides.iteritems(): if existing_package: if key not in package_data: package_data[key] = value else: if value is self.remove: package_data.pop(key, None) else: package_data[key] = value # timestamp defaults to now if not specified if not package_data.get("timestamp"): package_data["timestamp"] = int(time.time()) # format version is always set package_data["format_version"] = format_version # write out new package definition file package_file = ".".join([package_filename, package_extension]) filepath = os.path.join(pkg_base_path, package_file) with make_path_writable(pkg_base_path): with open_file_for_write(filepath, mode=self.package_file_mode) as f: dump_package_data(package_data, buf=f, format_=package_format) # delete the tmp 'building' file. if variant_version: filename = self.building_prefix + str(variant_version) filepath = os.path.join(family_path, filename) if os.path.exists(filepath): try: os.remove(filepath) except: pass # delete other stale building files; previous failed releases may have # left some around try: self._delete_stale_build_tagfiles(family_path) except: pass # touch the family dir, this keeps memcached resolves updated properly os.utime(family_path, None) # load new variant new_variant = None self.clear_caches() family = self.get_package_family(variant_name) if family: for package in self.iter_packages(family): if package.version == variant_version: for variant_ in self.iter_variants(package): if variant_.index == installed_variant_index: new_variant = variant_ break elif new_variant: break if not new_variant: raise RezSystemError("Internal failure - expected installed variant") return new_variant
def _copy_variant_payload(src_variant, dest_pkg_repo, shallow=False, follow_symlinks=False, overrides=None, verbose=False): # Get payload path of source variant. For some types (eg from a "memory" # type repo) there may not be a root. # variant_root = getattr(src_variant, "root", None) if not variant_root: raise PackageCopyError( "Cannot copy source variant %s - it is a type of variant that " "does not have a root.", src_variant.uri ) if not os.path.isdir(variant_root): raise PackageCopyError( "Cannot copy source variant %s - its root does not appear to " "be present on disk (%s).", src_variant.uri, variant_root ) dest_variant_name = overrides.get("name") or src_variant.name dest_variant_version = overrides.get("version") or src_variant.version # determine variant installation path dest_pkg_payload_path = dest_pkg_repo.get_package_payload_path( package_name=dest_variant_name, package_version=dest_variant_version ) if src_variant.subpath: variant_install_path = os.path.join(dest_pkg_payload_path, src_variant.subpath) else: variant_install_path = dest_pkg_payload_path # get ready for copy/symlinking copy_func = partial(replacing_copy, follow_symlinks=follow_symlinks) if shallow: maybe_symlink = replacing_symlink else: maybe_symlink = copy_func # possibly make install path temporarily writable last_dir = get_existing_path( variant_install_path, topmost_path=os.path.dirname(dest_pkg_payload_path)) if last_dir: ctxt = make_path_writable(last_dir) else: ctxt = with_noop() # copy the variant payload with ctxt: safe_makedirs(variant_install_path) # determine files not to copy skip_files = [] if src_variant.subpath: # Detect overlapped variants. This is the case where one variant subpath # might be A, and another is A/B. We must ensure that A/B is not created # as a symlink during shallow install of variant A - that would then # cause A/B payload to be installed back into original package, possibly # corrupting it. # # Here we detect this case, and create a list of dirs not to copy/link, # because they are in fact a subpath dir for another variant. # skip_files.extend(_get_overlapped_variant_dirs(src_variant)) else: # just skip package definition file for name in config.plugins.package_repository.filesystem.package_filenames: for fmt in (FileFormat.py, FileFormat.yaml): filename = name + '.' + fmt.extension skip_files.append(filename) # copy/link all topmost files within the variant root for name in os.listdir(variant_root): if name in skip_files: filepath = os.path.join(variant_root, name) if verbose: if src_variant.subpath: msg = ("Did not copy %s - this is part of an " "overlapping variant's root path.") else: msg = "Did not copy package definition file %s" print_info(msg, filepath) continue src_path = os.path.join(variant_root, name) dest_path = os.path.join(variant_install_path, name) if os.path.islink(src_path): copy_func(src_path, dest_path) else: maybe_symlink(src_path, dest_path) # copy permissions of source variant dirs onto dest src_package = src_variant.parent src_pkg_repo = src_package.repository src_pkg_payload_path = src_pkg_repo.get_package_payload_path( package_name=src_package.name, package_version=src_package.version ) shutil.copystat(src_pkg_payload_path, dest_pkg_payload_path) subpath = src_variant.subpath while subpath: src_path = os.path.join(src_pkg_payload_path, subpath) dest_path = os.path.join(dest_pkg_payload_path, subpath) shutil.copystat(src_path, dest_path) subpath = os.path.dirname(subpath)
def _add_new_requirement_packages(package, namespaces, requirements, force=False): """Add new Rez package requirements to a Rez package, if needed. If no import statements were changed then this function does nothing. After all, if the imports of a package were changed then there's no way a Rez package's requirements should be any different. Args: package (:class:`rez.packges_.DeveloperPackage`): Some Rez package whose requirements may change as a result of this function getting ran. namespaces (iter[str]): The Python dot-separated namespaces that a Rez package uses. In short, these are all of the import statements that a Rez package has inside of it and can be thought of as its "dependencies". The function uses this list to figure out if `user_namespaces` is actually still being imported. requirements (iter[tuple[:class:`rez.vendor.version.requirement.Requirement`, tuple[str]]]): Each Rez package that might get added to `package` and a series of Python namespaces that the Rez package defines. If there's any overlap between the package's namespaces and the full `namespaces` then that means that `package` depends on the Rez package and so it is added as a dependency to `package`. force (bool, optional): If True, change every requirement that already exists in `package` and `requirements`. If False, only change package requirements if `requirements` is in the list of changed `namespaces`. Default is False. Returns: bool: If `package` was changed as a result of this function. """ packages_to_add = set() if force: existing = set(requirement.name for requirement in package.requires or []) packages_to_add = set( str(package_) for package_, _ in requirements if package_.name in existing) else: for package_, package_namespaces in requirements: if not _is_package_needed(tuple(package_namespaces), namespaces): continue packages_to_add.add(str(package_)) if not packages_to_add: # Nothing to do so exit early. return False with open(package.filepath, "r") as handler: code = handler.read() new_code = api.add_to_attribute("requires", list(packages_to_add), code) with filesystem.make_path_writable( os.path.dirname(os.path.dirname(package.filepath))): with serialise.open_file_for_write(package.filepath) as handler: handler.write(new_code) return True
def _build_variant_base(self, variant, build_type, install_path=None, clean=False, install=False, **kwargs): # create build/install paths install_path = install_path or self.package.config.local_packages_path variant_install_path = self.get_package_install_path(install_path) variant_build_path = self.build_path if variant.subpath: variant_build_path = os.path.join(variant_build_path, variant.subpath) variant_install_path = os.path.join(variant_install_path, variant.subpath) # create directories (build, install) if clean and os.path.exists(variant_build_path): shutil.rmtree(variant_build_path) safe_makedirs(variant_build_path) # find last dir of installation path that exists, and possibly make it # writable during variant installation # last_dir = get_existing_path(variant_install_path, topmost_path=install_path) if last_dir: ctxt = make_path_writable(last_dir) else: ctxt = with_noop() with ctxt: if install: # inform package repo that a variant is about to be built/installed pkg_repo = package_repository_manager.get_repository(install_path) pkg_repo.pre_variant_install(variant.resource) if not os.path.exists(variant_install_path): safe_makedirs(variant_install_path) # Re-evaluate the variant, so that variables such as 'building' and # 'build_variant_index' are set, and any early-bound package attribs # are re-evaluated wrt these vars. This is done so that attribs such as # 'requires' can change depending on whether a build is occurring or not. # # Note that this re-evaluated variant is ONLY used here, for the purposes # of creating the build context. The variant that is actually installed # is the one evaluated where 'building' is False. # re_evaluated_package = variant.parent.get_reevaluated({ "building": True, "build_variant_index": variant.index or 0, "build_variant_requires": variant.variant_requires }) re_evaluated_variant = re_evaluated_package.get_variant(variant.index) # create build environment context, rxt_filepath = self.create_build_context( variant=re_evaluated_variant, build_type=build_type, build_path=variant_build_path) # run build system build_system_name = self.build_system.name() self._print("\nInvoking %s build system...", build_system_name) build_result = self.build_system.build( context=context, variant=variant, build_path=variant_build_path, install_path=variant_install_path, install=install, build_type=build_type) if not build_result.get("success"): raise BuildError("The %s build system failed." % build_system_name) if install: # install some files for debugging purposes extra_files = build_result.get("extra_files", []) if rxt_filepath: extra_files = extra_files + [rxt_filepath] for file_ in extra_files: copy_or_replace(file_, variant_install_path) # Install include modules. Note that this doesn't need to be done # multiple times, but for subsequent variants it has no effect. # self._install_include_modules(install_path) return build_result
def _copy_variant_payload(src_variant, dest_pkg_repo, shallow=False, follow_symlinks=False, overrides=None, verbose=False): # Get payload path of source variant. For some types (eg from a "memory" # type repo) there may not be a root. # variant_root = getattr(src_variant, "root", None) if not variant_root: raise PackageCopyError( "Cannot copy source variant %s - it is a type of variant that " "does not have a root." % src_variant.uri ) if not os.path.isdir(variant_root): raise PackageCopyError( "Cannot copy source variant %s - its root does not appear to " "be present on disk (%s)." % src_variant.uri, variant_root ) dest_variant_name = overrides.get("name") or src_variant.name dest_variant_version = overrides.get("version") or src_variant.version # determine variant installation path dest_pkg_payload_path = dest_pkg_repo.get_package_payload_path( package_name=dest_variant_name, package_version=dest_variant_version ) is_varianted = (src_variant.index is not None) src_variant_subpath = None if is_varianted: src_variant_subpath = src_variant._non_shortlinked_subpath variant_install_path = os.path.join( dest_pkg_payload_path, src_variant_subpath) else: variant_install_path = dest_pkg_payload_path # get ready for copy/symlinking copy_func = partial(replacing_copy, follow_symlinks=follow_symlinks) if shallow: maybe_symlink = replacing_symlink else: maybe_symlink = copy_func # possibly make install path temporarily writable last_dir = get_existing_path( variant_install_path, topmost_path=os.path.dirname(dest_pkg_payload_path)) if last_dir and config.make_package_temporarily_writable: ctxt = make_path_writable(last_dir) else: ctxt = with_noop() # copy the variant payload with ctxt: safe_makedirs(variant_install_path) # determine files not to copy skip_files = [] if is_varianted and not src_variant.parent.hashed_variants: # Detect overlapped variants. This is the case where one variant subpath # might be A, and another is A/B. We must ensure that A/B is not created # as a symlink during shallow install of variant A - that would then # cause A/B payload to be installed back into original package, possibly # corrupting it. # # Here we detect this case, and create a list of dirs not to copy/link, # because they are in fact a subpath dir for another variant. # # Note that for hashed variants, we don't do this check because overlapped # variants are not possible. # skip_files.extend(_get_overlapped_variant_dirs(src_variant)) else: # just skip package definition file for name in config.plugins.package_repository.filesystem.package_filenames: for fmt in (FileFormat.py, FileFormat.yaml): filename = name + '.' + fmt.extension skip_files.append(filename) # copy/link all topmost files within the variant root for name in os.listdir(variant_root): if name in skip_files: filepath = os.path.join(variant_root, name) if verbose and is_varianted: print_info( "Did not copy %s - this is part of an overlapping " "variant's root path.", filepath ) continue src_path = os.path.join(variant_root, name) dest_path = os.path.join(variant_install_path, name) if os.path.islink(src_path): copy_func(src_path, dest_path) else: maybe_symlink(src_path, dest_path) # copy permissions of source variant dirs onto dest src_package = src_variant.parent src_pkg_repo = src_package.repository src_pkg_payload_path = src_pkg_repo.get_package_payload_path( package_name=src_package.name, package_version=src_package.version ) shutil.copystat(src_pkg_payload_path, dest_pkg_payload_path) subpath = src_variant_subpath while subpath: src_path = os.path.join(src_pkg_payload_path, subpath) dest_path = os.path.join(dest_pkg_payload_path, subpath) shutil.copystat(src_path, dest_path) subpath = os.path.dirname(subpath) # create the variant shortlink if src_variant.parent.hashed_variants: try: # base _v dir base_shortlinks_path = os.path.join( dest_pkg_payload_path, src_package.config.variant_shortlinks_dirname ) safe_makedirs(base_shortlinks_path) # shortlink rel_variant_path = os.path.relpath( variant_install_path, base_shortlinks_path) create_unique_base26_symlink( base_shortlinks_path, rel_variant_path) except Exception as e: # Treat any error as warning - lack of shortlink is not # a breaking issue, it just means the variant root path # will be long. # print_warning( "Error creating variant shortlink for %s: %s: %s", variant_install_path, e.__class__.__name__, e )