def _copy_package_include_modules(src_package, dest_pkg_repo, overrides=None): src_include_modules_path = \ os.path.join(src_package.base, IncludeModuleManager.include_modules_subpath) if not os.path.exists(src_include_modules_path): return dest_package_name = overrides.get("name") or src_package.name dest_package_version = overrides.get("version") or src_package.version pkg_install_path = dest_pkg_repo.get_package_payload_path( package_name=dest_package_name, package_version=dest_package_version) dest_include_modules_path = \ os.path.join(pkg_install_path, IncludeModuleManager.include_modules_subpath) last_dir = get_existing_path( dest_include_modules_path, topmost_path=os.path.dirname(pkg_install_path)) if last_dir: ctxt = make_path_writable(last_dir) else: ctxt = with_noop() with ctxt: safe_makedirs(dest_include_modules_path) additive_copytree(src_include_modules_path, dest_include_modules_path)
def _copy_package_include_modules(src_package, dest_pkg_repo, overrides=None): src_include_modules_path = \ os.path.join(src_package.base, IncludeModuleManager.include_modules_subpath) if not os.path.exists(src_include_modules_path): return dest_package_name = overrides.get("name") or src_package.name dest_package_version = overrides.get("version") or src_package.version pkg_install_path = dest_pkg_repo.get_package_payload_path( package_name=dest_package_name, package_version=dest_package_version ) dest_include_modules_path = \ os.path.join(pkg_install_path, IncludeModuleManager.include_modules_subpath) last_dir = get_existing_path(dest_include_modules_path, topmost_path=os.path.dirname(pkg_install_path)) if last_dir: ctxt = make_path_writable(last_dir) else: ctxt = with_noop() with ctxt: safe_makedirs(dest_include_modules_path) additive_copytree(src_include_modules_path, dest_include_modules_path)
def _copy_variant_payload(src_variant, dest_pkg_repo, shallow=False, follow_symlinks=False, overrides=None, verbose=False): # Get payload path of source variant. For some types (eg from a "memory" # type repo) there may not be a root. # variant_root = getattr(src_variant, "root", None) if not variant_root: raise PackageCopyError( "Cannot copy source variant %s - it is a type of variant that " "does not have a root.", src_variant.uri) if not os.path.isdir(variant_root): raise PackageCopyError( "Cannot copy source variant %s - its root does not appear to " "be present on disk (%s).", src_variant.uri, variant_root) dest_variant_name = overrides.get("name") or src_variant.name dest_variant_version = overrides.get("version") or src_variant.version # determine variant installation path dest_pkg_payload_path = dest_pkg_repo.get_package_payload_path( package_name=dest_variant_name, package_version=dest_variant_version) if src_variant.subpath: variant_install_path = os.path.join(dest_pkg_payload_path, src_variant.subpath) else: variant_install_path = dest_pkg_payload_path # get ready for copy/symlinking copy_func = partial(replacing_copy, follow_symlinks=follow_symlinks) if shallow: maybe_symlink = replacing_symlink else: maybe_symlink = copy_func # possibly make install path temporarily writable last_dir = get_existing_path( variant_install_path, topmost_path=os.path.dirname(dest_pkg_payload_path)) if last_dir: ctxt = make_path_writable(last_dir) else: ctxt = with_noop() # copy the variant payload with ctxt: safe_makedirs(variant_install_path) # determine files not to copy skip_files = [] if src_variant.subpath: # Detect overlapped variants. This is the case where one variant subpath # might be A, and another is A/B. We must ensure that A/B is not created # as a symlink during shallow install of variant A - that would then # cause A/B payload to be installed back into original package, possibly # corrupting it. # # Here we detect this case, and create a list of dirs not to copy/link, # because they are in fact a subpath dir for another variant. # skip_files.extend(_get_overlapped_variant_dirs(src_variant)) else: # just skip package definition file for name in config.plugins.package_repository.filesystem.package_filenames: for fmt in (FileFormat.py, FileFormat.yaml): filename = name + '.' + fmt.extension skip_files.append(filename) # copy/link all topmost files within the variant root for name in os.listdir(variant_root): if name in skip_files: filepath = os.path.join(variant_root, name) if verbose: if src_variant.subpath: msg = ("Did not copy %s - this is part of an " "overlapping variant's root path.") else: msg = "Did not copy package definition file %s" print_info(msg, filepath) continue src_path = os.path.join(variant_root, name) dest_path = os.path.join(variant_install_path, name) if os.path.islink(src_path): copy_func(src_path, dest_path) else: maybe_symlink(src_path, dest_path) # copy permissions of source variant dirs onto dest src_package = src_variant.parent src_pkg_repo = src_package.repository src_pkg_payload_path = src_pkg_repo.get_package_payload_path( package_name=src_package.name, package_version=src_package.version) shutil.copystat(src_pkg_payload_path, dest_pkg_payload_path) subpath = src_variant.subpath while subpath: src_path = os.path.join(src_pkg_payload_path, subpath) dest_path = os.path.join(dest_pkg_payload_path, subpath) shutil.copystat(src_path, dest_path) subpath = os.path.dirname(subpath)
def _build_variant_base(self, variant, build_type, install_path=None, clean=False, install=False, **kwargs): # create build/install paths install_path = install_path or self.package.config.local_packages_path package_install_path = self.get_package_install_path(install_path) variant_build_path = self.build_path if variant.index is None: variant_install_path = package_install_path else: subpath = variant._non_shortlinked_subpath variant_build_path = os.path.join(variant_build_path, subpath) variant_install_path = os.path.join(package_install_path, subpath) # create directories (build, install) if clean and os.path.exists(variant_build_path): self._rmtree(variant_build_path) safe_makedirs(variant_build_path) # find last dir of installation path that exists, and possibly make it # writable during variant installation # last_dir = get_existing_path(variant_install_path, topmost_path=install_path) if last_dir: ctxt = make_path_writable(last_dir) else: ctxt = with_noop() with ctxt: if install: # inform package repo that a variant is about to be built/installed pkg_repo = package_repository_manager.get_repository( install_path) pkg_repo.pre_variant_install(variant.resource) if not os.path.exists(variant_install_path): safe_makedirs(variant_install_path) # if hashed variants are enabled, create the variant shortlink if variant.parent.hashed_variants: try: # create the dir containing all shortlinks base_shortlinks_path = os.path.join( package_install_path, variant.parent.config.variant_shortlinks_dirname) safe_makedirs(base_shortlinks_path) # create the shortlink rel_variant_path = os.path.relpath( variant_install_path, base_shortlinks_path) create_unique_base26_symlink(base_shortlinks_path, rel_variant_path) except Exception as e: # Treat any error as warning - lack of shortlink is not # a breaking issue, it just means the variant root path # will be long. # print_warning( "Error creating variant shortlink for %s: %s: %s", variant_install_path, e.__class__.__name__, e) # Re-evaluate the variant, so that variables such as 'building' and # 'build_variant_index' are set, and any early-bound package attribs # are re-evaluated wrt these vars. This is done so that attribs such as # 'requires' can change depending on whether a build is occurring or not. # # Note that this re-evaluated variant is ONLY used here, for the purposes # of creating the build context. The variant that is actually installed # is the one evaluated where 'building' is False. # re_evaluated_package = variant.parent.get_reevaluated({ "building": True, "build_variant_index": variant.index or 0, "build_variant_requires": variant.variant_requires }) re_evaluated_variant = re_evaluated_package.get_variant( variant.index) # create build environment (also creates build.rxt file) context, rxt_filepath = self.create_build_context( variant=re_evaluated_variant, build_type=build_type, build_path=variant_build_path) # list of extra files (build.rxt etc) that are installed if an # installation is taking place # extra_install_files = [rxt_filepath] # create variant.json file. This identifies which variant this is. # This is important for hashed variants, where it is not obvious # which variant is in which root path. The file is there for # debugging purposes only. # if variant.index is not None: data = { "index": variant.index, "data": variant.parent.data["variants"][variant.index] } filepath = os.path.join(variant_build_path, "variant.json") extra_install_files.append(filepath) with open(filepath, 'w') as f: json.dump(data, f, indent=2) # run build system build_system_name = self.build_system.name() self._print("\nInvoking %s build system...", build_system_name) build_result = self.build_system.build( context=context, variant=variant, build_path=variant_build_path, install_path=variant_install_path, install=install, build_type=build_type) if not build_result.get("success"): # delete the possibly partially installed variant payload if install: self._rmtree(variant_install_path) raise BuildError("The %s build system failed." % build_system_name) if install: # add some installation details to build result build_result.update({ "package_install_path": package_install_path, "variant_install_path": variant_install_path }) # the build system can also specify extra files that need to # be installed filepaths = build_result.get("extra_files") if filepaths: extra_install_files.extend(filepaths) # install extra files for file_ in extra_install_files: copy_or_replace(file_, variant_install_path) # Install include modules. Note that this doesn't need to be done # multiple times, but for subsequent variants it has no effect. # self._install_include_modules(install_path) return build_result
def _build_variant_base(self, variant, build_type, install_path=None, clean=False, install=False, **kwargs): # create build/install paths install_path = install_path or self.package.config.local_packages_path package_install_path = self.get_package_install_path(install_path) variant_build_path = self.build_path if variant.index is None: variant_install_path = package_install_path else: subpath = variant._non_shortlinked_subpath variant_build_path = os.path.join(variant_build_path, subpath) variant_install_path = os.path.join(package_install_path, subpath) # create directories (build, install) if clean and os.path.exists(variant_build_path): shutil.rmtree(variant_build_path) safe_makedirs(variant_build_path) # find last dir of installation path that exists, and possibly make it # writable during variant installation # last_dir = get_existing_path(variant_install_path, topmost_path=install_path) if last_dir: ctxt = make_path_writable(last_dir) else: ctxt = with_noop() with ctxt: if install: # inform package repo that a variant is about to be built/installed pkg_repo = package_repository_manager.get_repository(install_path) pkg_repo.pre_variant_install(variant.resource) if not os.path.exists(variant_install_path): safe_makedirs(variant_install_path) # if hashed variants are enabled, create the variant shortlink if variant.parent.hashed_variants: try: # create the dir containing all shortlinks base_shortlinks_path = os.path.join( package_install_path, variant.parent.config.variant_shortlinks_dirname ) safe_makedirs(base_shortlinks_path) # create the shortlink rel_variant_path = os.path.relpath( variant_install_path, base_shortlinks_path) create_unique_base26_symlink( base_shortlinks_path, rel_variant_path) except Exception as e: # Treat any error as warning - lack of shortlink is not # a breaking issue, it just means the variant root path # will be long. # print_warning( "Error creating variant shortlink for %s: %s: %s", variant_install_path, e.__class__.__name__, e ) # Re-evaluate the variant, so that variables such as 'building' and # 'build_variant_index' are set, and any early-bound package attribs # are re-evaluated wrt these vars. This is done so that attribs such as # 'requires' can change depending on whether a build is occurring or not. # # Note that this re-evaluated variant is ONLY used here, for the purposes # of creating the build context. The variant that is actually installed # is the one evaluated where 'building' is False. # re_evaluated_package = variant.parent.get_reevaluated({ "building": True, "build_variant_index": variant.index or 0, "build_variant_requires": variant.variant_requires }) re_evaluated_variant = re_evaluated_package.get_variant(variant.index) # create build environment context, rxt_filepath = self.create_build_context( variant=re_evaluated_variant, build_type=build_type, build_path=variant_build_path) # run build system build_system_name = self.build_system.name() self._print("\nInvoking %s build system...", build_system_name) build_result = self.build_system.build( context=context, variant=variant, build_path=variant_build_path, install_path=variant_install_path, install=install, build_type=build_type) if not build_result.get("success"): raise BuildError("The %s build system failed." % build_system_name) if install: # Install the 'variant.json' file, which identifies which variant # this is. This is important for hashed variants, where it is not # obvious which variant is in which root path. The file is there # for debugging purposes only. # if variant.index is not None: data = { "index": variant.index, "data": variant.parent.data["variants"][variant.index] } filepath = os.path.join(variant_install_path, "variant.json") with open(filepath, 'w') as f: json.dump(data, f, indent=2) # install some files for debugging purposes (incl build.rxt) extra_files = build_result.get("extra_files", []) if rxt_filepath: extra_files = extra_files + [rxt_filepath] for file_ in extra_files: copy_or_replace(file_, variant_install_path) # Install include modules. Note that this doesn't need to be done # multiple times, but for subsequent variants it has no effect. # self._install_include_modules(install_path) return build_result
def _copy_variant_payload(src_variant, dest_pkg_repo, shallow=False, follow_symlinks=False, overrides=None, verbose=False): # Get payload path of source variant. For some types (eg from a "memory" # type repo) there may not be a root. # variant_root = getattr(src_variant, "root", None) if not variant_root: raise PackageCopyError( "Cannot copy source variant %s - it is a type of variant that " "does not have a root.", src_variant.uri ) if not os.path.isdir(variant_root): raise PackageCopyError( "Cannot copy source variant %s - its root does not appear to " "be present on disk (%s).", src_variant.uri, variant_root ) dest_variant_name = overrides.get("name") or src_variant.name dest_variant_version = overrides.get("version") or src_variant.version # determine variant installation path dest_pkg_payload_path = dest_pkg_repo.get_package_payload_path( package_name=dest_variant_name, package_version=dest_variant_version ) if src_variant.subpath: variant_install_path = os.path.join(dest_pkg_payload_path, src_variant.subpath) else: variant_install_path = dest_pkg_payload_path # get ready for copy/symlinking copy_func = partial(replacing_copy, follow_symlinks=follow_symlinks) if shallow: maybe_symlink = replacing_symlink else: maybe_symlink = copy_func # possibly make install path temporarily writable last_dir = get_existing_path( variant_install_path, topmost_path=os.path.dirname(dest_pkg_payload_path)) if last_dir: ctxt = make_path_writable(last_dir) else: ctxt = with_noop() # copy the variant payload with ctxt: safe_makedirs(variant_install_path) # determine files not to copy skip_files = [] if src_variant.subpath: # Detect overlapped variants. This is the case where one variant subpath # might be A, and another is A/B. We must ensure that A/B is not created # as a symlink during shallow install of variant A - that would then # cause A/B payload to be installed back into original package, possibly # corrupting it. # # Here we detect this case, and create a list of dirs not to copy/link, # because they are in fact a subpath dir for another variant. # skip_files.extend(_get_overlapped_variant_dirs(src_variant)) else: # just skip package definition file for name in config.plugins.package_repository.filesystem.package_filenames: for fmt in (FileFormat.py, FileFormat.yaml): filename = name + '.' + fmt.extension skip_files.append(filename) # copy/link all topmost files within the variant root for name in os.listdir(variant_root): if name in skip_files: filepath = os.path.join(variant_root, name) if verbose: if src_variant.subpath: msg = ("Did not copy %s - this is part of an " "overlapping variant's root path.") else: msg = "Did not copy package definition file %s" print_info(msg, filepath) continue src_path = os.path.join(variant_root, name) dest_path = os.path.join(variant_install_path, name) if os.path.islink(src_path): copy_func(src_path, dest_path) else: maybe_symlink(src_path, dest_path) # copy permissions of source variant dirs onto dest src_package = src_variant.parent src_pkg_repo = src_package.repository src_pkg_payload_path = src_pkg_repo.get_package_payload_path( package_name=src_package.name, package_version=src_package.version ) shutil.copystat(src_pkg_payload_path, dest_pkg_payload_path) subpath = src_variant.subpath while subpath: src_path = os.path.join(src_pkg_payload_path, subpath) dest_path = os.path.join(dest_pkg_payload_path, subpath) shutil.copystat(src_path, dest_path) subpath = os.path.dirname(subpath)
def _build_variant_base(self, variant, build_type, install_path=None, clean=False, install=False, **kwargs): # create build/install paths install_path = install_path or self.package.config.local_packages_path variant_install_path = self.get_package_install_path(install_path) variant_build_path = self.build_path if variant.subpath: variant_build_path = os.path.join(variant_build_path, variant.subpath) variant_install_path = os.path.join(variant_install_path, variant.subpath) # create directories (build, install) if clean and os.path.exists(variant_build_path): shutil.rmtree(variant_build_path) safe_makedirs(variant_build_path) # find last dir of installation path that exists, and possibly make it # writable during variant installation # last_dir = get_existing_path(variant_install_path, topmost_path=install_path) if last_dir: ctxt = make_path_writable(last_dir) else: ctxt = with_noop() with ctxt: if install: # inform package repo that a variant is about to be built/installed pkg_repo = package_repository_manager.get_repository(install_path) pkg_repo.pre_variant_install(variant.resource) if not os.path.exists(variant_install_path): safe_makedirs(variant_install_path) # Re-evaluate the variant, so that variables such as 'building' and # 'build_variant_index' are set, and any early-bound package attribs # are re-evaluated wrt these vars. This is done so that attribs such as # 'requires' can change depending on whether a build is occurring or not. # # Note that this re-evaluated variant is ONLY used here, for the purposes # of creating the build context. The variant that is actually installed # is the one evaluated where 'building' is False. # re_evaluated_package = variant.parent.get_reevaluated({ "building": True, "build_variant_index": variant.index or 0, "build_variant_requires": variant.variant_requires }) re_evaluated_variant = re_evaluated_package.get_variant(variant.index) # create build environment context, rxt_filepath = self.create_build_context( variant=re_evaluated_variant, build_type=build_type, build_path=variant_build_path) # run build system build_system_name = self.build_system.name() self._print("\nInvoking %s build system...", build_system_name) build_result = self.build_system.build( context=context, variant=variant, build_path=variant_build_path, install_path=variant_install_path, install=install, build_type=build_type) if not build_result.get("success"): raise BuildError("The %s build system failed." % build_system_name) if install: # install some files for debugging purposes extra_files = build_result.get("extra_files", []) if rxt_filepath: extra_files = extra_files + [rxt_filepath] for file_ in extra_files: copy_or_replace(file_, variant_install_path) # Install include modules. Note that this doesn't need to be done # multiple times, but for subsequent variants it has no effect. # self._install_include_modules(install_path) return build_result
def _copy_variant_payload(src_variant, dest_pkg_repo, shallow=False, follow_symlinks=False, overrides=None, verbose=False): # Get payload path of source variant. For some types (eg from a "memory" # type repo) there may not be a root. # variant_root = getattr(src_variant, "root", None) if not variant_root: raise PackageCopyError( "Cannot copy source variant %s - it is a type of variant that " "does not have a root." % src_variant.uri ) if not os.path.isdir(variant_root): raise PackageCopyError( "Cannot copy source variant %s - its root does not appear to " "be present on disk (%s)." % src_variant.uri, variant_root ) dest_variant_name = overrides.get("name") or src_variant.name dest_variant_version = overrides.get("version") or src_variant.version # determine variant installation path dest_pkg_payload_path = dest_pkg_repo.get_package_payload_path( package_name=dest_variant_name, package_version=dest_variant_version ) is_varianted = (src_variant.index is not None) src_variant_subpath = None if is_varianted: src_variant_subpath = src_variant._non_shortlinked_subpath variant_install_path = os.path.join( dest_pkg_payload_path, src_variant_subpath) else: variant_install_path = dest_pkg_payload_path # get ready for copy/symlinking copy_func = partial(replacing_copy, follow_symlinks=follow_symlinks) if shallow: maybe_symlink = replacing_symlink else: maybe_symlink = copy_func # possibly make install path temporarily writable last_dir = get_existing_path( variant_install_path, topmost_path=os.path.dirname(dest_pkg_payload_path)) if last_dir and config.make_package_temporarily_writable: ctxt = make_path_writable(last_dir) else: ctxt = with_noop() # copy the variant payload with ctxt: safe_makedirs(variant_install_path) # determine files not to copy skip_files = [] if is_varianted and not src_variant.parent.hashed_variants: # Detect overlapped variants. This is the case where one variant subpath # might be A, and another is A/B. We must ensure that A/B is not created # as a symlink during shallow install of variant A - that would then # cause A/B payload to be installed back into original package, possibly # corrupting it. # # Here we detect this case, and create a list of dirs not to copy/link, # because they are in fact a subpath dir for another variant. # # Note that for hashed variants, we don't do this check because overlapped # variants are not possible. # skip_files.extend(_get_overlapped_variant_dirs(src_variant)) else: # just skip package definition file for name in config.plugins.package_repository.filesystem.package_filenames: for fmt in (FileFormat.py, FileFormat.yaml): filename = name + '.' + fmt.extension skip_files.append(filename) # copy/link all topmost files within the variant root for name in os.listdir(variant_root): if name in skip_files: filepath = os.path.join(variant_root, name) if verbose and is_varianted: print_info( "Did not copy %s - this is part of an overlapping " "variant's root path.", filepath ) continue src_path = os.path.join(variant_root, name) dest_path = os.path.join(variant_install_path, name) if os.path.islink(src_path): copy_func(src_path, dest_path) else: maybe_symlink(src_path, dest_path) # copy permissions of source variant dirs onto dest src_package = src_variant.parent src_pkg_repo = src_package.repository src_pkg_payload_path = src_pkg_repo.get_package_payload_path( package_name=src_package.name, package_version=src_package.version ) shutil.copystat(src_pkg_payload_path, dest_pkg_payload_path) subpath = src_variant_subpath while subpath: src_path = os.path.join(src_pkg_payload_path, subpath) dest_path = os.path.join(dest_pkg_payload_path, subpath) shutil.copystat(src_path, dest_path) subpath = os.path.dirname(subpath) # create the variant shortlink if src_variant.parent.hashed_variants: try: # base _v dir base_shortlinks_path = os.path.join( dest_pkg_payload_path, src_package.config.variant_shortlinks_dirname ) safe_makedirs(base_shortlinks_path) # shortlink rel_variant_path = os.path.relpath( variant_install_path, base_shortlinks_path) create_unique_base26_symlink( base_shortlinks_path, rel_variant_path) except Exception as e: # Treat any error as warning - lack of shortlink is not # a breaking issue, it just means the variant root path # will be long. # print_warning( "Error creating variant shortlink for %s: %s: %s", variant_install_path, e.__class__.__name__, e )