def _get_preprocessed(self, data): """ Returns: (DeveloperPackage, new_data) 2-tuple IF the preprocess function changed the package; otherwise None. """ from rez.serialise import process_python_objects from rez.utils.data_utils import get_dict_diff_str from copy import deepcopy def _get_package_level(): return getattr(self, "preprocess", None) def _get_global_level(): # load globally configured preprocess function package_preprocess_function = self.config.package_preprocess_function if not package_preprocess_function: return None elif isfunction(package_preprocess_function): preprocess_func = package_preprocess_function else: if '.' not in package_preprocess_function: print_error( "Setting 'package_preprocess_function' must be of " "form 'module[.module.module...].funcname'. Package " "preprocessing has not been applied.") return None elif isinstance(package_preprocess_function, basestring): if '.' not in package_preprocess_function: print_error( "Setting 'package_preprocess_function' must be of " "form 'module[.module.module...].funcname'. " "Package preprocessing has not been applied.") return None name, funcname = package_preprocess_function.rsplit('.', 1) try: module = __import__(name=name, fromlist=[funcname]) except Exception as e: print_error( "Failed to load preprocessing function '%s': %s" % (package_preprocess_function, str(e))) return None setattr(module, "InvalidPackageError", InvalidPackageError) preprocess_func = getattr(module, funcname) else: print_error("Invalid package_preprocess_function: %s" % package_preprocess_function) return None if not preprocess_func or not isfunction(preprocess_func): print_error("Function '%s' not found" % package_preprocess_function) return None return preprocess_func with add_sys_paths(config.package_definition_build_python_paths): preprocess_mode = PreprocessMode[ self.config.package_preprocess_mode] package_preprocess = _get_package_level() global_preprocess = _get_global_level() if preprocess_mode == PreprocessMode.after: preprocessors = [global_preprocess, package_preprocess] elif preprocess_mode == PreprocessMode.before: preprocessors = [package_preprocess, global_preprocess] else: preprocessors = [package_preprocess or global_preprocess] preprocessed_data = deepcopy(data) for preprocessor in preprocessors: if not preprocessor: continue level = "global" if preprocessor == global_preprocess else "local" print_info("Applying {0} preprocess function".format(level)) # apply preprocessing try: preprocessor(this=self, data=preprocessed_data) except InvalidPackageError: raise except Exception as e: print_error("Failed to apply preprocess: %s: %s" % (e.__class__.__name__, str(e))) return None # if preprocess added functions, these may need to be converted to # SourceCode instances preprocessed_data = process_python_objects(preprocessed_data) if preprocessed_data == data: return None # recreate package from modified package data package = create_package(self.name, preprocessed_data, package_cls=self.__class__) # print summary of changed package attributes txt = get_dict_diff_str( data, preprocessed_data, title="Package attributes were changed in preprocessing:") print_info(txt) return package, preprocessed_data
def _get_preprocessed(self, data): """ Returns: (DeveloperPackage, new_data) 2-tuple IFF the preprocess function changed the package; otherwise None. """ from rez.serialise import process_python_objects from rez.utils.data_utils import get_dict_diff_str from copy import deepcopy with add_sys_paths(config.package_definition_build_python_paths): preprocess_func = getattr(self, "preprocess", None) if preprocess_func: print_info("Applying preprocess from package.py") else: # load globally configured preprocess function dotted = self.config.package_preprocess_function if not dotted: return None if '.' not in dotted: print_error( "Setting 'package_preprocess_function' must be of " "form 'module[.module.module...].funcname'. Package " "preprocessing has not been applied.") return None name, funcname = dotted.rsplit('.', 1) try: module = __import__(name=name, fromlist=[funcname]) except Exception as e: print_error("Failed to load preprocessing function '%s': %s" % (dotted, str(e))) return None setattr(module, "InvalidPackageError", InvalidPackageError) preprocess_func = getattr(module, funcname) if not preprocess_func or not isfunction(isfunction): print_error("Function '%s' not found" % dotted) return None print_info("Applying preprocess function %s" % dotted) preprocessed_data = deepcopy(data) # apply preprocessing try: preprocess_func(this=self, data=preprocessed_data) except InvalidPackageError: raise except Exception as e: print_error("Failed to apply preprocess: %s: %s" % (e.__class__.__name__, str(e))) return None # if preprocess added functions, these may need to be converted to # SourceCode instances preprocessed_data = process_python_objects(preprocessed_data) if preprocessed_data == data: return None # recreate package from modified package data package = create_package(self.name, preprocessed_data, package_cls=self.__class__) # print summary of changed package attributes txt = get_dict_diff_str( data, preprocessed_data, title="Package attributes were changed in preprocessing:" ) print_info(txt) return package, preprocessed_data
def _create_variant(self, variant, dry_run=False, overrides=None): # special case overrides variant_name = overrides.get("name") or variant.name variant_version = overrides.get("version") or variant.version overrides = (overrides or {}).copy() overrides.pop("name", None) overrides.pop("version", None) # find or create the package family family = self.get_package_family(variant_name) if not family: family = self._create_family(variant_name) if isinstance(family, FileSystemCombinedPackageFamilyResource): raise NotImplementedError( "Cannot install variant into combined-style package file %r." % family.filepath) # find the package if it already exists existing_package = None for package in self.iter_packages(family): if package.version == variant_version: # during a build, the family/version dirs get created ahead of # time, which causes a 'missing package definition file' error. # This is fine, we can just ignore it and write the new file. try: package.validate_data() except PackageDefinitionFileMissing: break uuids = set([variant.uuid, package.uuid]) if len(uuids) > 1 and None not in uuids: raise ResourceError( "Cannot install variant %r into package %r - the " "packages are not the same (UUID mismatch)" % (variant, package)) existing_package = package if variant.index is None: if package.variants: raise ResourceError( "Attempting to install a package without variants " "(%r) into an existing package with variants (%r)" % (variant, package)) elif not package.variants: raise ResourceError( "Attempting to install a variant (%r) into an existing " "package without variants (%r)" % (variant, package)) existing_package_data = None release_data = {} # Need to treat 'config' as special case. In validated data, this is # converted to a Config object. We need it as the raw dict that you'd # see in a package.py. # def _get_package_data(pkg): data = pkg.validated_data() if hasattr(pkg, "_data"): raw_data = pkg._data else: raw_data = pkg.resource._data raw_config_data = raw_data.get('config') data.pop("config", None) if raw_config_data: data["config"] = raw_config_data return data def _remove_build_keys(obj): for key in package_build_only_keys: obj.pop(key, None) new_package_data = _get_package_data(variant.parent) new_package_data.pop("variants", None) new_package_data["name"] = variant_name if variant_version: new_package_data["version"] = variant_version package_changed = False _remove_build_keys(new_package_data) if existing_package: debug_print( "Found existing package for installation of variant %s: %s", variant.uri, existing_package.uri ) existing_package_data = _get_package_data(existing_package) _remove_build_keys(existing_package_data) # detect case where new variant introduces package changes outside of variant data_1 = existing_package_data.copy() data_2 = new_package_data.copy() for key in package_release_keys: data_2.pop(key, None) value = data_1.pop(key, None) if value is not None: release_data[key] = value for key in ("format_version", "base", "variants"): data_1.pop(key, None) data_2.pop(key, None) package_changed = (data_1 != data_2) if debug_print: if package_changed: from rez.utils.data_utils import get_dict_diff_str debug_print("Variant %s package data differs from package %s", variant.uri, existing_package.uri) txt = get_dict_diff_str(data_1, data_2, "Changes:") debug_print(txt) else: debug_print("Variant %s package data matches package %s", variant.uri, existing_package.uri) # check for existing installed variant existing_installed_variant = None installed_variant_index = None if existing_package: if variant.index is None: existing_installed_variant = \ self.iter_variants(existing_package).next() else: variant_requires = variant.variant_requires for variant_ in self.iter_variants(existing_package): variant_requires_ = existing_package.variants[variant_.index] if variant_requires_ == variant_requires: installed_variant_index = variant_.index existing_installed_variant = variant_ if existing_installed_variant: debug_print( "Variant %s already has installed equivalent: %s", variant.uri, existing_installed_variant.uri ) if dry_run: if not package_changed: return existing_installed_variant else: return None # construct package data for new installed package definition if existing_package: _, file_ = os.path.split(existing_package.filepath) package_filename, package_extension = os.path.splitext(file_) package_extension = package_extension[1:] package_format = FileFormat[package_extension] if package_changed: # graft together new package data, with existing package variants, # and other data that needs to stay unchanged (eg timestamp) package_data = new_package_data if variant.index is not None: package_data["variants"] = existing_package_data.get("variants", []) else: package_data = existing_package_data else: package_data = new_package_data package_filename = _settings.package_filenames[0] package_extension = "py" package_format = FileFormat.py # merge existing release data (if any) into the package. Note that when # this data becomes variant-specific, this step will no longer be needed package_data.update(release_data) # merge the new variant into the package if installed_variant_index is None and variant.index is not None: variant_requires = variant.variant_requires if not package_data.get("variants"): package_data["variants"] = [] package_data["variants"].append(variant_requires) installed_variant_index = len(package_data["variants"]) - 1 # a little data massaging is needed package_data.pop("base", None) # create version dir if it doesn't already exist family_path = os.path.join(self.location, variant_name) if variant_version: pkg_base_path = os.path.join(family_path, str(variant_version)) else: pkg_base_path = family_path if not os.path.exists(pkg_base_path): os.makedirs(pkg_base_path) # Apply overrides. # # If we're installing into an existing package, then existing attributes # in that package take precedence over `overrides`. If we're installing # to a new package, then `overrides` takes precedence always. # # This is done so that variants added to an existing package don't change # attributes such as 'timestamp' or release-related fields like 'revision'. # for key, value in overrides.iteritems(): if existing_package: if key not in package_data: package_data[key] = value else: if value is self.remove: package_data.pop(key, None) else: package_data[key] = value # timestamp defaults to now if not specified if not package_data.get("timestamp"): package_data["timestamp"] = int(time.time()) # format version is always set package_data["format_version"] = format_version # write out new package definition file package_file = ".".join([package_filename, package_extension]) filepath = os.path.join(pkg_base_path, package_file) with make_path_writable(pkg_base_path): with open_file_for_write(filepath, mode=self.package_file_mode) as f: dump_package_data(package_data, buf=f, format_=package_format) # delete the tmp 'building' file. if variant_version: filename = self.building_prefix + str(variant_version) filepath = os.path.join(family_path, filename) if os.path.exists(filepath): try: os.remove(filepath) except: pass # delete other stale building files; previous failed releases may have # left some around try: self._delete_stale_build_tagfiles(family_path) except: pass # touch the family dir, this keeps memcached resolves updated properly os.utime(family_path, None) # load new variant new_variant = None self.clear_caches() family = self.get_package_family(variant_name) if family: for package in self.iter_packages(family): if package.version == variant_version: for variant_ in self.iter_variants(package): if variant_.index == installed_variant_index: new_variant = variant_ break elif new_variant: break if not new_variant: raise RezSystemError("Internal failure - expected installed variant") return new_variant