def get_resource_from_handle(self, resource_handle, verify_repo=True): """Get a resource. Args: resource_handle (`ResourceHandle`): Handle of the resource. Returns: `PackageRepositoryResource` instance. """ if verify_repo: # we could fix the handle at this point, but handles should # always be made from repo.make_resource_handle... for now, # at least, error to catch any "incorrect" construction of # handles... if resource_handle.variables.get("repository_type") != self.name(): raise ResourceError( "repository_type mismatch - requested %r, " "repository_type is %r" % (resource_handle.variables["repository_type"], self.name())) if resource_handle.variables.get("location") != self.location: raise ResourceError( "location mismatch - requested %r, " "repository location is %r " % (resource_handle.variables["location"], self.location)) resource = self.pool.get_resource_from_handle(resource_handle) resource._repository = self return resource
def get_resource_from_handle(self, resource_handle, verify_repo=True): if verify_repo: repository_type = resource_handle.variables.get("repository_type") location = resource_handle.variables.get("location") if repository_type != self.name(): raise ResourceError("repository_type mismatch - requested %r, " "repository_type is %r" % (repository_type, self.name())) # It appears that sometimes, the handle location can differ to the # repo location even though they are the same path (different # mounts). We account for that here. # # https://github.com/nerdvegas/rez/pull/957 # if location != self.location: location = canonical_path(location, platform_) if location != self.location: raise ResourceError("location mismatch - requested %r, " "repository location is %r " % (location, self.location)) resource = self.pool.get_resource_from_handle(resource_handle) resource._repository = self return resource
def normalize_variables(cls, variables): if "repository_type" not in variables or "location" not in \ variables: raise ResourceError("%s resources require a repository_type and " "location" % cls.__name__) return super(PackageRepositoryResource, cls).normalize_variables( variables)
def _get_resource(self, resource_handle): resource_key = resource_handle.key resource_class = self.resource_classes.get(resource_key) if resource_class is None: raise ResourceError("Error getting resource from pool: Unknown " "resource type %r" % resource_key) return resource_class(resource_handle.variables)
def variant_requires(self): index = self.index if index is None: return [] else: try: return self.parent.variants[index] or [] except (IndexError, TypeError): raise ResourceError( "Unexpected error - variant %s cannot be found in its " "parent package %s" % (self.uri, self.parent.uri))
def _process(value): if isinstance(value, dict): for k, v in value.items(): new_value = _process(v) if new_value is _remove: del value[k] else: value[k] = new_value return value elif isfunction(value): if hasattr(value, "_early"): # run the function now, and replace with return value with add_sys_paths( config.package_definition_build_python_paths): func = value spec = getargspec(func) args = spec.args or [] if len(args) not in (0, 1): raise ResourceError("@early decorated function must " "take zero or one args only") if args: value_ = func(data) else: value_ = func() # process again in case this is a function returning a function return _process(value_) else: # if a rex function, the code has to be eval'd NOT as a function, # otherwise the globals dict doesn't get updated with any vars # defined in the code, and that means rex code like this: # # rr = 'test' # env.RR = '{rr}' # # ..won't work. It was never intentional that the above work, but # it does, so now we have to keep it so. # as_function = (value.__name__ not in package_rex_keys) return SourceCode(func=value, filepath=filepath, eval_as_function=as_function) elif ismodule(value): # modules cannot be installed as package attributes. They are present # in developer packages sometimes though - it's fine for a package # attribute to use an imported module at build time. # return _remove else: return value
def make_resource_handle(self, resource_key, **variables): """Create a `ResourceHandle` Nearly all `ResourceHandle` creation should go through here, because it gives the various resource classes a chance to normalize / standardize the resource handles, to improve caching / comparison / etc. """ if variables.get("repository_type", self.name()) != self.name(): raise ResourceError("repository_type mismatch - requested %r, " "repository_type is %r" % (variables["repository_type"], self.name())) variables["repository_type"] = self.name() if variables.get("location", self.location) != self.location: raise ResourceError("location mismatch - requested %r, repository " "location is %r" % (variables["location"], self.location)) variables["location"] = self.location resource_cls = self.pool.get_resource_class(resource_key) variables = resource_cls.normalize_variables(variables) return ResourceHandle(resource_key, variables)
def _subpath(self): if self.index is None: return None else: try: reqs = self.parent.variants[self.index] except IndexError: raise ResourceError( "Unexpected error - variant %s cannot be found in its " "parent package %s" % (self.uri, self.parent.uri)) dirs = [x.safe_str() for x in reqs] subpath = os.path.join(*dirs) return subpath
def register_resource(self, resource_class): resource_key = resource_class.key assert issubclass(resource_class, Resource) assert resource_key is not None cls_ = self.resource_classes.get(resource_key) if cls_: if cls_ == resource_class: return # already registered else: raise ResourceError( "Error registering resource class %s: Resource pool has " "already registered %r to %s" % (resource_class.__class__.__name__, resource_key, cls_.__class__.__name__)) self.resource_classes[resource_key] = resource_class
def load_py(stream, filepath=None): """Load python-formatted data from a stream. Args: stream (file-like object). Returns: dict. """ g = __builtins__.copy() scopes = ScopeContext() g['scope'] = scopes try: exec stream in g except Exception as e: import traceback frames = traceback.extract_tb(sys.exc_info()[2]) while filepath and frames and frames[0][0] != filepath: frames = frames[1:] msg = str(e) stack = ''.join(traceback.format_list(frames)).strip() if stack: msg += ":\n" + stack raise ResourceError(msg) result = {} excludes = set(('scope', '__builtins__')) for k, v in g.iteritems(): if k not in excludes and \ (k not in __builtins__ or __builtins__[k] != v): result[k] = v def _process_objects(data): for k, v in data.iteritems(): if isfunction(v): data[k] = SourceCode.from_function(v) elif isinstance(v, dict): _process_objects(v) return data result.update(scopes.to_dict()) result = _process_objects(result) return result
def load_py(stream, filepath=None): """Load python-formatted data from a stream. Args: stream (file-like object). Returns: dict. """ scopes = ScopeContext() g = dict(scope=scopes, early=early, late=late, include=include, ModifyList=ModifyList, InvalidPackageError=InvalidPackageError) try: exec stream in g except Exception as e: import traceback frames = traceback.extract_tb(sys.exc_info()[2]) while filepath and frames and frames[0][0] != filepath: frames = frames[1:] msg = "Problem loading %s: %s" % (filepath, str(e)) stack = ''.join(traceback.format_list(frames)).strip() if stack: msg += ":\n" + stack raise ResourceError(msg) result = {} excludes = set(('scope', 'InvalidPackageError', '__builtins__', 'early', 'late', 'include', 'ModifyList')) for k, v in g.iteritems(): if k not in excludes and \ (k not in __builtins__ or __builtins__[k] != v): result[k] = v result.update(scopes.to_dict()) result = process_python_objects(result, filepath=filepath) return result
def _check_class(resource, cls): if not isinstance(resource, cls): raise ResourceError("Expected %s, got %s" % (cls.__name__, resource.__class__.__name__))
def get_resource_class(self, resource_key): resource_class = self.resource_classes.get(resource_key) if resource_class is None: raise ResourceError("Error getting resource from pool: Unknown " "resource type %r" % resource_key) return resource_class
def _process(value): if isinstance(value, dict): for k, v in value.items(): value[k] = _process(v) return value elif isfunction(value): func = value if hasattr(func, "_early"): # run the function now, and replace with return value # # make a copy of the func with its own globals, and add 'this' import types fn = types.FunctionType(func.__code__, func.__globals__.copy(), name=func.__name__, argdefs=func.__defaults__, closure=func.__closure__) # apply globals fn.__globals__["this"] = EarlyThis(data) fn.__globals__.update(get_objects()) # execute the function args = py23.get_function_arg_names(func) if len(args) not in (0, 1): raise ResourceError("@early decorated function must " "take zero or one args only") if args: # this 'data' arg support isn't needed anymore, but I'm # supporting it til I know nobody is using it... # value_ = fn(data) else: value_ = fn() # process again in case this is a function returning a function return _process(value_) elif hasattr(func, "_late"): return SourceCode(func=func, filepath=filepath, eval_as_function=True) elif func.__name__ in package_rex_keys: # if a rex function, the code has to be eval'd NOT as a function, # otherwise the globals dict doesn't get updated with any vars # defined in the code, and that means rex code like this: # # rr = 'test' # env.RR = '{rr}' # # ..won't work. It was never intentional that the above work, but # it does, so now we have to keep it so. # return SourceCode(func=func, filepath=filepath, eval_as_function=False) else: # a normal function. Leave unchanged, it will be stripped after return func else: return value
def _create_variant(self, variant, dry_run=False, overrides=None): # special case overrides variant_name = overrides.get("name") or variant.name variant_version = overrides.get("version") or variant.version overrides = (overrides or {}).copy() overrides.pop("name", None) overrides.pop("version", None) # find or create the package family family = self.get_package_family(variant_name) if not family: family = self._create_family(variant_name) if isinstance(family, FileSystemCombinedPackageFamilyResource): raise NotImplementedError( "Cannot install variant into combined-style package file %r." % family.filepath) # find the package if it already exists existing_package = None for package in self.iter_packages(family): if package.version == variant_version: # during a build, the family/version dirs get created ahead of # time, which causes a 'missing package definition file' error. # This is fine, we can just ignore it and write the new file. try: package.validate_data() except PackageDefinitionFileMissing: break uuids = set([variant.uuid, package.uuid]) if len(uuids) > 1 and None not in uuids: raise ResourceError( "Cannot install variant %r into package %r - the " "packages are not the same (UUID mismatch)" % (variant, package)) existing_package = package if variant.index is None: if package.variants: raise ResourceError( "Attempting to install a package without variants " "(%r) into an existing package with variants (%r)" % (variant, package)) elif not package.variants: raise ResourceError( "Attempting to install a variant (%r) into an existing " "package without variants (%r)" % (variant, package)) existing_package_data = None release_data = {} # Need to treat 'config' as special case. In validated data, this is # converted to a Config object. We need it as the raw dict that you'd # see in a package.py. # def _get_package_data(pkg): data = pkg.validated_data() if hasattr(pkg, "_data"): raw_data = pkg._data else: raw_data = pkg.resource._data raw_config_data = raw_data.get('config') data.pop("config", None) if raw_config_data: data["config"] = raw_config_data return data def _remove_build_keys(obj): for key in package_build_only_keys: obj.pop(key, None) new_package_data = _get_package_data(variant.parent) new_package_data.pop("variants", None) new_package_data["name"] = variant_name if variant_version: new_package_data["version"] = variant_version package_changed = False _remove_build_keys(new_package_data) if existing_package: debug_print( "Found existing package for installation of variant %s: %s", variant.uri, existing_package.uri ) existing_package_data = _get_package_data(existing_package) _remove_build_keys(existing_package_data) # detect case where new variant introduces package changes outside of variant data_1 = existing_package_data.copy() data_2 = new_package_data.copy() for key in package_release_keys: data_2.pop(key, None) value = data_1.pop(key, None) if value is not None: release_data[key] = value for key in ("format_version", "base", "variants"): data_1.pop(key, None) data_2.pop(key, None) package_changed = (data_1 != data_2) if debug_print: if package_changed: from rez.utils.data_utils import get_dict_diff_str debug_print("Variant %s package data differs from package %s", variant.uri, existing_package.uri) txt = get_dict_diff_str(data_1, data_2, "Changes:") debug_print(txt) else: debug_print("Variant %s package data matches package %s", variant.uri, existing_package.uri) # check for existing installed variant existing_installed_variant = None installed_variant_index = None if existing_package: if variant.index is None: existing_installed_variant = \ self.iter_variants(existing_package).next() else: variant_requires = variant.variant_requires for variant_ in self.iter_variants(existing_package): variant_requires_ = existing_package.variants[variant_.index] if variant_requires_ == variant_requires: installed_variant_index = variant_.index existing_installed_variant = variant_ if existing_installed_variant: debug_print( "Variant %s already has installed equivalent: %s", variant.uri, existing_installed_variant.uri ) if dry_run: if not package_changed: return existing_installed_variant else: return None # construct package data for new installed package definition if existing_package: _, file_ = os.path.split(existing_package.filepath) package_filename, package_extension = os.path.splitext(file_) package_extension = package_extension[1:] package_format = FileFormat[package_extension] if package_changed: # graft together new package data, with existing package variants, # and other data that needs to stay unchanged (eg timestamp) package_data = new_package_data if variant.index is not None: package_data["variants"] = existing_package_data.get("variants", []) else: package_data = existing_package_data else: package_data = new_package_data package_filename = _settings.package_filenames[0] package_extension = "py" package_format = FileFormat.py # merge existing release data (if any) into the package. Note that when # this data becomes variant-specific, this step will no longer be needed package_data.update(release_data) # merge the new variant into the package if installed_variant_index is None and variant.index is not None: variant_requires = variant.variant_requires if not package_data.get("variants"): package_data["variants"] = [] package_data["variants"].append(variant_requires) installed_variant_index = len(package_data["variants"]) - 1 # a little data massaging is needed package_data.pop("base", None) # create version dir if it doesn't already exist family_path = os.path.join(self.location, variant_name) if variant_version: pkg_base_path = os.path.join(family_path, str(variant_version)) else: pkg_base_path = family_path if not os.path.exists(pkg_base_path): os.makedirs(pkg_base_path) # Apply overrides. # # If we're installing into an existing package, then existing attributes # in that package take precedence over `overrides`. If we're installing # to a new package, then `overrides` takes precedence always. # # This is done so that variants added to an existing package don't change # attributes such as 'timestamp' or release-related fields like 'revision'. # for key, value in overrides.iteritems(): if existing_package: if key not in package_data: package_data[key] = value else: if value is self.remove: package_data.pop(key, None) else: package_data[key] = value # timestamp defaults to now if not specified if not package_data.get("timestamp"): package_data["timestamp"] = int(time.time()) # format version is always set package_data["format_version"] = format_version # write out new package definition file package_file = ".".join([package_filename, package_extension]) filepath = os.path.join(pkg_base_path, package_file) with make_path_writable(pkg_base_path): with open_file_for_write(filepath, mode=self.package_file_mode) as f: dump_package_data(package_data, buf=f, format_=package_format) # delete the tmp 'building' file. if variant_version: filename = self.building_prefix + str(variant_version) filepath = os.path.join(family_path, filename) if os.path.exists(filepath): try: os.remove(filepath) except: pass # delete other stale building files; previous failed releases may have # left some around try: self._delete_stale_build_tagfiles(family_path) except: pass # touch the family dir, this keeps memcached resolves updated properly os.utime(family_path, None) # load new variant new_variant = None self.clear_caches() family = self.get_package_family(variant_name) if family: for package in self.iter_packages(family): if package.version == variant_version: for variant_ in self.iter_variants(package): if variant_.index == installed_variant_index: new_variant = variant_ break elif new_variant: break if not new_variant: raise RezSystemError("Internal failure - expected installed variant") return new_variant
def _create_variant(self, variant, dry_run=False, overrides=None): # find or create the package family family = self.get_package_family(variant.name) if not family: family = self._create_family(variant.name) if isinstance(family, FileSystemCombinedPackageFamilyResource): raise NotImplementedError( "Cannot install variant into combined-style package file %r." % family.filepath) # find the package if it already exists existing_package = None for package in self.iter_packages(family): if package.version == variant.version: # during a build, the family/version dirs get created ahead of # time, which causes a 'missing package definition file' error. # This is fine, we can just ignore it and write the new file. try: package.validate_data() except PackageDefinitionFileMissing: break uuids = set([variant.uuid, package.uuid]) if len(uuids) > 1 and None not in uuids: raise ResourceError( "Cannot install variant %r into package %r - the " "packages are not the same (UUID mismatch)" % (variant, package)) existing_package = package if variant.index is None: if package.variants: raise ResourceError( "Attempting to install a package without variants " "(%r) into an existing package with variants (%r)" % (variant, package)) elif not package.variants: raise ResourceError( "Attempting to install a variant (%r) into an existing " "package without variants (%r)" % (variant, package)) installed_variant_index = None existing_package_data = None existing_variants_data = None release_data = {} new_package_data = variant.parent.validated_data() new_package_data.pop("variants", None) package_changed = False if existing_package: existing_package_data = existing_package.validated_data() # detect case where new variant introduces package changes outside of variant data_1 = existing_package_data.copy() data_2 = new_package_data.copy() for key in package_release_keys: data_2.pop(key, None) value = data_1.pop(key, None) if value is not None: release_data[key] = value for key in ("base", "variants"): data_1.pop(key, None) data_2.pop(key, None) package_changed = (data_1 != data_2) # special case - installing a no-variant pkg into a no-variant pkg if existing_package and variant.index is None: if dry_run and not package_changed: variant_ = self.iter_variants(existing_package).next() return variant_ else: # just replace the package existing_package = None if existing_package: # see if variant already exists in package variant_requires = variant.variant_requires for variant_ in self.iter_variants(existing_package): variant_requires_ = existing_package.variants[variant_.index] if variant_requires_ == variant_requires: installed_variant_index = variant_.index if dry_run and not package_changed: return variant_ break parent_package = existing_package _, file_ = os.path.split(existing_package.filepath) package_filename, package_extension = os.path.splitext(file_) package_extension = package_extension[1:] package_format = FileFormat[package_extension] if package_changed: # graft together new package data, with existing package variants, # and other data that needs to stay unchanged (eg timestamp) package_data = new_package_data package_data["variants"] = existing_package_data.get( "variants", []) else: package_data = existing_package_data else: parent_package = variant.parent package_data = new_package_data package_filename = _settings.package_filenames[0] package_extension = "py" package_format = FileFormat.py if dry_run: return None # merge existing release data (if any) into the package. Note that when # this data becomes variant-specific, this step will no longer be needed package_data.update(release_data) # merge the new variant into the package if installed_variant_index is None and variant.index is not None: variant_requires = variant.variant_requires if not package_data.get("variants"): package_data["variants"] = [] package_data["variants"].append(variant_requires) installed_variant_index = len(package_data["variants"]) - 1 # a little data massaging is needed package_data["config"] = parent_package._data.get("config") package_data.pop("base", None) # create version dir and write out the new package definition file family_path = os.path.join(self.location, variant.name) if variant.version: path = os.path.join(family_path, str(variant.version)) else: path = family_path if not os.path.exists(path): os.makedirs(path) # add the timestamp overrides = overrides or {} overrides["timestamp"] = int(time.time()) # apply attribute overrides for key, value in overrides.iteritems(): if package_data.get(key) is None: package_data[key] = value package_file = ".".join([package_filename, package_extension]) filepath = os.path.join(path, package_file) with open_file_for_write(filepath) as f: dump_package_data(package_data, buf=f, format_=package_format) # touch the family dir, this keeps memcached resolves updated properly os.utime(family_path, None) # load new variant new_variant = None self.clear_caches() family = self.get_package_family(variant.name) if family: for package in self.iter_packages(family): if package.version == variant.version: for variant_ in self.iter_variants(package): if variant_.index == installed_variant_index: new_variant = variant_ break elif new_variant: break if not new_variant: raise RezSystemError( "Internal failure - expected installed variant") return new_variant