def get_output(self, style=OutputStyle.file): if self.manager is None: raise RezSystemError("You must call 'set_manager' on a Python rex " "interpreter before using it.") self.target_environ.update(self.manager.environ) return self.manager.environ
def apply_environ(self): """Apply changes to target environ. """ if self.manager is None: raise RezSystemError("You must call 'set_manager' on a Python rex " "interpreter before using it.") self.target_environ.update(self.manager.environ)
def command(opts, parser, extra_arg_groups=None): from rez.config import config from rez.utils.platform_ import platform_ from rez.exceptions import RezSystemError from rez.vendor import yaml from rez.vendor.yaml.error import YAMLError from rez.utils import py23 import os.path # we don't usually want warnings printed in a wrapped tool. But in cases # where we do (for debugging) we leave a backdoor - setting $REZ_QUIET=0 # will stop this warning suppression. if "REZ_QUIET" not in os.environ: config.override("quiet", True) yaml_file = os.path.abspath(opts.YAML) cli_args = opts.ARG for arg_group in (extra_arg_groups or []): cli_args.extend(arg_group) if platform_.name == "windows" and yaml_file.lower().endswith(".cmd"): with open(yaml_file) as f: content = "\n".join(f.readlines()[4:]) # strip batch script else: with open(yaml_file) as f: content = f.read() try: doc = yaml.load(content, Loader=yaml.FullLoader) except YAMLError as e: raise RezSystemError("Invalid executable file %s: %s" % (yaml_file, str(e))) func_name = doc["func_name"] nargs = doc.get("nargs", []) kwargs = doc.get("kwargs", {}) if isinstance(doc["module"], basestring): # refers to a rez module from rez.backport.importlib import import_module namespace = "rez.%s" % doc["module"] module = import_module(namespace) else: # refers to a rez plugin module from rez.plugin_managers import plugin_manager plugin_type, plugin_name = doc["module"] module = plugin_manager.get_plugin_module(plugin_type, plugin_name) target_func = getattr(module, func_name) func_args = py23.get_function_arg_names(target_func) if "_script" in func_args: kwargs["_script"] = yaml_file if "_cli_args" in func_args: kwargs["_cli_args"] = cli_args target_func(*nargs, **kwargs)
def find_pip(pip_version=None, python_version=None): """Find pip. Pip is searched in the following order: 1. Search for rezified python matching python version request; 2. If found, test if pip is present; 3. If pip is present, use it; 4. If not present, search for rezified pip (this is for backwards compatibility); 5. If rezified pip is found, use it; 6. If not, fall back to rez's python installation. Args: pip_version (str or `Version`): Version of pip to use, or latest if None. python_version (str or `Version`): Python version to use, or latest if None. Returns: 2-tuple: - str: Python executable. - `ResolvedContext`: Context containing pip, or None if we fell back to system pip. """ py_exe = None context = None found_pip_version = None valid_found = False for version in [pip_version, "latest"]: try: py_exe, found_pip_version, context = find_pip_from_context( python_version, pip_version=version ) valid_found = _check_found(py_exe, found_pip_version) if valid_found: break except BuildError as error: print_warning(str(error)) if not valid_found: import pip found_pip_version = pip.__version__ py_exe = sys.executable print_warning("Found no pip in any python and/or pip rez packages!") print_warning("Falling back to pip installed in rez own virtualenv:") logging_arguments = ( ("pip", found_pip_version, pip.__file__), ("python", ".".join(map(str, sys.version_info[:3])), py_exe), ) for warn_args in logging_arguments: print_warning("%10s: %s (%s)", *warn_args) if not _check_found(py_exe, found_pip_version, log_invalid=False): message = "pip{specifier} is required! Please update your pip." raise RezSystemError(message.format(specifier=PIP_SPECIFIER)) return py_exe, context
def find_pip(pip_version=None, python_version=None): """Find pip. Pip is searched in the following order: 1. Search for rezified python matching python version request; 2. If found, test if pip is present; 3. If pip is present, use it; 4. If not present, search for rezified pip (this is for backwards compatibility); 5. If rezified pip is found, use it; 6. If not, fall back to rez's python installation. Args: pip_version (str or `Version`): Version of pip to use, or latest if None. python_version (str or `Version`): Python version to use, or latest if None. Returns: 2-tuple: - str: Python executable. - `ResolvedContext`: Context containing pip, or None if we fell back to system pip. """ py_exe = None context = None py_exe, pip_version, context = find_pip_from_context( python_version, pip_version=pip_version ) if not py_exe: py_exe, pip_version, context = find_pip_from_context( python_version, pip_version=pip_version or "latest" ) if not py_exe: import pip pip_version = pip.__version__ py_exe = sys.executable print_warning( "Found no pip in python and pip package; " "falling back to pip installed in rez own virtualenv (version %s)", pip_version ) pip_major = pip_version.split('.')[0] if int(pip_major) < 19: raise RezSystemError("pip >= 19 is required! Please update your pip.") return py_exe, context
def find_pip(pip_version=None, python_version=None): """Find pip. Will revert to native pip installed with rez, if a pip rez package cannot be found. In this case, None is returned. Args: pip_version (str or `Version`): Version of pip to use, or latest if None. python_version (str or `Version`): Python version to use, or latest if None. Returns: 2-tuple: - str: Python executable. - `ResolvedContext`: Context containing pip, or None if we fell back to system pip. """ py_exe = "python" context = None # find pip, fall back to system if rez pip package not found try: context = create_context(pip_version, python_version) py_exe = context.which("python") except BuildError: # fall back on system pip py_exe = sys.executable print_info("Using %s -m pip", py_exe) # check version, must be >=19 if context: proc = context.execute_command( [py_exe, "-c", "import pip; print pip.__version__"], stdout=subprocess.PIPE) out, _ = proc.communicate() pip_version = out.strip() else: import pip pip_version = pip.__version__ pip_major = pip_version.split('.')[0] if int(pip_major) < 19: raise RezSystemError("pip >= 19 is required! Please update your pip.") return py_exe, context
def _err(msg): raise RezSystemError("Invalid executable file %s: %s" % (filepath, msg))
def _os(self): """ Note: We cannot replace this with 'distro.linux_distribution' in entirety as unfortunately there are slight differences. Eg our code gives 'Ubuntu-16.04' whereas distro gives 'ubuntu-16.04'. """ distributor = None release = None def _str(s): if (s.startswith("'") and s.endswith("'")) \ or (s.startswith('"') and s.endswith('"')): return s[1:-1] else: return s def _os(): if distributor and release: return "%s-%s" % (distributor, release) else: return None def _parse(txt, distributor_key, release_key): distributor_ = None release_ = None lines = txt.strip().split('\n') for line in lines: if line.startswith(distributor_key): s = line[len(distributor_key):].strip() distributor_ = _str(s) elif line.startswith(release_key): s = line[len(release_key):].strip() release_ = _str(s) return distributor_, release_ # first try parsing the /etc/lsb-release file file = "/etc/lsb-release" if os.path.isfile(file): with open(file) as f: txt = f.read() distributor, release = _parse(txt, "DISTRIB_ID=", "DISTRIB_RELEASE=") result = _os() if result: return result # next, try getting the output of the lsb_release program import subprocess p = Popen(['/usr/bin/env', 'lsb_release', '-a'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) txt = p.communicate()[0] if not p.returncode: distributor_, release_ = _parse(txt, "Distributor ID:", "Release:") if distributor_ and not distributor: distributor = distributor_ if release_ and not release: release = release_ result = _os() if result: return result # try to read the /etc/os-release file # this file contains OS specific data on linux # distributions # see https://www.freedesktop.org/software/systemd/man/os-release.html os_release = '/etc/os-release' if os.path.isfile(os_release): with open(os_release, 'r') as f: txt = f.read() distributor_, release_ = _parse(txt, "ID=", "VERSION_ID=") if distributor_ and not distributor: distributor = distributor_ if release_ and not release: release = release_ result = _os() if result: return result # use distro lib from rez.vendor.distro import distro parts = distro.linux_distribution(full_distribution_name=False) if parts[0] == '': raise RezSystemError("cannot detect operating system") return '-'.join(parts[:2])
def shell(self): """Get the current shell. @returns The current shell this process is running in. Examples: bash tcsh """ from rez.shells import get_shell_types shells = set(get_shell_types()) if not shells: raise RezSystemError("no shells available") if self.platform == "windows": return "cmd" else: import subprocess as sp shell = None # check parent process via ps try: args = ['ps', '-o', 'args=', '-p', str(os.getppid())] proc = sp.Popen(args, stdout=sp.PIPE) output = proc.communicate()[0] shell = os.path.basename(output.strip().split()[0]).replace( '-', '') except Exception: pass # check $SHELL if shell not in shells: shell = os.path.basename(os.getenv("SHELL", '')) # traverse parent procs via /proc/(pid)/status if shell not in shells: pid = str(os.getppid()) found = False while not found: try: file = os.path.join(os.sep, "proc", pid, "status") with open(file) as f: loc = f.read().split('\n') for line in loc: line = line.strip() toks = line.split() if len(toks) == 2: if toks[0] == "Name:": name = toks[1] if name in shells: shell = name found = True break elif toks[0] == "PPid:": pid = toks[1] except Exception: break if (shell not in shells) and ("sh" in shells): shell = "sh" # failed detection, fall back on 'sh' elif (shell not in shells) and ("bash" in shells): shell = "bash" # failed detection, fall back on 'bash' elif shell not in shells: shell = next(iter(shells)) # give up - just choose a shell # sh has to be handled as a special case if shell == "sh": if os.path.islink("/bin/sh"): path = os.readlink("/bin/sh") shell2 = os.path.split(path)[-1] if shell2 == "bash": # bash switches to sh-like shell when invoked as sh, # so we want to use the sh shell plugin pass elif shell2 == "dash": # dash doesn't have an sh emulation mode, so we have # to use the dash shell plugin if "dash" in shells: shell = "dash" else: # this isn't good! if "bash" in shells: shell = "bash" # fall back on bash else: shell = next(iter( shells)) # give up - just choose a shell # TODO: remove this when/if dash support added if shell == "dash": shell = "bash" return shell
def convert_dist(name, dest_path, make_variant=True, ignore_dirs=None, python_requirement="major_minor"): """Convert an already installed python distribution into a rez package. Args: dest_path (str): Where to put the rez package. The package will be created under dest_path/<NAME>/<VERSION>/. make_variant (bool): If True, makes a single variant in the rez package based on the MAJOR.MINOR version of python. ignore_dirs (bool): List of directory names to not copy from the dist. python_requirement (str): How the package should depend on python. One of: - "major": depend on python-X - "major_minor": depend on python-X.X - any other value: this string is used as the literal version range string. Returns: Install path of the new Rez package. """ dist = pkg_resources.get_distribution(name) pkg_name = convert_name(dist.project_name) pkg_version = convert_version(dist.version) if python_requirement == "major": pyver = str(sys.version_info[0]) elif python_requirement == "major_minor": pyver = '.'.join(str(x) for x in sys.version_info[:2]) else: pyver = python_requirement pypkg = "python-%s" % pyver pkg_requires = [] if not make_variant: pkg_requires.append(pypkg) for req in dist.requires(): pkg_requires += convert_requirement(req) pkg_path = _mkdirs(dest_path, pkg_name, pkg_version) pkg_file = os.path.join(pkg_path, "package.py") root_path = _mkdirs(pkg_path, pypkg) if make_variant else pkg_path basename = os.path.basename(dist.location) is_egg = (os.path.splitext(basename)[1] == ".egg") if os.path.isdir(dist.location): if is_egg: # this is an egg-dir for file in os.listdir(dist.location): fpath = os.path.join(dist.location, file) if os.path.isfile(fpath): shutil.copy(fpath, root_path) else: shutil.copytree(fpath, os.path.join(root_path, file), ignore=shutil.ignore_patterns(ignore_dirs)) else: # this is a site dir egginfo_dir = "%s.egg-info" % dist.egg_name() eggpath = os.path.join(dist.location, egginfo_dir) file = os.path.join(eggpath, "installed-files.txt") if not os.path.isfile(file): raise RezSystemError( "There is not enough information on disk to convert the " "python distribution '%s' into a Rez package. The distribution " "is installed to a common site, but the installed file " "information is not present." % name) with open(file) as f: installed_files = f.read().strip().split() dirs = set() files = set() for file in installed_files: path = os.path.join(eggpath, file) path = os.path.realpath(path) if os.path.isfile(path) and path.startswith(dist.location + os.sep): dir_ = os.path.dirname(path) if ignore_dirs: reldir = os.path.relpath(dir_, dist.location) if set(reldir.split(os.sep)) & set(ignore_dirs): continue files.add(path) dirs.add(dir_) def _dst(p): dst = os.path.relpath(p, dist.location) dst = os.path.join(root_path, dst) return os.path.realpath(dst) for dir_ in dirs: dst_dir = _dst(dir_) _mkdirs(dst_dir) for file in files: dst_file = _dst(file) shutil.copy(file, dst_file) else: # this is an egg-file import zipfile assert (is_egg and os.path.isfile(dist.location)) assert (zipfile.is_zipfile(dist.location)) z = zipfile.ZipFile(dist.location) z.extractall(root_path) variants_str = "[['%s']]" % pypkg if make_variant else '' content = textwrap.dedent(""" config_version = 0 name = '%(name)s' version = '%(version)s' %(variants)s requires = %(requires)s def commands(): env.PYTHONPATH.append('{this.root}') """ % dict(name=pkg_name, version=pkg_version, variants=variants_str, requires=str(pkg_requires))) content = content.strip() + '\n' with open(pkg_file, 'w') as f: f.write(content) return pkg_path
def spawn_shell(self, context_file, tmpdir, rcfile=None, norc=False, stdin=False, command=None, env=None, quiet=False, pre_command=None, add_rez=True, package_commands_sourced_first=None, **Popen_args): d = self.get_startup_sequence(rcfile, norc, bool(stdin), command) envvar = d["envvar"] files = d["files"] bind_files = d["bind_files"] do_rcfile = d["do_rcfile"] shell_command = None if package_commands_sourced_first is None: package_commands_sourced_first = config.package_commands_sourced_first def _record_shell(ex, files, bind_rez=True, print_msg=False): if bind_rez and package_commands_sourced_first: ex.source(context_file) for file_ in files: if os.path.exists(os.path.expanduser(file_)): ex.source(file_) if bind_rez and not package_commands_sourced_first: ex.source(context_file) if envvar: ex.unsetenv(envvar) if add_rez and bind_rez: ex.interpreter._bind_interactive_rez() if print_msg and add_rez and not quiet: ex.info('') ex.info('You are now in a rez-configured environment.') ex.info('') if system.is_production_rez_install: ex.command('rezolve context') def _write_shell(ex, filename): code = ex.get_output() target_file = os.path.join(tmpdir, filename) with open(target_file, 'w') as f: f.write(code) return target_file def _create_ex(): return RexExecutor(interpreter=self.new_shell(), parent_environ={}, add_default_namespaces=False) executor = _create_ex() if self.settings.prompt: newprompt = '${REZ_ENV_PROMPT}%s' % self.settings.prompt executor.interpreter._saferefenv('REZ_ENV_PROMPT') executor.env.REZ_ENV_PROMPT = newprompt if d["command"] is not None: _record_shell(executor, files=files) shell_command = d["command"] else: if d["stdin"]: assert (self.stdin_arg) shell_command = "%s %s" % (self.executable, self.stdin_arg) quiet = True elif do_rcfile: assert (self.rcfile_arg) shell_command = "%s %s" % (self.executable, self.rcfile_arg) else: shell_command = self.executable if do_rcfile: # hijack rcfile to insert our own script ex = _create_ex() _record_shell(ex, files=files, print_msg=(not quiet)) filename = "rcfile.%s" % self.file_extension() filepath = _write_shell(ex, filename) shell_command += " %s" % filepath elif envvar: # hijack env-var to insert our own script ex = _create_ex() _record_shell(ex, files=files, print_msg=(not quiet)) filename = "%s.%s" % (envvar, self.file_extension()) filepath = _write_shell(ex, filename) executor.setenv(envvar, filepath) else: # hijack $HOME to insert our own script files = [x for x in files if x not in bind_files ] + list(bind_files) if files: for file_ in files: if file_ in bind_files: bind_rez = True files_ = [file_] if d["source_bind_files"] else [] else: bind_rez = False files_ = [file_] ex = _create_ex() ex.setenv('HOME', os.environ.get('HOME', '')) _record_shell(ex, files=files_, bind_rez=bind_rez, print_msg=bind_rez) _write_shell(ex, os.path.basename(file_)) executor.setenv("HOME", tmpdir) # keep history if self.histfile and self.histvar: histfile = os.path.expanduser(self.histfile) if os.path.exists(histfile): executor.setenv(self.histvar, histfile) else: if config.warn("shell_startup"): print_warning( "WARNING: Could not configure environment from " "within the target shell (%s); this has been done " "in the parent process instead." % self.name()) executor.source(context_file) if shell_command: # an empty string means 'run no command and exit' executor.command(shell_command) executor.command("exit %s" % self.last_command_status) code = executor.get_output() target_file = os.path.join(tmpdir, "rez-shell.%s" % self.file_extension()) with open(target_file, 'w') as f: f.write(code) if d["stdin"] and stdin and (stdin is not True): Popen_args["stdin"] = stdin cmd = [] if pre_command: if isinstance(pre_command, basestring): cmd = pre_command.strip().split() else: cmd = pre_command cmd.extend([self.executable, target_file]) try: p = subprocess.Popen(cmd, env=env, **Popen_args) except Exception as e: cmd_str = ' '.join(map(pipes.quote, cmd)) raise RezSystemError("Error running command:\n%s\n%s" % (cmd_str, str(e))) return p
def _os(self): distributor = None release = None def _str(s): if (s.startswith("'") and s.endswith("'")) \ or (s.startswith('"') and s.endswith('"')): return s[1:-1] else: return s def _os(): if distributor and release: return "%s-%s" % (distributor, release) else: return None def _parse(txt, distributor_key, release_key): distributor_ = None release_ = None lines = txt.strip().split('\n') for line in lines: if line.startswith(distributor_key): s = line[len(distributor_key):].strip() distributor_ = _str(s) elif line.startswith(release_key): s = line[len(release_key):].strip() release_ = _str(s) return distributor_, release_ # first try parsing the /etc/lsb-release file file = "/etc/lsb-release" if os.path.isfile(file): with open(file) as f: txt = f.read() distributor, release = _parse(txt, "DISTRIB_ID=", "DISTRIB_RELEASE=") result = _os() if result: return result # next, try getting the output of the lsb_release program import subprocess p = subprocess.Popen(['/usr/bin/env', 'lsb_release', '-a'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) txt = p.communicate()[0] if not p.returncode: distributor_, release_ = _parse(txt, "Distributor ID:", "Release:") if distributor_ and not distributor: distributor = distributor_ if release_ and not release: release = release_ result = _os() if result: return result # last, use python's dist detection. It is known to return incorrect # info on some systems though try: distributor_, release_, _ = platform.linux_distribution() except: distributor_, release_, _ = platform.dist() if distributor_ and not distributor: distributor = distributor_ if release_ and not release: release = release_ result = _os() if result: return result # last resort, accept missing release if distributor: return distributor # give up raise RezSystemError("cannot detect operating system")
def _create_variant(self, variant, dry_run=False, overrides=None): # special case overrides variant_name = overrides.get("name") or variant.name variant_version = overrides.get("version") or variant.version overrides = (overrides or {}).copy() overrides.pop("name", None) overrides.pop("version", None) # find or create the package family family = self.get_package_family(variant_name) if not family: family = self._create_family(variant_name) if isinstance(family, FileSystemCombinedPackageFamilyResource): raise NotImplementedError( "Cannot install variant into combined-style package file %r." % family.filepath) # find the package if it already exists existing_package = None for package in self.iter_packages(family): if package.version == variant_version: # during a build, the family/version dirs get created ahead of # time, which causes a 'missing package definition file' error. # This is fine, we can just ignore it and write the new file. try: package.validate_data() except PackageDefinitionFileMissing: break uuids = set([variant.uuid, package.uuid]) if len(uuids) > 1 and None not in uuids: raise ResourceError( "Cannot install variant %r into package %r - the " "packages are not the same (UUID mismatch)" % (variant, package)) existing_package = package if variant.index is None: if package.variants: raise ResourceError( "Attempting to install a package without variants " "(%r) into an existing package with variants (%r)" % (variant, package)) elif not package.variants: raise ResourceError( "Attempting to install a variant (%r) into an existing " "package without variants (%r)" % (variant, package)) existing_package_data = None release_data = {} # Need to treat 'config' as special case. In validated data, this is # converted to a Config object. We need it as the raw dict that you'd # see in a package.py. # def _get_package_data(pkg): data = pkg.validated_data() if hasattr(pkg, "_data"): raw_data = pkg._data else: raw_data = pkg.resource._data raw_config_data = raw_data.get('config') data.pop("config", None) if raw_config_data: data["config"] = raw_config_data return data def _remove_build_keys(obj): for key in package_build_only_keys: obj.pop(key, None) new_package_data = _get_package_data(variant.parent) new_package_data.pop("variants", None) new_package_data["name"] = variant_name if variant_version: new_package_data["version"] = variant_version package_changed = False _remove_build_keys(new_package_data) if existing_package: debug_print( "Found existing package for installation of variant %s: %s", variant.uri, existing_package.uri ) existing_package_data = _get_package_data(existing_package) _remove_build_keys(existing_package_data) # detect case where new variant introduces package changes outside of variant data_1 = existing_package_data.copy() data_2 = new_package_data.copy() for key in package_release_keys: data_2.pop(key, None) value = data_1.pop(key, None) if value is not None: release_data[key] = value for key in ("format_version", "base", "variants"): data_1.pop(key, None) data_2.pop(key, None) package_changed = (data_1 != data_2) if debug_print: if package_changed: from rez.utils.data_utils import get_dict_diff_str debug_print("Variant %s package data differs from package %s", variant.uri, existing_package.uri) txt = get_dict_diff_str(data_1, data_2, "Changes:") debug_print(txt) else: debug_print("Variant %s package data matches package %s", variant.uri, existing_package.uri) # check for existing installed variant existing_installed_variant = None installed_variant_index = None if existing_package: if variant.index is None: existing_installed_variant = \ self.iter_variants(existing_package).next() else: variant_requires = variant.variant_requires for variant_ in self.iter_variants(existing_package): variant_requires_ = existing_package.variants[variant_.index] if variant_requires_ == variant_requires: installed_variant_index = variant_.index existing_installed_variant = variant_ if existing_installed_variant: debug_print( "Variant %s already has installed equivalent: %s", variant.uri, existing_installed_variant.uri ) if dry_run: if not package_changed: return existing_installed_variant else: return None # construct package data for new installed package definition if existing_package: _, file_ = os.path.split(existing_package.filepath) package_filename, package_extension = os.path.splitext(file_) package_extension = package_extension[1:] package_format = FileFormat[package_extension] if package_changed: # graft together new package data, with existing package variants, # and other data that needs to stay unchanged (eg timestamp) package_data = new_package_data if variant.index is not None: package_data["variants"] = existing_package_data.get("variants", []) else: package_data = existing_package_data else: package_data = new_package_data package_filename = _settings.package_filenames[0] package_extension = "py" package_format = FileFormat.py # merge existing release data (if any) into the package. Note that when # this data becomes variant-specific, this step will no longer be needed package_data.update(release_data) # merge the new variant into the package if installed_variant_index is None and variant.index is not None: variant_requires = variant.variant_requires if not package_data.get("variants"): package_data["variants"] = [] package_data["variants"].append(variant_requires) installed_variant_index = len(package_data["variants"]) - 1 # a little data massaging is needed package_data.pop("base", None) # create version dir if it doesn't already exist family_path = os.path.join(self.location, variant_name) if variant_version: pkg_base_path = os.path.join(family_path, str(variant_version)) else: pkg_base_path = family_path if not os.path.exists(pkg_base_path): os.makedirs(pkg_base_path) # Apply overrides. # # If we're installing into an existing package, then existing attributes # in that package take precedence over `overrides`. If we're installing # to a new package, then `overrides` takes precedence always. # # This is done so that variants added to an existing package don't change # attributes such as 'timestamp' or release-related fields like 'revision'. # for key, value in overrides.iteritems(): if existing_package: if key not in package_data: package_data[key] = value else: if value is self.remove: package_data.pop(key, None) else: package_data[key] = value # timestamp defaults to now if not specified if not package_data.get("timestamp"): package_data["timestamp"] = int(time.time()) # format version is always set package_data["format_version"] = format_version # write out new package definition file package_file = ".".join([package_filename, package_extension]) filepath = os.path.join(pkg_base_path, package_file) with make_path_writable(pkg_base_path): with open_file_for_write(filepath, mode=self.package_file_mode) as f: dump_package_data(package_data, buf=f, format_=package_format) # delete the tmp 'building' file. if variant_version: filename = self.building_prefix + str(variant_version) filepath = os.path.join(family_path, filename) if os.path.exists(filepath): try: os.remove(filepath) except: pass # delete other stale building files; previous failed releases may have # left some around try: self._delete_stale_build_tagfiles(family_path) except: pass # touch the family dir, this keeps memcached resolves updated properly os.utime(family_path, None) # load new variant new_variant = None self.clear_caches() family = self.get_package_family(variant_name) if family: for package in self.iter_packages(family): if package.version == variant_version: for variant_ in self.iter_variants(package): if variant_.index == installed_variant_index: new_variant = variant_ break elif new_variant: break if not new_variant: raise RezSystemError("Internal failure - expected installed variant") return new_variant
def _create_variant(self, variant, dry_run=False, overrides=None): # find or create the package family family = self.get_package_family(variant.name) if not family: family = self._create_family(variant.name) if isinstance(family, FileSystemCombinedPackageFamilyResource): raise NotImplementedError( "Cannot install variant into combined-style package file %r." % family.filepath) # find the package if it already exists existing_package = None for package in self.iter_packages(family): if package.version == variant.version: # during a build, the family/version dirs get created ahead of # time, which causes a 'missing package definition file' error. # This is fine, we can just ignore it and write the new file. try: package.validate_data() except PackageDefinitionFileMissing: break uuids = set([variant.uuid, package.uuid]) if len(uuids) > 1 and None not in uuids: raise ResourceError( "Cannot install variant %r into package %r - the " "packages are not the same (UUID mismatch)" % (variant, package)) existing_package = package if variant.index is None: if package.variants: raise ResourceError( "Attempting to install a package without variants " "(%r) into an existing package with variants (%r)" % (variant, package)) elif not package.variants: raise ResourceError( "Attempting to install a variant (%r) into an existing " "package without variants (%r)" % (variant, package)) installed_variant_index = None existing_package_data = None existing_variants_data = None release_data = {} new_package_data = variant.parent.validated_data() new_package_data.pop("variants", None) package_changed = False if existing_package: existing_package_data = existing_package.validated_data() # detect case where new variant introduces package changes outside of variant data_1 = existing_package_data.copy() data_2 = new_package_data.copy() for key in package_release_keys: data_2.pop(key, None) value = data_1.pop(key, None) if value is not None: release_data[key] = value for key in ("base", "variants"): data_1.pop(key, None) data_2.pop(key, None) package_changed = (data_1 != data_2) # special case - installing a no-variant pkg into a no-variant pkg if existing_package and variant.index is None: if dry_run and not package_changed: variant_ = self.iter_variants(existing_package).next() return variant_ else: # just replace the package existing_package = None if existing_package: # see if variant already exists in package variant_requires = variant.variant_requires for variant_ in self.iter_variants(existing_package): variant_requires_ = existing_package.variants[variant_.index] if variant_requires_ == variant_requires: installed_variant_index = variant_.index if dry_run and not package_changed: return variant_ break parent_package = existing_package _, file_ = os.path.split(existing_package.filepath) package_filename, package_extension = os.path.splitext(file_) package_extension = package_extension[1:] package_format = FileFormat[package_extension] if package_changed: # graft together new package data, with existing package variants, # and other data that needs to stay unchanged (eg timestamp) package_data = new_package_data package_data["variants"] = existing_package_data.get( "variants", []) else: package_data = existing_package_data else: parent_package = variant.parent package_data = new_package_data package_filename = _settings.package_filenames[0] package_extension = "py" package_format = FileFormat.py if dry_run: return None # merge existing release data (if any) into the package. Note that when # this data becomes variant-specific, this step will no longer be needed package_data.update(release_data) # merge the new variant into the package if installed_variant_index is None and variant.index is not None: variant_requires = variant.variant_requires if not package_data.get("variants"): package_data["variants"] = [] package_data["variants"].append(variant_requires) installed_variant_index = len(package_data["variants"]) - 1 # a little data massaging is needed package_data["config"] = parent_package._data.get("config") package_data.pop("base", None) # create version dir and write out the new package definition file family_path = os.path.join(self.location, variant.name) if variant.version: path = os.path.join(family_path, str(variant.version)) else: path = family_path if not os.path.exists(path): os.makedirs(path) # add the timestamp overrides = overrides or {} overrides["timestamp"] = int(time.time()) # apply attribute overrides for key, value in overrides.iteritems(): if package_data.get(key) is None: package_data[key] = value package_file = ".".join([package_filename, package_extension]) filepath = os.path.join(path, package_file) with open_file_for_write(filepath) as f: dump_package_data(package_data, buf=f, format_=package_format) # touch the family dir, this keeps memcached resolves updated properly os.utime(family_path, None) # load new variant new_variant = None self.clear_caches() family = self.get_package_family(variant.name) if family: for package in self.iter_packages(family): if package.version == variant.version: for variant_ in self.iter_variants(package): if variant_.index == installed_variant_index: new_variant = variant_ break elif new_variant: break if not new_variant: raise RezSystemError( "Internal failure - expected installed variant") return new_variant
def _create_variant(self, variant, dry_run=False, overrides=None): # special case overrides variant_name = overrides.get("name") or variant.name variant_version = overrides.get("version") or variant.version overrides = (overrides or {}).copy() overrides.pop("name", None) overrides.pop("version", None) # Need to treat 'config' as special case. In validated data, this is # converted to a Config object. We need it as the raw dict that you'd # see in a package.py. # def _get_package_data(pkg): data = pkg.validated_data() if hasattr(pkg, "_data"): raw_data = pkg._data else: raw_data = pkg.resource._data raw_config_data = raw_data.get("config") data.pop("config", None) if raw_config_data: data["config"] = raw_config_data return data def _remove_build_keys(obj): for key in package_build_only_keys: obj.pop(key, None) package_data = _get_package_data(variant.parent) package_data.pop("variants", None) package_data["name"] = variant_name if variant_version: package_data["version"] = variant_version _remove_build_keys(package_data) installed_variant_index = None if dry_run: return None # a little data massaging is needed package_data.pop("base", None) # Apply overrides for key, value in overrides.items(): if value is self.remove: package_data.pop(key, None) else: package_data[key] = value # timestamp defaults to now if not specified if not package_data.get("timestamp"): package_data["timestamp"] = int(time.time()) # format version is always set package_data["format_version"] = 2 # Late binding functions added # Stop if package is unversioned and config does not allow that if (not package_data["version"] and not config.allow_unversioned_packages): raise PackageMetadataError("Unversioned package is not allowed.") # Upsert to database date = datetime.datetime.now() family_name = package_data["name"] version_string = str(package_data["version"]) document = package_family_document(family_name, date) filter_ = { "type": "family", "_id": family_name, } self.collection.update_one(filter_, {"$set": document}, upsert=True) document = package_document(family_name, date, version_string, package_to_dict(package_data)) filter_ = { "type": "package", "family": family_name, "version": version_string, } self.collection.update_one(filter_, {"$set": document}, upsert=True) # load new variant new_variant = None self.clear_caches() family = self.get_package_family(variant_name) if family: for package in self.iter_packages(family): if package.version == variant_version: for variant_ in self.iter_variants(package): if variant_.index == installed_variant_index: new_variant = variant_ break elif new_variant: break if not new_variant: raise RezSystemError( "Internal failure - expected installed variant") return new_variant