def get_output(self, style=OutputStyle.file):
if self.manager is None:
raise RezSystemError("You must call 'set_manager' on a Python rex "
"interpreter before using it.")
self.target_environ.update(self.manager.environ)
return self.manager.environ
def apply_environ(self):
"""Apply changes to target environ.
"""
if self.manager is None:
raise RezSystemError("You must call 'set_manager' on a Python rex "
"interpreter before using it.")
self.target_environ.update(self.manager.environ)
def command(opts, parser, extra_arg_groups=None):
from rez.config import config
from rez.utils.platform_ import platform_
from rez.exceptions import RezSystemError
from rez.vendor import yaml
from rez.vendor.yaml.error import YAMLError
from rez.utils import py23
import os.path
# we don't usually want warnings printed in a wrapped tool. But in cases
# where we do (for debugging) we leave a backdoor - setting $REZ_QUIET=0
# will stop this warning suppression.
if "REZ_QUIET" not in os.environ:
config.override("quiet", True)
yaml_file = os.path.abspath(opts.YAML)
cli_args = opts.ARG
for arg_group in (extra_arg_groups or []):
cli_args.extend(arg_group)
if platform_.name == "windows" and yaml_file.lower().endswith(".cmd"):
with open(yaml_file) as f:
content = "\n".join(f.readlines()[4:]) # strip batch script
else:
with open(yaml_file) as f:
content = f.read()
try:
doc = yaml.load(content, Loader=yaml.FullLoader)
except YAMLError as e:
raise RezSystemError("Invalid executable file %s: %s" %
(yaml_file, str(e)))
func_name = doc["func_name"]
nargs = doc.get("nargs", [])
kwargs = doc.get("kwargs", {})
if isinstance(doc["module"], basestring):
# refers to a rez module
from rez.backport.importlib import import_module
namespace = "rez.%s" % doc["module"]
module = import_module(namespace)
else:
# refers to a rez plugin module
from rez.plugin_managers import plugin_manager
plugin_type, plugin_name = doc["module"]
module = plugin_manager.get_plugin_module(plugin_type, plugin_name)
target_func = getattr(module, func_name)
func_args = py23.get_function_arg_names(target_func)
if "_script" in func_args:
kwargs["_script"] = yaml_file
if "_cli_args" in func_args:
kwargs["_cli_args"] = cli_args
target_func(*nargs, **kwargs)
def find_pip(pip_version=None, python_version=None):
"""Find pip.
Pip is searched in the following order:
1. Search for rezified python matching python version request;
2. If found, test if pip is present;
3. If pip is present, use it;
4. If not present, search for rezified pip (this is for backwards compatibility);
5. If rezified pip is found, use it;
6. If not, fall back to rez's python installation.
Args:
pip_version (str or `Version`): Version of pip to use, or latest if None.
python_version (str or `Version`): Python version to use, or latest if
None.
Returns:
2-tuple:
- str: Python executable.
- `ResolvedContext`: Context containing pip, or None if we fell back
to system pip.
"""
py_exe = None
context = None
found_pip_version = None
valid_found = False
for version in [pip_version, "latest"]:
try:
py_exe, found_pip_version, context = find_pip_from_context(
python_version, pip_version=version
)
valid_found = _check_found(py_exe, found_pip_version)
if valid_found:
break
except BuildError as error:
print_warning(str(error))
if not valid_found:
import pip
found_pip_version = pip.__version__
py_exe = sys.executable
print_warning("Found no pip in any python and/or pip rez packages!")
print_warning("Falling back to pip installed in rez own virtualenv:")
logging_arguments = (
("pip", found_pip_version, pip.__file__),
("python", ".".join(map(str, sys.version_info[:3])), py_exe),
)
for warn_args in logging_arguments:
print_warning("%10s: %s (%s)", *warn_args)
if not _check_found(py_exe, found_pip_version, log_invalid=False):
message = "pip{specifier} is required! Please update your pip."
raise RezSystemError(message.format(specifier=PIP_SPECIFIER))
return py_exe, context
def find_pip(pip_version=None, python_version=None):
"""Find pip.
Pip is searched in the following order:
1. Search for rezified python matching python version request;
2. If found, test if pip is present;
3. If pip is present, use it;
4. If not present, search for rezified pip (this is for backwards compatibility);
5. If rezified pip is found, use it;
6. If not, fall back to rez's python installation.
Args:
pip_version (str or `Version`): Version of pip to use, or latest if None.
python_version (str or `Version`): Python version to use, or latest if
None.
Returns:
2-tuple:
- str: Python executable.
- `ResolvedContext`: Context containing pip, or None if we fell back
to system pip.
"""
py_exe = None
context = None
py_exe, pip_version, context = find_pip_from_context(
python_version,
pip_version=pip_version
)
if not py_exe:
py_exe, pip_version, context = find_pip_from_context(
python_version,
pip_version=pip_version or "latest"
)
if not py_exe:
import pip
pip_version = pip.__version__
py_exe = sys.executable
print_warning(
"Found no pip in python and pip package; "
"falling back to pip installed in rez own virtualenv (version %s)",
pip_version
)
pip_major = pip_version.split('.')[0]
if int(pip_major) < 19:
raise RezSystemError("pip >= 19 is required! Please update your pip.")
return py_exe, context
def find_pip(pip_version=None, python_version=None):
"""Find pip.
Will revert to native pip installed with rez, if a pip rez package cannot
be found. In this case, None is returned.
Args:
pip_version (str or `Version`): Version of pip to use, or latest if None.
python_version (str or `Version`): Python version to use, or latest if
None.
Returns:
2-tuple:
- str: Python executable.
- `ResolvedContext`: Context containing pip, or None if we fell back
to system pip.
"""
py_exe = "python"
context = None
# find pip, fall back to system if rez pip package not found
try:
context = create_context(pip_version, python_version)
py_exe = context.which("python")
except BuildError:
# fall back on system pip
py_exe = sys.executable
print_info("Using %s -m pip", py_exe)
# check version, must be >=19
if context:
proc = context.execute_command(
[py_exe, "-c", "import pip; print pip.__version__"],
stdout=subprocess.PIPE)
out, _ = proc.communicate()
pip_version = out.strip()
else:
import pip
pip_version = pip.__version__
pip_major = pip_version.split('.')[0]
if int(pip_major) < 19:
raise RezSystemError("pip >= 19 is required! Please update your pip.")
return py_exe, context
def _err(msg):
raise RezSystemError("Invalid executable file %s: %s" %
(filepath, msg))
def _os(self):
"""
Note: We cannot replace this with 'distro.linux_distribution' in
entirety as unfortunately there are slight differences. Eg our code
gives 'Ubuntu-16.04' whereas distro gives 'ubuntu-16.04'.
"""
distributor = None
release = None
def _str(s):
if (s.startswith("'") and s.endswith("'")) \
or (s.startswith('"') and s.endswith('"')):
return s[1:-1]
else:
return s
def _os():
if distributor and release:
return "%s-%s" % (distributor, release)
else:
return None
def _parse(txt, distributor_key, release_key):
distributor_ = None
release_ = None
lines = txt.strip().split('\n')
for line in lines:
if line.startswith(distributor_key):
s = line[len(distributor_key):].strip()
distributor_ = _str(s)
elif line.startswith(release_key):
s = line[len(release_key):].strip()
release_ = _str(s)
return distributor_, release_
# first try parsing the /etc/lsb-release file
file = "/etc/lsb-release"
if os.path.isfile(file):
with open(file) as f:
txt = f.read()
distributor, release = _parse(txt, "DISTRIB_ID=",
"DISTRIB_RELEASE=")
result = _os()
if result:
return result
# next, try getting the output of the lsb_release program
import subprocess
p = Popen(['/usr/bin/env', 'lsb_release', '-a'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True)
txt = p.communicate()[0]
if not p.returncode:
distributor_, release_ = _parse(txt, "Distributor ID:", "Release:")
if distributor_ and not distributor:
distributor = distributor_
if release_ and not release:
release = release_
result = _os()
if result:
return result
# try to read the /etc/os-release file
# this file contains OS specific data on linux
# distributions
# see https://www.freedesktop.org/software/systemd/man/os-release.html
os_release = '/etc/os-release'
if os.path.isfile(os_release):
with open(os_release, 'r') as f:
txt = f.read()
distributor_, release_ = _parse(txt, "ID=", "VERSION_ID=")
if distributor_ and not distributor:
distributor = distributor_
if release_ and not release:
release = release_
result = _os()
if result:
return result
# use distro lib
from rez.vendor.distro import distro
parts = distro.linux_distribution(full_distribution_name=False)
if parts[0] == '':
raise RezSystemError("cannot detect operating system")
return '-'.join(parts[:2])
def shell(self):
"""Get the current shell.
@returns The current shell this process is running in. Examples:
bash
tcsh
"""
from rez.shells import get_shell_types
shells = set(get_shell_types())
if not shells:
raise RezSystemError("no shells available")
if self.platform == "windows":
return "cmd"
else:
import subprocess as sp
shell = None
# check parent process via ps
try:
args = ['ps', '-o', 'args=', '-p', str(os.getppid())]
proc = sp.Popen(args, stdout=sp.PIPE)
output = proc.communicate()[0]
shell = os.path.basename(output.strip().split()[0]).replace(
'-', '')
except Exception:
pass
# check $SHELL
if shell not in shells:
shell = os.path.basename(os.getenv("SHELL", ''))
# traverse parent procs via /proc/(pid)/status
if shell not in shells:
pid = str(os.getppid())
found = False
while not found:
try:
file = os.path.join(os.sep, "proc", pid, "status")
with open(file) as f:
loc = f.read().split('\n')
for line in loc:
line = line.strip()
toks = line.split()
if len(toks) == 2:
if toks[0] == "Name:":
name = toks[1]
if name in shells:
shell = name
found = True
break
elif toks[0] == "PPid:":
pid = toks[1]
except Exception:
break
if (shell not in shells) and ("sh" in shells):
shell = "sh" # failed detection, fall back on 'sh'
elif (shell not in shells) and ("bash" in shells):
shell = "bash" # failed detection, fall back on 'bash'
elif shell not in shells:
shell = next(iter(shells)) # give up - just choose a shell
# sh has to be handled as a special case
if shell == "sh":
if os.path.islink("/bin/sh"):
path = os.readlink("/bin/sh")
shell2 = os.path.split(path)[-1]
if shell2 == "bash":
# bash switches to sh-like shell when invoked as sh,
# so we want to use the sh shell plugin
pass
elif shell2 == "dash":
# dash doesn't have an sh emulation mode, so we have
# to use the dash shell plugin
if "dash" in shells:
shell = "dash"
else:
# this isn't good!
if "bash" in shells:
shell = "bash" # fall back on bash
else:
shell = next(iter(
shells)) # give up - just choose a shell
# TODO: remove this when/if dash support added
if shell == "dash":
shell = "bash"
return shell
def convert_dist(name,
dest_path,
make_variant=True,
ignore_dirs=None,
python_requirement="major_minor"):
"""Convert an already installed python distribution into a rez package.
Args:
dest_path (str): Where to put the rez package. The package will be
created under dest_path/<NAME>/<VERSION>/.
make_variant (bool): If True, makes a single variant in the rez package
based on the MAJOR.MINOR version of python.
ignore_dirs (bool): List of directory names to not copy from the dist.
python_requirement (str): How the package should depend on python.
One of:
- "major": depend on python-X
- "major_minor": depend on python-X.X
- any other value: this string is used as the literal version
range string.
Returns:
Install path of the new Rez package.
"""
dist = pkg_resources.get_distribution(name)
pkg_name = convert_name(dist.project_name)
pkg_version = convert_version(dist.version)
if python_requirement == "major":
pyver = str(sys.version_info[0])
elif python_requirement == "major_minor":
pyver = '.'.join(str(x) for x in sys.version_info[:2])
else:
pyver = python_requirement
pypkg = "python-%s" % pyver
pkg_requires = []
if not make_variant:
pkg_requires.append(pypkg)
for req in dist.requires():
pkg_requires += convert_requirement(req)
pkg_path = _mkdirs(dest_path, pkg_name, pkg_version)
pkg_file = os.path.join(pkg_path, "package.py")
root_path = _mkdirs(pkg_path, pypkg) if make_variant else pkg_path
basename = os.path.basename(dist.location)
is_egg = (os.path.splitext(basename)[1] == ".egg")
if os.path.isdir(dist.location):
if is_egg:
# this is an egg-dir
for file in os.listdir(dist.location):
fpath = os.path.join(dist.location, file)
if os.path.isfile(fpath):
shutil.copy(fpath, root_path)
else:
shutil.copytree(fpath,
os.path.join(root_path, file),
ignore=shutil.ignore_patterns(ignore_dirs))
else:
# this is a site dir
egginfo_dir = "%s.egg-info" % dist.egg_name()
eggpath = os.path.join(dist.location, egginfo_dir)
file = os.path.join(eggpath, "installed-files.txt")
if not os.path.isfile(file):
raise RezSystemError(
"There is not enough information on disk to convert the "
"python distribution '%s' into a Rez package. The distribution "
"is installed to a common site, but the installed file "
"information is not present." % name)
with open(file) as f:
installed_files = f.read().strip().split()
dirs = set()
files = set()
for file in installed_files:
path = os.path.join(eggpath, file)
path = os.path.realpath(path)
if os.path.isfile(path) and path.startswith(dist.location +
os.sep):
dir_ = os.path.dirname(path)
if ignore_dirs:
reldir = os.path.relpath(dir_, dist.location)
if set(reldir.split(os.sep)) & set(ignore_dirs):
continue
files.add(path)
dirs.add(dir_)
def _dst(p):
dst = os.path.relpath(p, dist.location)
dst = os.path.join(root_path, dst)
return os.path.realpath(dst)
for dir_ in dirs:
dst_dir = _dst(dir_)
_mkdirs(dst_dir)
for file in files:
dst_file = _dst(file)
shutil.copy(file, dst_file)
else:
# this is an egg-file
import zipfile
assert (is_egg and os.path.isfile(dist.location))
assert (zipfile.is_zipfile(dist.location))
z = zipfile.ZipFile(dist.location)
z.extractall(root_path)
variants_str = "[['%s']]" % pypkg if make_variant else ''
content = textwrap.dedent("""
config_version = 0
name = '%(name)s'
version = '%(version)s'
%(variants)s
requires = %(requires)s
def commands():
env.PYTHONPATH.append('{this.root}')
""" % dict(name=pkg_name,
version=pkg_version,
variants=variants_str,
requires=str(pkg_requires)))
content = content.strip() + '\n'
with open(pkg_file, 'w') as f:
f.write(content)
return pkg_path
def spawn_shell(self,
context_file,
tmpdir,
rcfile=None,
norc=False,
stdin=False,
command=None,
env=None,
quiet=False,
pre_command=None,
add_rez=True,
package_commands_sourced_first=None,
**Popen_args):
d = self.get_startup_sequence(rcfile, norc, bool(stdin), command)
envvar = d["envvar"]
files = d["files"]
bind_files = d["bind_files"]
do_rcfile = d["do_rcfile"]
shell_command = None
if package_commands_sourced_first is None:
package_commands_sourced_first = config.package_commands_sourced_first
def _record_shell(ex, files, bind_rez=True, print_msg=False):
if bind_rez and package_commands_sourced_first:
ex.source(context_file)
for file_ in files:
if os.path.exists(os.path.expanduser(file_)):
ex.source(file_)
if bind_rez and not package_commands_sourced_first:
ex.source(context_file)
if envvar:
ex.unsetenv(envvar)
if add_rez and bind_rez:
ex.interpreter._bind_interactive_rez()
if print_msg and add_rez and not quiet:
ex.info('')
ex.info('You are now in a rez-configured environment.')
ex.info('')
if system.is_production_rez_install:
ex.command('rezolve context')
def _write_shell(ex, filename):
code = ex.get_output()
target_file = os.path.join(tmpdir, filename)
with open(target_file, 'w') as f:
f.write(code)
return target_file
def _create_ex():
return RexExecutor(interpreter=self.new_shell(),
parent_environ={},
add_default_namespaces=False)
executor = _create_ex()
if self.settings.prompt:
newprompt = '${REZ_ENV_PROMPT}%s' % self.settings.prompt
executor.interpreter._saferefenv('REZ_ENV_PROMPT')
executor.env.REZ_ENV_PROMPT = newprompt
if d["command"] is not None:
_record_shell(executor, files=files)
shell_command = d["command"]
else:
if d["stdin"]:
assert (self.stdin_arg)
shell_command = "%s %s" % (self.executable, self.stdin_arg)
quiet = True
elif do_rcfile:
assert (self.rcfile_arg)
shell_command = "%s %s" % (self.executable, self.rcfile_arg)
else:
shell_command = self.executable
if do_rcfile:
# hijack rcfile to insert our own script
ex = _create_ex()
_record_shell(ex, files=files, print_msg=(not quiet))
filename = "rcfile.%s" % self.file_extension()
filepath = _write_shell(ex, filename)
shell_command += " %s" % filepath
elif envvar:
# hijack env-var to insert our own script
ex = _create_ex()
_record_shell(ex, files=files, print_msg=(not quiet))
filename = "%s.%s" % (envvar, self.file_extension())
filepath = _write_shell(ex, filename)
executor.setenv(envvar, filepath)
else:
# hijack $HOME to insert our own script
files = [x for x in files if x not in bind_files
] + list(bind_files)
if files:
for file_ in files:
if file_ in bind_files:
bind_rez = True
files_ = [file_] if d["source_bind_files"] else []
else:
bind_rez = False
files_ = [file_]
ex = _create_ex()
ex.setenv('HOME', os.environ.get('HOME', ''))
_record_shell(ex,
files=files_,
bind_rez=bind_rez,
print_msg=bind_rez)
_write_shell(ex, os.path.basename(file_))
executor.setenv("HOME", tmpdir)
# keep history
if self.histfile and self.histvar:
histfile = os.path.expanduser(self.histfile)
if os.path.exists(histfile):
executor.setenv(self.histvar, histfile)
else:
if config.warn("shell_startup"):
print_warning(
"WARNING: Could not configure environment from "
"within the target shell (%s); this has been done "
"in the parent process instead." % self.name())
executor.source(context_file)
if shell_command: # an empty string means 'run no command and exit'
executor.command(shell_command)
executor.command("exit %s" % self.last_command_status)
code = executor.get_output()
target_file = os.path.join(tmpdir,
"rez-shell.%s" % self.file_extension())
with open(target_file, 'w') as f:
f.write(code)
if d["stdin"] and stdin and (stdin is not True):
Popen_args["stdin"] = stdin
cmd = []
if pre_command:
if isinstance(pre_command, basestring):
cmd = pre_command.strip().split()
else:
cmd = pre_command
cmd.extend([self.executable, target_file])
try:
p = subprocess.Popen(cmd, env=env, **Popen_args)
except Exception as e:
cmd_str = ' '.join(map(pipes.quote, cmd))
raise RezSystemError("Error running command:\n%s\n%s" %
(cmd_str, str(e)))
return p
def _os(self):
distributor = None
release = None
def _str(s):
if (s.startswith("'") and s.endswith("'")) \
or (s.startswith('"') and s.endswith('"')):
return s[1:-1]
else:
return s
def _os():
if distributor and release:
return "%s-%s" % (distributor, release)
else:
return None
def _parse(txt, distributor_key, release_key):
distributor_ = None
release_ = None
lines = txt.strip().split('\n')
for line in lines:
if line.startswith(distributor_key):
s = line[len(distributor_key):].strip()
distributor_ = _str(s)
elif line.startswith(release_key):
s = line[len(release_key):].strip()
release_ = _str(s)
return distributor_, release_
# first try parsing the /etc/lsb-release file
file = "/etc/lsb-release"
if os.path.isfile(file):
with open(file) as f:
txt = f.read()
distributor, release = _parse(txt, "DISTRIB_ID=",
"DISTRIB_RELEASE=")
result = _os()
if result:
return result
# next, try getting the output of the lsb_release program
import subprocess
p = subprocess.Popen(['/usr/bin/env', 'lsb_release', '-a'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
txt = p.communicate()[0]
if not p.returncode:
distributor_, release_ = _parse(txt, "Distributor ID:", "Release:")
if distributor_ and not distributor:
distributor = distributor_
if release_ and not release:
release = release_
result = _os()
if result:
return result
# last, use python's dist detection. It is known to return incorrect
# info on some systems though
try:
distributor_, release_, _ = platform.linux_distribution()
except:
distributor_, release_, _ = platform.dist()
if distributor_ and not distributor:
distributor = distributor_
if release_ and not release:
release = release_
result = _os()
if result:
return result
# last resort, accept missing release
if distributor:
return distributor
# give up
raise RezSystemError("cannot detect operating system")
def _create_variant(self, variant, dry_run=False, overrides=None):
# special case overrides
variant_name = overrides.get("name") or variant.name
variant_version = overrides.get("version") or variant.version
overrides = (overrides or {}).copy()
overrides.pop("name", None)
overrides.pop("version", None)
# find or create the package family
family = self.get_package_family(variant_name)
if not family:
family = self._create_family(variant_name)
if isinstance(family, FileSystemCombinedPackageFamilyResource):
raise NotImplementedError(
"Cannot install variant into combined-style package file %r."
% family.filepath)
# find the package if it already exists
existing_package = None
for package in self.iter_packages(family):
if package.version == variant_version:
# during a build, the family/version dirs get created ahead of
# time, which causes a 'missing package definition file' error.
# This is fine, we can just ignore it and write the new file.
try:
package.validate_data()
except PackageDefinitionFileMissing:
break
uuids = set([variant.uuid, package.uuid])
if len(uuids) > 1 and None not in uuids:
raise ResourceError(
"Cannot install variant %r into package %r - the "
"packages are not the same (UUID mismatch)"
% (variant, package))
existing_package = package
if variant.index is None:
if package.variants:
raise ResourceError(
"Attempting to install a package without variants "
"(%r) into an existing package with variants (%r)"
% (variant, package))
elif not package.variants:
raise ResourceError(
"Attempting to install a variant (%r) into an existing "
"package without variants (%r)" % (variant, package))
existing_package_data = None
release_data = {}
# Need to treat 'config' as special case. In validated data, this is
# converted to a Config object. We need it as the raw dict that you'd
# see in a package.py.
#
def _get_package_data(pkg):
data = pkg.validated_data()
if hasattr(pkg, "_data"):
raw_data = pkg._data
else:
raw_data = pkg.resource._data
raw_config_data = raw_data.get('config')
data.pop("config", None)
if raw_config_data:
data["config"] = raw_config_data
return data
def _remove_build_keys(obj):
for key in package_build_only_keys:
obj.pop(key, None)
new_package_data = _get_package_data(variant.parent)
new_package_data.pop("variants", None)
new_package_data["name"] = variant_name
if variant_version:
new_package_data["version"] = variant_version
package_changed = False
_remove_build_keys(new_package_data)
if existing_package:
debug_print(
"Found existing package for installation of variant %s: %s",
variant.uri, existing_package.uri
)
existing_package_data = _get_package_data(existing_package)
_remove_build_keys(existing_package_data)
# detect case where new variant introduces package changes outside of variant
data_1 = existing_package_data.copy()
data_2 = new_package_data.copy()
for key in package_release_keys:
data_2.pop(key, None)
value = data_1.pop(key, None)
if value is not None:
release_data[key] = value
for key in ("format_version", "base", "variants"):
data_1.pop(key, None)
data_2.pop(key, None)
package_changed = (data_1 != data_2)
if debug_print:
if package_changed:
from rez.utils.data_utils import get_dict_diff_str
debug_print("Variant %s package data differs from package %s",
variant.uri, existing_package.uri)
txt = get_dict_diff_str(data_1, data_2, "Changes:")
debug_print(txt)
else:
debug_print("Variant %s package data matches package %s",
variant.uri, existing_package.uri)
# check for existing installed variant
existing_installed_variant = None
installed_variant_index = None
if existing_package:
if variant.index is None:
existing_installed_variant = \
self.iter_variants(existing_package).next()
else:
variant_requires = variant.variant_requires
for variant_ in self.iter_variants(existing_package):
variant_requires_ = existing_package.variants[variant_.index]
if variant_requires_ == variant_requires:
installed_variant_index = variant_.index
existing_installed_variant = variant_
if existing_installed_variant:
debug_print(
"Variant %s already has installed equivalent: %s",
variant.uri, existing_installed_variant.uri
)
if dry_run:
if not package_changed:
return existing_installed_variant
else:
return None
# construct package data for new installed package definition
if existing_package:
_, file_ = os.path.split(existing_package.filepath)
package_filename, package_extension = os.path.splitext(file_)
package_extension = package_extension[1:]
package_format = FileFormat[package_extension]
if package_changed:
# graft together new package data, with existing package variants,
# and other data that needs to stay unchanged (eg timestamp)
package_data = new_package_data
if variant.index is not None:
package_data["variants"] = existing_package_data.get("variants", [])
else:
package_data = existing_package_data
else:
package_data = new_package_data
package_filename = _settings.package_filenames[0]
package_extension = "py"
package_format = FileFormat.py
# merge existing release data (if any) into the package. Note that when
# this data becomes variant-specific, this step will no longer be needed
package_data.update(release_data)
# merge the new variant into the package
if installed_variant_index is None and variant.index is not None:
variant_requires = variant.variant_requires
if not package_data.get("variants"):
package_data["variants"] = []
package_data["variants"].append(variant_requires)
installed_variant_index = len(package_data["variants"]) - 1
# a little data massaging is needed
package_data.pop("base", None)
# create version dir if it doesn't already exist
family_path = os.path.join(self.location, variant_name)
if variant_version:
pkg_base_path = os.path.join(family_path, str(variant_version))
else:
pkg_base_path = family_path
if not os.path.exists(pkg_base_path):
os.makedirs(pkg_base_path)
# Apply overrides.
#
# If we're installing into an existing package, then existing attributes
# in that package take precedence over `overrides`. If we're installing
# to a new package, then `overrides` takes precedence always.
#
# This is done so that variants added to an existing package don't change
# attributes such as 'timestamp' or release-related fields like 'revision'.
#
for key, value in overrides.iteritems():
if existing_package:
if key not in package_data:
package_data[key] = value
else:
if value is self.remove:
package_data.pop(key, None)
else:
package_data[key] = value
# timestamp defaults to now if not specified
if not package_data.get("timestamp"):
package_data["timestamp"] = int(time.time())
# format version is always set
package_data["format_version"] = format_version
# write out new package definition file
package_file = ".".join([package_filename, package_extension])
filepath = os.path.join(pkg_base_path, package_file)
with make_path_writable(pkg_base_path):
with open_file_for_write(filepath, mode=self.package_file_mode) as f:
dump_package_data(package_data, buf=f, format_=package_format)
# delete the tmp 'building' file.
if variant_version:
filename = self.building_prefix + str(variant_version)
filepath = os.path.join(family_path, filename)
if os.path.exists(filepath):
try:
os.remove(filepath)
except:
pass
# delete other stale building files; previous failed releases may have
# left some around
try:
self._delete_stale_build_tagfiles(family_path)
except:
pass
# touch the family dir, this keeps memcached resolves updated properly
os.utime(family_path, None)
# load new variant
new_variant = None
self.clear_caches()
family = self.get_package_family(variant_name)
if family:
for package in self.iter_packages(family):
if package.version == variant_version:
for variant_ in self.iter_variants(package):
if variant_.index == installed_variant_index:
new_variant = variant_
break
elif new_variant:
break
if not new_variant:
raise RezSystemError("Internal failure - expected installed variant")
return new_variant
def _create_variant(self, variant, dry_run=False, overrides=None):
# find or create the package family
family = self.get_package_family(variant.name)
if not family:
family = self._create_family(variant.name)
if isinstance(family, FileSystemCombinedPackageFamilyResource):
raise NotImplementedError(
"Cannot install variant into combined-style package file %r." %
family.filepath)
# find the package if it already exists
existing_package = None
for package in self.iter_packages(family):
if package.version == variant.version:
# during a build, the family/version dirs get created ahead of
# time, which causes a 'missing package definition file' error.
# This is fine, we can just ignore it and write the new file.
try:
package.validate_data()
except PackageDefinitionFileMissing:
break
uuids = set([variant.uuid, package.uuid])
if len(uuids) > 1 and None not in uuids:
raise ResourceError(
"Cannot install variant %r into package %r - the "
"packages are not the same (UUID mismatch)" %
(variant, package))
existing_package = package
if variant.index is None:
if package.variants:
raise ResourceError(
"Attempting to install a package without variants "
"(%r) into an existing package with variants (%r)"
% (variant, package))
elif not package.variants:
raise ResourceError(
"Attempting to install a variant (%r) into an existing "
"package without variants (%r)" % (variant, package))
installed_variant_index = None
existing_package_data = None
existing_variants_data = None
release_data = {}
new_package_data = variant.parent.validated_data()
new_package_data.pop("variants", None)
package_changed = False
if existing_package:
existing_package_data = existing_package.validated_data()
# detect case where new variant introduces package changes outside of variant
data_1 = existing_package_data.copy()
data_2 = new_package_data.copy()
for key in package_release_keys:
data_2.pop(key, None)
value = data_1.pop(key, None)
if value is not None:
release_data[key] = value
for key in ("base", "variants"):
data_1.pop(key, None)
data_2.pop(key, None)
package_changed = (data_1 != data_2)
# special case - installing a no-variant pkg into a no-variant pkg
if existing_package and variant.index is None:
if dry_run and not package_changed:
variant_ = self.iter_variants(existing_package).next()
return variant_
else:
# just replace the package
existing_package = None
if existing_package:
# see if variant already exists in package
variant_requires = variant.variant_requires
for variant_ in self.iter_variants(existing_package):
variant_requires_ = existing_package.variants[variant_.index]
if variant_requires_ == variant_requires:
installed_variant_index = variant_.index
if dry_run and not package_changed:
return variant_
break
parent_package = existing_package
_, file_ = os.path.split(existing_package.filepath)
package_filename, package_extension = os.path.splitext(file_)
package_extension = package_extension[1:]
package_format = FileFormat[package_extension]
if package_changed:
# graft together new package data, with existing package variants,
# and other data that needs to stay unchanged (eg timestamp)
package_data = new_package_data
package_data["variants"] = existing_package_data.get(
"variants", [])
else:
package_data = existing_package_data
else:
parent_package = variant.parent
package_data = new_package_data
package_filename = _settings.package_filenames[0]
package_extension = "py"
package_format = FileFormat.py
if dry_run:
return None
# merge existing release data (if any) into the package. Note that when
# this data becomes variant-specific, this step will no longer be needed
package_data.update(release_data)
# merge the new variant into the package
if installed_variant_index is None and variant.index is not None:
variant_requires = variant.variant_requires
if not package_data.get("variants"):
package_data["variants"] = []
package_data["variants"].append(variant_requires)
installed_variant_index = len(package_data["variants"]) - 1
# a little data massaging is needed
package_data["config"] = parent_package._data.get("config")
package_data.pop("base", None)
# create version dir and write out the new package definition file
family_path = os.path.join(self.location, variant.name)
if variant.version:
path = os.path.join(family_path, str(variant.version))
else:
path = family_path
if not os.path.exists(path):
os.makedirs(path)
# add the timestamp
overrides = overrides or {}
overrides["timestamp"] = int(time.time())
# apply attribute overrides
for key, value in overrides.iteritems():
if package_data.get(key) is None:
package_data[key] = value
package_file = ".".join([package_filename, package_extension])
filepath = os.path.join(path, package_file)
with open_file_for_write(filepath) as f:
dump_package_data(package_data, buf=f, format_=package_format)
# touch the family dir, this keeps memcached resolves updated properly
os.utime(family_path, None)
# load new variant
new_variant = None
self.clear_caches()
family = self.get_package_family(variant.name)
if family:
for package in self.iter_packages(family):
if package.version == variant.version:
for variant_ in self.iter_variants(package):
if variant_.index == installed_variant_index:
new_variant = variant_
break
elif new_variant:
break
if not new_variant:
raise RezSystemError(
"Internal failure - expected installed variant")
return new_variant
def _create_variant(self, variant, dry_run=False, overrides=None):
# special case overrides
variant_name = overrides.get("name") or variant.name
variant_version = overrides.get("version") or variant.version
overrides = (overrides or {}).copy()
overrides.pop("name", None)
overrides.pop("version", None)
# Need to treat 'config' as special case. In validated data, this is
# converted to a Config object. We need it as the raw dict that you'd
# see in a package.py.
#
def _get_package_data(pkg):
data = pkg.validated_data()
if hasattr(pkg, "_data"):
raw_data = pkg._data
else:
raw_data = pkg.resource._data
raw_config_data = raw_data.get("config")
data.pop("config", None)
if raw_config_data:
data["config"] = raw_config_data
return data
def _remove_build_keys(obj):
for key in package_build_only_keys:
obj.pop(key, None)
package_data = _get_package_data(variant.parent)
package_data.pop("variants", None)
package_data["name"] = variant_name
if variant_version:
package_data["version"] = variant_version
_remove_build_keys(package_data)
installed_variant_index = None
if dry_run:
return None
# a little data massaging is needed
package_data.pop("base", None)
# Apply overrides
for key, value in overrides.items():
if value is self.remove:
package_data.pop(key, None)
else:
package_data[key] = value
# timestamp defaults to now if not specified
if not package_data.get("timestamp"):
package_data["timestamp"] = int(time.time())
# format version is always set
package_data["format_version"] = 2 # Late binding functions added
# Stop if package is unversioned and config does not allow that
if (not package_data["version"]
and not config.allow_unversioned_packages):
raise PackageMetadataError("Unversioned package is not allowed.")
# Upsert to database
date = datetime.datetime.now()
family_name = package_data["name"]
version_string = str(package_data["version"])
document = package_family_document(family_name, date)
filter_ = {
"type": "family",
"_id": family_name,
}
self.collection.update_one(filter_, {"$set": document}, upsert=True)
document = package_document(family_name, date, version_string,
package_to_dict(package_data))
filter_ = {
"type": "package",
"family": family_name,
"version": version_string,
}
self.collection.update_one(filter_, {"$set": document}, upsert=True)
# load new variant
new_variant = None
self.clear_caches()
family = self.get_package_family(variant_name)
if family:
for package in self.iter_packages(family):
if package.version == variant_version:
for variant_ in self.iter_variants(package):
if variant_.index == installed_variant_index:
new_variant = variant_
break
elif new_variant:
break
if not new_variant:
raise RezSystemError(
"Internal failure - expected installed variant")
return new_variant
