def make_symlinks(paths, directory, force=False):
"""Make symlinks for all of the given Rez-specific paths and place them under `directory`.
Args:
paths (iter[str]):
Each installed Rez package path to generate symlinks for.
directory (str):
The absolute path to a directory on-disk to place the
symlinks under. This path can later be fed directly into
REZ_PACKAGES_PATH to reproduce the environment.
force (bool, optional):
If True, delete any symlinks that may already exist.
If False and symlinks exist, normal exceptions will be raised.
Default is False.
"""
for path in paths:
package = finder.get_nearest_rez_package(path)
root = inspection.get_packages_path_from_package(package)
relative_directory = os.path.relpath(path, root)
destination_directory = os.path.join(directory, relative_directory)
destination_root = os.path.dirname(destination_directory)
if not os.path.isdir(destination_root):
os.makedirs(destination_root)
if force and os.path.exists(destination_directory):
os.remove(destination_directory)
os.symlink(path, destination_directory)
def _make_package_with_contents(root, name, version, create_package):
directory = os.path.join(root, name)
os.makedirs(directory)
create_package(directory, name, version)
return finder.get_nearest_rez_package(directory)
def test_non_api_documentation(self):
"""Find Sphinx documentation, even if it isn't labelled as API documentation."""
contents = textwrap.dedent("""\
name = "some_package"
version = "1.0.0"
description = "A package with help information"
help = [
["some_documentation", "https://google.com"],
["another_one", "https://sphinx-code-include.readthedocs.io/en/latest"],
]
""")
directory = tempfile.mkdtemp()
root = os.path.join(directory, "some_package")
os.makedirs(root)
with open(os.path.join(root, "package.py"), "w") as handler:
handler.write(contents)
self.delete_item_later(directory)
package = finder.get_nearest_rez_package(root)
self.assertEqual("https://google.com",
url_help.find_package_documentation(package))
def _verify_source_package(self, directory, variants):
"""Make sure the the Rez package which ``rez_pip_boy`` converted has the expected files.
Args:
directory (str): The path leading to a source Rez package.
It should contain one rezbuild.py file and one package.py file.
variants (list[list[str]]): All Rez variations to take into account.
e.g. [["python-2.7"]].
"""
directory = os.path.expanduser(os.path.expandvars(directory))
rezbuild = os.path.join(
directory,
cli._BUILD_FILE_NAME # pylint: disable=protected-access
)
with open(rezbuild, "r") as handler:
rezbuild_code = handler.read()
package = finder.get_nearest_rez_package(directory)
self.assertIsNotNone(package)
self.assertEqual(_BUILD_COMMAND_CODE, rezbuild_code)
package_variants = [
list(map(str, variant)) for variant in package.variants or []
]
self.assertEqual(variants, package_variants)
self.assertEqual("python {root}/rezbuild.py", package.build_command)
return package
def test_build_requires(self):
"""Include `build_requires` in the Rez resolve."""
directory = _make_test_packages()
build_directory = tempfile.mkdtemp(
suffix="_BuildRequires_test_private_build_requires"
)
atexit.register(functools.partial(shutil.rmtree, build_directory))
package_directory = os.path.join(directory, "package_b")
creator.build(
finder.get_nearest_rez_package(package_directory),
build_directory,
quiet=True,
packages_path=[directory],
)
with _keep_pwd():
os.chdir(package_directory)
result = _test(
"directory --directory '{directory}' unittest_* --build-requires".format(
directory=os.path.join(build_directory, "package_b", "1.1.0"),
),
directory,
build_directory=build_directory,
)
self.assertEqual({"build_a-1.2.0", "package_a", "package_b-1.1.0"}, result)
def _make_version_package(path, text):
directory = os.path.dirname(path)
if not os.path.isdir(directory):
os.makedirs(directory)
with open(path, "w") as handler:
handler.write(text)
os.makedirs(os.path.join(directory, "python"))
open(os.path.join(directory, "python", "some_module.py"),
"a").close()
with open(os.path.join(directory, "rezbuild.py"), "w") as handler:
handler.write(
textwrap.dedent("""\
import os
import shutil
def main(source, install):
shutil.copytree(
os.path.join(source, "python"),
os.path.join(install, "python"),
)
main(
os.environ["REZ_BUILD_SOURCE_PATH"],
os.environ["REZ_BUILD_INSTALL_PATH"],
)
"""))
return finder.get_nearest_rez_package(directory)
def test_source_variant(self, run_command, has_documentation):
"""Create a source (non-built) Rez package that has 1+ variants and run a command on it.
Args:
run_command (:class:`mock.MagicMock`):
A replacement for the function that would normally run
as part of the commands that run on a Rez package. If
this function gets run, we know that this test passes.
"""
has_documentation.side_effect = [False, True]
packages = self._setup_test(
run_command, package_common.make_source_variant_python_package)
roots = list(
set(
inspection.get_packages_path_from_package(package)
for package in packages))
directory = tempfile.mkdtemp(suffix="_some_build_location")
self.delete_item_later(directory)
build_package = package_common.make_build_python_package(
textwrap.dedent("""\
name = "project_b"
version = "2.0.0"
revision = {
"push_url": "fake_git_repo",
}
"""),
"project_b",
"2.0.0",
directory,
)
build_package = finder.get_nearest_rez_package(
os.path.join(build_package))
build_root = inspection.get_packages_path_from_package(build_package)
paths = roots + [build_root]
with rez_configuration.patch_packages_path(paths):
self._test(
(
set(),
[],
[
# Important note: "project_b" doesn't actually already
# have documentation. It's a result of `side_effect`, above.
#
worker.Skip(
build_package,
finder.get_package_root(build_package),
"Python package already has documentation.",
)
],
),
paths,
)
self.assertEqual(1, run_command.call_count)
def _has_versioned_parent_package_folder():
"""bool: Check if this unittest suite is being run from a built Rez package."""
package = finder.get_nearest_rez_package(_CURRENT_DIRECTORY)
root = finder.get_package_root(package)
folder = os.path.basename(root)
return folder == os.environ["REZ_{name}_VERSION".format(
name=package.name.upper())]
def test_in_valid_context(self):
"""Make sure :func:`rez_lint.inspection.in_valid_context` works as expected."""
package = finder.get_nearest_rez_package(_CURRENT_DIRECTORY)
package_ = collections.namedtuple("Package", "name version")
fake_package = package_("foo", "6.6.6")
self.assertTrue(inspection.in_valid_context(package))
self.assertFalse(inspection.in_valid_context(fake_package))
def _make_built_package(root):
directory = os.path.join(root, "1.0.0")
os.makedirs(directory)
with open(os.path.join(directory, "package.py"), "w") as handler:
handler.write(
textwrap.dedent("""\
name = "something"
version = "1.0.0"
"""))
return finder.get_nearest_rez_package(directory)
def test_mix(self, run_command):
"""Run command on only to the Rez packages that need it.
Args:
run_command (:class:`mock.MagicMock`):
A replacement for the function that would normally run
as part of the commands that run on a Rez package. If
this function gets run, we know that this test passes.
"""
run_command.return_value = ""
root = os.path.join(tempfile.mkdtemp(), "test_folder")
os.makedirs(root)
self.delete_item_later(root)
packages = [
package_common.make_package("project_a", root,
package_common.make_source_package),
package_common.make_package(
"project_b",
root,
package_common.make_source_python_package,
dependencies={"project_a-1+<2"},
),
]
repository, packages, remote_root = package_common.make_fake_repository(
packages, root)
self.delete_item_later(repository.working_dir)
self.delete_item_later(remote_root)
release_path = _release_packages(packages)
self.delete_item_later(release_path)
package = finder.get_nearest_rez_package(
os.path.join(release_path, "project_a", "1.0.0"))
self._test(
(
set(),
[],
[
worker.Skip(
package,
finder.get_package_root(package),
"Rez Package does not define Python packages / modules.",
)
],
),
[release_path],
)
self.assertEqual(1, run_command.call_count)
def test_allow_current_context(self):
"""Check that has_python_package only creates a context when needed."""
package = finder.get_nearest_rez_package(_CURRENT_DIRECTORY)
inspection.in_valid_context = _check_called(
inspection.in_valid_context)
self.assertTrue(
inspection.has_python_package(package,
allow_current_context=False))
self.assertFalse(inspection.in_valid_context.was_run)
self.assertTrue(
inspection.has_python_package(package, allow_current_context=True))
self.assertTrue(inspection.in_valid_context.was_run)
def _needs_replacement(package, user_namespaces):
"""Figure out if the Rez package has Python files in it that :class:`MoveImports` can act upon.
The logic goes like this:
- A Rez source python package is assumed to have all its files available
- Return whether namespaces are found
- If the Rez python package is a released package though,
we can't just assume all Python files are available.
- The built Rez package may output an .egg or .whl file, for example
- So if namespaces are found, then return True
- If no namespaces are found, clone the package's repository and try again
Args:
package (:class:`rez.packages_.Package`):
Some Rez package (source or released package) to check for Python imports.
user_namespaces (set[str]):
Python dot-separated namespaces which a user is trying to
replace. If any of the found namespaces match these then
it means `package` must have at least one of its modules
replaced.
Returns:
bool:
If any of the user-provided Python namespaces were found
inside of the given `package`.
"""
root = finder.get_package_root(package)
package = finder.get_nearest_rez_package(root)
namespaces = set()
for path in move_break_api.expand_paths(root):
if path == package.filepath:
continue
namespaces.update(move_break_api.get_namespaces(path))
if namespaces.intersection(user_namespaces):
return True
if not inspection.is_built_package(package):
return False
repository = repository_area.get_repository(package)
repository_package = repository_area.get_package(repository.working_dir,
package.name)
return _needs_replacement(repository_package, user_namespaces)
def test_recursive(self):
"""Make sure that getting plugins recursively works."""
package = finder.get_nearest_rez_package(_CURRENT_DIRECTORY)
found, invalids = cli._find_rez_packages( # pylint: disable=protected-access
finder.get_package_root(package),
recursive=True,
)
found_package = next(iter((found)))
self.assertEqual(1, len(found))
self.assertTrue(bool(found))
self.assertEqual(found_package.name, package.name)
self.assertEqual(found_package.filepath, package.filepath)
self.assertEqual(found_package.version, package.version)
self.assertEqual(set(), invalids)
def _guess_repository_from_symlinks(directory):
"""Find a built Rez package's repository by tracing through its symlinks.
Rez packages are frequently built using symlinks when users are
working locally. It's not "standard-practice" for Rez work but it's
a very common one. So, if the package is built but not released
and no repository can be found, try to use symlinks to "find" a
repository for that package.
Args:
directory (str):
The folder to search for symlinked Rez/repositories, recursively.
Raises:
RuntimeError:
If no symlinks were found or multiple Rez packages were found through symlinks.
Returns:
:class:`git.Repo`: The found repository.
"""
symlinks = set()
for root, folders, files in os.walk(directory):
for path in folders + files:
full = os.path.join(root, path)
if os.path.islink(full):
symlinks.add(full)
if not symlinks:
message = 'Directory "{directory}" has no symlinks.'.format(directory=directory)
_LOGGER.debug(message)
raise RuntimeError(message)
packages = {finder.get_nearest_rez_package(path).name for path in symlinks}
if len(packages) != 1:
raise RuntimeError(
'More than one Rez package "{packages}" was found.'
"".format(packages=packages)
)
path = next(iter(symlinks))
real_path = os.path.realpath(os.readlink(path))
return git.Repo(real_path, search_parent_directories=True)
def test_invalid_absolute_decrement_minor(self):
"""Don't set the minor version, explicitly because it is a negative value."""
directory = tempfile.mkdtemp(suffix="_test_increment_minor")
self.delete_item_later(directory)
with open(os.path.join(directory, "package.py"), "w") as handler:
handler.write(
textwrap.dedent("""\
name = "foo"
version = "12.5.7"
"""))
package = finder.get_nearest_rez_package(directory)
with self.assertRaises(ValueError):
rez_bump_api.bump(package, minor=-4, absolute=True)
def test_invalid_001(self):
"""Check that there is always at least some version information provided."""
directory = tempfile.mkdtemp(suffix="_test_increment_minor")
self.delete_item_later(directory)
with open(os.path.join(directory, "package.py"), "w") as handler:
handler.write(
textwrap.dedent("""\
name = "foo"
version = "12.5.7"
"""))
package = finder.get_nearest_rez_package(directory)
with self.assertRaises(ValueError):
rez_bump_api.bump(package)
def _bump(package, increment, new_dependencies):
rez_bump_api.bump(package, **{increment: 1})
with open(package.filepath, "r") as handler:
code = handler.read()
new_code = api.add_to_attribute("requires", new_dependencies, code)
with filesystem.make_path_writable(
os.path.dirname(os.path.dirname(package.filepath))):
with serialise.open_file_for_write(package.filepath) as handler:
handler.write(new_code)
root = finder.get_package_root(package)
return finder.get_nearest_rez_package(root)
def test_no_requires(self):
"""Allow the build requires flags even if the Rez package doesn't define them."""
directory = _make_test_packages()
build_directory = tempfile.mkdtemp(
suffix="_BuildRequires_test_private_build_requires"
)
atexit.register(functools.partial(shutil.rmtree, build_directory))
package_directory = os.path.join(directory, "package_a")
creator.build(
finder.get_nearest_rez_package(package_directory),
build_directory,
quiet=True,
packages_path=[directory],
)
with _keep_pwd():
os.chdir(package_directory)
result = _test(
" ".join(
[
"directory",
"--directory",
"'{directory}'".format(
directory=os.path.join(
build_directory,
"package_a",
"1.0.0",
)
),
"name_A",
"--build-requires",
"--private-build-requires",
]
),
directory,
build_directory=build_directory,
)
self.assertEqual({"package_a-1.0.0", "package_d-1.1.0"}, result)
def test_source_with_variants(self):
"""A Rez source package that contains a Python package should return true.
If that source package has variants, as long as the variant contains
a Python module, it shouldn't need to be built and still return True.
"""
root = tempfile.mkdtemp(suffix="_a_rez_source_package_with_a_variant")
self.delete_item_later(root)
root = os.path.join(root, "some_package")
os.makedirs(root)
with open(os.path.join(root, "package.py"), "w") as handler:
handler.write(
textwrap.dedent("""\
name = "some_package"
version = "1.0.0"
variants = [["python-2"]]
def commands():
import os
env.PYTHONPATH.append(os.path.join("{root}", "python"))
"""))
python_root = os.path.join(root, "python", "some_package")
os.makedirs(python_root)
open(os.path.join(python_root, "__init__.py"), "a").close()
open(os.path.join(python_root, "some_module.py"), "a").close()
local.LocalBuildProcess.build = _check_called(
local.LocalBuildProcess.build)
package = finder.get_nearest_rez_package(root)
self.assertTrue(
inspection.has_python_package(
package,
paths=[root] + config.packages_path, # pylint: disable=no-member
allow_build=False,
))
self.assertFalse(local.LocalBuildProcess.build.was_run)
def test_build(self):
"""A build Rez package should return True if it has a Python module inside of it."""
root = tempfile.mkdtemp(
suffix="_a_rez_source_package_with_no_python_modules_until_build")
self.delete_item_later(root)
root = os.path.join(root, "some_package")
os.makedirs(root)
with open(os.path.join(root, "package.py"), "w") as handler:
template = textwrap.dedent("""\
name = "some_package"
version = "1.0.0"
requires = ["python-{sys.version_info.major}"]
build_command = "python {{root}}/rezbuild.py {{install}}"
def commands():
import os
env.PYTHONPATH.append(os.path.join("{{root}}", "python"))
""")
handler.write(template.format(sys=sys))
with open(os.path.join(root, "rezbuild.py"), "w") as handler:
handler.write(_get_rezbuild_text())
python_root = os.path.join(root, "python", "some_package")
os.makedirs(python_root)
open(os.path.join(python_root, "__init__.py"), "a").close()
package = finder.get_nearest_rez_package(root)
install_root = tempfile.mkdtemp(suffix="_installed_rez_package")
self.delete_item_later(install_root)
build_package = creator.build(package, install_root, quiet=True)
self.assertTrue(
inspection.has_python_package(
build_package,
paths=[root] + config.packages_path, # pylint: disable=no-member
allow_build=True,
))
def _make_rez_package(name, package_name, text, root):
"""Create a package.py or package.yaml Rez package file.
Args:
name (str): The Rez package family name.
package_name (str): Use "package.py" or "package.yaml" here.
text (str): The contents of the created file.
root (str): A directory where the newly created file will be written to.
Returns:
:class:`rez.packages_.DeveloperPackage`: The generated Rez package.
"""
directory = os.path.join(root, name)
os.makedirs(directory)
with open(os.path.join(directory, package_name), "w") as handler:
handler.write(text)
return finder.get_nearest_rez_package(directory)
def test_source(self):
"""Return True if a Rez package with at least one Python module is found."""
root = tempfile.mkdtemp(
suffix="_a_rez_source_package_that_has_at_least_1_python_module")
self.delete_item_later(root)
root = os.path.join(root, "some_package")
os.makedirs(root)
with open(os.path.join(root, "package.py"), "w") as handler:
handler.write(
textwrap.dedent("""\
name = "some_package"
version = "1.0.0"
def commands():
import os
env.PYTHONPATH.append(os.path.join("{root}", "python"))
"""))
python_root = os.path.join(root, "python", "some_package")
os.makedirs(python_root)
open(os.path.join(python_root, "__init__.py"), "a").close()
open(os.path.join(python_root, "some_module.py"), "a").close()
local.LocalBuildProcess.build = _check_called(
local.LocalBuildProcess.build)
package = finder.get_nearest_rez_package(root)
self.assertTrue(
inspection.has_python_package(
package,
paths=[root] + config.packages_path, # pylint: disable=no-member
allow_build=False,
))
self.assertFalse(local.LocalBuildProcess.build.was_run)
def _verify_installed_package(self, directory):
"""Build a Rez package and make sure it builds correctly.
Args:
directory (str): The absolute path where a Rez source package is defined.
"""
install_directory = tempfile.mkdtemp(
prefix="rez_pip_boy_", suffix="_verify_installed_package")
atexit.register(functools.partial(shutil.rmtree, install_directory))
package = finder.get_nearest_rez_package(directory)
creator.build(package, install_directory, quiet=True)
installed_package_directory = os.path.join(install_directory,
package.name,
str(package.version))
self.assertTrue(
os.path.isfile(
os.path.join(installed_package_directory, "package.py")))
self.assertFalse(
os.path.isfile(
os.path.join(installed_package_directory, "rezbuild.py")))
def test_normalize_and_increment_002(self):
"""Reset all of the number tokens below whatever was incremented."""
directory = tempfile.mkdtemp(suffix="_test_normalize_and_increment")
self.delete_item_later(directory)
with open(os.path.join(directory, "package.py"), "w") as handler:
handler.write(
textwrap.dedent("""\
name = "foo"
version = "12.5.beta.7"
"""))
package = finder.get_nearest_rez_package(directory)
rez_bump_api.bump(package, minor=1, normalize=True)
with open(package.filepath, "r") as handler:
code = handler.read()
expected = textwrap.dedent("""\
name = "foo"
version = "12.6.beta.0"
""")
self.assertEquals(expected, code)
def test_from_package_pwd(self):
"""Get the resolve from a package's directory."""
directory = _make_test_packages()
package_directory = os.path.join(directory, "package_b")
build_directory = tempfile.mkdtemp(suffix="_Run_test_from_package_pwd")
atexit.register(functools.partial(shutil.rmtree, build_directory))
creator.build(
finder.get_nearest_rez_package(package_directory),
build_directory,
quiet=True,
)
with _keep_pwd():
os.chdir(package_directory)
result = _test(
"directory unittest_*",
directory,
build_directory=build_directory,
)
self.assertEqual({"package_a", "package_b-1.1.0"}, result)
def test_from_package_directory(self):
"""Get the resolve from a package's directory."""
directory = _make_test_packages()
build_directory = tempfile.mkdtemp(suffix="_Run_test_from_package_directory")
atexit.register(functools.partial(shutil.rmtree, build_directory))
package_directory = os.path.join(directory, "package_b")
creator.build(
finder.get_nearest_rez_package(package_directory),
build_directory,
quiet=True,
)
result = _test(
"directory --directory '{directory}' unittest_*".format(
directory=os.path.join(build_directory, "package_b", "1.1.0"),
),
directory,
build_directory=build_directory,
)
self.assertEqual({"package_a", "package_b-1.1.0"}, result)
def test_decrement_minor(self):
"""Lower the value of a package's version without knowing what that version is."""
directory = tempfile.mkdtemp(suffix="_test_increment_minor")
self.delete_item_later(directory)
with open(os.path.join(directory, "package.py"), "w") as handler:
handler.write(
textwrap.dedent("""\
name = "foo"
version = "12.5.7"
"""))
package = finder.get_nearest_rez_package(directory)
rez_bump_api.bump(package, minor=-4)
with open(package.filepath, "r") as handler:
code = handler.read()
expected = textwrap.dedent("""\
name = "foo"
version = "12.1.7"
""")
self.assertEquals(expected, code)
def test_absolute_increment_minor(self):
"""Set the minor version, explicitly."""
directory = tempfile.mkdtemp(suffix="_test_increment_minor")
self.delete_item_later(directory)
with open(os.path.join(directory, "package.py"), "w") as handler:
handler.write(
textwrap.dedent("""\
name = "foo"
version = "12.5.7"
"""))
package = finder.get_nearest_rez_package(directory)
rez_bump_api.bump(package, minor=3, absolute=True)
with open(package.filepath, "r") as handler:
code = handler.read()
expected = textwrap.dedent("""\
name = "foo"
version = "12.3.7"
""")
self.assertEquals(expected, code)
def _resolve_arguments(arguments):
"""Change user-provided arguments into something that this Python package can use.
Args:
arguments (:class:`argparse.Namespace`): The user-provided arguments from command-line.
Raises:
EnvironmentError: If no Rez package could be found from the user's input.
Returns:
tuple[:class:`rez.developer_package.DeveloperPackage`, set[str], str]:
- The "source (current) Rez package". This package will be
modified if called by the "fix" command.
- The files / folders within the user's temporary "build Rez package"
- The temporary directory where the "build Rez package" is written to.
"""
current_root = os.getcwd()
path = arguments.package
if not os.path.isabs(arguments.package):
path = os.path.normcase(os.path.join(current_root, path))
if os.path.isfile(path):
_LOGGER.debug(
'Path "%s" must be a directory leading to a Rez package.', path)
path = os.path.dirname(path)
package = finder.get_nearest_rez_package(path)
if not package:
raise EnvironmentError(
'Path "{path}" is not inside of a Rez package version. '
"Please CD into a Rez package and try again.".format(path=path))
return package
