def install_as_rez_package(repo_path): """Install the current rez installation as a rez package. Note: This is very similar to 'rez-bind rez', however rez-bind is intended for deprecation. Rez itself is a special case. Args: repo_path (str): Repository to install the rez package into. """ def commands(): env.PYTHONPATH.append('{this.root}') # noqa def make_root(variant, root): # copy source rez_path = rez.__path__[0] site_path = os.path.dirname(rez_path) rezplugins_path = os.path.join(site_path, "rezplugins") shutil.copytree(rez_path, os.path.join(root, "rez")) shutil.copytree(rezplugins_path, os.path.join(root, "rezplugins")) variant = system.variant variant.append("python-{0.major}.{0.minor}".format(sys.version_info)) with make_package("rez", repo_path, make_root=make_root) as pkg: pkg.version = rez.__version__ pkg.commands = commands pkg.variants = [variant] print('') print("Success! Rez was installed to %s/rez/%s" % (repo_path, rez.__version__))
def bind(path, version_range=None, opts=None, parser=None): exe_path = getattr(opts, 'exe', None) # gcc gcc_path = find_exe('gcc', filepath=exe_path) gcc_version = extract_version(gcc_path, ['-dumpfullversion', '-dumpversion']) log("binding gcc: %s" % gcc_path) # g++ gpp_path = find_exe('g++', filepath=exe_path) gpp_version = extract_version(gpp_path, ['-dumpfullversion', '-dumpversion']) log("binding g++: %s" % gpp_path) if gcc_version != gpp_version: raise RezBindError("gcc version different than g++ can not continue") # create directories and symlink gcc and g++ def make_root(variant, root): bin_path = make_dirs(root, 'bin') gcc_link_path = os.path.join(bin_path, 'gcc') platform_.symlink(gcc_path, gcc_link_path) gpp_link_path = os.path.join(bin_path, 'g++') platform_.symlink(gpp_path, gpp_link_path) with make_package('gcc', path, make_root=make_root) as pkg: pkg.version = gcc_version pkg.tools = ['gcc', 'g++'] pkg.commands = commands pkg.variants = [system.variant] return pkg.installed_variants
def _make_package(help_=tuple()): """Make a Rez package with the given help attribute for testing. Args: help_ (str or list[list[str, str]]): A single help string or a list of display + help string pairs. The left of the pair is the text users might see. The right side is either an absolute or relative path on-disk or a website URL. Returns: :class:`rez.packages.DeveloperPackage`: The generated package. """ directory = tempfile.mkdtemp(suffix="_make_package") atexit.register(functools.partial(shutil.rmtree, directory)) name = "does_not_matter" version = "1.0.0" with package_maker.make_package(name, directory) as maker: maker.name = name maker.version = version maker.help = help_ return packages_.get_developer_package( os.path.join(directory, name, version))
def bind(path, version_range=None, opts=None, parser=None): rez_version = Version(rez.__version__) check_version(rez_version, version_range) rez_major_version = rez_version.trim(1) rez_major_minor_version = rez_version.trim(2) next_major = int(str(rez_major_version)) + 1 rez_req_str = "rez-%s+<%d" % (str(rez_major_minor_version), next_major) gui_lib = getattr(opts, "gui_lib", "") def make_root(variant, root): # copy source rez_path = rez.__path__[0] site_path = os.path.dirname(rez_path) rezgui_path = os.path.join(site_path, "rezgui") shutil.copytree(rezgui_path, os.path.join(root, "rezgui")) # create rez-gui executable binpath = make_dirs(root, "bin") filepath = os.path.join(binpath, "rez-gui") create_executable_script(filepath, rez_gui_source) # create package with make_package("rezgui", path, make_root=make_root) as pkg: pkg.version = rez_version pkg.variants = [system.variant] pkg.commands = commands pkg.tools = ["rez-gui"] pkg.requires = [rez_req_str] if gui_lib: pkg.requires.append(gui_lib) return pkg.installed_variants
def install_as_rez_package(repo_path, pkg_name='rez'): """Install the current rez installation as a rez package. Note: This is very similar to 'rez-bind rez', however rez-bind is intended for deprecation. Rez itself is a special case. Args: repo_path (str): Repository to install the rez package into. pkg_name (str): Rez package's package name (Default: rez). """ def commands(): env.PYTHONPATH.append('{this.root}') # noqa def make_root(variant, root): # copy source rez_path = rez.__path__[0] site_path = os.path.dirname(rez_path) rezplugins_path = os.path.join(site_path, "rezplugins") shutil.copytree(rez_path, os.path.join(root, "rez")) shutil.copytree(rezplugins_path, os.path.join(root, "rezplugins")) variant = system.variant variant.append("python-{0.major}.{0.minor}".format(sys.version_info)) with make_package(pkg_name, repo_path, make_root=make_root) as pkg: pkg.version = rez.__version__ pkg.commands = commands pkg.variants = [variant] print("installing rez as Python package under", repo_path) print('') for installed_variant in pkg.installed_variants: install_path = installed_variant.base print("SUCCESS! Rez Python package was installed to", install_path)
def bind(path, version_range=None, opts=None, parser=None): version = Version(system.arch) check_version(version, version_range) with make_package("arch", path) as pkg: pkg.version = version return pkg.installed_variants
def bind(path, version_range=None, opts=None, parser=None): version = Version(system.os) check_version(version, version_range) with make_package("os", path) as pkg: pkg.version = version pkg.requires = [ "platform-%s" % system.platform, "arch-%s" % system.arch ] return pkg.installed_variants
def bind(path, version_range=None, opts=None, parser=None): # find executable, determine version exepath = find_exe("python", opts.exe) code = "import sys; print('.'.join(str(x) for x in sys.version_info))" version = extract_version(exepath, ["-c", code]) check_version(version, version_range) log("binding python: %s" % exepath) # find builtin modules builtin_paths = {} entries = [("lib", "os"), ("extra", "setuptools")] for dirname, module_name in entries: success, out, err = run_python_command( ["import %s" % module_name, "print(%s.__file__)" % module_name]) if success: pypath = os.path.dirname(out) if os.path.basename(pypath) == module_name: pypath = os.path.dirname(pypath) if pypath not in builtin_paths.values(): builtin_paths[dirname] = pypath # make the package # def make_root(variant, root): binpath = make_dirs(root, "bin") link = os.path.join(binpath, "python") platform_.symlink(exepath, link) if builtin_paths: pypath = make_dirs(root, "python") for dirname, srcpath in builtin_paths.items(): destpath = os.path.join(pypath, dirname) log("Copying builtins from %s to %s..." % (srcpath, destpath)) shutil.copytree(srcpath, destpath) with make_package("python", path, make_root=make_root) as pkg: pkg.version = version pkg.tools = ["python"] pkg.commands = commands pkg.variants = [system.variant] if builtin_paths: pkg.post_commands = post_commands return pkg.installed_variants
def bind(path, version_range=None, opts=None, parser=None): exepath = find_exe("cmake", getattr(opts, "exe", None)) version = extract_version(exepath, "--version", word_index=2 if os.name == 'nt' else -1) check_version(version, version_range) def make_root(variant, root): binpath = make_dirs(root, "bin") link = os.path.join(binpath, "cmake") platform_.symlink(exepath, link) with make_package("cmake", path, make_root=make_root) as pkg: pkg.version = version pkg.tools = ["cmake"] pkg.commands = commands pkg.variants = [system.variant] return pkg.installed_variants
def bind(path, version_range=None, opts=None, parser=None): version = Version("1.0") check_version(version, version_range) def make_root(variant, root): binpath = make_dirs(root, "bin") filepath = os.path.join(binpath, "hello_world") create_executable_script( filepath, hello_world_source, py_script_mode=ExecutableScriptMode.platform_specific) with make_package("hello_world", path, make_root=make_root) as pkg: pkg.version = version pkg.tools = ["hello_world"] pkg.commands = commands return pkg.installed_variants
def bind(path, version_range=None, opts=None, parser=None): version = rez.__version__ check_version(version, version_range) def make_root(variant, root): # copy source rez_path = rez.__path__[0] site_path = os.path.dirname(rez_path) rezplugins_path = os.path.join(site_path, "rezplugins") shutil.copytree(rez_path, os.path.join(root, "rez")) shutil.copytree(rezplugins_path, os.path.join(root, "rezplugins")) with make_package("rez", path, make_root=make_root) as pkg: pkg.version = version pkg.commands = commands pkg.requires = ["python-2.7+<4"] pkg.variants = [system.variant] return pkg.installed_variants
def install_as_production_package(install_py, repo_path, pkg_name='rez'): """Install a production rez installation as a rez package. Note: This is very similar to 'rez-bind rez', however rez-bind is intended for deprecation. Rez itself is a special case. Args: install_py (list[str]): Command line args to call "install.py". repo_path (str): Repository to install the rez package into. pkg_name (str): Rez package's package name (Default: rez). """ def commands(): import os bin_folder = "Scripts" if os.name == "nt" else "bin" env.PATH.append(os.path.join("{this.root}", bin_folder, "rez")) env.PYTHONPATH.append(os.path.join("{this.root}", "python")) def make_root(variant, root): subprocess.check_call(list(install_py) + [root]) # copy source rez_path = rez.__path__[0] site_path = os.path.dirname(rez_path) rezplugins_path = os.path.join(site_path, "rezplugins") py_root = os.path.join(root, "python") os.mkdir(py_root) shutil.copytree(rez_path, os.path.join(py_root, "rez")) shutil.copytree(rezplugins_path, os.path.join(py_root, "rezplugins")) print('Setting up "%s"' % os.path.join(variant.base, 'package.py')) with make_package(pkg_name, repo_path, make_root=make_root) as pkg: # Production Python venv probably works for same platform + arch combo pkg.variants = [system.variant[:-1]] pkg.version = rez.__version__ pkg.commands = commands print('SUCCESS! After activated Rez, you can then:') print('rez env %s' % pkg_name)
def bind(path, version_range=None, opts=None, parser=None): version = Version(system.platform) check_version(version, version_range) def post_commands(): """Setup default XDG_* environment variables. https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html#variables """ import os xdg_defaults = ( ('XDG_DATA_HOME', ['$HOME/.local/share']), ('XDG_DATA_DIRS', ['/usr/local/share', '/usr/share']), ('XDG_CONFIG_HOME', ['$HOME/.config']), ('XDG_CONFIG_DIRS', ['/etc/xdg']), ('XDG_CACHE_HOME', ['$HOME/.cache']), ) for xdg_var, defaults in xdg_defaults: invalid_var = undefined(xdg_var) or not str(env[xdg_var]) paths = [] if invalid_var else str(env[xdg_var]).split(os.pathsep) append = len(defaults) != 1 for default_path in defaults: expanded = expandvars(default_path) if not (default_path in paths or expanded in paths): if append: env[xdg_var].append(expanded) else: env[xdg_var] = expanded with make_package("platform", path) as pkg: pkg.version = version pkg.relocatable = True if system.platform == 'linux': pkg.post_commands = post_commands return pkg.installed_variants
def pip_install_package(source_name, pip_version=None, python_version=None, mode=InstallMode.min_deps, release=False, prefix=None, extra_args=None): """Install a pip-compatible python package as a rez package. Args: source_name (str): Name of package or archive/url containing the pip package source. This is the same as the arg you would pass to the 'pip install' command. pip_version (str or `Version`): Version of pip to use to perform the install, uses latest if None. python_version (str or `Version`): Python version to use to perform the install, and subsequently have the resulting rez package depend on. mode (`InstallMode`): Installation mode, determines how dependencies are managed. release (bool): If True, install as a released package; otherwise, it will be installed as a local package. extra_args (List[str]): Additional options to the pip install command. Returns: 2-tuple: List of `Variant`: Installed variants; List of `Variant`: Skipped variants (already installed). """ installed_variants = [] skipped_variants = [] py_exe, context = find_pip(pip_version, python_version) print_info("Installing %r with pip taken from %r", source_name, py_exe) # TODO: should check if packages_path is writable before continuing with pip # if prefix is not None: packages_path = prefix else: packages_path = (config.release_packages_path if release else config.local_packages_path) targetpath = mkdtemp(suffix="-rez", prefix="pip-") if context and config.debug("package_release"): buf = StringIO() print("\n\npackage download environment:", file=buf) context.print_info(buf) _log(buf.getvalue()) # Build pip commandline cmd = [py_exe, "-m", "pip", "install"] _extra_args = extra_args or config.pip_extra_args or [] if "--no-use-pep517" not in _extra_args: cmd.append("--use-pep517") if not _option_present(_extra_args, "-t", "--target"): cmd.append("--target=%s" % targetpath) if mode == InstallMode.no_deps and "--no-deps" not in _extra_args: cmd.append("--no-deps") cmd.extend(_extra_args) cmd.append(source_name) # run pip # # Note: https://github.com/pypa/pip/pull/3934. If/when this PR is merged, # it will allow explicit control of where to put bin files. # _cmd(context=context, command=cmd) # determine version of python in use if context is None: # since we had to use system pip, we have to assume system python version py_ver_str = '.'.join(map(str, sys.version_info)) py_ver = Version(py_ver_str) else: python_variant = context.get_resolved_package("python") py_ver = python_variant.version # Collect resulting python packages using distlib distribution_path = DistributionPath([targetpath]) distributions = list(distribution_path.get_distributions()) dist_names = [x.name for x in distributions] def log_append_pkg_variants(pkg_maker): template = '{action} [{package.qualified_name}] {package.uri}{suffix}' actions_variants = [ ( print_info, 'Installed', installed_variants, pkg_maker.installed_variants or [], ), ( print_debug, 'Skipped', skipped_variants, pkg_maker.skipped_variants or [], ), ] for print_, action, variants, pkg_variants in actions_variants: for variant in pkg_variants: variants.append(variant) package = variant.parent suffix = (' (%s)' % variant.subpath) if variant.subpath else '' print_(template.format(**locals())) # get list of package and dependencies for distribution in distributions: # convert pip requirements into rez requirements rez_requires = get_rez_requirements(installed_dist=distribution, python_version=py_ver, name_casings=dist_names) # log the pip -> rez requirements translation, for debugging _log("Pip to rez requirements translation information for " + distribution.name_and_version + ":\n" + pformat({ "pip": { "run_requires": map(str, distribution.run_requires) }, "rez": rez_requires })) # determine where pip files need to be copied into rez package src_dst_lut = _get_distribution_files_mapping(distribution, targetpath) # build tools list tools = [] for relpath in src_dst_lut.values(): dir_, filename = os.path.split(relpath) if dir_ == "bin": tools.append(filename) # Sanity warning to see if any files will be copied if not src_dst_lut: message = 'No source files exist for {}!' if not _verbose: message += '\nTry again with rez-pip --verbose ...' print_warning(message.format(distribution.name_and_version)) def make_root(variant, path): """Using distlib to iterate over all installed files of the current distribution to copy files to the target directory of the rez package variant """ for rel_src, rel_dest in src_dst_lut.items(): src = os.path.join(targetpath, rel_src) dest = os.path.join(path, rel_dest) if not os.path.exists(os.path.dirname(dest)): os.makedirs(os.path.dirname(dest)) shutil.copyfile(src, dest) if _is_exe(src): shutil.copystat(src, dest) # create the rez package name = pip_to_rez_package_name(distribution.name) version = pip_to_rez_version(distribution.version) requires = rez_requires["requires"] variant_requires = rez_requires["variant_requires"] metadata = rez_requires["metadata"] with make_package(name, packages_path, make_root=make_root) as pkg: # basics (version etc) pkg.version = version if distribution.metadata.summary: pkg.description = distribution.metadata.summary # requirements and variants if requires: pkg.requires = requires if variant_requires: pkg.variants = [variant_requires] # commands commands = [] commands.append("env.PYTHONPATH.append('{root}/python')") if tools: pkg.tools = tools commands.append("env.PATH.append('{root}/bin')") pkg.commands = '\n'.join(commands) # Make the package use hashed variants. This is required because we # can't control what ends up in its variants, and that can easily # include problematic chars (>, +, ! etc). # TODO: https://github.com/nerdvegas/rez/issues/672 # pkg.hashed_variants = True # add some custom attributes to retain pip-related info pkg.pip_name = distribution.name_and_version pkg.from_pip = True pkg.is_pure_python = metadata["is_pure_python"] distribution_metadata = distribution.metadata.todict() help_ = [] if "home_page" in distribution_metadata: help_.append(["Home Page", distribution_metadata["home_page"]]) if "download_url" in distribution_metadata: help_.append( ["Source Code", distribution_metadata["download_url"]]) if help_: pkg.help = help_ if "author" in distribution_metadata: author = distribution_metadata["author"] if "author_email" in distribution_metadata: author += ' ' + distribution_metadata["author_email"] pkg.authors = [author] log_append_pkg_variants(pkg) # cleanup shutil.rmtree(targetpath) # print summary # if installed_variants: print_info("%d packages were installed.", len(installed_variants)) else: print_warning("NO packages were installed.") if skipped_variants: print_warning( "%d packages were already installed.", len(skipped_variants), ) return installed_variants, skipped_variants
def bind(name, path, import_name=None, version_range=None, version=None, requires=None, pure_python=None, tools=None, extra_module_names=[], extra_attrs={}): import_name = import_name or name if version is None: version = get_version_in_python( name, ["import %s" % import_name, "print(%s.__version__)" % import_name]) check_version(version, version_range) py_major_minor = '.'.join(str(x) for x in sys.version_info[:2]) py_req = "python-%s" % py_major_minor found_tools = {} if pure_python is None: raise NotImplementedError # detect elif pure_python: variant = [py_req] else: variant = system.variant + [py_req] for tool in (tools or []): try: src = find_exe(tool) found_tools[tool] = src log("found tool '%s': %s" % (tool, src)) except RezBindError as e: print_warning(str(e)) def make_root(variant, root): pypath = make_dirs(root, "python") copy_module(import_name, pypath) if found_tools: binpath = make_dirs(root, "bin") for tool, src in sorted(found_tools.items()): dest = os.path.join(binpath, tool) shutil.copy2(src, dest) for name_ in extra_module_names: copy_module(name_, pypath) with make_package(name, path, make_root=make_root) as pkg: pkg.version = version pkg.variants = [variant] if requires: pkg.requires = requires if found_tools: pkg.tools = list(found_tools) pkg.commands = commands_with_bin else: pkg.commands = commands for key, value in extra_attrs.items(): pkg[key] = value return pkg.installed_variants
def pip_install_package(source_name, pip_version=None, python_version=None, mode=InstallMode.min_deps, release=False, prefix=None, extra_args=None): """Install a pip-compatible python package as a rez package. Args: source_name (str): Name of package or archive/url containing the pip package source. This is the same as the arg you would pass to the 'pip install' command. pip_version (str or `Version`): Version of pip to use to perform the install, uses latest if None. python_version (str or `Version`): Python version to use to perform the install, and subsequently have the resulting rez package depend on. mode (`InstallMode`): Installation mode, determines how dependencies are managed. release (bool): If True, install as a released package; otherwise, it will be installed as a local package. extra_args (List[str]): Additional options to the pip install command. Returns: 2-tuple: List of `Variant`: Installed variants; List of `Variant`: Skipped variants (already installed). """ installed_variants = [] skipped_variants = [] py_exe, context = find_pip(pip_version, python_version) print_info("Installing %r with pip taken from %r", source_name, py_exe) # TODO: should check if packages_path is writable before continuing with pip # if prefix is not None: packages_path = prefix else: packages_path = (config.release_packages_path if release else config.local_packages_path) tmpdir = mkdtemp(suffix="-rez", prefix="pip-") stagingdir = os.path.join(tmpdir, "rez_staging") stagingsep = "".join([os.path.sep, "rez_staging", os.path.sep]) destpath = os.path.join(stagingdir, "python") # TODO use binpath once https://github.com/pypa/pip/pull/3934 is approved binpath = os.path.join(stagingdir, "bin") if context and config.debug("package_release"): buf = StringIO() print("\n\npackage download environment:", file=buf) context.print_info(buf) _log(buf.getvalue()) # Build pip commandline cmd = [py_exe, "-m", "pip", "install"] _extra_args = extra_args or config.pip_extra_args or [] if "--no-use-pep517" not in _extra_args: cmd.append("--use-pep517") if not _option_present(_extra_args, "-t", "--target"): cmd.append("--target=%s" % destpath) if mode == InstallMode.no_deps and "--no-deps" not in _extra_args: cmd.append("--no-deps") cmd.extend(_extra_args) cmd.append(source_name) # run pip _cmd(context=context, command=cmd) # determine version of python in use if context is None: # since we had to use system pip, we have to assume system python version py_ver_str = '.'.join(map(str, sys.version_info)) py_ver = Version(py_ver_str) else: python_variant = context.get_resolved_package("python") py_ver = python_variant.version # moving bin folder to expected relative location as per wheel RECORD files staged_binpath = os.path.join(destpath, "bin") if os.path.isdir(staged_binpath): shutil.move(os.path.join(destpath, "bin"), binpath) # Collect resulting python packages using distlib distribution_path = DistributionPath([destpath]) distributions = list(distribution_path.get_distributions()) dist_names = [x.name for x in distributions] # get list of package and dependencies for distribution in distributions: # convert pip requirements into rez requirements rez_requires = get_rez_requirements(installed_dist=distribution, python_version=py_ver, name_casings=dist_names) # log the pip -> rez translation, for debugging _log("Pip to rez translation information for " + distribution.name_and_version + ":\n" + pformat({ "pip": { "run_requires": map(str, distribution.run_requires) }, "rez": rez_requires })) # iterate over installed files and determine dest filepaths tools = [] src_dst_lut = {} for installed_file in distribution.list_installed_files(): # distlib expects the script files to be located in ../../bin/ # when in fact ../bin seems to be the resulting path after the # installation as such we need to point the bin files to the # expected location to match wheel RECORD files installed_filepath = os.path.normpath(installed_file[0]) bin_prefix = os.path.join('..', '..', 'bin') + os.sep if installed_filepath.startswith(bin_prefix): # account for extra parentdir as explained above installed = os.path.join(destpath, '_', installed_filepath) else: installed = os.path.join(destpath, installed_filepath) source_file = os.path.normpath(installed) if os.path.exists(source_file): destination_file = os.path.relpath(source_file, stagingdir) exe = False if is_exe(source_file) and destination_file.startswith("bin" + os.sep): _file = os.path.basename(destination_file) tools.append(_file) exe = True src_dst_lut[source_file] = [destination_file, exe] else: _log("Source file does not exist: " + source_file + "!") def make_root(variant, path): """Using distlib to iterate over all installed files of the current distribution to copy files to the target directory of the rez package variant """ for source_file, data in src_dst_lut.items(): destination_file, exe = data destination_file = os.path.normpath( os.path.join(path, destination_file)) if not os.path.exists(os.path.dirname(destination_file)): os.makedirs(os.path.dirname(destination_file)) shutil.copyfile(source_file, destination_file) if exe: shutil.copystat(source_file, destination_file) # create the rez package name = pip_to_rez_package_name(distribution.name) version = pip_to_rez_version(distribution.version) requires = rez_requires["requires"] variant_requires = rez_requires["variant_requires"] metadata = rez_requires["metadata"] with make_package(name, packages_path, make_root=make_root) as pkg: # basics (version etc) pkg.version = version if distribution.metadata.summary: pkg.description = distribution.metadata.summary # requirements and variants if requires: pkg.requires = requires if variant_requires: pkg.variants = [variant_requires] # commands commands = [] commands.append("env.PYTHONPATH.append('{root}/python')") if tools: pkg.tools = tools commands.append("env.PATH.append('{root}/bin')") pkg.commands = '\n'.join(commands) # Make the package use hashed variants. This is required because we # can't control what ends up in its variants, and that can easily # include problematic chars (>, +, ! etc). # TODO: https://github.com/nerdvegas/rez/issues/672 # pkg.hashed_variants = True # add some custom attributes to retain pip-related info pkg.pip_name = distribution.name_and_version pkg.from_pip = True pkg.is_pure_python = metadata["is_pure_python"] installed_variants.extend(pkg.installed_variants or []) skipped_variants.extend(pkg.skipped_variants or []) # cleanup shutil.rmtree(tmpdir) return installed_variants, skipped_variants