def setUpClass(cls):
TempdirMixin.setUpClass()
cls.settings = dict()
cls.tempdir = tempfile.mkdtemp()
python = which("python")
assert python, "No Python found"
result = subprocess.check_output(
[python, "--version"],
universal_newlines=True,
stderr=subprocess.STDOUT,
)
_, version = result.rstrip().split(" ", 1)
version = version.split()[-1]
version = int(version[0])
with make_package("python", cls.tempdir) as maker:
PATH = os.path.dirname(python).replace("\\", "/")
maker.version = str(version)
maker.commands = "\n".join(["env.PATH.prepend(r'%s')" % PATH])
cls.context = ResolvedContext(["python"], package_paths=[cls.tempdir])
cls.python_version = version
def bind(path, version_range=None, opts=None, parser=None):
rez_version = Version(rez.__version__)
check_version(rez_version, version_range)
rez_major_version = rez_version.trim(1)
rez_major_minor_version = rez_version.trim(2)
next_major = int(str(rez_major_version)) + 1
rez_req_str = "rez-%s+<%d" % (str(rez_major_minor_version), next_major)
gui_lib = getattr(opts, "gui_lib", "")
def make_root(variant, root):
# copy source
rez_path = rez.__path__[0]
site_path = os.path.dirname(rez_path)
rezgui_path = os.path.join(site_path, "rezgui")
shutil.copytree(rezgui_path, os.path.join(root, "rezgui"))
# create rez-gui executable
binpath = make_dirs(root, "bin")
filepath = os.path.join(binpath, "rez-gui")
create_executable_script(filepath, rez_gui_source)
# create package
with make_package("rezgui", path, make_root=make_root) as pkg:
pkg.version = rez_version
pkg.variants = [system.variant]
pkg.commands = commands
pkg.tools = ["rez-gui"]
pkg.requires = [rez_req_str]
if gui_lib:
pkg.requires.append(gui_lib)
return "rezgui", rez_version
def bind(path, version_range=None, opts=None, parser=None):
exe_path = getattr(opts, 'exe', None)
# gcc
gcc_path = find_exe('gcc', filepath=exe_path)
gcc_version = extract_version(gcc_path,
['-dumpfullversion', '-dumpversion'])
log("binding gcc: %s" % gcc_path)
# g++
gpp_path = find_exe('g++', filepath=exe_path)
gpp_version = extract_version(gpp_path,
['-dumpfullversion', '-dumpversion'])
log("binding g++: %s" % gpp_path)
if gcc_version != gpp_version:
raise RezBindError("gcc version different than g++ can not continue")
# create directories and symlink gcc and g++
def make_root(variant, root):
bin_path = make_dirs(root, 'bin')
gcc_link_path = os.path.join(bin_path, 'gcc')
platform_.symlink(gcc_path, gcc_link_path)
gpp_link_path = os.path.join(bin_path, 'g++')
platform_.symlink(gpp_path, gpp_link_path)
with make_package('gcc', path, make_root=make_root) as pkg:
pkg.version = gcc_version
pkg.tools = ['gcc', 'g++']
pkg.commands = commands
pkg.variants = [system.variant]
return pkg.installed_variants
def make_package(definition, build_path):
'''Install our predefined Rez package.py file to a separate build folder.
Args:
definition (module): The package.py to make a copy to into `build_path`.
build_path (str): The absolute path to install `definition` to.
Returns:
`rez.package_maker__.PackageMaker`: The created package.
'''
with package_maker.make_package(definition.name,
build_path,
skip_existing=True) as pkg:
mirror('authors', definition, pkg, default=[getpass.getuser()])
mirror('commands', definition, pkg)
mirror('description', definition, pkg)
mirror('help', definition, pkg, default='')
mirror('name', definition, pkg)
mirror('requires', definition, pkg, default=[])
mirror('timestamp', definition, pkg)
mirror('tools', definition, pkg)
# mirror('uuid', definition, pkg, default=str(uuid.uuid4()))
pkg.version = rez_version.Version(definition.version)
return pkg
def bind(path, version_range=None, opts=None, parser=None):
# find executable, determine version
if opts and opts.exe:
exepath = find_exe("python", opts.exe)
code = "import sys; print '.'.join(str(x) for x in sys.version_info)"
version = extract_version(exepath, ["-c", code])
else:
exepath = sys.executable
strver = '.'.join(str(x) for x in sys.version_info[:3])
version = Version(strver)
check_version(version, version_range)
def make_root(variant, root):
binpath = make_dirs(root, "bin")
link = os.path.join(binpath, "python")
platform_.symlink(exepath, link)
with make_package("python", path, make_root=make_root) as pkg:
pkg.version = version
pkg.tools = ["python"]
pkg.commands = commands
pkg.variants = [system.variant]
return "python", version
def bind(path, version_range=None, opts=None, parser=None):
# find executable, determine version
exepath = find_exe("python", opts.exe)
code = "import sys;print('.'.join(str(x) for x in sys.version_info))"
version = extract_version(exepath, ["-c", code])
check_version(version, version_range)
log("binding python: %s" % exepath)
def make_root(variant, root):
bindir = make_dirs(root, "bin")
if os.name == "nt":
fname = os.path.join(bindir, "python.bat")
with open(fname, "w") as f:
f.write(bat.format(python=exepath))
else:
fname = os.path.join(bindir, "python")
with open(fname, "w") as f:
f.write(sh.format(python=exepath))
# Make executable
st = os.stat(fname)
os.chmod(fname, st.st_mode | stat.S_IEXEC)
with make_package("python", path, make_root=make_root) as pkg:
pkg.version = version
pkg.tools = ["python"]
pkg.commands = commands
pkg.variants = [system.variant]
pkg.exe = exepath
return pkg.installed_variants
def bind(path, version_range=None, opts=None, parser=None):
rez_version = Version(rez.__version__)
check_version(rez_version, version_range)
rez_major_version = rez_version.trim(1)
rez_major_minor_version = rez_version.trim(2)
next_major = int(str(rez_major_version)) + 1
rez_req_str = "rez-%s+<%d" % (str(rez_major_minor_version), next_major)
gui_lib = getattr(opts, "gui_lib", "")
def make_root(variant, root):
# copy source
rez_path = rez.__path__[0]
site_path = os.path.dirname(rez_path)
rezgui_path = os.path.join(site_path, "rezgui")
shutil.copytree(rezgui_path, os.path.join(root, "rezgui"))
# create rez-gui executable
binpath = make_dirs(root, "bin")
filepath = os.path.join(binpath, "rez-gui")
create_executable_script(filepath, rez_gui_source)
# create package
with make_package("rezgui", path, make_root=make_root) as pkg:
pkg.version = rez_version
pkg.variants = [system.variant]
pkg.commands = commands
pkg.tools = ["rez-gui"]
pkg.requires = [rez_req_str]
if gui_lib:
pkg.requires.append(gui_lib)
return "rezgui", rez_version
def bind(path, version_range=None, opts=None, parser=None):
# find executable, determine version
if opts and opts.exe:
exepath = find_exe("python", opts.exe)
code = "import sys; print '.'.join(str(x) for x in sys.version_info)"
version = extract_version(exepath, ["-c", code])
else:
exepath = sys.executable
strver = '.'.join(str(x) for x in sys.version_info[:3])
version = Version(strver)
check_version(version, version_range)
def make_root(variant, root):
binpath = make_dirs(root, "bin")
link = os.path.join(binpath, "python")
platform_.symlink(exepath, link)
with make_package("python", path, make_root=make_root) as pkg:
pkg.version = version
pkg.tools = ["python"]
pkg.commands = commands
pkg.variants = [system.variant]
return "python", version
def bind(path, version_range=None, opts=None, parser=None):
version = Version(system.platform)
check_version(version, version_range)
with make_package("platform", path) as pkg:
pkg.version = version
return "platform", version
def bind(path, version_range=None, opts=None, parser=None):
version = Version(system.arch)
check_version(version, version_range)
with make_package("arch", path) as pkg:
pkg.version = version
return ("arch", version)
def bind(path, version_range=None, opts=None, parser=None):
version = Version(system.arch)
check_version(version, version_range)
with make_package("arch", path) as pkg:
pkg.version = version
return ("arch", version)
def bind(path, version_range=None, opts=None, parser=None):
version = Version(system.os)
check_version(version, version_range)
with make_package("os", path) as pkg:
pkg.version = version
pkg.requires = ["platform-%s" % system.platform, "arch-%s" % system.arch]
return pkg.installed_variants
def bind(path, version_range=None, opts=None, parser=None):
versions = ["AMD64", "AMD86"]
for version in versions:
version = Version(version)
check_version(version, version_range)
with make_package("target", path) as pkg:
pkg.version = version
return ("target", versions)
def bind(path, version_range=None, opts=None, parser=None):
version = Version(system.os)
check_version(version, version_range)
with make_package("os", path) as pkg:
pkg.version = version
pkg.requires = [
"platform-%s" % system.platform,
"arch-%s" % system.arch
]
return "os", version
def bind(path, version_range=None, opts=None, parser=None):
# find executable, determine version
exepath = find_exe("python", opts.exe)
code = "import sys; print '.'.join(str(x) for x in sys.version_info)"
version = extract_version(exepath, ["-c", code])
check_version(version, version_range)
log("binding python: %s" % exepath)
# find builtin modules
builtin_paths = {}
entries = [("lib", "os"),
("extra", "setuptools")]
for dirname, module_name in entries:
success, out, err = run_python_command([
"import %s" % module_name,
"print %s.__file__" % module_name])
if success:
pypath = os.path.dirname(out)
if os.path.basename(pypath) == module_name:
pypath = os.path.dirname(pypath)
if pypath not in builtin_paths.values():
builtin_paths[dirname] = pypath
# make the package
#
def make_root(variant, root):
binpath = make_dirs(root, "bin")
link = os.path.join(binpath, "python")
platform_.symlink(exepath, link)
if builtin_paths:
pypath = make_dirs(root, "python")
for dirname, srcpath in builtin_paths.iteritems():
destpath = os.path.join(pypath, dirname)
log("Copying builtins from %s to %s..." % (srcpath, destpath))
shutil.copytree(srcpath, destpath)
with make_package("python", path, make_root=make_root) as pkg:
pkg.version = version
pkg.tools = ["python"]
pkg.commands = commands
pkg.variants = [system.variant]
if builtin_paths:
pkg.post_commands = post_commands
return pkg.installed_variants
def bind(path, version_range=None, opts=None, parser=None):
version = Version("1.0")
check_version(version, version_range)
def make_root(variant, root):
binpath = make_dirs(root, "bin")
filepath = os.path.join(binpath, "hello_world")
create_executable_script(filepath, hello_world_source)
with make_package("hello_world", path, make_root=make_root) as pkg:
pkg.version = version
pkg.tools = ["hello_world"]
pkg.commands = commands
return pkg.installed_variants
def bind(path, version_range=None, opts=None, parser=None):
version = Version("1.0")
check_version(version, version_range)
def make_root(variant, root):
binpath = make_dirs(root, "bin")
filepath = os.path.join(binpath, "hello_world")
create_executable_script(filepath, hello_world_source)
with make_package("hello_world", path, make_root=make_root) as pkg:
pkg.version = version
pkg.tools = ["hello_world"]
pkg.commands = commands
return pkg.installed_variants
def bind(path, version_range=None, opts=None, parser=None):
exepath = find_exe("cmake", getattr(opts, "exe", None))
version = extract_version(exepath, "--version")
check_version(version, version_range)
def make_root(variant, root):
binpath = make_dirs(root, "bin")
link = os.path.join(binpath, "cmake")
platform_.symlink(exepath, link)
with make_package("cmake", path, make_root=make_root) as pkg:
pkg.version = version
pkg.tools = ["cmake"]
pkg.commands = commands
pkg.variants = [system.variant]
return "cmake", version
def bind(path, version_range=None, opts=None, parser=None):
exepath = find_exe("cmake", getattr(opts, "exe", None))
version = extract_version(exepath, "--version")
check_version(version, version_range)
def make_root(variant, root):
binpath = make_dirs(root, "bin")
link = os.path.join(binpath, "cmake")
platform_.symlink(exepath, link)
with make_package("cmake", path, make_root=make_root) as pkg:
pkg.version = version
pkg.tools = ["cmake"]
pkg.commands = commands
pkg.variants = [system.variant]
return "cmake", version
def bind(path, version_range=None, opts=None, parser=None):
is_win64 = False
if os.environ.get('PROCESSOR_ARCHITECTURE','x86') != 'x86':
is_win64 = True
if os.environ.get('PROCESSOR_ARCHITEW6432'):
is_win64 = True
if os.environ.get('ProgramW6432'):
is_win64 = True
msvcs = enumerate_msvc(is_win64)
if not msvcs:
version = getattr(opts, "version", None)
if not version:
_msvc_root = getattr(opts, "root", None)
if not _msvc_root:
print "can't find msvc root, use --root and pass the directory to find vcvarsall.bat"
exit(1)
else:
vcvarsall = os.path.join(_msvc_root, "vcvarsall.bat")
if not os.path.isfile(vcvarsall):
print "can't find:", vcvarsall
exit(1)
version = os.path.split(_msvc_root)[-2].split()[-1]
msvcs[version] = _msvc_root
for variant in ['target-AMD64', 'target-AMD86']:
variants = system.variant+[variant]
for version, msvc_root in msvcs.iteritems():
check_version(version, version_range)
def make_root(variant, root):
binpath = make_dirs(root, "bin")
bat = os.path.join(binpath, "setupmsvc.bat")
with open(bat, "w") as f:
f.write(setupmsvc.format(msvc_root=msvc_root))
with make_package("msvc", path, make_root=make_root) as pkg:
pkg.version = version
pkg.tools = ["setupmsvc"]
pkg.commands = commands
pkg.variants = [variants]
return "msvc", msvcs.keys()
def bind(path, version_range=None, opts=None, parser=None):
version = rez.__version__
check_version(version, version_range)
def make_root(variant, root):
# copy source
rez_path = rez.__path__[0]
site_path = os.path.dirname(rez_path)
rezplugins_path = os.path.join(site_path, "rezplugins")
shutil.copytree(rez_path, os.path.join(root, "rez"))
shutil.copytree(rezplugins_path, os.path.join(root, "rezplugins"))
with make_package("rez", path, make_root=make_root) as pkg:
pkg.version = version
pkg.commands = commands
pkg.requires = ["python-2.6+<3"]
pkg.variants = [system.variant]
return "rez", version
def bind(path, version_range=None, opts=None, parser=None):
# find executable, determine version
exepath = find_exe("python", opts.exe)
code = "import sys;print('.'.join(str(x) for x in sys.version_info))"
version = extract_version(exepath, ["-c", code])
check_version(version, version_range)
log("binding python: %s" % exepath)
def make_root(variant, root):
make_dirs(root, "python")
with make_package("python", path, make_root=make_root) as pkg:
pkg.version = version
pkg.tools = ["python"]
pkg.commands = commands
pkg.variants = [system.variant]
pkg.exe = exepath
return pkg.installed_variants
def bind(path, version_range=None, opts=None, parser=None):
# find executable, determine version
if opts and opts.root:
bin_path = opts.root
else:
possible_paths = [r'C:\Program Files (x86)\QuickTime']
for root_path in possible_paths:
if os.path.exists(root_path):
bin_path = root_path
break
else:
raise EnvironmentError(
'Unable to find Quicktime on this system in the paths; {}'.format(
', '.join(possible_paths)
)
)
if platform_.name == 'windows':
from win32api import GetFileVersionInfo, LOWORD, HIWORD
try:
info = GetFileVersionInfo(os.path.join(bin_path, 'QuickTimePlayer.exe'), "\\")
ms = info['FileVersionMS']
ls = info['FileVersionLS']
version = Version('{}.{}.{}.{}'.format(HIWORD(ms), LOWORD(ms), HIWORD(ls), LOWORD(ls)))
except:
raise EnvironmentError('Unknown version')
else:
raise EnvironmentError('Only binds on windows at the moment')
check_version(version, version_range)
with make_package("quicktime", path) as pkg:
pkg.version = version
if platform_.name == 'windows':
pkg.tools = ["QuickTimePlayer.exe"]
pkg.commands = win_commands(bin_path)
else:
raise EnvironmentError('Only binds on windows at the moment')
pkg.variants = [system.variant]
return "quicktime", version
def bind(path, version_range=None, opts=None, parser=None):
if opts and opts.pypkg:
py_package_name = opts.pypkg
if opts.name:
package_name = opts.name
else:
package_name = py_package_name.replace('-', '_')
else:
raise ValueError('A package name needs to be specified')
# connect to PyPi and get the released package versions
pypi_client = xmlrpclib.ServerProxy('https://pypi.python.org/pypi')
package_versions = sorted(pypi_client.package_releases(py_package_name), key=lambda v: Version(v))
if len(package_versions) < 1:
raise ValueError('Invalid package name: {}'.format(package_name))
# get the version
if opts.version:
version = next((v for v in package_versions if v == opts.version), None)
if version is None:
raise ValueError('Invalid version: {}, valid versions: {}'.format(opts.version, package_versions))
else:
version = package_versions[-1]
check_version(Version(version), version_range)
# get the source distribution, this could be better and try to install wheels as well
release_info = next(
(r for r in pypi_client.release_urls(py_package_name, version) if r['packagetype'] == 'sdist'),
None
)
if release_info is None:
raise ValueError(
'No source distribution (we don\'t support wheels at the moment) for {} {}'.format(py_package_name, version)
)
# clear the temp dir if it exists
tmp_dir = os.path.join(platform_.tmpdir, package_name)
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
os.mkdir(tmp_dir)
# download file to tmp dir
handle = urllib2.urlopen(release_info['url'])
src_archive = os.path.join(tmp_dir, release_info['filename'])
with open(src_archive, 'wb') as src_archive_fp:
src_archive_fp.write(handle.read())
# decompress the file
package_dir = None
if release_info['filename'].endswith('.zip'):
zfile = zipfile.ZipFile(src_archive, 'r')
zfile.extractall(tmp_dir)
package_dir = src_archive.replace('.zip', '')
elif release_info['filename'].endswith('.tar.gz'):
tfile = tarfile.open(src_archive, 'r:gz')
tfile.extractall(tmp_dir)
package_dir = src_archive.replace('.tar.gz', '')
elif release_info['filename'].endswith('.tgz'):
tfile = tarfile.open(src_archive, 'r:gz')
tfile.extractall(tmp_dir)
package_dir = src_archive.replace('.tgz', '')
else:
raise ValueError('Package file downloaded cannot be decompressed: {}'.format(src_archive))
setup_file = os.path.join(package_dir, 'setup.py')
if not os.path.exists(setup_file):
raise RuntimeError(
'Package does not contain a "setup.py" file in the root of the archive: {}'.format(py_package_name)
)
def setup_dummy(*args, **kwargs):
global SETUPTOOLS_ARGS
SETUPTOOLS_ARGS = {}
if 'scripts' in kwargs:
SETUPTOOLS_ARGS['scripts'] = kwargs['scripts']
if 'description' in kwargs:
SETUPTOOLS_ARGS['description'] = kwargs['description']
if 'long_description' in kwargs:
SETUPTOOLS_ARGS['long_description'] = kwargs['long_description']
if 'author' in kwargs:
SETUPTOOLS_ARGS['author'] = kwargs['author']
if 'author_email' in kwargs:
SETUPTOOLS_ARGS['author_email'] = kwargs['author_email']
if 'install_requires' in kwargs:
SETUPTOOLS_ARGS['install_requires'] = kwargs['install_requires']
if 'setup_requires' in kwargs:
SETUPTOOLS_ARGS['setup_requires'] = kwargs['setup_requires']
if 'version' in kwargs:
SETUPTOOLS_ARGS['version'] = kwargs['version']
else:
raise RuntimeError('A package must have a version.')
# monkey patch the setup function so we can steal the arguments passed to it
setuptools.setup = setup_dummy
distutils.core.setup = setup_dummy
cwd = os.getcwd()
os.chdir(package_dir)
sys.path.extend([package_dir])
old_args = sys.argv[:]
try:
_ = imp.load_source('{}_setup'.format(package_name), setup_file)
except Exception as e:
import traceback
traceback.print_exc()
raise e
sys.argv = old_args[:]
os.chdir(cwd)
sys.path.remove(package_dir)
def parse_setup_requirements(req_str):
'''
e.g. "arrow >= 0.4.4, < 1" -> 'arrow-0.4.4+<1
:param req_str: requirement in the setuptools format
:return: requirement in the rez format
'''
if not any((op in req_str for op in ['<', '>', '='])):
# even simpler "arrow" req
return req_str.replace('-', '_').strip()
if '==' in req_str:
# just a simple "arrow == 0.4.4" req
req_pgk_name, req_version = req_str.split('==')
return '{}-{}'.format(req_pgk_name.replace('-', '_').strip(), req_version.strip())
complex_req_match = re.match(r'^(?P<pkg_name>[a-zA-Z0-9\.-_]+) *(?P<reqs>[\.,<>=0-9) ]+)$', req_str.strip())
if not complex_req_match:
raise RuntimeError('Unable to parse requirement; {}'.format(req_str))
req_pgk_name = complex_req_match.group('pkg_name')
req_pgk_req_str = complex_req_match.group('reqs')
requirements = map(lambda s: s.strip(), req_pgk_req_str.split(','))
versions_rewrite = []
for sub_requirement_str in requirements:
sub_requirement_match = re.match(r'([<>=]+?) ?([0-9a-zA-Z\.]+)', sub_requirement_str)
operation = sub_requirement_match.group(1).strip()
req_version = sub_requirement_match.group(2).strip()
if operation == '>=':
versions_rewrite.insert(0, '{}+'.format(req_version))
elif operation == '<':
versions_rewrite.append('<{}'.format(req_version))
else:
raise RuntimeError('Unable to translate operation: "{}" in requirement: {}'.format(operation, req_str))
return '{}-{}'.format(req_pgk_name.replace('-', '_'), ''.join(versions_rewrite))
def make_root(variant, root):
binpath = make_dirs(root, "bin")
pythonpath = make_dirs(root, "python")
headerpath = make_dirs(root, "include")
install_cmd = ' '.join([
'pip', 'install',
'--no-deps', '--ignore-installed', '--verbose', '--verbose', '--verbose',
'--global-option="--verbose"',
'--install-option="--install-headers={}"'.format(headerpath),
'--install-option="--install-purelib={}"'.format(pythonpath),
'--install-option="--install-platlib={}"'.format(pythonpath),
'--install-option="--install-scripts={}"'.format(binpath),
py_package_name
])
try:
subprocess.check_output(
install_cmd,
cwd=package_dir
)
except subprocess.CalledProcessError as _:
raise RuntimeError('Bind failed to install python package with command: {}'.format(install_cmd))
with make_package(package_name, path, make_root=make_root) as pkg:
pkg.version = version
if 'description' in SETUPTOOLS_ARGS:
if len(SETUPTOOLS_ARGS['description']) < 150:
pkg.nice_name = SETUPTOOLS_ARGS['description']
else:
pkg.nice_name = SETUPTOOLS_ARGS['description'][:147] + '...'
if 'long_description' in SETUPTOOLS_ARGS:
pkg.description = SETUPTOOLS_ARGS['long_description']
author_parts = []
if 'author' in SETUPTOOLS_ARGS:
author_parts.append(SETUPTOOLS_ARGS['author'])
if 'author_email' in SETUPTOOLS_ARGS:
author_parts.append(SETUPTOOLS_ARGS['author_email'])
if len(author_parts) > 0:
pkg.authors = [' '.join(author_parts)]
else:
pkg.authors = []
if 'install_requires' in SETUPTOOLS_ARGS:
pkg.requires = [parse_setup_requirements(req) for req in SETUPTOOLS_ARGS['install_requires']]
else:
pkg.requires = []
if 'setup_requires' in SETUPTOOLS_ARGS:
pkg.build_requires = [parse_setup_requirements(req) for req in SETUPTOOLS_ARGS['setup_requires']]
if 'scripts' in SETUPTOOLS_ARGS:
pkg.tools = [os.path.split(script)[-1] for script in SETUPTOOLS_ARGS['scripts']]
pkg.commands = commands
return package_name, Version(version)
def pip_install_package(source_name,
pip_version=None,
python_version=None,
mode=InstallMode.min_deps,
release=False):
"""Install a pip-compatible python package as a rez package.
Args:
source_name (str): Name of package or archive/url containing the pip
package source. This is the same as the arg you would pass to
the 'pip install' command.
pip_version (str or `Version`): Version of pip to use to perform the
install, uses latest if None.
python_version (str or `Version`): Python version to use to perform the
install, and subsequently have the resulting rez package depend on.
mode (`InstallMode`): Installation mode, determines how dependencies are
managed.
release (bool): If True, install as a released package; otherwise, it
will be installed as a local package.
Returns:
2-tuple:
List of `Variant`: Installed variants;
List of `Variant`: Skipped variants (already installed).
"""
installed_variants = []
skipped_variants = []
pip_exe, context = find_pip(pip_version, python_version)
# TODO: should check if packages_path is writable before continuing with pip
#
packages_path = (config.release_packages_path
if release else config.local_packages_path)
tmpdir = mkdtemp(suffix="-rez", prefix="pip-")
stagingdir = os.path.join(tmpdir, "rez_staging")
stagingsep = "".join([os.path.sep, "rez_staging", os.path.sep])
destpath = os.path.join(stagingdir, "python")
# TODO use binpath once https://github.com/pypa/pip/pull/3934 is approved
binpath = os.path.join(stagingdir, "bin")
if context and config.debug("package_release"):
buf = StringIO()
print("\n\npackage download environment:", file=buf)
context.print_info(buf)
_log(buf.getvalue())
# Build pip commandline
cmd = [pip_exe, "install", "--use-pep517", "--target=%s" % destpath]
if mode == InstallMode.no_deps:
cmd.append("--no-deps")
cmd.append(source_name)
_cmd(context=context, command=cmd)
_system = System()
def pure_python_package(installed_dist):
true_table = {"true": True, "false": False}
packages = pkg_resources.find_distributions(destpath)
dist = next(
(package
for package in packages if package.key == installed_dist.key),
None)
wheel_data = dist.get_metadata('WHEEL')
# see https://www.python.org/dev/peps/pep-0566/#json-compatible-metadata
wheel_data = Parser().parsestr(wheel_data)
# see https://www.python.org/dev/peps/pep-0427/#what-s-the-deal-with-purelib-vs-platlib
return true_table[wheel_data["Root-Is-Purelib"]]
# Collect resulting python packages using distlib
distribution_path = DistributionPath([destpath])
distributions = [d for d in distribution_path.get_distributions()]
# moving bin folder to expected relative location as per wheel RECORD files
staged_binpath = os.path.join(destpath, "bin")
if os.path.isdir(staged_binpath):
shutil.move(os.path.join(destpath, "bin"), binpath)
for distribution in distribution_path.get_distributions():
requirements = []
if distribution.metadata.run_requires:
# Handle requirements. Currently handles conditional environment based
# requirements and normal requirements
# TODO: Handle optional requirements?
for requirement in distribution.metadata.run_requires:
if "environment" in requirement:
if interpret(requirement["environment"]):
requirements.extend(
_get_dependencies(requirement, distributions))
elif "extra" in requirement:
# Currently ignoring optional requirements
pass
else:
requirements.extend(
_get_dependencies(requirement, distributions))
tools = []
src_dst_lut = {}
for installed_file in distribution.list_installed_files():
# distlib expects the script files to be located in ../../bin/
# when in fact ../bin seems to be the resulting path after the
# installation as such we need to point the bin files to the
# expected location to match wheel RECORD files
installed_filepath = os.path.normpath(installed_file[0])
bin_prefix = os.path.join('..', '..', 'bin') + os.sep
if installed_filepath.startswith(bin_prefix):
# account for extra parentdir as explained above
installed = os.path.join(destpath, '_', installed_filepath)
else:
installed = os.path.join(destpath, installed_filepath)
source_file = os.path.normpath(installed)
if os.path.exists(source_file):
destination_file = os.path.relpath(source_file, stagingdir)
exe = False
if is_exe(source_file) and destination_file.startswith("bin" +
os.sep):
_file = os.path.basename(destination_file)
tools.append(_file)
exe = True
data = [destination_file, exe]
src_dst_lut[source_file] = data
else:
_log("Source file does not exist: " + source_file + "!")
def make_root(variant, path):
"""Using distlib to iterate over all installed files of the current
distribution to copy files to the target directory of the rez package
variant
"""
for source_file, data in src_dst_lut.items():
destination_file, exe = data
destination_file = os.path.normpath(
os.path.join(path, destination_file))
if not os.path.exists(os.path.dirname(destination_file)):
os.makedirs(os.path.dirname(destination_file))
shutil.copyfile(source_file, destination_file)
if exe:
shutil.copystat(source_file, destination_file)
# determine variant requirements
variant_reqs = []
pure = pure_python_package(distribution)
if not pure:
variant_reqs.append("platform-%s" % _system.platform)
variant_reqs.append("arch-%s" % _system.arch)
# Add the python version requirement. Note that we specify python to
# minor version because of environment markers - these often mean that
# you cannot use a loose python requirement (ie major version only)
# because then the package requirements would not be correct for all
# versions of python within that major version.
#
# This is not perfect. It means that often we will overspecify the required
# python version; and theoretically there could be markers that specify
# python down to the patch version. However, accurately varianting on
# python based on markers may be overly complicated, and may also
# result in odd varianting cases.
#
# https://www.python.org/dev/peps/pep-0508/#environment-markers
#
if context is None:
# since we had to use system pip, we have to assume system python version
py_ver = '.'.join(map(str, sys.version_info[:2]))
else:
python_variant = context.get_resolved_package("python")
py_ver = python_variant.version.trim(2)
variant_reqs.append("python-%s" % py_ver)
name = pip_to_rez_package_name(distribution)
with make_package(name, packages_path, make_root=make_root) as pkg:
pkg.version = pip_to_rez_version(distribution.version)
if distribution.metadata.summary:
pkg.description = distribution.metadata.summary
pkg.variants = [variant_reqs]
if requirements:
pkg.requires = requirements
commands = []
commands.append("env.PYTHONPATH.append('{root}/python')")
if tools:
pkg.tools = tools
commands.append("env.PATH.append('{root}/bin')")
pkg.commands = '\n'.join(commands)
installed_variants.extend(pkg.installed_variants or [])
skipped_variants.extend(pkg.skipped_variants or [])
# cleanup
shutil.rmtree(tmpdir)
return installed_variants, skipped_variants
def bind(path, version_range=None, opts=None, parser=None):
if platform_.name != 'windows':
raise EnvironmentError('Only binds on windows at the moment')
# find executable, determine version
if opts and opts.root:
bin_path = opts.root
else:
installed_versions = []
autodesk_root = r'C:\Program Files\Autodesk'
for app_folder in os.listdir(autodesk_root):
if app_folder.startswith('3ds Max'):
installed_versions.append(
(app_folder,
Version(app_folder.replace('3ds Max', '').strip())))
if len(installed_versions) < 1:
raise EnvironmentError(
'Unable to find any installed version of 3ds Max under "{}"'.
format(autodesk_root))
app_folder, version = sorted(installed_versions,
key=lambda v: v[1])[-1]
bin_path = os.path.join(autodesk_root, app_folder)
from win32api import GetFileVersionInfo, LOWORD, HIWORD
try:
info = GetFileVersionInfo(os.path.join(bin_path, '3dsmax.exe'), "\\")
ms = info['FileVersionMS']
ls = info['FileVersionLS']
version = Version('{}.{}.{}.{}'.format(HIWORD(ms), LOWORD(ms),
HIWORD(ls), LOWORD(ls)))
except:
raise EnvironmentError('Unknown version')
check_version(version, version_range)
def make_root(resources_path, variant, path):
import shutil
shutil.copy(os.path.join(resources_path, '3dsmax_icon.png'),
os.path.join(path, '3dsmax_icon.png'))
make_root_partial = functools.partial(
make_root,
resource_filename(Requirement.parse('rez'), "rez/bind/resources"))
with make_package("3dsmax", path, make_root=make_root_partial) as pkg:
pkg.version = version
pkg.tools = ["3dsmax"]
pkg.description = '3D DCC application'
pkg.authors = ['Autodesk']
pkg.requires = ['roaming_user']
pkg.nice_name = '3DS Max'
pkg.tools_info = {
'3dsmax': {
'command': [
'start', '3DSMax', '/D', '%ADSK_3DSMAX_x64_2015%', '/wait',
'/B', '3dsmax.exe'
],
'nice_name':
'3ds Max',
'priority':
89,
'icon':
'{root}/3dsmax_icon.png',
'launch_in_prompt':
False
}
}
pkg.commands = win_commands(bin_path)
pkg.variants = [system.variant]
return "3dsmax", version
def bind(path, version_range=None, opts=None, parser=None):
if platform_.name != 'windows':
raise EnvironmentError('Only binds on windows at the moment')
# find executable, determine version
if opts and opts.root:
bin_path = opts.root
try:
info = GetFileVersionInfo(os.path.join(bin_path, 'Photoshop.exe'),
"\\")
ms = info['FileVersionMS']
ls = info['FileVersionLS']
version = Version('{}.{}.{}.{}'.format(HIWORD(ms), LOWORD(ms),
HIWORD(ls), LOWORD(ls)))
except:
raise EnvironmentError('Unknown version')
else:
installed_versions = []
adobe_root = r'C:\Program Files\Adobe'
app_folder_prefix = 'Adobe Photoshop'
for app_folder in os.listdir(adobe_root):
if app_folder.startswith(app_folder_prefix):
app_exe = os.path.join(adobe_root, app_folder, 'Photoshop.exe')
if os.path.exists(app_exe):
try:
info = GetFileVersionInfo(app_exe, "\\")
ms = info['FileVersionMS']
ls = info['FileVersionLS']
version = Version('{}.{}.{}.{}'.format(
HIWORD(ms), LOWORD(ms), HIWORD(ls), LOWORD(ls)))
except:
raise EnvironmentError('Unknown version')
installed_versions.append((app_folder, version))
if len(installed_versions) < 1:
raise EnvironmentError(
'Unable to find any installed version of 3ds Max under "{}"'.
format(adobe_root))
app_folder, version = sorted(installed_versions,
key=lambda v: v[1])[-1]
bin_path = os.path.join(adobe_root, app_folder)
check_version(version, version_range)
def make_root(resources_path, variant, path):
import shutil
shutil.copy(os.path.join(resources_path, 'Photoshop_icon.png'),
os.path.join(path, 'Photoshop_icon.png'))
make_root_partial = functools.partial(
make_root,
resource_filename(Requirement.parse('rez'), "rez/bind/resources"))
with make_package("photoshop", path, make_root=make_root_partial) as pkg:
pkg.version = version
pkg.tools = ["photoshop"]
pkg.description = 'Painting Application'
pkg.authors = ['Adobe']
pkg.requires = []
pkg.nice_name = 'Photoshop'
pkg.tools_info = {
'photoshop': {
'command': [
'start', 'Photoshop', '/D', '%REZ_PHOTOSHOP_ROOT%',
'/wait', '/B', 'Photoshop.exe'
],
'nice_name':
'Photoshop',
'priority':
89,
'icon':
'{root}/Photoshop_icon.png',
'launch_in_prompt':
False
}
}
pkg.commands = win_commands(bin_path)
pkg.variants = [system.variant]
return "photoshop", version
def pip_install_package(source_name, python_version=None,
release=False, no_deps=False,
prefix=None, auto_variants=True,
variants=None):
"""Install a pip-compatible python package as a rez package.
Args:
source_name (str): Name of package or archive/url containing the pip
package source. This is the same as the arg you would pass to
the 'pip install' command.
python_version (str or `Version`): Python version to use to perform the
install, and subsequently have the resulting rez package depend on.
prefix (str, optional): Override install location to here,
similar to `rez build --prefix`
no_deps (bool, optional): The `pip --no-deps` argument
auto_variants (bool, optional): Compute variants from the PyPI
classifiers portion of setup()
release (bool): If True, install as a released package; otherwise, it
will be installed as a local package.
prefix (str, optional): Override release path with this absolute path
Returns:
2-tuple:
List of `Variant`: Installed variants;
List of `Variant`: Skipped variants (already installed).
"""
installed_variants = []
skipped_variants = []
if prefix is not None:
config.release_packages_path = prefix
# TODO: should check if packages_path is writable
# before continuing with pip
#
packages_path = (config.release_packages_path if release
else config.local_packages_path)
tmpdir = mkdtemp(suffix="-rez", prefix="pip-")
stagingdir = os.path.join(tmpdir, "rez_staging")
stagingsep = "".join([os.path.sep, "rez_staging", os.path.sep])
destpath = os.path.join(stagingdir, "python")
python_exe, context = find_python(python_version)
if context and config.debug("package_release"):
buf = StringIO()
print >> buf, "\n\npackage download environment:"
context.print_info(buf)
_log(buf.getvalue())
# Build pip commandline
cmd = [
python_exe, "-m", "pip", "install",
"--target", destpath,
# Only ever consider wheels, anything else is ancient
"--use-pep517",
# Handle case where the Python distribution used alongside
# pip already has a package installed in its `site-packages/` dir.
"--ignore-installed",
]
if no_deps:
# Delegate the installation of dependencies to the user
# This is important, as each dependency may have different
# requirements of its own, and variants to go with it.
cmd.append("--no-deps")
cmd.append(source_name)
_cmd(context=context, command=cmd)
# Collect resulting python packages using distlib
distribution_path = DistributionPath([destpath])
distributions = [d for d in distribution_path.get_distributions()]
for distribution in distribution_path.get_distributions():
requirements = []
if distribution.metadata.run_requires:
# Handle requirements. Currently handles
# conditional environment based
# requirements and normal requirements
# TODO: Handle optional requirements?
for requirement in distribution.metadata.run_requires:
if "environment" in requirement:
if interpret(requirement["environment"]):
requirements.extend(_get_dependencies(
requirement, distributions))
elif "extra" in requirement:
# Currently ignoring optional requirements
pass
else:
requirements.extend(_get_dependencies(
requirement, distributions))
tools = []
src_dst_lut = {}
files = distribution.list_installed_files()
for installed_file in files:
source_file = os.path.join(destpath, installed_file[0])
source_file = os.path.normpath(source_file)
if os.path.exists(source_file):
destination_file = source_file.split(stagingsep)[1]
exe = False
starts_with_bin = destination_file.startswith(
"%s%s" % ("bin", os.path.sep)
)
if is_exe(source_file) and starts_with_bin:
_, _file = os.path.split(destination_file)
tools.append(_file)
exe = True
data = [destination_file, exe]
src_dst_lut[source_file] = data
else:
_log("Source file does not exist: " + source_file + "!")
def make_root(variant, path):
"""Using distlib to iterate over all installed files of the current
distribution to copy files to the target directory of the rez
package variant
"""
for source_file, data in src_dst_lut.items():
destination_file, exe = data
destination_file = os.path.join(path, destination_file)
destination_file = os.path.normpath(destination_file)
if not os.path.exists(os.path.dirname(destination_file)):
os.makedirs(os.path.dirname(destination_file))
shutil.copyfile(source_file, destination_file)
if exe:
shutil.copystat(source_file, destination_file)
name, _ = parse_name_and_version(distribution.name_and_version)
name = distribution.name[0:len(name)].replace("-", "_")
# determine variant requirements
variants_ = variants or []
if (not variants_) and auto_variants:
variants_.extend(wheel_to_variants(distribution))
if variants_:
print_info("'%s' - Automatically detected variants: %s" % (
name, ", ".join(variants_))
)
with make_package(name, packages_path, make_root=make_root) as pkg:
pkg.version = distribution.version
if distribution.metadata.summary:
pkg.description = distribution.metadata.summary
if variants_:
pkg.variants = [variants_]
if requirements:
pkg.requires = requirements
commands = []
commands.append("env.PYTHONPATH.append('{root}/python')")
if tools:
pkg.tools = tools
commands.append("env.PATH.append('{root}/bin')")
pkg.commands = '\n'.join(commands)
installed_variants.extend(pkg.installed_variants or [])
skipped_variants.extend(pkg.skipped_variants or [])
# cleanup
shutil.rmtree(tmpdir)
return installed_variants, skipped_variants
def bind(name,
path,
import_name=None,
version_range=None,
version=None,
requires=None,
pure_python=None,
tools=None,
extra_module_names=[],
extra_attrs={}):
import_name = import_name or name
if version is None:
version = get_version_in_python(
name,
["import %s" % import_name,
"print %s.__version__" % import_name])
check_version(version, version_range)
py_major_minor = '.'.join(str(x) for x in sys.version_info[:2])
py_req = "python-%s" % py_major_minor
found_tools = {}
if pure_python is None:
raise NotImplementedError # detect
elif pure_python:
variant = [py_req]
else:
variant = system.variant + [py_req]
for tool in (tools or []):
try:
src = find_exe(tool)
found_tools[tool] = src
log("found tool '%s': %s" % (tool, src))
except RezBindError as e:
print_warning(str(e))
def make_root(variant, root):
pypath = make_dirs(root, "python")
copy_module(import_name, pypath)
if found_tools:
binpath = make_dirs(root, "bin")
for tool, src in sorted(found_tools.items()):
dest = os.path.join(binpath, tool)
shutil.copy2(src, dest)
for name_ in extra_module_names:
copy_module(name_, pypath)
with make_package(name, path, make_root=make_root) as pkg:
pkg.version = version
pkg.variants = [variant]
if requires:
pkg.requires = requires
if found_tools:
pkg.tools = list(found_tools)
pkg.commands = commands_with_bin
else:
pkg.commands = commands
for key, value in extra_attrs.iteritems():
pkg[key] = value
return pkg.installed_variants
def bind(path, version_range=None, opts=None, parser=None):
if platform_.name != 'windows':
raise EnvironmentError('Only binds on windows at the moment')
# # find executable, determine version
# if opts and opts.root:
# root_path = opts.root
# try:
# info = GetFileVersionInfo(os.path.join(root_path, 'bin', 'deadlinecommand.exe'), "\\")
# ms = info['FileVersionMS']
# ls = info['FileVersionLS']
# version = Version('{}.{}.{}.{}'.format(HIWORD(ms), LOWORD(ms), HIWORD(ls), LOWORD(ls)))
# except:
# raise EnvironmentError('Unknown version')
# else:
installed_versions = []
thinkbox_root = r'C:\Program Files\Thinkbox'
app_folder_prefix = 'Deadline7' # previous value was 'Deadline' with no number
for app_folder in os.listdir(thinkbox_root):
if app_folder.startswith(app_folder_prefix):
app_exe = os.path.join(thinkbox_root, app_folder, 'bin', 'deadlinecommand.exe')
if os.path.exists(app_exe):
try:
info = GetFileVersionInfo(app_exe, "\\")
ms = info['FileVersionMS']
ls = info['FileVersionLS']
version = Version('{}.{}.{}.{}'.format(HIWORD(ms), LOWORD(ms), HIWORD(ls), LOWORD(ls)))
except:
raise EnvironmentError('Unknown version')
installed_versions.append(
(app_folder, version)
)
if len(installed_versions) < 1:
raise EnvironmentError(
'Unable to find any installed version of Deadline under "{}"'.format(
thinkbox_root
)
)
app_folder, version = sorted(installed_versions, key=lambda v: v[1])[-1]
root_path = os.path.join(thinkbox_root, app_folder, 'bin')
check_version(version, version_range)
def make_root(resources_path, variant, path):
import shutil
shutil.copy(
os.path.join(resources_path, 'deadline_monitor_icon.png'),
os.path.join(path, 'deadline_monitor_icon.png')
)
make_root_partial = functools.partial(make_root, resource_filename(Requirement.parse('rez'), "rez/bind/resources"))
with make_package("deadline", path, make_root=make_root_partial) as pkg:
pkg.version = version
pkg.tools = ["monitor"]
pkg.description = 'Render Manager'
pkg.authors = ['Thinkbox']
pkg.requires = []
pkg.nice_name = 'Deadline'
pkg.tools_info = {
'monitor': {
'command': ['deadlinemonitor.exe'],
'nice_name': 'Monitor',
'priority': 89,
'icon': '{root}/deadline_monitor_icon.png',
'launch_in_prompt': False
}
}
pkg.commands = win_commands(root_path, version.major)
pkg.variants = [system.variant]
return "deadline", version
def bind(name, path, import_name=None, version_range=None, version=None,
requires=None, pure_python=None, tools=None, extra_module_names=[],
extra_attrs={}):
import_name = import_name or name
if version is None:
version = get_version_in_python(
name,
["import %s" % import_name,
"print %s.__version__" % import_name])
check_version(version, version_range)
py_major_minor = '.'.join(str(x) for x in sys.version_info[:2])
py_req = "python-%s" % py_major_minor
found_tools = {}
if pure_python is None:
raise NotImplementedError # detect
elif pure_python:
variant = [py_req]
else:
variant = system.variant + [py_req]
for tool in (tools or []):
try:
src = find_exe(tool)
found_tools[tool] = src
log("found tool '%s': %s" % (tool, src))
except RezBindError as e:
print_warning(str(e))
def make_root(variant, root):
pypath = make_dirs(root, "python")
copy_module(import_name, pypath)
if found_tools:
binpath = make_dirs(root, "bin")
for tool, src in sorted(found_tools.items()):
dest = os.path.join(binpath, tool)
shutil.copy2(src, dest)
for name_ in extra_module_names:
copy_module(name_, pypath)
with make_package(name, path, make_root=make_root) as pkg:
pkg.version = version
pkg.variants = [variant]
if requires:
pkg.requires = requires
if found_tools:
pkg.tools = list(found_tools)
pkg.commands = commands_with_bin
else:
pkg.commands = commands
for key, value in extra_attrs.iteritems():
pkg[key] = value
return pkg.installed_variants
def bind(path, version_range=None, opts=None, parser=None):
if platform_.name != 'windows':
raise EnvironmentError('Only binds on windows at the moment')
# find executable, determine version
if opts and opts.root:
bin_path = opts.root
else:
installed_versions = []
autodesk_root = r'C:\Program Files\Autodesk'
for app_folder in os.listdir(autodesk_root):
if app_folder.startswith('Mudbox'):
installed_versions.append(
(app_folder, Version(app_folder.replace('Mudbox', '').strip()))
)
if len(installed_versions) < 1:
raise EnvironmentError(
'Unable to find any installed version of Mudbox under "{}"'.format(
autodesk_root
)
)
app_folder, version = sorted(installed_versions, key=lambda v: v[1])[-1]
bin_path = os.path.join(autodesk_root, app_folder)
from win32api import GetFileVersionInfo, LOWORD, HIWORD
try:
info = GetFileVersionInfo(os.path.join(bin_path, 'mudbox.exe'), "\\")
ms = info['FileVersionMS']
ls = info['FileVersionLS']
version = Version('{}.{}.{}.{}'.format(HIWORD(ms), LOWORD(ms), HIWORD(ls), LOWORD(ls)))
except:
raise EnvironmentError('Unknown version')
check_version(version, version_range)
def make_root(resources_path, variant, path):
import shutil
shutil.copy(
os.path.join(resources_path, 'mudbox_icon.png'),
os.path.join(path, 'mudbox_icon.png')
)
make_root_partial = functools.partial(make_root, resource_filename(Requirement.parse('rez'), "rez/bind/resources"))
with make_package("mudbox", path, make_root=make_root_partial) as pkg:
pkg.version = version
pkg.tools = ["mudbox"]
pkg.description = 'Scupting application'
pkg.authors = ['Autodesk']
pkg.requires = ['roaming_user']
pkg.nice_name = 'Mudbox'
pkg.tools_info = {
'mudbox': {
'command': ['mudbox'],
'nice_name': 'Mudbox',
'priority': 89,
'icon': '{root}/mudbox_icon.png',
'launch_in_prompt': False
}
}
pkg.commands = win_commands(bin_path)
pkg.variants = [system.variant]
return "mudbox", version
def pip_install_package(source_name, pip_version=None, python_version=None,
mode=InstallMode.min_deps, release=False):
"""Install a pip-compatible python package as a rez package.
Args:
source_name (str): Name of package or archive/url containing the pip
package source. This is the same as the arg you would pass to
the 'pip install' command.
pip_version (str or `Version`): Version of pip to use to perform the
install, uses latest if None.
python_version (str or `Version`): Python version to use to perform the
install, and subsequently have the resulting rez package depend on.
mode (`InstallMode`): Installation mode, determines how dependencies are
managed.
release (bool): If True, install as a released package; otherwise, it
will be installed as a local package.
Returns:
2-tuple:
List of `Variant`: Installed variants;
List of `Variant`: Skipped variants (already installed).
"""
installed_variants = []
skipped_variants = []
pip_exe, context = find_pip(pip_version, python_version)
# TODO: should check if packages_path is writable before continuing with pip
#
packages_path = (config.release_packages_path if release
else config.local_packages_path)
tmpdir = mkdtemp(suffix="-rez", prefix="pip-")
stagingdir = os.path.join(tmpdir, "rez_staging")
stagingsep = "".join([os.path.sep, "rez_staging", os.path.sep])
destpath = os.path.join(stagingdir, "python")
binpath = os.path.join(stagingdir, "bin")
incpath = os.path.join(stagingdir, "include")
datapath = stagingdir
if context and config.debug("package_release"):
buf = StringIO()
print >> buf, "\n\npackage download environment:"
context.print_info(buf)
_log(buf.getvalue())
# Build pip commandline
cmd = [pip_exe, "install",
"--install-option=--install-lib=%s" % destpath,
"--install-option=--install-scripts=%s" % binpath,
"--install-option=--install-headers=%s" % incpath,
"--install-option=--install-data=%s" % datapath]
if mode == InstallMode.no_deps:
cmd.append("--no-deps")
cmd.append(source_name)
_cmd(context=context, command=cmd)
_system = System()
# Collect resulting python packages using distlib
distribution_path = DistributionPath([destpath], include_egg=True)
distributions = [d for d in distribution_path.get_distributions()]
for distribution in distribution_path.get_distributions():
requirements = []
if distribution.metadata.run_requires:
# Handle requirements. Currently handles conditional environment based
# requirements and normal requirements
# TODO: Handle optional requirements?
for requirement in distribution.metadata.run_requires:
if "environment" in requirement:
if interpret(requirement["environment"]):
requirements.extend(_get_dependencies(requirement, distributions))
elif "extra" in requirement:
# Currently ignoring optional requirements
pass
else:
requirements.extend(_get_dependencies(requirement, distributions))
tools = []
src_dst_lut = {}
for installed_file in distribution.list_installed_files(allow_fail=True):
source_file = os.path.normpath(os.path.join(destpath, installed_file[0]))
if os.path.exists(source_file):
destination_file = installed_file[0].split(stagingsep)[1]
exe = False
if is_exe(source_file) and \
destination_file.startswith("%s%s" % ("bin", os.path.sep)):
_, _file = os.path.split(destination_file)
tools.append(_file)
exe = True
data = [destination_file, exe]
src_dst_lut[source_file] = data
else:
_log("Source file does not exist: " + source_file + "!")
def make_root(variant, path):
"""Using distlib to iterate over all installed files of the current
distribution to copy files to the target directory of the rez package
variant
"""
for source_file, data in src_dst_lut.items():
destination_file, exe = data
destination_file = os.path.normpath(os.path.join(path, destination_file))
if not os.path.exists(os.path.dirname(destination_file)):
os.makedirs(os.path.dirname(destination_file))
shutil.copyfile(source_file, destination_file)
if exe:
shutil.copystat(source_file, destination_file)
# determine variant requirements
# TODO detect if platform/arch/os necessary, no if pure python
variant_reqs = []
variant_reqs.append("platform-%s" % _system.platform)
variant_reqs.append("arch-%s" % _system.arch)
variant_reqs.append("os-%s" % _system.os)
if context is None:
# since we had to use system pip, we have to assume system python version
py_ver = '.'.join(map(str, sys.version_info[:2]))
else:
python_variant = context.get_resolved_package("python")
py_ver = python_variant.version.trim(2)
variant_reqs.append("python-%s" % py_ver)
name, _ = parse_name_and_version(distribution.name_and_version)
name = distribution.name[0:len(name)].replace("-", "_")
with make_package(name, packages_path, make_root=make_root) as pkg:
pkg.version = distribution.version
if distribution.metadata.summary:
pkg.description = distribution.metadata.summary
pkg.variants = [variant_reqs]
if requirements:
pkg.requires = requirements
commands = []
commands.append("env.PYTHONPATH.append('{root}/python')")
if tools:
pkg.tools = tools
commands.append("env.PATH.append('{root}/bin')")
pkg.commands = '\n'.join(commands)
installed_variants.extend(pkg.installed_variants or [])
skipped_variants.extend(pkg.skipped_variants or [])
# cleanup
shutil.rmtree(tmpdir)
return installed_variants, skipped_variants
def pip_install_package(source_name,
pip_version=None,
python_version=None,
mode=InstallMode.min_deps,
release=False):
"""Install a pip-compatible python package as a rez package.
Args:
source_name (str): Name of package or archive/url containing the pip
package source. This is the same as the arg you would pass to
the 'pip install' command.
pip_version (str or `Version`): Version of pip to use to perform the
install, uses latest if None.
python_version (str or `Version`): Python version to use to perform the
install, and subsequently have the resulting rez package depend on.
mode (`InstallMode`): Installation mode, determines how dependencies are
managed.
release (bool): If True, install as a released package; otherwise, it
will be installed as a local package.
Returns:
2-tuple:
List of `Variant`: Installed variants;
List of `Variant`: Skipped variants (already installed).
"""
installed_variants = []
skipped_variants = []
py_exe, context = find_pip(pip_version, python_version)
# TODO: should check if packages_path is writable before continuing with pip
#
packages_path = (config.release_packages_path
if release else config.local_packages_path)
tmpdir = mkdtemp(suffix="-rez", prefix="pip-")
stagingdir = os.path.join(tmpdir, "rez_staging")
stagingsep = "".join([os.path.sep, "rez_staging", os.path.sep])
destpath = os.path.join(stagingdir, "python")
# TODO use binpath once https://github.com/pypa/pip/pull/3934 is approved
binpath = os.path.join(stagingdir, "bin")
if context and config.debug("package_release"):
buf = StringIO()
print("\n\npackage download environment:", file=buf)
context.print_info(buf)
_log(buf.getvalue())
# Build pip commandline
cmd = [
py_exe, "-m", "pip", "install", "--use-pep517",
"--target=%s" % destpath
]
if mode == InstallMode.no_deps:
cmd.append("--no-deps")
cmd.append(source_name)
_cmd(context=context, command=cmd)
_system = System()
# determine version of python in use
if context is None:
# since we had to use system pip, we have to assume system python version
py_ver_str = '.'.join(map(str, sys.version_info))
py_ver = Version(py_ver_str)
else:
python_variant = context.get_resolved_package("python")
py_ver = python_variant.version
# moving bin folder to expected relative location as per wheel RECORD files
staged_binpath = os.path.join(destpath, "bin")
if os.path.isdir(staged_binpath):
shutil.move(os.path.join(destpath, "bin"), binpath)
# Collect resulting python packages using distlib
distribution_path = DistributionPath([destpath])
distributions = list(distribution_path.get_distributions())
dist_names = [x.name for x in distributions]
# get list of package and dependencies
for distribution in distributions:
# convert pip requirements into rez requirements
rez_requires = get_rez_requirements(installed_dist=distribution,
python_version=py_ver,
name_casings=dist_names)
# log the pip -> rez translation, for debugging
_log("Pip to rez translation information for " +
distribution.name_and_version + ":\n" +
pformat({
"pip": {
"run_requires": map(str, distribution.run_requires)
},
"rez": rez_requires
}))
# iterate over installed files and determine dest filepaths
tools = []
src_dst_lut = {}
for installed_file in distribution.list_installed_files():
# distlib expects the script files to be located in ../../bin/
# when in fact ../bin seems to be the resulting path after the
# installation as such we need to point the bin files to the
# expected location to match wheel RECORD files
installed_filepath = os.path.normpath(installed_file[0])
bin_prefix = os.path.join('..', '..', 'bin') + os.sep
if installed_filepath.startswith(bin_prefix):
# account for extra parentdir as explained above
installed = os.path.join(destpath, '_', installed_filepath)
else:
installed = os.path.join(destpath, installed_filepath)
source_file = os.path.normpath(installed)
if os.path.exists(source_file):
destination_file = os.path.relpath(source_file, stagingdir)
exe = False
if is_exe(source_file) and destination_file.startswith("bin" +
os.sep):
_file = os.path.basename(destination_file)
tools.append(_file)
exe = True
src_dst_lut[source_file] = [destination_file, exe]
else:
_log("Source file does not exist: " + source_file + "!")
def make_root(variant, path):
"""Using distlib to iterate over all installed files of the current
distribution to copy files to the target directory of the rez package
variant
"""
for source_file, data in src_dst_lut.items():
destination_file, exe = data
destination_file = os.path.normpath(
os.path.join(path, destination_file))
if not os.path.exists(os.path.dirname(destination_file)):
os.makedirs(os.path.dirname(destination_file))
shutil.copyfile(source_file, destination_file)
if exe:
shutil.copystat(source_file, destination_file)
# create the rez package
name = pip_to_rez_package_name(distribution.name)
version = pip_to_rez_version(distribution.version)
requires = rez_requires["requires"]
variant_requires = rez_requires["variant_requires"]
metadata = rez_requires["metadata"]
with make_package(name, packages_path, make_root=make_root) as pkg:
# basics (version etc)
pkg.version = version
if distribution.metadata.summary:
pkg.description = distribution.metadata.summary
# requirements and variants
if requires:
pkg.requires = requires
if variant_requires:
pkg.variants = [variant_requires]
# commands
commands = []
commands.append("env.PYTHONPATH.append('{root}/python')")
if tools:
pkg.tools = tools
commands.append("env.PATH.append('{root}/bin')")
pkg.commands = '\n'.join(commands)
# Make the package use hashed variants. This is required because we
# can't control what ends up in its variants, and that can easily
# include problematic chars (>, +, ! etc).
# TODO: https://github.com/nerdvegas/rez/issues/672
#
pkg.hashed_variants = True
# add some custom attributes to retain pip-related info
pkg.pip_name = distribution.name_and_version
pkg.from_pip = True
pkg.is_pure_python = metadata["is_pure_python"]
installed_variants.extend(pkg.installed_variants or [])
skipped_variants.extend(pkg.skipped_variants or [])
# cleanup
shutil.rmtree(tmpdir)
return installed_variants, skipped_variants
def bind(path, version_range=None, opts=None, parser=None):
if opts and opts.pypkg:
py_package_name = opts.pypkg
if opts.name:
package_name = opts.name
else:
package_name = py_package_name.replace('-', '_')
else:
raise ValueError('A package name needs to be specified')
# connect to PyPi and get the released package versions
pypi_client = xmlrpclib.ServerProxy('https://pypi.python.org/pypi')
package_versions = sorted(pypi_client.package_releases(py_package_name),
key=lambda v: Version(v))
if len(package_versions) < 1:
raise ValueError('Invalid package name: {}'.format(package_name))
# get the version
if opts.version:
version = next((v for v in package_versions if v == opts.version),
None)
if version is None:
raise ValueError('Invalid version: {}, valid versions: {}'.format(
opts.version, package_versions))
else:
version = package_versions[-1]
check_version(Version(version), version_range)
# get the source distribution, this could be better and try to install wheels as well
release_info = next(
(r for r in pypi_client.release_urls(py_package_name, version)
if r['packagetype'] == 'sdist'), None)
if release_info is None:
raise ValueError(
'No source distribution (we don\'t support wheels at the moment) for {} {}'
.format(py_package_name, version))
# clear the temp dir if it exists
tmp_dir = os.path.join(platform_.tmpdir, package_name)
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
os.mkdir(tmp_dir)
# download file to tmp dir
handle = urllib2.urlopen(release_info['url'])
src_archive = os.path.join(tmp_dir, release_info['filename'])
with open(src_archive, 'wb') as src_archive_fp:
src_archive_fp.write(handle.read())
# decompress the file
package_dir = None
if release_info['filename'].endswith('.zip'):
zfile = zipfile.ZipFile(src_archive, 'r')
zfile.extractall(tmp_dir)
package_dir = src_archive.replace('.zip', '')
elif release_info['filename'].endswith('.tar.gz'):
tfile = tarfile.open(src_archive, 'r:gz')
tfile.extractall(tmp_dir)
package_dir = src_archive.replace('.tar.gz', '')
elif release_info['filename'].endswith('.tgz'):
tfile = tarfile.open(src_archive, 'r:gz')
tfile.extractall(tmp_dir)
package_dir = src_archive.replace('.tgz', '')
else:
raise ValueError(
'Package file downloaded cannot be decompressed: {}'.format(
src_archive))
setup_file = os.path.join(package_dir, 'setup.py')
if not os.path.exists(setup_file):
raise RuntimeError(
'Package does not contain a "setup.py" file in the root of the archive: {}'
.format(py_package_name))
def setup_dummy(*args, **kwargs):
global SETUPTOOLS_ARGS
SETUPTOOLS_ARGS = {}
if 'scripts' in kwargs:
SETUPTOOLS_ARGS['scripts'] = kwargs['scripts']
if 'description' in kwargs:
SETUPTOOLS_ARGS['description'] = kwargs['description']
if 'long_description' in kwargs:
SETUPTOOLS_ARGS['long_description'] = kwargs['long_description']
if 'author' in kwargs:
SETUPTOOLS_ARGS['author'] = kwargs['author']
if 'author_email' in kwargs:
SETUPTOOLS_ARGS['author_email'] = kwargs['author_email']
if 'install_requires' in kwargs:
SETUPTOOLS_ARGS['install_requires'] = kwargs['install_requires']
if 'setup_requires' in kwargs:
SETUPTOOLS_ARGS['setup_requires'] = kwargs['setup_requires']
if 'version' in kwargs:
SETUPTOOLS_ARGS['version'] = kwargs['version']
else:
raise RuntimeError('A package must have a version.')
# monkey patch the setup function so we can steal the arguments passed to it
setuptools.setup = setup_dummy
distutils.core.setup = setup_dummy
cwd = os.getcwd()
os.chdir(package_dir)
sys.path.extend([package_dir])
old_args = sys.argv[:]
try:
_ = imp.load_source('{}_setup'.format(package_name), setup_file)
except Exception as e:
import traceback
traceback.print_exc()
raise e
sys.argv = old_args[:]
os.chdir(cwd)
sys.path.remove(package_dir)
def parse_setup_requirements(req_str):
'''
e.g. "arrow >= 0.4.4, < 1" -> 'arrow-0.4.4+<1
:param req_str: requirement in the setuptools format
:return: requirement in the rez format
'''
if not any((op in req_str for op in ['<', '>', '='])):
# even simpler "arrow" req
return req_str.replace('-', '_').strip()
if '==' in req_str:
# just a simple "arrow == 0.4.4" req
req_pgk_name, req_version = req_str.split('==')
return '{}-{}'.format(
req_pgk_name.replace('-', '_').strip(), req_version.strip())
complex_req_match = re.match(
r'^(?P<pkg_name>[a-zA-Z0-9\.-_]+) *(?P<reqs>[\.,<>=0-9) ]+)$',
req_str.strip())
if not complex_req_match:
raise RuntimeError(
'Unable to parse requirement; {}'.format(req_str))
req_pgk_name = complex_req_match.group('pkg_name')
req_pgk_req_str = complex_req_match.group('reqs')
requirements = map(lambda s: s.strip(), req_pgk_req_str.split(','))
versions_rewrite = []
for sub_requirement_str in requirements:
sub_requirement_match = re.match(r'([<>=]+?) ?([0-9a-zA-Z\.]+)',
sub_requirement_str)
operation = sub_requirement_match.group(1).strip()
req_version = sub_requirement_match.group(2).strip()
if operation == '>=':
versions_rewrite.insert(0, '{}+'.format(req_version))
elif operation == '<':
versions_rewrite.append('<{}'.format(req_version))
else:
raise RuntimeError(
'Unable to translate operation: "{}" in requirement: {}'.
format(operation, req_str))
return '{}-{}'.format(req_pgk_name.replace('-', '_'),
''.join(versions_rewrite))
def make_root(variant, root):
binpath = make_dirs(root, "bin")
pythonpath = make_dirs(root, "python")
headerpath = make_dirs(root, "include")
install_cmd = ' '.join([
'pip', 'install', '--no-deps', '--ignore-installed', '--verbose',
'--verbose', '--verbose', '--global-option="--verbose"',
'--install-option="--install-headers={}"'.format(headerpath),
'--install-option="--install-purelib={}"'.format(pythonpath),
'--install-option="--install-platlib={}"'.format(pythonpath),
'--install-option="--install-scripts={}"'.format(binpath),
py_package_name
])
try:
subprocess.check_output(install_cmd, cwd=package_dir)
except subprocess.CalledProcessError as _:
raise RuntimeError(
'Bind failed to install python package with command: {}'.
format(install_cmd))
with make_package(package_name, path, make_root=make_root) as pkg:
pkg.version = version
if 'description' in SETUPTOOLS_ARGS:
if len(SETUPTOOLS_ARGS['description']) < 150:
pkg.nice_name = SETUPTOOLS_ARGS['description']
else:
pkg.nice_name = SETUPTOOLS_ARGS['description'][:147] + '...'
if 'long_description' in SETUPTOOLS_ARGS:
pkg.description = SETUPTOOLS_ARGS['long_description']
author_parts = []
if 'author' in SETUPTOOLS_ARGS:
author_parts.append(SETUPTOOLS_ARGS['author'])
if 'author_email' in SETUPTOOLS_ARGS:
author_parts.append(SETUPTOOLS_ARGS['author_email'])
if len(author_parts) > 0:
pkg.authors = [' '.join(author_parts)]
else:
pkg.authors = []
if 'install_requires' in SETUPTOOLS_ARGS:
pkg.requires = [
parse_setup_requirements(req)
for req in SETUPTOOLS_ARGS['install_requires']
]
else:
pkg.requires = []
if 'setup_requires' in SETUPTOOLS_ARGS:
pkg.build_requires = [
parse_setup_requirements(req)
for req in SETUPTOOLS_ARGS['setup_requires']
]
if 'scripts' in SETUPTOOLS_ARGS:
pkg.tools = [
os.path.split(script)[-1]
for script in SETUPTOOLS_ARGS['scripts']
]
pkg.commands = commands
return package_name, Version(version)
def bind(path, version_range=None, opts=None, parser=None):
if platform_.name != 'windows':
raise EnvironmentError('Only binds on windows at the moment')
# find executable, determine version
if opts and opts.root:
bin_path = opts.root
try:
info = GetFileVersionInfo(os.path.join(bin_path, 'Support Files', 'AfterFX.exe'), "\\")
ms = info['FileVersionMS']
ls = info['FileVersionLS']
version = Version('{}.{}.{}.{}'.format(HIWORD(ms), LOWORD(ms), HIWORD(ls), LOWORD(ls)))
except:
raise EnvironmentError('Unknown version')
else:
installed_versions = []
adobe_root = r'C:\Program Files\Adobe'
app_folder_prefix = 'Adobe After Effects'
for app_folder in os.listdir(adobe_root):
if app_folder.startswith(app_folder_prefix):
app_exe = os.path.join(adobe_root, app_folder, 'Support Files', 'AfterFX.exe')
if os.path.exists(app_exe):
try:
info = GetFileVersionInfo(app_exe, "\\")
ms = info['FileVersionMS']
ls = info['FileVersionLS']
version = Version('{}.{}.{}.{}'.format(HIWORD(ms), LOWORD(ms), HIWORD(ls), LOWORD(ls)))
except:
raise EnvironmentError('Unknown version')
installed_versions.append(
(app_folder, version)
)
if len(installed_versions) < 1:
raise EnvironmentError(
'Unable to find any installed version of After Effects under "{}"'.format(
adobe_root
)
)
app_folder, version = sorted(installed_versions, key=lambda v: v[1])[-1]
bin_path = os.path.join(adobe_root, app_folder)
check_version(version, version_range)
def make_root(resources_path, variant, path):
import shutil
shutil.copy(
os.path.join(resources_path, 'aftereffects_icon.png'),
os.path.join(path, 'aftereffects_icon.png')
)
make_root_partial = functools.partial(make_root, resource_filename(Requirement.parse('rez'), "rez/bind/resources"))
with make_package("aftereffects", path, make_root=make_root_partial) as pkg:
pkg.version = version
pkg.tools = ["aftereffects"]
pkg.description = 'Animation and effects application'
pkg.authors = ['Adobe']
pkg.requires = []
pkg.nice_name = 'After Effects'
pkg.tools_info = {
'aftereffects': {
'command': ['start', '^"AfterEffects^"', '/D', '%REZ_PHOTOSHOP_ROOT%', '/wait', '/B', 'AfterFX.exe'],
'nice_name': 'AfterEffects',
'priority': 89,
'icon': '{root}/aftereffects_icon.png',
'launch_in_prompt': False
}
}
pkg.commands = win_commands(bin_path)
pkg.variants = [system.variant]
return "aftereffects", version
def bind(path, version_range=None, opts=None, parser=None):
if platform_.name != 'windows':
raise EnvironmentError('Only binds on windows at the moment')
# # find executable, determine version
# if opts and opts.root:
# root_path = opts.root
# try:
# info = GetFileVersionInfo(os.path.join(root_path, 'bin', 'deadlinecommand.exe'), "\\")
# ms = info['FileVersionMS']
# ls = info['FileVersionLS']
# version = Version('{}.{}.{}.{}'.format(HIWORD(ms), LOWORD(ms), HIWORD(ls), LOWORD(ls)))
# except:
# raise EnvironmentError('Unknown version')
# else:
installed_versions = []
thinkbox_root = r'C:\Program Files\Thinkbox'
app_folder_prefix = 'Deadline7' # previous value was 'Deadline' with no number
for app_folder in os.listdir(thinkbox_root):
if app_folder.startswith(app_folder_prefix):
app_exe = os.path.join(thinkbox_root, app_folder, 'bin',
'deadlinecommand.exe')
if os.path.exists(app_exe):
try:
info = GetFileVersionInfo(app_exe, "\\")
ms = info['FileVersionMS']
ls = info['FileVersionLS']
version = Version('{}.{}.{}.{}'.format(
HIWORD(ms), LOWORD(ms), HIWORD(ls), LOWORD(ls)))
except:
raise EnvironmentError('Unknown version')
installed_versions.append((app_folder, version))
if len(installed_versions) < 1:
raise EnvironmentError(
'Unable to find any installed version of Deadline under "{}"'.
format(thinkbox_root))
app_folder, version = sorted(installed_versions, key=lambda v: v[1])[-1]
root_path = os.path.join(thinkbox_root, app_folder, 'bin')
check_version(version, version_range)
def make_root(resources_path, variant, path):
import shutil
shutil.copy(os.path.join(resources_path, 'deadline_monitor_icon.png'),
os.path.join(path, 'deadline_monitor_icon.png'))
make_root_partial = functools.partial(
make_root,
resource_filename(Requirement.parse('rez'), "rez/bind/resources"))
with make_package("deadline", path, make_root=make_root_partial) as pkg:
pkg.version = version
pkg.tools = ["monitor"]
pkg.description = 'Render Manager'
pkg.authors = ['Thinkbox']
pkg.requires = []
pkg.nice_name = 'Deadline'
pkg.tools_info = {
'monitor': {
'command': ['deadlinemonitor.exe'],
'nice_name': 'Monitor',
'priority': 89,
'icon': '{root}/deadline_monitor_icon.png',
'launch_in_prompt': False
}
}
pkg.commands = win_commands(root_path, version.major)
pkg.variants = [system.variant]
return "deadline", version
