def load_plugins(self):
import pkgutil
from rez.backport.importlib import import_module
type_module_name = 'rezplugins.' + self.type_name
package = import_module(type_module_name)
# on import, the `__path__` variable of the imported package is extended
# to include existing directories on the plugin search path (via
# extend_path, above). this means that `walk_packages` will walk over all
# modules on the search path at the same level (.e.g in a
# 'rezplugins/type_name' sub-directory).
paths = [package.__path__] if isinstance(package.__path__, basestring) \
else package.__path__
for path in paths:
for loader, modname, ispkg in pkgutil.walk_packages(
[path], package.__name__ + '.'):
if loader is not None:
plugin_name = modname.split('.')[-1]
if plugin_name.startswith('_'):
continue
if config.debug("plugins"):
print_debug("loading %s plugin at %s: %s..."
% (self.type_name, path, modname))
try:
# load_module will force reload the module if it's
# already loaded, so check for that
module = sys.modules.get(modname)
if module is None:
module = loader.find_module(modname).load_module(modname)
if hasattr(module, 'register_plugin') and \
hasattr(module.register_plugin, '__call__'):
plugin_class = module.register_plugin()
if plugin_class != None:
self.register_plugin(plugin_name, plugin_class, module)
else:
if config.debug("plugins"):
print_warning(
"'register_plugin' function at %s: %s did not return a class."
% (path, modname))
else:
if config.debug("plugins"):
print_warning(
"no 'register_plugin' function at %s: %s"
% (path, modname))
# delete from sys.modules?
except Exception as e:
nameish = modname.split('.')[-1]
self.failed_plugins[nameish] = str(e)
if config.debug("plugins"):
import traceback
from StringIO import StringIO
out = StringIO()
traceback.print_exc(file=out)
print_debug(out.getvalue())
# load config
data, _ = _load_config_from_filepaths([os.path.join(path, "rezconfig")])
deep_update(self.config_data, data)
def load_plugins(self):
import pkgutil
from rez.backport.importlib import import_module
type_module_name = 'rezplugins.' + self.type_name
package = import_module(type_module_name)
# on import, the `__path__` variable of the imported package is extended
# to include existing directories on the plugin search path (via
# extend_path, above). this means that `walk_packages` will walk over all
# modules on the search path at the same level (.e.g in a
# 'rezplugins/type_name' sub-directory).
paths = [package.__path__] if isinstance(package.__path__, basestring) \
else package.__path__
for path in paths:
for loader, modname, ispkg in pkgutil.walk_packages(
[path], package.__name__ + '.'):
if loader is not None:
plugin_name = modname.split('.')[-1]
if plugin_name.startswith('_'):
continue
if config.debug("plugins"):
print_debug("loading %s plugin at %s: %s..."
% (self.type_name, path, modname))
try:
# load_module will force reload the module if it's
# already loaded, so check for that
module = sys.modules.get(modname)
if module is None:
module = loader.find_module(modname).load_module(modname)
if hasattr(module, 'register_plugin') and \
hasattr(module.register_plugin, '__call__'):
plugin_class = module.register_plugin()
if plugin_class != None:
self.register_plugin(plugin_name, plugin_class, module)
else:
if config.debug("plugins"):
print_warning(
"'register_plugin' function at %s: %s did not return a class."
% (path, modname))
else:
if config.debug("plugins"):
print_warning(
"no 'register_plugin' function at %s: %s"
% (path, modname))
# delete from sys.modules?
except Exception as e:
nameish = modname.split('.')[-1]
self.failed_plugins[nameish] = str(e)
if config.debug("plugins"):
import traceback
from StringIO import StringIO
out = StringIO()
traceback.print_exc(file=out)
print_debug(out.getvalue())
# load config
data, _ = _load_config_from_filepaths([os.path.join(path, "rezconfig")])
deep_update(self.config_data, data)
def set_pika_log_level():
mod_name = "rez.vendor.pika"
if config.debug("context_tracking"):
logging.getLogger(mod_name).setLevel(logging.DEBUG)
else:
logging.getLogger(mod_name).setLevel(logging.WARNING)
def _data(self):
if not self.schema:
return None
data = self._load()
if config.debug("resources"):
print_debug("Loaded resource: %s" % str(self))
return data
def _data(self):
if not self.schema:
return None
data = self._load()
if config.debug("resources"):
print_debug("Loaded resource: %s" % str(self))
return data
def load_plugin_cmd():
"""Load subcommand from command type plugin
The command type plugin module should have attribute `command_behavior`,
and the value must be a dict if provided. For example:
# in your command plugin module
command_behavior = {
"hidden": False, # (bool): default False
"arg_mode": None, # (str): "passthrough", "grouped", default None
}
If the attribute not present, default behavior will be given.
"""
from rez.config import config
from rez.utils.logging_ import print_debug
from rez.plugin_managers import plugin_manager
ext_plugins = dict()
for plugin_name in plugin_manager.get_plugins("command"):
module = plugin_manager.get_plugin_module("command", plugin_name)
behavior = getattr(module, "command_behavior", None)
if behavior is None:
behavior = dict()
if config.debug("plugins"):
print_debug("Attribute 'command_behavior' not found in plugin "
"module %s, registering with default behavior." %
module.__name__)
try:
data = behavior.copy()
data.update({"module_name": module.__name__})
ext_plugins[plugin_name] = data
except Exception:
if config.debug("plugins"):
import traceback
from rez.vendor.six.six import StringIO
out = StringIO()
traceback.print_exc(file=out)
print_debug(out.getvalue())
return ext_plugins
def load_module(self, name, package):
from hashlib import sha1
from rez.config import config # avoiding circular import
from rez.developer_package import DeveloperPackage
# in rare cases, a @late bound function may get called before the
# package is built. An example is 'requires' and the other requires-like
# functions. These need to be evaluated before a build, but it does also
# make sense to sometimes implement these as late-bound functions. We
# detect this case here, and load the modules from the original (pre-
# copied into package payload) location.
#
if isinstance(package, DeveloperPackage):
# load sourcefile from original location
path = config.package_definition_python_path
filepath = os.path.join(path, "%s.py" % name)
if not os.path.exists(filepath):
return None
with open(filepath, "rb") as f:
hash_str = sha1(f.read().strip()).hexdigest()
else:
# load sourcefile that's been copied into package install payload
path = os.path.join(package.base, self.include_modules_subpath)
pathname = os.path.join(path, "%s.py" % name)
hashname = os.path.join(path, "%s.sha1" % name)
if os.path.isfile(pathname) and os.path.isfile(hashname):
with open(hashname, "r") as f:
hash_str = f.readline()
filepath = pathname
else:
# Fallback for backward compat
pathname = os.path.join(path, "%s-*.py" % name)
hashnames = glob(pathname)
if not hashnames:
return None
filepath = hashnames[0]
hash_str = filepath.rsplit('-', 1)[-1].split('.', 1)[0]
# End, for details of backward compat,
# see https://github.com/nerdvegas/rez/issues/934
# and https://github.com/nerdvegas/rez/pull/935
module = self.modules.get(hash_str)
if module is not None:
return module
if config.debug("file_loads"):
print_debug("Loading include sourcefile: %s" % filepath)
module = py23.load_module_from_file(name, filepath)
self.modules[hash_str] = module
return module
def append_if_valid(dir_):
if os.path.isdir(dir_):
subdir = os.path.normcase(os.path.join(dir_, pname))
initfile = os.path.join(subdir, init_py)
if subdir not in path and os.path.isfile(initfile):
path.append(subdir)
elif config.debug("plugins"):
print_debug("skipped nonexistant rez plugin path: %s" % dir_)
def _load_file(filepath, format_, update_data_callback):
load_func = load_functions[format_]
if config.debug("file_loads"):
print_debug("Loading file: %s" % filepath)
with open(filepath) as f:
result = load_func(f, filepath=filepath)
if update_data_callback:
result = update_data_callback(format_, result)
return result
def _load_file(filepath, format_, update_data_callback):
load_func = load_functions[format_]
if config.debug("file_loads"):
print_debug("Loading file: %s" % filepath)
with open(filepath) as f:
result = load_func(f, filepath=filepath)
if update_data_callback:
result = update_data_callback(format_, result)
return result
def load_module(self, name, package):
from rez.config import config # avoiding circular import
from rez.developer_package import DeveloperPackage
# in rare cases, a @late bound function may get called before the
# package is built. An example is 'requires' and the other requires-like
# functions. These need to be evaluated before a build, but it does also
# make sense to sometimes implement these as late-bound functions. We
# detect this case here, and load the modules from the original (pre-
# copied into package payload) location.
#
if isinstance(package, DeveloperPackage):
from hashlib import sha1
# load sourcefile from original location
path = config.package_definition_python_path
filepath = os.path.join(path, "%s.py" % name)
if not os.path.exists(filepath):
return None
with open(filepath) as f:
txt = f.read().strip()
hash_str = sha1(txt).hexdigest()
else:
# load sourcefile that's been copied into package install payload
path = os.path.join(package.base, self.include_modules_subpath)
pathname = os.path.join(path, "%s-*.py" % name)
pathnames = glob(pathname)
if not pathnames:
return None
filepath = pathnames[0]
hash_str = filepath.rsplit('-', 1)[-1].split('.', 1)[0]
module = self.modules.get(hash_str)
if module is not None:
return module
if config.debug("file_loads"):
print_debug("Loading include sourcefile: %s" % filepath)
with open(filepath) as f:
module = imp.load_source(name, filepath, f)
self.modules[hash_str] = module
return module
def load_module(self, name, package):
from rez.config import config # avoiding circular import
from rez.developer_package import DeveloperPackage
# in rare cases, a @late bound function may get called before the
# package is built. An example is 'requires' and the other requires-like
# functions. These need to be evaluated before a build, but it does also
# make sense to sometimes implement these as late-bound functions. We
# detect this case here, and load the modules from the original (pre-
# copied into package payload) location.
#
if isinstance(package, DeveloperPackage):
from hashlib import sha1
# load sourcefile from original location
path = config.package_definition_python_path
filepath = os.path.join(path, "%s.py" % name)
if not os.path.exists(filepath):
return None
with open(filepath) as f:
txt = f.read().strip()
hash_str = sha1(txt).hexdigest()
else:
# load sourcefile that's been copied into package install payload
path = os.path.join(package.base, self.include_modules_subpath)
pathname = os.path.join(path, "%s-*.py" % name)
pathnames = glob(pathname)
if not pathnames:
return None
filepath = pathnames[0]
hash_str = filepath.rsplit('-', 1)[-1].split('.', 1)[0]
module = self.modules.get(hash_str)
if module is not None:
return module
if config.debug("file_loads"):
print_debug("Loading include sourcefile: %s" % filepath)
with open(filepath) as f:
module = imp.load_source(name, filepath, f)
self.modules[hash_str] = module
return module
def publish_message(self, data):
if not self.settings.host:
print_error("Did not publish message, host is not specified")
return
routing_key = self.settings.exchange_routing_key
print("Publishing AMQP message on %s..." % routing_key)
publish_message(host=self.settings.host,
amqp_settings=self.settings,
routing_key=routing_key,
data=data)
if config.debug("package_release"):
print_debug("Published message: %s" % (data))
def publish_message(self, data):
if not self.settings.host:
print_error("Did not publish message, host is not specified")
return
try:
conn = Connection(host=self.settings.host,
port=self.settings.port,
userid=self.settings.userid,
password=self.settings.password,
connect_timeout=self.settings.connect_timeout)
except socket.error as e:
print_error("Cannot connect to the message broker: %s" % (e))
return
channel = conn.channel()
# Declare the exchange
try:
channel.exchange_declare(
self.settings.exchange_name,
self.settings.exchange_type,
durable=self.settings.exchange_durable,
auto_delete=self.settings.exchange_auto_delete)
except Exception as e:
print_error("Failed to declare an exchange: %s" % (e))
return
# build the message
msg = basic_message.Message(
body=json.dumps(data),
delivery_mode=self.settings.message_delivery_mode,
content_type="application/json",
content_encoding="utf-8")
routing_key = self.settings.exchange_routing_key
print "Publishing AMQP message on %s..." % routing_key
# publish the message
try:
channel.basic_publish(msg, self.settings.exchange_name,
routing_key)
except Exception as e:
print_error("Failed to publish message: %s" % (e))
return
if config.debug("package_release"):
print_debug("Published message: %s" % (data))
def extend_path(path, name):
"""Extend a package's path.
Intended use is to place the following code in a package's __init__.py:
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
This will add to the package's __path__ all subdirectories of
directories on sys.path named after the package. This is useful
if one wants to distribute different parts of a single logical
package as multiple directories.
If the input path is not a list (as is the case for frozen
packages) it is returned unchanged. The input path is not
modified; an extended copy is returned. Items are only appended
to the copy at the end.
It is assumed that 'plugin_path' is a sequence. Items of 'plugin_path'
that are not (unicode or 8-bit) strings referring to existing
directories are ignored. Unicode items of sys.path that cause
errors when used as filenames may cause this function to raise an
exception (in line with os.path.isdir() behavior).
"""
if not isinstance(path, list):
# This could happen e.g. when this is called from inside a
# frozen package. Return the path unchanged in that case.
return path
pname = os.path.join(*name.split('.')) # Reconstitute as relative path
# Just in case os.extsep != '.'
init_py = "__init__" + os.extsep + "py"
path = path[:]
for dir in config.plugin_path:
if not os.path.isdir(dir):
if config.debug("plugins"):
print_debug("skipped nonexistant rez plugin path: %s" % dir)
continue
subdir = os.path.join(dir, pname)
# XXX This may still add duplicate entries to path on
# case-insensitive filesystems
initfile = os.path.join(subdir, init_py)
if subdir not in path and os.path.isfile(initfile):
path.append(subdir)
return path
def extend_path(path, name):
"""Extend a package's path.
Intended use is to place the following code in a package's __init__.py:
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
This will add to the package's __path__ all subdirectories of
directories on sys.path named after the package. This is useful
if one wants to distribute different parts of a single logical
package as multiple directories.
If the input path is not a list (as is the case for frozen
packages) it is returned unchanged. The input path is not
modified; an extended copy is returned. Items are only appended
to the copy at the end.
It is assumed that 'plugin_path' is a sequence. Items of 'plugin_path'
that are not (unicode or 8-bit) strings referring to existing
directories are ignored. Unicode items of sys.path that cause
errors when used as filenames may cause this function to raise an
exception (in line with os.path.isdir() behavior).
"""
if not isinstance(path, list):
# This could happen e.g. when this is called from inside a
# frozen package. Return the path unchanged in that case.
return path
pname = os.path.join(*name.split('.')) # Reconstitute as relative path
# Just in case os.extsep != '.'
init_py = "__init__" + os.extsep + "py"
path = path[:]
for dir in config.plugin_path:
if not os.path.isdir(dir):
if config.debug("plugins"):
print_debug("skipped nonexistant rez plugin path: %s" % dir)
continue
subdir = os.path.join(dir, pname)
# XXX This may still add duplicate entries to path on
# case-insensitive filesystems
initfile = os.path.join(subdir, init_py)
if subdir not in path and os.path.isfile(initfile):
path.append(subdir)
return path
def publish_message(self, data):
if not self.settings.host:
print_error("Did not publish message, host is not specified")
return
try:
conn = Connection(host=self.settings.host,
port=self.settings.port,
userid=self.settings.userid,
password=self.settings.password,
connect_timeout=self.settings.connect_timeout)
except socket.error as e:
print_error("Cannot connect to the message broker: %s" % (e))
return
channel = conn.channel()
# Declare the exchange
try:
channel.exchange_declare(
self.settings.exchange_name,
self.settings.exchange_type,
durable=self.settings.exchange_durable,
auto_delete=self.settings.exchange_auto_delete)
except Exception as e:
print_error("Failed to declare an exchange: %s" % (e))
return
# build the message
msg = basic_message.Message(
body=json.dumps(data),
delivery_mode=self.settings.message_delivery_mode,
content_type="application/json",
content_encoding="utf-8")
routing_key = self.settings.exchange_routing_key
print "Publishing AMQP message on %s..." % routing_key
# publish the message
try:
channel.basic_publish(msg, self.settings.exchange_name, routing_key)
except Exception as e:
print_error("Failed to publish message: %s" % (e))
return
if config.debug("package_release"):
print_debug("Published message: %s" % (data))
def _release(self, commands, errors=None):
for conf in commands:
if self.settings.print_commands or config.debug("package_release"):
from subprocess import list2cmdline
toks = [conf["command"]] + conf.get("args", [])
msg = "running command: %s" % list2cmdline(toks)
if self.settings.print_commands:
print msg
else:
print_debug(msg)
if not self.execute_command(cmd_name=conf.get("command"),
cmd_arguments=conf.get("args"),
user=conf.get("user"),
errors=errors):
if self.settings.stop_on_error:
return
def _execute_commands(self, commands, install_path, package, errors=None):
release_dict = dict(path=install_path)
formatter = scoped_formatter(system=system,
release=release_dict,
package=package)
for conf in commands:
program = conf["command"]
env_ = None
env = conf.get("env")
if env:
env_ = os.environ.copy()
env_.update(env)
args = conf.get("args", [])
args = [formatter.format(x) for x in args]
args = [expandvars(x, environ=env_) for x in args]
if self.settings.print_commands or config.debug("package_release"):
from subprocess import list2cmdline
toks = [program] + args
msgs = []
msgs.append("running command: %s" % list2cmdline(toks))
if env:
for key, value in env.iteritems():
msgs.append(" with: %s=%s" % (key, value))
if self.settings.print_commands:
print '\n'.join(msgs)
else:
for msg in msgs:
print_debug(msg)
if not self.execute_command(cmd_name=program,
cmd_arguments=args,
user=conf.get("user"),
errors=errors,
env=env_):
if self.settings.stop_on_error:
return
def _execute_commands(self, commands, install_path, package, errors=None):
release_dict = dict(path=install_path)
formatter = scoped_formatter(system=system,
release=release_dict,
package=package)
for conf in commands:
program = conf["command"]
env_ = None
env = conf.get("env")
if env:
env_ = os.environ.copy()
env_.update(env)
args = conf.get("args", [])
args = [formatter.format(x) for x in args]
args = [expandvars(x, environ=env_) for x in args]
if self.settings.print_commands or config.debug("package_release"):
from subprocess import list2cmdline
toks = [program] + args
msgs = []
msgs.append("running command: %s" % list2cmdline(toks))
if env:
for key, value in env.iteritems():
msgs.append(" with: %s=%s" % (key, value))
if self.settings.print_commands:
print '\n'.join(msgs)
else:
for msg in msgs:
print_debug(msg)
if not self.execute_command(cmd_name=program,
cmd_arguments=args,
user=conf.get("user"),
errors=errors,
env=env_):
if self.settings.stop_on_error:
return
def _execute_commands(self, commands, install_path, package, errors=None,
variants=None):
release_dict = dict(path=install_path)
variant_infos = []
if variants:
for variant in variants:
if isinstance(variant, (int, long)):
variant_infos.append(variant)
else:
package = variant.parent
var_dict = dict(variant.resource.variables)
# using '%s' will preserve potential str/unicode nature
var_dict['variant_requires'] = ['%s' % x
for x in variant.resource.variant_requires]
variant_infos.append(var_dict)
formatter = scoped_formatter(system=system,
release=release_dict,
package=package,
variants=variant_infos,
num_variants=len(variant_infos))
for conf in commands:
program = conf["command"]
env_ = None
env = conf.get("env")
if env:
env_ = os.environ.copy()
env_.update(env)
# If we have, ie, a list, and format_pretty is True, it will be printed
# as "1 2 3" instead of "[1, 2, 3]"
formatter.__dict__['format_pretty'] = conf.get(
"pretty_args", True)
args = conf.get("args", [])
args = [formatter.format(x) for x in args]
args = [expandvars(x, environ=env_) for x in args]
if self.settings.print_commands or config.debug("package_release"):
from subprocess import list2cmdline
toks = [program] + args
msgs = []
msgs.append("running command: %s" % list2cmdline(toks))
if env:
for key, value in env.iteritems():
msgs.append(" with: %s=%s" % (key, value))
if self.settings.print_commands:
print '\n'.join(msgs)
else:
for msg in msgs:
print_debug(msg)
if not self.execute_command(cmd_name=program,
cmd_arguments=args,
user=conf.get("user"),
errors=errors,
env=env_):
if self.settings.stop_on_error:
return
def load_plugins(self):
import pkgutil
from rez.backport.importlib import import_module
type_module_name = 'rezplugins.' + self.type_name
package = import_module(type_module_name)
# on import, the `__path__` variable of the imported package is extended
# to include existing directories on the plugin search path (via
# extend_path, above). this means that `walk_packages` will walk over all
# modules on the search path at the same level (.e.g in a
# 'rezplugins/type_name' sub-directory).
paths = [package.__path__] if isinstance(package.__path__, basestring) \
else package.__path__
# reverse plugin path order, so that custom plugins have a chance to
# be found before the builtin plugins (from /rezplugins).
paths = reversed(paths)
for path in paths:
if config.debug("plugins"):
print_debug("searching plugin path %s...", path)
for importer, modname, ispkg in pkgutil.iter_modules(
[path], package.__name__ + '.'):
if importer is None:
continue
plugin_name = modname.split('.')[-1]
if plugin_name.startswith('_') or plugin_name == 'rezconfig':
continue
if plugin_name in self.plugin_modules:
# same named plugins will have identical module name,
# which will just reuse previous imported module from
# `sys.modules` below. skipping the rest of the process
# for good.
if config.debug("plugins"):
print_warning(
"skipped same named %s plugin at %s: %s" %
(self.type_name, path, modname))
continue
if config.debug("plugins"):
print_debug("loading %s plugin at %s: %s..." %
(self.type_name, path, modname))
try:
# nerdvegas/rez#218
# load_module will force reload the module if it's
# already loaded, so check for that
plugin_module = sys.modules.get(modname)
if plugin_module is None:
loader = importer.find_module(modname)
plugin_module = loader.load_module(modname)
elif os.path.dirname(plugin_module.__file__) != path:
if config.debug("plugins"):
# this should not happen but if it does, tell why.
print_warning(
"plugin module %s is not loaded from current "
"load path but reused from previous imported "
"path: %s" % (modname, plugin_module.__file__))
if (hasattr(plugin_module, "register_plugin")
and callable(plugin_module.register_plugin)):
plugin_class = plugin_module.register_plugin()
if plugin_class is not None:
self.register_plugin(plugin_name, plugin_class,
plugin_module)
else:
if config.debug("plugins"):
print_warning(
"'register_plugin' function at %s: %s did "
"not return a class." % (path, modname))
else:
if config.debug("plugins"):
print_warning(
"no 'register_plugin' function at %s: %s" %
(path, modname))
# delete from sys.modules?
except Exception as e:
nameish = modname.split('.')[-1]
self.failed_plugins[nameish] = str(e)
if config.debug("plugins"):
import traceback
from rez.vendor.six.six import StringIO
out = StringIO()
traceback.print_exc(file=out)
print_debug(out.getvalue())
# load config
data, _ = _load_config_from_filepaths(
[os.path.join(path, "rezconfig")])
deep_update(self.config_data, data)
def pip_install_package(source_name,
pip_version=None,
python_version=None,
mode=InstallMode.min_deps,
release=False):
"""Install a pip-compatible python package as a rez package.
Args:
source_name (str): Name of package or archive/url containing the pip
package source. This is the same as the arg you would pass to
the 'pip install' command.
pip_version (str or `Version`): Version of pip to use to perform the
install, uses latest if None.
python_version (str or `Version`): Python version to use to perform the
install, and subsequently have the resulting rez package depend on.
mode (`InstallMode`): Installation mode, determines how dependencies are
managed.
release (bool): If True, install as a released package; otherwise, it
will be installed as a local package.
Returns:
2-tuple:
List of `Variant`: Installed variants;
List of `Variant`: Skipped variants (already installed).
"""
installed_variants = []
skipped_variants = []
py_exe, context = find_pip(pip_version, python_version)
# TODO: should check if packages_path is writable before continuing with pip
#
packages_path = (config.release_packages_path
if release else config.local_packages_path)
tmpdir = mkdtemp(suffix="-rez", prefix="pip-")
stagingdir = os.path.join(tmpdir, "rez_staging")
stagingsep = "".join([os.path.sep, "rez_staging", os.path.sep])
destpath = os.path.join(stagingdir, "python")
# TODO use binpath once https://github.com/pypa/pip/pull/3934 is approved
binpath = os.path.join(stagingdir, "bin")
if context and config.debug("package_release"):
buf = StringIO()
print("\n\npackage download environment:", file=buf)
context.print_info(buf)
_log(buf.getvalue())
# Build pip commandline
cmd = [
py_exe, "-m", "pip", "install", "--use-pep517",
"--target=%s" % destpath
]
if mode == InstallMode.no_deps:
cmd.append("--no-deps")
cmd.append(source_name)
_cmd(context=context, command=cmd)
_system = System()
# determine version of python in use
if context is None:
# since we had to use system pip, we have to assume system python version
py_ver_str = '.'.join(map(str, sys.version_info))
py_ver = Version(py_ver_str)
else:
python_variant = context.get_resolved_package("python")
py_ver = python_variant.version
# moving bin folder to expected relative location as per wheel RECORD files
staged_binpath = os.path.join(destpath, "bin")
if os.path.isdir(staged_binpath):
shutil.move(os.path.join(destpath, "bin"), binpath)
# Collect resulting python packages using distlib
distribution_path = DistributionPath([destpath])
distributions = list(distribution_path.get_distributions())
dist_names = [x.name for x in distributions]
# get list of package and dependencies
for distribution in distributions:
# convert pip requirements into rez requirements
rez_requires = get_rez_requirements(installed_dist=distribution,
python_version=py_ver,
name_casings=dist_names)
# log the pip -> rez translation, for debugging
_log("Pip to rez translation information for " +
distribution.name_and_version + ":\n" +
pformat({
"pip": {
"run_requires": map(str, distribution.run_requires)
},
"rez": rez_requires
}))
# iterate over installed files and determine dest filepaths
tools = []
src_dst_lut = {}
for installed_file in distribution.list_installed_files():
# distlib expects the script files to be located in ../../bin/
# when in fact ../bin seems to be the resulting path after the
# installation as such we need to point the bin files to the
# expected location to match wheel RECORD files
installed_filepath = os.path.normpath(installed_file[0])
bin_prefix = os.path.join('..', '..', 'bin') + os.sep
if installed_filepath.startswith(bin_prefix):
# account for extra parentdir as explained above
installed = os.path.join(destpath, '_', installed_filepath)
else:
installed = os.path.join(destpath, installed_filepath)
source_file = os.path.normpath(installed)
if os.path.exists(source_file):
destination_file = os.path.relpath(source_file, stagingdir)
exe = False
if is_exe(source_file) and destination_file.startswith("bin" +
os.sep):
_file = os.path.basename(destination_file)
tools.append(_file)
exe = True
src_dst_lut[source_file] = [destination_file, exe]
else:
_log("Source file does not exist: " + source_file + "!")
def make_root(variant, path):
"""Using distlib to iterate over all installed files of the current
distribution to copy files to the target directory of the rez package
variant
"""
for source_file, data in src_dst_lut.items():
destination_file, exe = data
destination_file = os.path.normpath(
os.path.join(path, destination_file))
if not os.path.exists(os.path.dirname(destination_file)):
os.makedirs(os.path.dirname(destination_file))
shutil.copyfile(source_file, destination_file)
if exe:
shutil.copystat(source_file, destination_file)
# create the rez package
name = pip_to_rez_package_name(distribution.name)
version = pip_to_rez_version(distribution.version)
requires = rez_requires["requires"]
variant_requires = rez_requires["variant_requires"]
metadata = rez_requires["metadata"]
with make_package(name, packages_path, make_root=make_root) as pkg:
# basics (version etc)
pkg.version = version
if distribution.metadata.summary:
pkg.description = distribution.metadata.summary
# requirements and variants
if requires:
pkg.requires = requires
if variant_requires:
pkg.variants = [variant_requires]
# commands
commands = []
commands.append("env.PYTHONPATH.append('{root}/python')")
if tools:
pkg.tools = tools
commands.append("env.PATH.append('{root}/bin')")
pkg.commands = '\n'.join(commands)
# Make the package use hashed variants. This is required because we
# can't control what ends up in its variants, and that can easily
# include problematic chars (>, +, ! etc).
# TODO: https://github.com/nerdvegas/rez/issues/672
#
pkg.hashed_variants = True
# add some custom attributes to retain pip-related info
pkg.pip_name = distribution.name_and_version
pkg.from_pip = True
pkg.is_pure_python = metadata["is_pure_python"]
installed_variants.extend(pkg.installed_variants or [])
skipped_variants.extend(pkg.skipped_variants or [])
# cleanup
shutil.rmtree(tmpdir)
return installed_variants, skipped_variants
def pip_install_package(source_name,
pip_version=None,
python_version=None,
mode=InstallMode.min_deps,
release=False,
prefix=None,
extra_args=None):
"""Install a pip-compatible python package as a rez package.
Args:
source_name (str): Name of package or archive/url containing the pip
package source. This is the same as the arg you would pass to
the 'pip install' command.
pip_version (str or `Version`): Version of pip to use to perform the
install, uses latest if None.
python_version (str or `Version`): Python version to use to perform the
install, and subsequently have the resulting rez package depend on.
mode (`InstallMode`): Installation mode, determines how dependencies are
managed.
release (bool): If True, install as a released package; otherwise, it
will be installed as a local package.
extra_args (List[str]): Additional options to the pip install command.
Returns:
2-tuple:
List of `Variant`: Installed variants;
List of `Variant`: Skipped variants (already installed).
"""
installed_variants = []
skipped_variants = []
py_exe, context = find_pip(pip_version, python_version)
print_info("Installing %r with pip taken from %r", source_name, py_exe)
# TODO: should check if packages_path is writable before continuing with pip
#
if prefix is not None:
packages_path = prefix
else:
packages_path = (config.release_packages_path
if release else config.local_packages_path)
targetpath = mkdtemp(suffix="-rez", prefix="pip-")
if context and config.debug("package_release"):
buf = StringIO()
print("\n\npackage download environment:", file=buf)
context.print_info(buf)
_log(buf.getvalue())
# Build pip commandline
cmd = [py_exe, "-m", "pip", "install"]
_extra_args = extra_args or config.pip_extra_args or []
if "--no-use-pep517" not in _extra_args:
cmd.append("--use-pep517")
if not _option_present(_extra_args, "-t", "--target"):
cmd.append("--target=%s" % targetpath)
if mode == InstallMode.no_deps and "--no-deps" not in _extra_args:
cmd.append("--no-deps")
cmd.extend(_extra_args)
cmd.append(source_name)
# run pip
#
# Note: https://github.com/pypa/pip/pull/3934. If/when this PR is merged,
# it will allow explicit control of where to put bin files.
#
_cmd(context=context, command=cmd)
# determine version of python in use
if context is None:
# since we had to use system pip, we have to assume system python version
py_ver_str = '.'.join(map(str, sys.version_info))
py_ver = Version(py_ver_str)
else:
python_variant = context.get_resolved_package("python")
py_ver = python_variant.version
# Collect resulting python packages using distlib
distribution_path = DistributionPath([targetpath])
distributions = list(distribution_path.get_distributions())
dist_names = [x.name for x in distributions]
def log_append_pkg_variants(pkg_maker):
template = '{action} [{package.qualified_name}] {package.uri}{suffix}'
actions_variants = [
(
print_info,
'Installed',
installed_variants,
pkg_maker.installed_variants or [],
),
(
print_debug,
'Skipped',
skipped_variants,
pkg_maker.skipped_variants or [],
),
]
for print_, action, variants, pkg_variants in actions_variants:
for variant in pkg_variants:
variants.append(variant)
package = variant.parent
suffix = (' (%s)' % variant.subpath) if variant.subpath else ''
print_(template.format(**locals()))
# get list of package and dependencies
for distribution in distributions:
# convert pip requirements into rez requirements
rez_requires = get_rez_requirements(installed_dist=distribution,
python_version=py_ver,
name_casings=dist_names)
# log the pip -> rez requirements translation, for debugging
_log("Pip to rez requirements translation information for " +
distribution.name_and_version + ":\n" +
pformat({
"pip": {
"run_requires": map(str, distribution.run_requires)
},
"rez": rez_requires
}))
# determine where pip files need to be copied into rez package
src_dst_lut = _get_distribution_files_mapping(distribution, targetpath)
# build tools list
tools = []
for relpath in src_dst_lut.values():
dir_, filename = os.path.split(relpath)
if dir_ == "bin":
tools.append(filename)
# Sanity warning to see if any files will be copied
if not src_dst_lut:
message = 'No source files exist for {}!'
if not _verbose:
message += '\nTry again with rez-pip --verbose ...'
print_warning(message.format(distribution.name_and_version))
def make_root(variant, path):
"""Using distlib to iterate over all installed files of the current
distribution to copy files to the target directory of the rez package
variant
"""
for rel_src, rel_dest in src_dst_lut.items():
src = os.path.join(targetpath, rel_src)
dest = os.path.join(path, rel_dest)
if not os.path.exists(os.path.dirname(dest)):
os.makedirs(os.path.dirname(dest))
shutil.copyfile(src, dest)
if _is_exe(src):
shutil.copystat(src, dest)
# create the rez package
name = pip_to_rez_package_name(distribution.name)
version = pip_to_rez_version(distribution.version)
requires = rez_requires["requires"]
variant_requires = rez_requires["variant_requires"]
metadata = rez_requires["metadata"]
with make_package(name, packages_path, make_root=make_root) as pkg:
# basics (version etc)
pkg.version = version
if distribution.metadata.summary:
pkg.description = distribution.metadata.summary
# requirements and variants
if requires:
pkg.requires = requires
if variant_requires:
pkg.variants = [variant_requires]
# commands
commands = []
commands.append("env.PYTHONPATH.append('{root}/python')")
if tools:
pkg.tools = tools
commands.append("env.PATH.append('{root}/bin')")
pkg.commands = '\n'.join(commands)
# Make the package use hashed variants. This is required because we
# can't control what ends up in its variants, and that can easily
# include problematic chars (>, +, ! etc).
# TODO: https://github.com/nerdvegas/rez/issues/672
#
pkg.hashed_variants = True
# add some custom attributes to retain pip-related info
pkg.pip_name = distribution.name_and_version
pkg.from_pip = True
pkg.is_pure_python = metadata["is_pure_python"]
distribution_metadata = distribution.metadata.todict()
help_ = []
if "home_page" in distribution_metadata:
help_.append(["Home Page", distribution_metadata["home_page"]])
if "download_url" in distribution_metadata:
help_.append(
["Source Code", distribution_metadata["download_url"]])
if help_:
pkg.help = help_
if "author" in distribution_metadata:
author = distribution_metadata["author"]
if "author_email" in distribution_metadata:
author += ' ' + distribution_metadata["author_email"]
pkg.authors = [author]
log_append_pkg_variants(pkg)
# cleanup
shutil.rmtree(targetpath)
# print summary
#
if installed_variants:
print_info("%d packages were installed.", len(installed_variants))
else:
print_warning("NO packages were installed.")
if skipped_variants:
print_warning(
"%d packages were already installed.",
len(skipped_variants),
)
return installed_variants, skipped_variants
cmd_str = ' '.join(quote(x) for x in command)
_log("running: %s" % cmd_str)
if context is None:
p = Popen(command)
else:
p = context.execute_shell(command=command, block=False)
with p:
p.wait()
if p.returncode:
raise BuildError("Failed to download source with pip: %s" % cmd_str)
_verbose = config.debug("package_release")
def _log(msg):
if _verbose:
print_debug(msg)
# Copyright 2013-2016 Allan Johns.
#
# This library is free software: you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
def pip_install_package(source_name, pip_version=None, python_version=None,
mode=InstallMode.min_deps, release=False):
"""Install a pip-compatible python package as a rez package.
Args:
source_name (str): Name of package or archive/url containing the pip
package source. This is the same as the arg you would pass to
the 'pip install' command.
pip_version (str or `Version`): Version of pip to use to perform the
install, uses latest if None.
python_version (str or `Version`): Python version to use to perform the
install, and subsequently have the resulting rez package depend on.
mode (`InstallMode`): Installation mode, determines how dependencies are
managed.
release (bool): If True, install as a released package; otherwise, it
will be installed as a local package.
Returns:
2-tuple:
List of `Variant`: Installed variants;
List of `Variant`: Skipped variants (already installed).
"""
installed_variants = []
skipped_variants = []
pip_exe, context = find_pip(pip_version, python_version)
# TODO: should check if packages_path is writable before continuing with pip
#
packages_path = (config.release_packages_path if release
else config.local_packages_path)
tmpdir = mkdtemp(suffix="-rez", prefix="pip-")
stagingdir = os.path.join(tmpdir, "rez_staging")
stagingsep = "".join([os.path.sep, "rez_staging", os.path.sep])
destpath = os.path.join(stagingdir, "python")
binpath = os.path.join(stagingdir, "bin")
incpath = os.path.join(stagingdir, "include")
datapath = stagingdir
if context and config.debug("package_release"):
buf = StringIO()
print >> buf, "\n\npackage download environment:"
context.print_info(buf)
_log(buf.getvalue())
# Build pip commandline
cmd = [pip_exe, "install",
"--install-option=--install-lib=%s" % destpath,
"--install-option=--install-scripts=%s" % binpath,
"--install-option=--install-headers=%s" % incpath,
"--install-option=--install-data=%s" % datapath]
if mode == InstallMode.no_deps:
cmd.append("--no-deps")
cmd.append(source_name)
_cmd(context=context, command=cmd)
_system = System()
# Collect resulting python packages using distlib
distribution_path = DistributionPath([destpath], include_egg=True)
distributions = [d for d in distribution_path.get_distributions()]
for distribution in distribution_path.get_distributions():
requirements = []
if distribution.metadata.run_requires:
# Handle requirements. Currently handles conditional environment based
# requirements and normal requirements
# TODO: Handle optional requirements?
for requirement in distribution.metadata.run_requires:
if "environment" in requirement:
if interpret(requirement["environment"]):
requirements.extend(_get_dependencies(requirement, distributions))
elif "extra" in requirement:
# Currently ignoring optional requirements
pass
else:
requirements.extend(_get_dependencies(requirement, distributions))
tools = []
src_dst_lut = {}
for installed_file in distribution.list_installed_files(allow_fail=True):
source_file = os.path.normpath(os.path.join(destpath, installed_file[0]))
if os.path.exists(source_file):
destination_file = installed_file[0].split(stagingsep)[1]
exe = False
if is_exe(source_file) and \
destination_file.startswith("%s%s" % ("bin", os.path.sep)):
_, _file = os.path.split(destination_file)
tools.append(_file)
exe = True
data = [destination_file, exe]
src_dst_lut[source_file] = data
else:
_log("Source file does not exist: " + source_file + "!")
def make_root(variant, path):
"""Using distlib to iterate over all installed files of the current
distribution to copy files to the target directory of the rez package
variant
"""
for source_file, data in src_dst_lut.items():
destination_file, exe = data
destination_file = os.path.normpath(os.path.join(path, destination_file))
if not os.path.exists(os.path.dirname(destination_file)):
os.makedirs(os.path.dirname(destination_file))
shutil.copyfile(source_file, destination_file)
if exe:
shutil.copystat(source_file, destination_file)
# determine variant requirements
# TODO detect if platform/arch/os necessary, no if pure python
variant_reqs = []
variant_reqs.append("platform-%s" % _system.platform)
variant_reqs.append("arch-%s" % _system.arch)
variant_reqs.append("os-%s" % _system.os)
if context is None:
# since we had to use system pip, we have to assume system python version
py_ver = '.'.join(map(str, sys.version_info[:2]))
else:
python_variant = context.get_resolved_package("python")
py_ver = python_variant.version.trim(2)
variant_reqs.append("python-%s" % py_ver)
name, _ = parse_name_and_version(distribution.name_and_version)
name = distribution.name[0:len(name)].replace("-", "_")
with make_package(name, packages_path, make_root=make_root) as pkg:
pkg.version = distribution.version
if distribution.metadata.summary:
pkg.description = distribution.metadata.summary
pkg.variants = [variant_reqs]
if requirements:
pkg.requires = requirements
commands = []
commands.append("env.PYTHONPATH.append('{root}/python')")
if tools:
pkg.tools = tools
commands.append("env.PATH.append('{root}/bin')")
pkg.commands = '\n'.join(commands)
installed_variants.extend(pkg.installed_variants or [])
skipped_variants.extend(pkg.skipped_variants or [])
# cleanup
shutil.rmtree(tmpdir)
return installed_variants, skipped_variants
def log(msg):
if config.debug("bind_modules"):
print_debug(msg)
def _execute_commands(self,
commands,
install_path,
package,
errors=None,
variants=None):
release_dict = dict(path=install_path)
variant_infos = []
if variants:
for variant in variants:
if isinstance(variant, (int, long)):
variant_infos.append(variant)
else:
package = variant.parent
var_dict = dict(variant.resource.variables)
# using '%s' will preserve potential str/unicode nature
var_dict['variant_requires'] = [
'%s' % x for x in variant.resource.variant_requires
]
variant_infos.append(var_dict)
formatter = scoped_formatter(system=system,
release=release_dict,
package=package,
variants=variant_infos,
num_variants=len(variant_infos))
for conf in commands:
program = conf["command"]
env_ = None
env = conf.get("env")
if env:
env_ = os.environ.copy()
env_.update(env)
# If we have, ie, a list, and format_pretty is True, it will be printed
# as "1 2 3" instead of "[1, 2, 3]"
formatter.__dict__['format_pretty'] = conf.get("pretty_args", True)
args = conf.get("args", [])
args = [formatter.format(x) for x in args]
args = [expandvars(x, environ=env_) for x in args]
if self.settings.print_commands or config.debug("package_release"):
from subprocess import list2cmdline
toks = [program] + args
msgs = []
msgs.append("running command: %s" % list2cmdline(toks))
if env:
for key, value in env.iteritems():
msgs.append(" with: %s=%s" % (key, value))
if self.settings.print_commands:
print('\n'.join(msgs))
else:
for msg in msgs:
print_debug(msg)
if not self.execute_command(cmd_name=program,
cmd_arguments=args,
user=conf.get("user"),
errors=errors,
env=env_):
if self.settings.stop_on_error:
return
def convert_old_commands(commands, annotate=True):
"""Converts old-style package commands into equivalent Rex code."""
from rez.config import config
from rez.utils.logging_ import print_debug
def _repl(s):
return s.replace('\\"', '"')
def _encode(s):
# this replaces all occurrances of '\"' with '"', *except* for those
# occurrances of '\"' that are within double quotes, which themselves
# are not escaped. In other words, the string:
# 'hey "there \"you\" " \"dude\" '
# ..will convert to:
# 'hey "there \"you\" " "dude" '
s_new = ''
prev_i = 0
for m in within_unescaped_quotes_regex.finditer(s):
s_ = s[prev_i:m.start()]
s_new += _repl(s_)
s_new += s[m.start():m.end()]
prev_i = m.end()
s_ = s[prev_i:]
s_new += _repl(s_)
return repr(s_new)
loc = []
for cmd in commands:
if annotate:
txt = "OLD COMMAND: %s" % cmd
line = "comment(%s)" % _encode(txt)
loc.append(line)
cmd = convert_old_command_expansions(cmd)
toks = cmd.strip().split()
try:
if toks[0] == "export":
var, value = cmd.split(' ', 1)[1].split('=', 1)
for bookend in ('"', "'"):
if value.startswith(bookend) and value.endswith(bookend):
value = value[1:-1]
break
# As the only old-style commands were Linux/Bash based,
# we assume using the default separator ":" is ok - we don't
# need to use os.pathsep as we don't expected to see a
# Windows path here.
separator = config.env_var_separators.get(var, ":")
# This is a special case. We don't want to include "';'" in
# our env var separators map as it's not really the correct
# behaviour/something we want to promote. It's included here for
# backwards compatibility only, and to not propogate elsewhere.
if var == "CMAKE_MODULE_PATH":
value = value.replace("'%s'" % separator, separator)
value = value.replace('"%s"' % separator, separator)
value = value.replace(":", separator)
parts = value.split(separator)
parts = [x for x in parts if x]
if len(parts) > 1:
idx = None
var1 = "$%s" % var
var2 = "${%s}" % var
if var1 in parts:
idx = parts.index(var1)
elif var2 in parts:
idx = parts.index(var2)
if idx in (0, len(parts) - 1):
func = "appendenv" if idx == 0 else "prependenv"
parts = parts[1:] if idx == 0 else parts[:-1]
val = separator.join(parts)
loc.append("%s('%s', %s)" % (func, var, _encode(val)))
continue
loc.append("setenv('%s', %s)" % (var, _encode(value)))
elif toks[0].startswith('#'):
loc.append("comment(%s)" % _encode(' '.join(toks[1:])))
elif toks[0] == "alias":
match = re.search("alias (?P<key>.*?)=(?P<value>.*)", cmd)
key = match.groupdict()['key'].strip()
value = match.groupdict()['value'].strip()
if (value.startswith('"') and value.endswith('"')) or \
(value.startswith("'") and value.endswith("'")):
value = value[1:-1]
loc.append("alias('%s', %s)" % (key, _encode(value)))
else:
# assume we can execute this as a straight command
loc.append("command(%s)" % _encode(cmd))
except:
# if anything goes wrong, just fall back to bash command
loc.append("command(%s)" % _encode(cmd))
rex_code = '\n'.join(loc)
if config.debug("old_commands"):
br = '-' * 80
msg = textwrap.dedent("""
%s
OLD COMMANDS:
%s
NEW COMMANDS:
%s
%s
""") % (br, '\n'.join(commands), rex_code, br)
print_debug(msg)
return rex_code
def pip_install_package(source_name,
pip_version=None,
python_version=None,
mode=InstallMode.min_deps,
release=False):
"""Install a pip-compatible python package as a rez package.
Args:
source_name (str): Name of package or archive/url containing the pip
package source. This is the same as the arg you would pass to
the 'pip install' command.
pip_version (str or `Version`): Version of pip to use to perform the
install, uses latest if None.
python_version (str or `Version`): Python version to use to perform the
install, and subsequently have the resulting rez package depend on.
mode (`InstallMode`): Installation mode, determines how dependencies are
managed.
release (bool): If True, install as a released package; otherwise, it
will be installed as a local package.
Returns:
2-tuple:
List of `Variant`: Installed variants;
List of `Variant`: Skipped variants (already installed).
"""
installed_variants = []
skipped_variants = []
pip_exe, context = find_pip(pip_version, python_version)
# TODO: should check if packages_path is writable before continuing with pip
#
packages_path = (config.release_packages_path
if release else config.local_packages_path)
tmpdir = mkdtemp(suffix="-rez", prefix="pip-")
stagingdir = os.path.join(tmpdir, "rez_staging")
stagingsep = "".join([os.path.sep, "rez_staging", os.path.sep])
destpath = os.path.join(stagingdir, "python")
# TODO use binpath once https://github.com/pypa/pip/pull/3934 is approved
binpath = os.path.join(stagingdir, "bin")
if context and config.debug("package_release"):
buf = StringIO()
print("\n\npackage download environment:", file=buf)
context.print_info(buf)
_log(buf.getvalue())
# Build pip commandline
cmd = [pip_exe, "install", "--use-pep517", "--target=%s" % destpath]
if mode == InstallMode.no_deps:
cmd.append("--no-deps")
cmd.append(source_name)
_cmd(context=context, command=cmd)
_system = System()
def pure_python_package(installed_dist):
true_table = {"true": True, "false": False}
packages = pkg_resources.find_distributions(destpath)
dist = next(
(package
for package in packages if package.key == installed_dist.key),
None)
wheel_data = dist.get_metadata('WHEEL')
# see https://www.python.org/dev/peps/pep-0566/#json-compatible-metadata
wheel_data = Parser().parsestr(wheel_data)
# see https://www.python.org/dev/peps/pep-0427/#what-s-the-deal-with-purelib-vs-platlib
return true_table[wheel_data["Root-Is-Purelib"]]
# Collect resulting python packages using distlib
distribution_path = DistributionPath([destpath])
distributions = [d for d in distribution_path.get_distributions()]
# moving bin folder to expected relative location as per wheel RECORD files
staged_binpath = os.path.join(destpath, "bin")
if os.path.isdir(staged_binpath):
shutil.move(os.path.join(destpath, "bin"), binpath)
for distribution in distribution_path.get_distributions():
requirements = []
if distribution.metadata.run_requires:
# Handle requirements. Currently handles conditional environment based
# requirements and normal requirements
# TODO: Handle optional requirements?
for requirement in distribution.metadata.run_requires:
if "environment" in requirement:
if interpret(requirement["environment"]):
requirements.extend(
_get_dependencies(requirement, distributions))
elif "extra" in requirement:
# Currently ignoring optional requirements
pass
else:
requirements.extend(
_get_dependencies(requirement, distributions))
tools = []
src_dst_lut = {}
for installed_file in distribution.list_installed_files():
# distlib expects the script files to be located in ../../bin/
# when in fact ../bin seems to be the resulting path after the
# installation as such we need to point the bin files to the
# expected location to match wheel RECORD files
installed_filepath = os.path.normpath(installed_file[0])
bin_prefix = os.path.join('..', '..', 'bin') + os.sep
if installed_filepath.startswith(bin_prefix):
# account for extra parentdir as explained above
installed = os.path.join(destpath, '_', installed_filepath)
else:
installed = os.path.join(destpath, installed_filepath)
source_file = os.path.normpath(installed)
if os.path.exists(source_file):
destination_file = os.path.relpath(source_file, stagingdir)
exe = False
if is_exe(source_file) and destination_file.startswith("bin" +
os.sep):
_file = os.path.basename(destination_file)
tools.append(_file)
exe = True
data = [destination_file, exe]
src_dst_lut[source_file] = data
else:
_log("Source file does not exist: " + source_file + "!")
def make_root(variant, path):
"""Using distlib to iterate over all installed files of the current
distribution to copy files to the target directory of the rez package
variant
"""
for source_file, data in src_dst_lut.items():
destination_file, exe = data
destination_file = os.path.normpath(
os.path.join(path, destination_file))
if not os.path.exists(os.path.dirname(destination_file)):
os.makedirs(os.path.dirname(destination_file))
shutil.copyfile(source_file, destination_file)
if exe:
shutil.copystat(source_file, destination_file)
# determine variant requirements
variant_reqs = []
pure = pure_python_package(distribution)
if not pure:
variant_reqs.append("platform-%s" % _system.platform)
variant_reqs.append("arch-%s" % _system.arch)
# Add the python version requirement. Note that we specify python to
# minor version because of environment markers - these often mean that
# you cannot use a loose python requirement (ie major version only)
# because then the package requirements would not be correct for all
# versions of python within that major version.
#
# This is not perfect. It means that often we will overspecify the required
# python version; and theoretically there could be markers that specify
# python down to the patch version. However, accurately varianting on
# python based on markers may be overly complicated, and may also
# result in odd varianting cases.
#
# https://www.python.org/dev/peps/pep-0508/#environment-markers
#
if context is None:
# since we had to use system pip, we have to assume system python version
py_ver = '.'.join(map(str, sys.version_info[:2]))
else:
python_variant = context.get_resolved_package("python")
py_ver = python_variant.version.trim(2)
variant_reqs.append("python-%s" % py_ver)
name = pip_to_rez_package_name(distribution)
with make_package(name, packages_path, make_root=make_root) as pkg:
pkg.version = pip_to_rez_version(distribution.version)
if distribution.metadata.summary:
pkg.description = distribution.metadata.summary
pkg.variants = [variant_reqs]
if requirements:
pkg.requires = requirements
commands = []
commands.append("env.PYTHONPATH.append('{root}/python')")
if tools:
pkg.tools = tools
commands.append("env.PATH.append('{root}/bin')")
pkg.commands = '\n'.join(commands)
installed_variants.extend(pkg.installed_variants or [])
skipped_variants.extend(pkg.skipped_variants or [])
# cleanup
shutil.rmtree(tmpdir)
return installed_variants, skipped_variants
def convert_old_commands(commands, annotate=True):
"""Converts old-style package commands into equivalent Rex code."""
from rez.config import config
from rez.utils.logging_ import print_debug
def _repl(s):
return s.replace('\\"', '"')
def _encode(s):
# this replaces all occurrances of '\"' with '"', *except* for those
# occurrances of '\"' that are within double quotes, which themselves
# are not escaped. In other words, the string:
# 'hey "there \"you\" " \"dude\" '
# ..will convert to:
# 'hey "there \"you\" " "dude" '
s_new = ''
prev_i = 0
for m in within_unescaped_quotes_regex.finditer(s):
s_ = s[prev_i:m.start()]
s_new += _repl(s_)
s_new += s[m.start():m.end()]
prev_i = m.end()
s_ = s[prev_i:]
s_new += _repl(s_)
return repr(s_new)
loc = []
for cmd in commands:
if annotate:
txt = "OLD COMMAND: %s" % cmd
line = "comment(%s)" % _encode(txt)
loc.append(line)
cmd = convert_old_command_expansions(cmd)
toks = cmd.strip().split()
if toks[0] == "export":
var, value = cmd.split(' ', 1)[1].split('=', 1)
for bookend in ('"', "'"):
if value.startswith(bookend) and value.endswith(bookend):
value = value[1:-1]
break
# As the only old-style commands were Linux/Bash based,
# we assume using the default separator ":" is ok - we don't
# need to use os.pathsep as we don't expected to see a
# Windows path here.
separator = config.env_var_separators.get(var, ":")
# This is a special case. We don't want to include "';'" in
# our env var separators map as it's not really the correct
# behaviour/something we want to promote. It's included here for
# backwards compatibility only, and to not propogate elsewhere.
if var == "CMAKE_MODULE_PATH":
value = value.replace("'%s'" % separator, separator)
value = value.replace('"%s"' % separator, separator)
value = value.replace(":", separator)
parts = value.split(separator)
parts = [x for x in parts if x]
if len(parts) > 1:
idx = None
var1 = "$%s" % var
var2 = "${%s}" % var
if var1 in parts:
idx = parts.index(var1)
elif var2 in parts:
idx = parts.index(var2)
if idx in (0, len(parts) - 1):
func = "appendenv" if idx == 0 else "prependenv"
parts = parts[1:] if idx == 0 else parts[:-1]
val = separator.join(parts)
loc.append("%s('%s', %s)" % (func, var, _encode(val)))
continue
loc.append("setenv('%s', %s)" % (var, _encode(value)))
elif toks[0].startswith('#'):
loc.append("comment(%s)" % _encode(' '.join(toks[1:])))
elif toks[0] == "alias":
match = re.search("alias (?P<key>.*?)=(?P<value>.*)", cmd)
key = match.groupdict()['key'].strip()
value = match.groupdict()['value'].strip()
if (value.startswith('"') and value.endswith('"')) or \
(value.startswith("'") and value.endswith("'")):
value = value[1:-1]
loc.append("alias('%s', %s)" % (key, _encode(value)))
else:
# assume we can execute this as a straight command
loc.append("command(%s)" % _encode(cmd))
rex_code = '\n'.join(loc)
if config.debug("old_commands"):
br = '-' * 80
msg = textwrap.dedent(
"""
%s
OLD COMMANDS:
%s
NEW COMMANDS:
%s
%s
""") % (br, '\n'.join(commands), rex_code, br)
print_debug(msg)
return rex_code
def log(msg):
if config.debug("bind_modules"):
print_debug(msg)
def pip_install_package(source_name, python_version=None,
release=False, no_deps=False,
prefix=None, auto_variants=True,
variants=None):
"""Install a pip-compatible python package as a rez package.
Args:
source_name (str): Name of package or archive/url containing the pip
package source. This is the same as the arg you would pass to
the 'pip install' command.
python_version (str or `Version`): Python version to use to perform the
install, and subsequently have the resulting rez package depend on.
prefix (str, optional): Override install location to here,
similar to `rez build --prefix`
no_deps (bool, optional): The `pip --no-deps` argument
auto_variants (bool, optional): Compute variants from the PyPI
classifiers portion of setup()
release (bool): If True, install as a released package; otherwise, it
will be installed as a local package.
prefix (str, optional): Override release path with this absolute path
Returns:
2-tuple:
List of `Variant`: Installed variants;
List of `Variant`: Skipped variants (already installed).
"""
installed_variants = []
skipped_variants = []
if prefix is not None:
config.release_packages_path = prefix
# TODO: should check if packages_path is writable
# before continuing with pip
#
packages_path = (config.release_packages_path if release
else config.local_packages_path)
tmpdir = mkdtemp(suffix="-rez", prefix="pip-")
stagingdir = os.path.join(tmpdir, "rez_staging")
stagingsep = "".join([os.path.sep, "rez_staging", os.path.sep])
destpath = os.path.join(stagingdir, "python")
python_exe, context = find_python(python_version)
if context and config.debug("package_release"):
buf = StringIO()
print >> buf, "\n\npackage download environment:"
context.print_info(buf)
_log(buf.getvalue())
# Build pip commandline
cmd = [
python_exe, "-m", "pip", "install",
"--target", destpath,
# Only ever consider wheels, anything else is ancient
"--use-pep517",
# Handle case where the Python distribution used alongside
# pip already has a package installed in its `site-packages/` dir.
"--ignore-installed",
]
if no_deps:
# Delegate the installation of dependencies to the user
# This is important, as each dependency may have different
# requirements of its own, and variants to go with it.
cmd.append("--no-deps")
cmd.append(source_name)
_cmd(context=context, command=cmd)
# Collect resulting python packages using distlib
distribution_path = DistributionPath([destpath])
distributions = [d for d in distribution_path.get_distributions()]
for distribution in distribution_path.get_distributions():
requirements = []
if distribution.metadata.run_requires:
# Handle requirements. Currently handles
# conditional environment based
# requirements and normal requirements
# TODO: Handle optional requirements?
for requirement in distribution.metadata.run_requires:
if "environment" in requirement:
if interpret(requirement["environment"]):
requirements.extend(_get_dependencies(
requirement, distributions))
elif "extra" in requirement:
# Currently ignoring optional requirements
pass
else:
requirements.extend(_get_dependencies(
requirement, distributions))
tools = []
src_dst_lut = {}
files = distribution.list_installed_files()
for installed_file in files:
source_file = os.path.join(destpath, installed_file[0])
source_file = os.path.normpath(source_file)
if os.path.exists(source_file):
destination_file = source_file.split(stagingsep)[1]
exe = False
starts_with_bin = destination_file.startswith(
"%s%s" % ("bin", os.path.sep)
)
if is_exe(source_file) and starts_with_bin:
_, _file = os.path.split(destination_file)
tools.append(_file)
exe = True
data = [destination_file, exe]
src_dst_lut[source_file] = data
else:
_log("Source file does not exist: " + source_file + "!")
def make_root(variant, path):
"""Using distlib to iterate over all installed files of the current
distribution to copy files to the target directory of the rez
package variant
"""
for source_file, data in src_dst_lut.items():
destination_file, exe = data
destination_file = os.path.join(path, destination_file)
destination_file = os.path.normpath(destination_file)
if not os.path.exists(os.path.dirname(destination_file)):
os.makedirs(os.path.dirname(destination_file))
shutil.copyfile(source_file, destination_file)
if exe:
shutil.copystat(source_file, destination_file)
name, _ = parse_name_and_version(distribution.name_and_version)
name = distribution.name[0:len(name)].replace("-", "_")
# determine variant requirements
variants_ = variants or []
if (not variants_) and auto_variants:
variants_.extend(wheel_to_variants(distribution))
if variants_:
print_info("'%s' - Automatically detected variants: %s" % (
name, ", ".join(variants_))
)
with make_package(name, packages_path, make_root=make_root) as pkg:
pkg.version = distribution.version
if distribution.metadata.summary:
pkg.description = distribution.metadata.summary
if variants_:
pkg.variants = [variants_]
if requirements:
pkg.requires = requirements
commands = []
commands.append("env.PYTHONPATH.append('{root}/python')")
if tools:
pkg.tools = tools
commands.append("env.PATH.append('{root}/bin')")
pkg.commands = '\n'.join(commands)
installed_variants.extend(pkg.installed_variants or [])
skipped_variants.extend(pkg.skipped_variants or [])
# cleanup
shutil.rmtree(tmpdir)
return installed_variants, skipped_variants
def _cmd(context, command):
cmd_str = ' '.join(quote(x) for x in command)
_log("running: %s" % cmd_str)
if context is None:
p = popen(command)
else:
p = context.execute_shell(command=command, block=False)
p.wait()
if p.returncode:
raise BuildError("Failed to download source with pip: %s" % cmd_str)
_verbose = config.debug("package_release")
def _log(msg):
if _verbose:
print_debug(msg)
# Copyright 2013-2016 Allan Johns.
#
# This library is free software: you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,