def _check_found(py_exe, version_text, log_invalid=True):
"""Check the Python and pip version text found.
Args:
py_exe (str or None): Python executable path found, if any.
version_text (str or None): Pip version found, if any.
log_invalid (bool): Whether to log messages if found invalid.
Returns:
bool: Python is OK and pip version fits against ``PIP_SPECIFIER``.
"""
is_valid = True
message = "Needs pip%s, but found '%s' for Python '%s'"
if version_text is None or not py_exe:
is_valid = False
if log_invalid:
print_debug(message, PIP_SPECIFIER, version_text, py_exe)
elif PackagingVersion(version_text) not in PIP_SPECIFIER:
is_valid = False
if log_invalid:
print_warning(message, PIP_SPECIFIER, version_text, py_exe)
return is_valid
def load_plugins(self):
import pkgutil
from rez.backport.importlib import import_module
type_module_name = 'rezplugins.' + self.type_name
package = import_module(type_module_name)
# on import, the `__path__` variable of the imported package is extended
# to include existing directories on the plugin search path (via
# extend_path, above). this means that `walk_packages` will walk over all
# modules on the search path at the same level (.e.g in a
# 'rezplugins/type_name' sub-directory).
paths = [package.__path__] if isinstance(package.__path__, basestring) \
else package.__path__
for path in paths:
for loader, modname, ispkg in pkgutil.walk_packages(
[path], package.__name__ + '.'):
if loader is not None:
plugin_name = modname.split('.')[-1]
if plugin_name.startswith('_'):
continue
if config.debug("plugins"):
print_debug("loading %s plugin at %s: %s..."
% (self.type_name, path, modname))
try:
# load_module will force reload the module if it's
# already loaded, so check for that
module = sys.modules.get(modname)
if module is None:
module = loader.find_module(modname).load_module(modname)
if hasattr(module, 'register_plugin') and \
hasattr(module.register_plugin, '__call__'):
plugin_class = module.register_plugin()
if plugin_class != None:
self.register_plugin(plugin_name, plugin_class, module)
else:
if config.debug("plugins"):
print_warning(
"'register_plugin' function at %s: %s did not return a class."
% (path, modname))
else:
if config.debug("plugins"):
print_warning(
"no 'register_plugin' function at %s: %s"
% (path, modname))
# delete from sys.modules?
except Exception as e:
nameish = modname.split('.')[-1]
self.failed_plugins[nameish] = str(e)
if config.debug("plugins"):
import traceback
from StringIO import StringIO
out = StringIO()
traceback.print_exc(file=out)
print_debug(out.getvalue())
# load config
data, _ = _load_config_from_filepaths([os.path.join(path, "rezconfig")])
deep_update(self.config_data, data)
def get_changelog(self, previous_revision=None, max_revisions=None):
prev_commit = None
if previous_revision is not None:
try:
prev_commit = previous_revision["commit"]
except:
if self.package.config.debug("package_release"):
print_debug("couldn't determine previous commit from: %r" %
previous_revision)
args = ["log"]
if max_revisions:
args.extend(["-n", str(max_revisions)])
if prev_commit:
# git returns logs to last common ancestor, so even if previous
# release was from a different branch, this is ok
try:
commit_range = "%s..HEAD" % prev_commit
stdout = self.git("log", commit_range)
except ReleaseVCSError:
# Special case where the sha stored in the latest version does not exists due to the
# git ->github migration where we rewrote the history of the repos
stdout = self.git("log")
else:
stdout = self.git("log")
return '\n'.join(stdout)
def _cmd(self, *nargs):
"""Convenience function for executing a program such as 'git' etc."""
cmd_str = ' '.join(map(quote, nargs))
if self.package.config.debug("package_release"):
print_debug("Running command: %s" % cmd_str)
p = Popen(nargs,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=self.pkg_root,
text=True)
out, err = p.communicate()
if p.returncode:
print_debug("command stdout:")
print_debug(out)
print_debug("command stderr:")
print_debug(err)
raise ReleaseVCSError("command failed: %s\n%s" % (cmd_str, err))
out = out.strip()
if out:
return [x.rstrip() for x in out.split('\n')]
else:
return []
def load_plugins(self):
import pkgutil
from rez.backport.importlib import import_module
type_module_name = 'rezplugins.' + self.type_name
package = import_module(type_module_name)
# on import, the `__path__` variable of the imported package is extended
# to include existing directories on the plugin search path (via
# extend_path, above). this means that `walk_packages` will walk over all
# modules on the search path at the same level (.e.g in a
# 'rezplugins/type_name' sub-directory).
paths = [package.__path__] if isinstance(package.__path__, basestring) \
else package.__path__
for path in paths:
for loader, modname, ispkg in pkgutil.walk_packages(
[path], package.__name__ + '.'):
if loader is not None:
plugin_name = modname.split('.')[-1]
if plugin_name.startswith('_'):
continue
if config.debug("plugins"):
print_debug("loading %s plugin at %s: %s..."
% (self.type_name, path, modname))
try:
# load_module will force reload the module if it's
# already loaded, so check for that
module = sys.modules.get(modname)
if module is None:
module = loader.find_module(modname).load_module(modname)
if hasattr(module, 'register_plugin') and \
hasattr(module.register_plugin, '__call__'):
plugin_class = module.register_plugin()
if plugin_class != None:
self.register_plugin(plugin_name, plugin_class, module)
else:
if config.debug("plugins"):
print_warning(
"'register_plugin' function at %s: %s did not return a class."
% (path, modname))
else:
if config.debug("plugins"):
print_warning(
"no 'register_plugin' function at %s: %s"
% (path, modname))
# delete from sys.modules?
except Exception as e:
nameish = modname.split('.')[-1]
self.failed_plugins[nameish] = str(e)
if config.debug("plugins"):
import traceback
from StringIO import StringIO
out = StringIO()
traceback.print_exc(file=out)
print_debug(out.getvalue())
# load config
data, _ = _load_config_from_filepaths([os.path.join(path, "rezconfig")])
deep_update(self.config_data, data)
def get_changelog(self, previous_revision=None, max_revisions=None):
prev_commit = None
if previous_revision is not None:
try:
prev_commit = previous_revision["commit"]
except:
if self.package.config.debug("package_release"):
print_debug("couldn't determine previous commit from: %r" %
previous_revision)
args = ["log"]
if max_revisions:
args.extend(["-l", str(max_revisions)])
if prev_commit:
# git behavior is to simply print the log from the last common
# ancsestor... which is apparently desired. so we'll mimic that
# however, we want to print in order from most recent to oldest,
# because:
# a) if the log gets truncated, we want to cut off the
# oldest commits, not the current one, and
# b) this mimics the order they're printed in git
# c) this mimics the order they're printed if you have no
# previous_revision, and just do "hg log"
# d) if max_revisions is giving, want limiting will only take the
# most recent N entries
commit_range = "reverse(ancestor(%s, .)::.)" % prev_commit
args.extend(["-r", commit_range])
stdout = self.hg(*args)
return '\n'.join(stdout)
def get_changelog(self, previous_revision=None, max_revisions=None):
prev_commit = None
if previous_revision is not None:
try:
prev_commit = previous_revision["commit"]
except:
if self.package.config.debug("package_release"):
print_debug("couldn't determine previous commit from: %r"
% previous_revision)
args = ["log"]
if max_revisions:
args.extend(["-l", str(max_revisions)])
if prev_commit:
# git behavior is to simply print the log from the last common
# ancsestor... which is apparently desired. so we'll mimic that
# however, we want to print in order from most recent to oldest,
# because:
# a) if the log gets truncated, we want to cut off the
# oldest commits, not the current one, and
# b) this mimics the order they're printed in git
# c) this mimics the order they're printed if you have no
# previous_revision, and just do "hg log"
# d) if max_revisions is giving, want limiting will only take the
# most recent N entries
commit_range = "reverse(ancestor(%s, .)::.)" % prev_commit
args.extend(["-r", commit_range])
stdout = self.hg(*args)
return '\n'.join(stdout)
def get_changelog(self, previous_revision=None, max_revisions=None):
prev_commit = None
if previous_revision is not None:
try:
prev_commit = previous_revision["commit"]
except:
if self.package.config.debug("package_release"):
print_debug("couldn't determine previous commit from: %r"
% previous_revision)
args = ["log"]
if max_revisions:
args.extend(["-n", str(max_revisions)])
if prev_commit:
# git returns logs to last common ancestor, so even if previous
# release was from a different branch, this is ok
try:
commit_range = "%s..HEAD" % prev_commit
stdout = self.git("log", commit_range)
except ReleaseVCSError:
# Special case where the sha stored in the latest version does not exists due to the
# git ->github migration where we rewrote the history of the repos
stdout = self.git("log")
else:
stdout = self.git("log")
return '\n'.join(stdout)
def _data(self):
if not self.schema:
return None
data = self._load()
if config.debug("resources"):
print_debug("Loaded resource: %s" % str(self))
return data
def _data(self):
if not self.schema:
return None
data = self._load()
if config.debug("resources"):
print_debug("Loaded resource: %s" % str(self))
return data
def load_module(self, name, package):
from hashlib import sha1
from rez.config import config # avoiding circular import
from rez.developer_package import DeveloperPackage
# in rare cases, a @late bound function may get called before the
# package is built. An example is 'requires' and the other requires-like
# functions. These need to be evaluated before a build, but it does also
# make sense to sometimes implement these as late-bound functions. We
# detect this case here, and load the modules from the original (pre-
# copied into package payload) location.
#
if isinstance(package, DeveloperPackage):
# load sourcefile from original location
path = config.package_definition_python_path
filepath = os.path.join(path, "%s.py" % name)
if not os.path.exists(filepath):
return None
with open(filepath, "rb") as f:
hash_str = sha1(f.read().strip()).hexdigest()
else:
# load sourcefile that's been copied into package install payload
path = os.path.join(package.base, self.include_modules_subpath)
pathname = os.path.join(path, "%s.py" % name)
hashname = os.path.join(path, "%s.sha1" % name)
if os.path.isfile(pathname) and os.path.isfile(hashname):
with open(hashname, "r") as f:
hash_str = f.readline()
filepath = pathname
else:
# Fallback for backward compat
pathname = os.path.join(path, "%s-*.py" % name)
hashnames = glob(pathname)
if not hashnames:
return None
filepath = hashnames[0]
hash_str = filepath.rsplit('-', 1)[-1].split('.', 1)[0]
# End, for details of backward compat,
# see https://github.com/nerdvegas/rez/issues/934
# and https://github.com/nerdvegas/rez/pull/935
module = self.modules.get(hash_str)
if module is not None:
return module
if config.debug("file_loads"):
print_debug("Loading include sourcefile: %s" % filepath)
module = py23.load_module_from_file(name, filepath)
self.modules[hash_str] = module
return module
def append_if_valid(dir_):
if os.path.isdir(dir_):
subdir = os.path.normcase(os.path.join(dir_, pname))
initfile = os.path.join(subdir, init_py)
if subdir not in path and os.path.isfile(initfile):
path.append(subdir)
elif config.debug("plugins"):
print_debug("skipped nonexistant rez plugin path: %s" % dir_)
def create_release_hooks(names, source_path):
hooks = []
for name in names:
try:
hook = create_release_hook(name, source_path)
hooks.append(hook)
except Exception:
import traceback
print_warning("Release hook '%s' is not available." % name)
print_debug(traceback.format_exc())
return hooks
def _load_file(filepath, format_, update_data_callback):
load_func = load_functions[format_]
if config.debug("file_loads"):
print_debug("Loading file: %s" % filepath)
with open(filepath) as f:
result = load_func(f, filepath=filepath)
if update_data_callback:
result = update_data_callback(format_, result)
return result
def _load_file(filepath, format_, update_data_callback):
load_func = load_functions[format_]
if config.debug("file_loads"):
print_debug("Loading file: %s" % filepath)
with open(filepath) as f:
result = load_func(f, filepath=filepath)
if update_data_callback:
result = update_data_callback(format_, result)
return result
def load_module(self, name, package):
from rez.config import config # avoiding circular import
from rez.developer_package import DeveloperPackage
# in rare cases, a @late bound function may get called before the
# package is built. An example is 'requires' and the other requires-like
# functions. These need to be evaluated before a build, but it does also
# make sense to sometimes implement these as late-bound functions. We
# detect this case here, and load the modules from the original (pre-
# copied into package payload) location.
#
if isinstance(package, DeveloperPackage):
from hashlib import sha1
# load sourcefile from original location
path = config.package_definition_python_path
filepath = os.path.join(path, "%s.py" % name)
if not os.path.exists(filepath):
return None
with open(filepath) as f:
txt = f.read().strip()
hash_str = sha1(txt).hexdigest()
else:
# load sourcefile that's been copied into package install payload
path = os.path.join(package.base, self.include_modules_subpath)
pathname = os.path.join(path, "%s-*.py" % name)
pathnames = glob(pathname)
if not pathnames:
return None
filepath = pathnames[0]
hash_str = filepath.rsplit('-', 1)[-1].split('.', 1)[0]
module = self.modules.get(hash_str)
if module is not None:
return module
if config.debug("file_loads"):
print_debug("Loading include sourcefile: %s" % filepath)
with open(filepath) as f:
module = imp.load_source(name, filepath, f)
self.modules[hash_str] = module
return module
def load_module(self, name, package):
from rez.config import config # avoiding circular import
from rez.developer_package import DeveloperPackage
# in rare cases, a @late bound function may get called before the
# package is built. An example is 'requires' and the other requires-like
# functions. These need to be evaluated before a build, but it does also
# make sense to sometimes implement these as late-bound functions. We
# detect this case here, and load the modules from the original (pre-
# copied into package payload) location.
#
if isinstance(package, DeveloperPackage):
from hashlib import sha1
# load sourcefile from original location
path = config.package_definition_python_path
filepath = os.path.join(path, "%s.py" % name)
if not os.path.exists(filepath):
return None
with open(filepath) as f:
txt = f.read().strip()
hash_str = sha1(txt).hexdigest()
else:
# load sourcefile that's been copied into package install payload
path = os.path.join(package.base, self.include_modules_subpath)
pathname = os.path.join(path, "%s-*.py" % name)
pathnames = glob(pathname)
if not pathnames:
return None
filepath = pathnames[0]
hash_str = filepath.rsplit('-', 1)[-1].split('.', 1)[0]
module = self.modules.get(hash_str)
if module is not None:
return module
if config.debug("file_loads"):
print_debug("Loading include sourcefile: %s" % filepath)
with open(filepath) as f:
module = imp.load_source(name, filepath, f)
self.modules[hash_str] = module
return module
def test_print(self):
"""Test valid msg and nargs combinations for print_*."""
for msg in ("Hello", "Hello %s", "Hello %s %s"):
logging_.print_debug(msg)
logging_.print_info(msg)
logging_.print_warning(msg)
logging_.print_error(msg)
logging_.print_critical(msg)
for nargs in ([], ["foo"], ["foo", "bar"]):
logging_.print_debug(msg, *nargs)
logging_.print_info(msg, *nargs)
logging_.print_warning(msg, *nargs)
logging_.print_error(msg, *nargs)
logging_.print_critical(msg, *nargs)
def test_print(self):
"""Test valid msg and nargs combinations for print_*."""
for msg in ("Hello", "Hello %s", "Hello %s %s"):
logging_.print_debug(msg)
logging_.print_info(msg)
logging_.print_warning(msg)
logging_.print_error(msg)
logging_.print_critical(msg)
for nargs in ([], ["foo"], ["foo", "bar"]):
logging_.print_debug(msg, *nargs)
logging_.print_info(msg, *nargs)
logging_.print_warning(msg, *nargs)
logging_.print_error(msg, *nargs)
logging_.print_critical(msg, *nargs)
def publish_message(self, data):
if not self.settings.host:
print_error("Did not publish message, host is not specified")
return
routing_key = self.settings.exchange_routing_key
print("Publishing AMQP message on %s..." % routing_key)
publish_message(host=self.settings.host,
amqp_settings=self.settings,
routing_key=routing_key,
data=data)
if config.debug("package_release"):
print_debug("Published message: %s" % (data))
def extend_path(path, name):
"""Extend a package's path.
Intended use is to place the following code in a package's __init__.py:
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
This will add to the package's __path__ all subdirectories of
directories on sys.path named after the package. This is useful
if one wants to distribute different parts of a single logical
package as multiple directories.
If the input path is not a list (as is the case for frozen
packages) it is returned unchanged. The input path is not
modified; an extended copy is returned. Items are only appended
to the copy at the end.
It is assumed that 'plugin_path' is a sequence. Items of 'plugin_path'
that are not (unicode or 8-bit) strings referring to existing
directories are ignored. Unicode items of sys.path that cause
errors when used as filenames may cause this function to raise an
exception (in line with os.path.isdir() behavior).
"""
if not isinstance(path, list):
# This could happen e.g. when this is called from inside a
# frozen package. Return the path unchanged in that case.
return path
pname = os.path.join(*name.split('.')) # Reconstitute as relative path
# Just in case os.extsep != '.'
init_py = "__init__" + os.extsep + "py"
path = path[:]
for dir in config.plugin_path:
if not os.path.isdir(dir):
if config.debug("plugins"):
print_debug("skipped nonexistant rez plugin path: %s" % dir)
continue
subdir = os.path.join(dir, pname)
# XXX This may still add duplicate entries to path on
# case-insensitive filesystems
initfile = os.path.join(subdir, init_py)
if subdir not in path and os.path.isfile(initfile):
path.append(subdir)
return path
def extend_path(path, name):
"""Extend a package's path.
Intended use is to place the following code in a package's __init__.py:
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
This will add to the package's __path__ all subdirectories of
directories on sys.path named after the package. This is useful
if one wants to distribute different parts of a single logical
package as multiple directories.
If the input path is not a list (as is the case for frozen
packages) it is returned unchanged. The input path is not
modified; an extended copy is returned. Items are only appended
to the copy at the end.
It is assumed that 'plugin_path' is a sequence. Items of 'plugin_path'
that are not (unicode or 8-bit) strings referring to existing
directories are ignored. Unicode items of sys.path that cause
errors when used as filenames may cause this function to raise an
exception (in line with os.path.isdir() behavior).
"""
if not isinstance(path, list):
# This could happen e.g. when this is called from inside a
# frozen package. Return the path unchanged in that case.
return path
pname = os.path.join(*name.split('.')) # Reconstitute as relative path
# Just in case os.extsep != '.'
init_py = "__init__" + os.extsep + "py"
path = path[:]
for dir in config.plugin_path:
if not os.path.isdir(dir):
if config.debug("plugins"):
print_debug("skipped nonexistant rez plugin path: %s" % dir)
continue
subdir = os.path.join(dir, pname)
# XXX This may still add duplicate entries to path on
# case-insensitive filesystems
initfile = os.path.join(subdir, init_py)
if subdir not in path and os.path.isfile(initfile):
path.append(subdir)
return path
def publish_message(self, data):
if not self.settings.host:
print_error("Did not publish message, host is not specified")
return
try:
conn = Connection(host=self.settings.host,
port=self.settings.port,
userid=self.settings.userid,
password=self.settings.password,
connect_timeout=self.settings.connect_timeout)
except socket.error as e:
print_error("Cannot connect to the message broker: %s" % (e))
return
channel = conn.channel()
# Declare the exchange
try:
channel.exchange_declare(
self.settings.exchange_name,
self.settings.exchange_type,
durable=self.settings.exchange_durable,
auto_delete=self.settings.exchange_auto_delete)
except Exception as e:
print_error("Failed to declare an exchange: %s" % (e))
return
# build the message
msg = basic_message.Message(
body=json.dumps(data),
delivery_mode=self.settings.message_delivery_mode,
content_type="application/json",
content_encoding="utf-8")
routing_key = self.settings.exchange_routing_key
print "Publishing AMQP message on %s..." % routing_key
# publish the message
try:
channel.basic_publish(msg, self.settings.exchange_name,
routing_key)
except Exception as e:
print_error("Failed to publish message: %s" % (e))
return
if config.debug("package_release"):
print_debug("Published message: %s" % (data))
def _cmd(self, *nargs):
"""Convenience function for executing a program such as 'git' etc."""
cmd_str = ' '.join(nargs)
if self.package.config.debug("package_release"):
print_debug("Running command: %s" % cmd_str)
p = subprocess.Popen(nargs, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, cwd=self.pkg_root)
out, err = p.communicate()
if p.returncode:
raise ReleaseVCSError("command failed: %s\n%s" % (cmd_str, err))
out = out.strip()
if out:
return [x.rstrip() for x in out.split('\n')]
else:
return []
def publish_message(self, data):
if not self.settings.host:
print_error("Did not publish message, host is not specified")
return
try:
conn = Connection(host=self.settings.host,
port=self.settings.port,
userid=self.settings.userid,
password=self.settings.password,
connect_timeout=self.settings.connect_timeout)
except socket.error as e:
print_error("Cannot connect to the message broker: %s" % (e))
return
channel = conn.channel()
# Declare the exchange
try:
channel.exchange_declare(
self.settings.exchange_name,
self.settings.exchange_type,
durable=self.settings.exchange_durable,
auto_delete=self.settings.exchange_auto_delete)
except Exception as e:
print_error("Failed to declare an exchange: %s" % (e))
return
# build the message
msg = basic_message.Message(
body=json.dumps(data),
delivery_mode=self.settings.message_delivery_mode,
content_type="application/json",
content_encoding="utf-8")
routing_key = self.settings.exchange_routing_key
print "Publishing AMQP message on %s..." % routing_key
# publish the message
try:
channel.basic_publish(msg, self.settings.exchange_name, routing_key)
except Exception as e:
print_error("Failed to publish message: %s" % (e))
return
if config.debug("package_release"):
print_debug("Published message: %s" % (data))
def _release(self, commands, errors=None):
for conf in commands:
if self.settings.print_commands or config.debug("package_release"):
from subprocess import list2cmdline
toks = [conf["command"]] + conf.get("args", [])
msg = "running command: %s" % list2cmdline(toks)
if self.settings.print_commands:
print msg
else:
print_debug(msg)
if not self.execute_command(cmd_name=conf.get("command"),
cmd_arguments=conf.get("args"),
user=conf.get("user"),
errors=errors):
if self.settings.stop_on_error:
return
def post_release(self, user, install_path, variants, **kwargs):
# note that the package we use here is the *installed* package, not the
# developer package (self.package). Otherwise, attributes such as 'root'
# will be None
errors = []
if variants:
package = variants[0].parent
else:
package = self.package
self._execute_commands(self.settings.post_release_commands,
install_path=install_path,
package=package,
errors=errors)
if errors:
print_debug("The following post-release commands failed:\n"
+ '\n\n'.join(errors))
def load_plugin_cmd():
"""Load subcommand from command type plugin
The command type plugin module should have attribute `command_behavior`,
and the value must be a dict if provided. For example:
# in your command plugin module
command_behavior = {
"hidden": False, # (bool): default False
"arg_mode": None, # (str): "passthrough", "grouped", default None
}
If the attribute not present, default behavior will be given.
"""
from rez.config import config
from rez.utils.logging_ import print_debug
from rez.plugin_managers import plugin_manager
ext_plugins = dict()
for plugin_name in plugin_manager.get_plugins("command"):
module = plugin_manager.get_plugin_module("command", plugin_name)
behavior = getattr(module, "command_behavior", None)
if behavior is None:
behavior = dict()
if config.debug("plugins"):
print_debug("Attribute 'command_behavior' not found in plugin "
"module %s, registering with default behavior." %
module.__name__)
try:
data = behavior.copy()
data.update({"module_name": module.__name__})
ext_plugins[plugin_name] = data
except Exception:
if config.debug("plugins"):
import traceback
from rez.vendor.six.six import StringIO
out = StringIO()
traceback.print_exc(file=out)
print_debug(out.getvalue())
return ext_plugins
def post_release(self, user, install_path, variants, **kwargs):
# note that the package we use here is the *installed* package, not the
# developer package (self.package). Otherwise, attributes such as 'root'
# will be None
errors = []
if variants:
package = variants[0].parent
else:
package = self.package
self._execute_commands(self.settings.post_release_commands,
install_path=install_path,
package=package,
errors=errors)
if errors:
print_debug("The following post-release commands failed:\n" +
'\n\n'.join(errors))
def get_changelog(self, previous_revision=None):
prev_commit = None
if previous_revision is not None:
try:
prev_commit = previous_revision["commit"]
except:
if self.package.config.debug("package_release"):
print_debug("couldn't determine previous commit from: %r"
% previous_revision)
if prev_commit:
# git returns logs to last common ancestor, so even if previous
# release was from a different branch, this is ok
commit_range = "%s..HEAD" % prev_commit
stdout = self.git("log", commit_range)
else:
stdout = self.git("log")
return '\n'.join(stdout)
def get_changelog(self, previous_revision=None):
prev_commit = None
if previous_revision is not None:
try:
prev_commit = previous_revision["commit"]
except:
if self.package.config.debug("package_release"):
print_debug("couldn't determine previous commit from: %r" %
previous_revision)
if prev_commit:
# git behavior is to simply print the log from the last common
# ancsestor... which is apparently desired. so we'll mimic that
commit_range = "ancestor(%s, .)::." % prev_commit
stdout = self.hg("log", "-r", commit_range)
else:
stdout = self.hg("log")
return '\n'.join(stdout)
def get_changelog(self, previous_revision=None):
prev_commit = None
if previous_revision is not None:
try:
prev_commit = previous_revision["commit"]
except:
if self.package.config.debug("package_release"):
print_debug("couldn't determine previous commit from: %r" % previous_revision)
if prev_commit:
# git returns logs to last common ancestor, so even if previous
# release was from a different branch, this is ok
commit_range = "%s..HEAD" % prev_commit
stdout = self.git("log", commit_range)
else:
stdout = self.git("log")
return "\n".join(stdout)
def get_changelog(self, previous_revision=None):
prev_commit = None
if previous_revision is not None:
try:
prev_commit = previous_revision["commit"]
except:
if self.package.config.debug("package_release"):
print_debug("couldn't determine previous commit from: %r"
% previous_revision)
if prev_commit:
# git behavior is to simply print the log from the last common
# ancsestor... which is apparently desired. so we'll mimic that
commit_range = "ancestor(%s, .)::." % prev_commit
stdout = self.hg("log", "-r", commit_range)
else:
stdout = self.hg("log")
return '\n'.join(stdout)
def _execute_commands(self, commands, install_path, package, errors=None):
release_dict = dict(path=install_path)
formatter = scoped_formatter(system=system,
release=release_dict,
package=package)
for conf in commands:
program = conf["command"]
env_ = None
env = conf.get("env")
if env:
env_ = os.environ.copy()
env_.update(env)
args = conf.get("args", [])
args = [formatter.format(x) for x in args]
args = [expandvars(x, environ=env_) for x in args]
if self.settings.print_commands or config.debug("package_release"):
from subprocess import list2cmdline
toks = [program] + args
msgs = []
msgs.append("running command: %s" % list2cmdline(toks))
if env:
for key, value in env.iteritems():
msgs.append(" with: %s=%s" % (key, value))
if self.settings.print_commands:
print '\n'.join(msgs)
else:
for msg in msgs:
print_debug(msg)
if not self.execute_command(cmd_name=program,
cmd_arguments=args,
user=conf.get("user"),
errors=errors,
env=env_):
if self.settings.stop_on_error:
return
def _execute_commands(self, commands, install_path, package, errors=None):
release_dict = dict(path=install_path)
formatter = scoped_formatter(system=system,
release=release_dict,
package=package)
for conf in commands:
program = conf["command"]
env_ = None
env = conf.get("env")
if env:
env_ = os.environ.copy()
env_.update(env)
args = conf.get("args", [])
args = [formatter.format(x) for x in args]
args = [expandvars(x, environ=env_) for x in args]
if self.settings.print_commands or config.debug("package_release"):
from subprocess import list2cmdline
toks = [program] + args
msgs = []
msgs.append("running command: %s" % list2cmdline(toks))
if env:
for key, value in env.iteritems():
msgs.append(" with: %s=%s" % (key, value))
if self.settings.print_commands:
print '\n'.join(msgs)
else:
for msg in msgs:
print_debug(msg)
if not self.execute_command(cmd_name=program,
cmd_arguments=args,
user=conf.get("user"),
errors=errors,
env=env_):
if self.settings.stop_on_error:
return
def get_changelog(self, previous_revision=None, max_revisions=None):
prev_commit = None
if previous_revision is not None:
try:
prev_commit = previous_revision["commit"]
except:
if self.package.config.debug("package_release"):
print_debug("couldn't determine previous commit from: %r"
% previous_revision)
args = ["log"]
if max_revisions:
args.extend(["-n", str(max_revisions)])
if prev_commit:
# git returns logs to last common ancestor, so even if previous
# release was from a different branch, this is ok
commit_range = "%s..HEAD" % prev_commit
args.append(commit_range)
stdout = self.git(*args)
return '\n'.join(stdout)
def get_changelog(self, previous_revision=None, max_revisions=None):
prev_commit = None
if previous_revision is not None:
try:
prev_commit = previous_revision["commit"]
except:
if self.package.config.debug("package_release"):
print_debug("couldn't determine previous commit from: %r" %
previous_revision)
args = ["log"]
if max_revisions:
args.extend(["-n", str(max_revisions)])
if prev_commit:
# git returns logs to last common ancestor, so even if previous
# release was from a different branch, this is ok
commit_range = "%s..HEAD" % prev_commit
args.append(commit_range)
stdout = self.git(*args)
return '\n'.join(stdout)
def _execute_commands(self,
commands,
install_path,
package,
errors=None,
variants=None):
release_dict = dict(path=install_path)
variant_infos = []
if variants:
for variant in variants:
if isinstance(variant, (int, long)):
variant_infos.append(variant)
else:
package = variant.parent
var_dict = dict(variant.resource.variables)
# using '%s' will preserve potential str/unicode nature
var_dict['variant_requires'] = [
'%s' % x for x in variant.resource.variant_requires
]
variant_infos.append(var_dict)
formatter = scoped_formatter(system=system,
release=release_dict,
package=package,
variants=variant_infos,
num_variants=len(variant_infos))
for conf in commands:
program = conf["command"]
env_ = None
env = conf.get("env")
if env:
env_ = os.environ.copy()
env_.update(env)
# If we have, ie, a list, and format_pretty is True, it will be printed
# as "1 2 3" instead of "[1, 2, 3]"
formatter.__dict__['format_pretty'] = conf.get("pretty_args", True)
args = conf.get("args", [])
args = [formatter.format(x) for x in args]
args = [expandvars(x, environ=env_) for x in args]
if self.settings.print_commands or config.debug("package_release"):
from subprocess import list2cmdline
toks = [program] + args
msgs = []
msgs.append("running command: %s" % list2cmdline(toks))
if env:
for key, value in env.iteritems():
msgs.append(" with: %s=%s" % (key, value))
if self.settings.print_commands:
print('\n'.join(msgs))
else:
for msg in msgs:
print_debug(msg)
if not self.execute_command(cmd_name=program,
cmd_arguments=args,
user=conf.get("user"),
errors=errors,
env=env_):
if self.settings.stop_on_error:
return
def _log(msg):
if _verbose:
print_debug(msg)
def log(msg):
if config.debug("bind_modules"):
print_debug(msg)
def _log(msg):
if _verbose:
print_debug(msg)
def find_pip_from_context(python_version, pip_version=None):
"""Find pip from rez context.
Args:
python_version (str or `Version`): Python version to use
pip_version (str or `Version`): Version of pip to use, or latest.
Returns:
3-tuple:
- str: Python executable or None if we fell back to system pip.
- str: Pip version or None if we fell back to system pip.
- `ResolvedContext`: Context containing pip, or None if we fell back
to system pip.
"""
target = "python"
package_request = []
if python_version:
ver = Version(str(python_version))
python_major_minor_ver = ver.trim(2)
else:
# use latest major.minor
package = get_latest_package("python")
if package:
python_major_minor_ver = package.version.trim(2)
else:
raise BuildError("Found no python package.")
python_package = "python-%s" % str(python_major_minor_ver)
package_request.append(python_package)
if pip_version:
target = "pip"
if pip_version == "latest":
package_request.append("pip")
else:
package_request.append("pip-%s" % str(pip_version))
print_info("Trying to use pip from %s package", target)
try:
context = ResolvedContext(package_request)
except (PackageFamilyNotFoundError, PackageNotFoundError):
print_debug("No rez package called %s found", target)
return None, None, None
py_exe_name = "python"
if platform_.name != "windows":
# Python < 2 on Windows doesn't have versionned executable.
py_exe_name += str(python_major_minor_ver.trim(1))
py_exe = context.which(py_exe_name)
proc = context.execute_command(
# -E and -s are used to isolate the environment as much as possible.
# See python --help for more details. We absolutely don't want to get
# pip from the user home.
[
py_exe, "-E", "-s", "-c",
"import pip, sys; sys.stdout.write(pip.__version__)"
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True)
out, err = proc.communicate()
if proc.returncode:
print_debug("Failed to get pip from package %s", target)
print_debug(out)
print_debug(err)
return None, None, None
pip_version = out.strip()
variant = context.get_resolved_package(target)
package = variant.parent
print_info("Found pip-%s inside %s. Will use it with %s", pip_version,
package.uri, py_exe)
return py_exe, pip_version, context
def _execute_commands(self, commands, install_path, package, errors=None,
variants=None):
release_dict = dict(path=install_path)
variant_infos = []
if variants:
for variant in variants:
if isinstance(variant, (int, long)):
variant_infos.append(variant)
else:
package = variant.parent
var_dict = dict(variant.resource.variables)
# using '%s' will preserve potential str/unicode nature
var_dict['variant_requires'] = ['%s' % x
for x in variant.resource.variant_requires]
variant_infos.append(var_dict)
formatter = scoped_formatter(system=system,
release=release_dict,
package=package,
variants=variant_infos,
num_variants=len(variant_infos))
for conf in commands:
program = conf["command"]
env_ = None
env = conf.get("env")
if env:
env_ = os.environ.copy()
env_.update(env)
# If we have, ie, a list, and format_pretty is True, it will be printed
# as "1 2 3" instead of "[1, 2, 3]"
formatter.__dict__['format_pretty'] = conf.get(
"pretty_args", True)
args = conf.get("args", [])
args = [formatter.format(x) for x in args]
args = [expandvars(x, environ=env_) for x in args]
if self.settings.print_commands or config.debug("package_release"):
from subprocess import list2cmdline
toks = [program] + args
msgs = []
msgs.append("running command: %s" % list2cmdline(toks))
if env:
for key, value in env.iteritems():
msgs.append(" with: %s=%s" % (key, value))
if self.settings.print_commands:
print '\n'.join(msgs)
else:
for msg in msgs:
print_debug(msg)
if not self.execute_command(cmd_name=program,
cmd_arguments=args,
user=conf.get("user"),
errors=errors,
env=env_):
if self.settings.stop_on_error:
return
def load_plugins(self):
import pkgutil
from rez.backport.importlib import import_module
type_module_name = 'rezplugins.' + self.type_name
package = import_module(type_module_name)
# on import, the `__path__` variable of the imported package is extended
# to include existing directories on the plugin search path (via
# extend_path, above). this means that `walk_packages` will walk over all
# modules on the search path at the same level (.e.g in a
# 'rezplugins/type_name' sub-directory).
paths = [package.__path__] if isinstance(package.__path__, basestring) \
else package.__path__
# reverse plugin path order, so that custom plugins have a chance to
# be found before the builtin plugins (from /rezplugins).
paths = reversed(paths)
for path in paths:
if config.debug("plugins"):
print_debug("searching plugin path %s...", path)
for importer, modname, ispkg in pkgutil.iter_modules(
[path], package.__name__ + '.'):
if importer is None:
continue
plugin_name = modname.split('.')[-1]
if plugin_name.startswith('_') or plugin_name == 'rezconfig':
continue
if plugin_name in self.plugin_modules:
# same named plugins will have identical module name,
# which will just reuse previous imported module from
# `sys.modules` below. skipping the rest of the process
# for good.
if config.debug("plugins"):
print_warning(
"skipped same named %s plugin at %s: %s" %
(self.type_name, path, modname))
continue
if config.debug("plugins"):
print_debug("loading %s plugin at %s: %s..." %
(self.type_name, path, modname))
try:
# nerdvegas/rez#218
# load_module will force reload the module if it's
# already loaded, so check for that
plugin_module = sys.modules.get(modname)
if plugin_module is None:
loader = importer.find_module(modname)
plugin_module = loader.load_module(modname)
elif os.path.dirname(plugin_module.__file__) != path:
if config.debug("plugins"):
# this should not happen but if it does, tell why.
print_warning(
"plugin module %s is not loaded from current "
"load path but reused from previous imported "
"path: %s" % (modname, plugin_module.__file__))
if (hasattr(plugin_module, "register_plugin")
and callable(plugin_module.register_plugin)):
plugin_class = plugin_module.register_plugin()
if plugin_class is not None:
self.register_plugin(plugin_name, plugin_class,
plugin_module)
else:
if config.debug("plugins"):
print_warning(
"'register_plugin' function at %s: %s did "
"not return a class." % (path, modname))
else:
if config.debug("plugins"):
print_warning(
"no 'register_plugin' function at %s: %s" %
(path, modname))
# delete from sys.modules?
except Exception as e:
nameish = modname.split('.')[-1]
self.failed_plugins[nameish] = str(e)
if config.debug("plugins"):
import traceback
from rez.vendor.six.six import StringIO
out = StringIO()
traceback.print_exc(file=out)
print_debug(out.getvalue())
# load config
data, _ = _load_config_from_filepaths(
[os.path.join(path, "rezconfig")])
deep_update(self.config_data, data)
def log(msg):
if config.debug("bind_modules"):
print_debug(msg)
def convert_old_commands(commands, annotate=True):
"""Converts old-style package commands into equivalent Rex code."""
from rez.config import config
from rez.utils.logging_ import print_debug
def _repl(s):
return s.replace('\\"', '"')
def _encode(s):
# this replaces all occurrances of '\"' with '"', *except* for those
# occurrances of '\"' that are within double quotes, which themselves
# are not escaped. In other words, the string:
# 'hey "there \"you\" " \"dude\" '
# ..will convert to:
# 'hey "there \"you\" " "dude" '
s_new = ''
prev_i = 0
for m in within_unescaped_quotes_regex.finditer(s):
s_ = s[prev_i:m.start()]
s_new += _repl(s_)
s_new += s[m.start():m.end()]
prev_i = m.end()
s_ = s[prev_i:]
s_new += _repl(s_)
return repr(s_new)
loc = []
for cmd in commands:
if annotate:
txt = "OLD COMMAND: %s" % cmd
line = "comment(%s)" % _encode(txt)
loc.append(line)
cmd = convert_old_command_expansions(cmd)
toks = cmd.strip().split()
try:
if toks[0] == "export":
var, value = cmd.split(' ', 1)[1].split('=', 1)
for bookend in ('"', "'"):
if value.startswith(bookend) and value.endswith(bookend):
value = value[1:-1]
break
# As the only old-style commands were Linux/Bash based,
# we assume using the default separator ":" is ok - we don't
# need to use os.pathsep as we don't expected to see a
# Windows path here.
separator = config.env_var_separators.get(var, ":")
# This is a special case. We don't want to include "';'" in
# our env var separators map as it's not really the correct
# behaviour/something we want to promote. It's included here for
# backwards compatibility only, and to not propogate elsewhere.
if var == "CMAKE_MODULE_PATH":
value = value.replace("'%s'" % separator, separator)
value = value.replace('"%s"' % separator, separator)
value = value.replace(":", separator)
parts = value.split(separator)
parts = [x for x in parts if x]
if len(parts) > 1:
idx = None
var1 = "$%s" % var
var2 = "${%s}" % var
if var1 in parts:
idx = parts.index(var1)
elif var2 in parts:
idx = parts.index(var2)
if idx in (0, len(parts) - 1):
func = "appendenv" if idx == 0 else "prependenv"
parts = parts[1:] if idx == 0 else parts[:-1]
val = separator.join(parts)
loc.append("%s('%s', %s)" % (func, var, _encode(val)))
continue
loc.append("setenv('%s', %s)" % (var, _encode(value)))
elif toks[0].startswith('#'):
loc.append("comment(%s)" % _encode(' '.join(toks[1:])))
elif toks[0] == "alias":
match = re.search("alias (?P<key>.*?)=(?P<value>.*)", cmd)
key = match.groupdict()['key'].strip()
value = match.groupdict()['value'].strip()
if (value.startswith('"') and value.endswith('"')) or \
(value.startswith("'") and value.endswith("'")):
value = value[1:-1]
loc.append("alias('%s', %s)" % (key, _encode(value)))
else:
# assume we can execute this as a straight command
loc.append("command(%s)" % _encode(cmd))
except:
# if anything goes wrong, just fall back to bash command
loc.append("command(%s)" % _encode(cmd))
rex_code = '\n'.join(loc)
if config.debug("old_commands"):
br = '-' * 80
msg = textwrap.dedent("""
%s
OLD COMMANDS:
%s
NEW COMMANDS:
%s
%s
""") % (br, '\n'.join(commands), rex_code, br)
print_debug(msg)
return rex_code
def convert_old_commands(commands, annotate=True):
"""Converts old-style package commands into equivalent Rex code."""
from rez.config import config
from rez.utils.logging_ import print_debug
def _repl(s):
return s.replace('\\"', '"')
def _encode(s):
# this replaces all occurrances of '\"' with '"', *except* for those
# occurrances of '\"' that are within double quotes, which themselves
# are not escaped. In other words, the string:
# 'hey "there \"you\" " \"dude\" '
# ..will convert to:
# 'hey "there \"you\" " "dude" '
s_new = ''
prev_i = 0
for m in within_unescaped_quotes_regex.finditer(s):
s_ = s[prev_i:m.start()]
s_new += _repl(s_)
s_new += s[m.start():m.end()]
prev_i = m.end()
s_ = s[prev_i:]
s_new += _repl(s_)
return repr(s_new)
loc = []
for cmd in commands:
if annotate:
txt = "OLD COMMAND: %s" % cmd
line = "comment(%s)" % _encode(txt)
loc.append(line)
cmd = convert_old_command_expansions(cmd)
toks = cmd.strip().split()
if toks[0] == "export":
var, value = cmd.split(' ', 1)[1].split('=', 1)
for bookend in ('"', "'"):
if value.startswith(bookend) and value.endswith(bookend):
value = value[1:-1]
break
# As the only old-style commands were Linux/Bash based,
# we assume using the default separator ":" is ok - we don't
# need to use os.pathsep as we don't expected to see a
# Windows path here.
separator = config.env_var_separators.get(var, ":")
# This is a special case. We don't want to include "';'" in
# our env var separators map as it's not really the correct
# behaviour/something we want to promote. It's included here for
# backwards compatibility only, and to not propogate elsewhere.
if var == "CMAKE_MODULE_PATH":
value = value.replace("'%s'" % separator, separator)
value = value.replace('"%s"' % separator, separator)
value = value.replace(":", separator)
parts = value.split(separator)
parts = [x for x in parts if x]
if len(parts) > 1:
idx = None
var1 = "$%s" % var
var2 = "${%s}" % var
if var1 in parts:
idx = parts.index(var1)
elif var2 in parts:
idx = parts.index(var2)
if idx in (0, len(parts) - 1):
func = "appendenv" if idx == 0 else "prependenv"
parts = parts[1:] if idx == 0 else parts[:-1]
val = separator.join(parts)
loc.append("%s('%s', %s)" % (func, var, _encode(val)))
continue
loc.append("setenv('%s', %s)" % (var, _encode(value)))
elif toks[0].startswith('#'):
loc.append("comment(%s)" % _encode(' '.join(toks[1:])))
elif toks[0] == "alias":
match = re.search("alias (?P<key>.*?)=(?P<value>.*)", cmd)
key = match.groupdict()['key'].strip()
value = match.groupdict()['value'].strip()
if (value.startswith('"') and value.endswith('"')) or \
(value.startswith("'") and value.endswith("'")):
value = value[1:-1]
loc.append("alias('%s', %s)" % (key, _encode(value)))
else:
# assume we can execute this as a straight command
loc.append("command(%s)" % _encode(cmd))
rex_code = '\n'.join(loc)
if config.debug("old_commands"):
br = '-' * 80
msg = textwrap.dedent(
"""
%s
OLD COMMANDS:
%s
NEW COMMANDS:
%s
%s
""") % (br, '\n'.join(commands), rex_code, br)
print_debug(msg)
return rex_code
