def get_mapping(rel_src):
topdir = rel_src.split(os.sep)[0]
# Special case - dist-info files. These are all in a '<pkgname>-<version>.dist-info'
# dir. We keep this dir and place it in the root 'python' dir of the rez package.
#
if topdir.endswith(".dist-info"):
rel_dest = os.path.join("python", rel_src)
return (rel_src, rel_dest)
# Remapping of other installed files according to manifest
if topdir == os.pardir:
for remap in config.pip_install_remaps:
path = remap['record_path']
if re.search(path, rel_src):
pip_subpath = re.sub(path, remap['pip_install'], rel_src)
rez_subpath = re.sub(path, remap['rez_install'], rel_src)
return (pip_subpath, rez_subpath)
tokenised_path = rel_src.replace(os.pardir, '{pardir}')
tokenised_path = tokenised_path.replace(os.sep, '{sep}')
dist_record = '{dist.name}-{dist.version}.dist-info{os.sep}RECORD'
dist_record = dist_record.format(dist=distribution, os=os)
try_this_message = r"""
Unknown source file in {0}! '{1}'
To resolve, try:
1. Manually install the pip package using 'pip install --target'
to a temporary location.
2. See where '{1}'
actually got installed to by pip, RELATIVE to --target location
3. Create a new rule to 'pip_install_remaps' configuration like:
{{
"record_path": r"{2}",
"pip_install": r"<RELATIVE path pip installed to in 2.>",
"rez_install": r"<DESTINATION sub-path in rez package>",
}}
4. Try rez-pip install again.
If path remapping is not enough, consider submitting a new issue
via https://github.com/nerdvegas/rez/issues/new
""".format(dist_record, rel_src, tokenised_path)
print_error(dedent(try_this_message).lstrip())
raise IOError(
89, # errno.EDESTADDRREQ : Destination address required
"Don't know what to do with relative path in {0}, see "
"above error message for".format(dist_record),
rel_src,
)
# At this point the file should be <pkg-name>/..., so we put
# into 'python' subdir in rez package.
#
rel_dest = os.path.join("python", rel_src)
return (rel_src, rel_dest)
def send_email(self, subject, body):
if not self.settings.recipients:
return # nothing to do, sending email to nobody
if not self.settings.smtp_host:
print_warning("did not send release email: "
"SMTP host is not specified")
return
recipients = self.get_recipients()
if not recipients:
return
print "Sending release email to:"
print '\n'.join("- %s" % x for x in recipients)
msg = MIMEText(body)
msg["Subject"] = subject
msg["From"] = self.settings.sender
msg["To"] = str(',').join(recipients)
try:
s = smtplib.SMTP(self.settings.smtp_host, self.settings.smtp_port)
s.sendmail(from_addr=self.settings.sender,
to_addrs=recipients,
msg=msg.as_string())
print 'Email(s) sent.'
except Exception, e:
print_error("release email delivery failed: %s" % str(e))
def send_email(self, subject, body):
if not self.settings.recipients:
return # nothing to do, sending email to nobody
if not self.settings.smtp_host:
print_warning("did not send release email: "
"SMTP host is not specified")
return
recipients = self.get_recipients()
if not recipients:
return
print("Sending release email to:")
print('\n'.join("- %s" % x for x in recipients))
msg = MIMEText(body)
msg["Subject"] = subject
msg["From"] = self.settings.sender
msg["To"] = str(',').join(recipients)
try:
s = smtplib.SMTP(self.settings.smtp_host, self.settings.smtp_port)
s.sendmail(from_addr=self.settings.sender,
to_addrs=recipients,
msg=msg.as_string())
print('Email(s) sent.')
except Exception as e:
print_error("release email delivery failed: %s" % str(e))
def _get(key, fn):
try:
value = fn()
doc[key] = value
return (value is not None)
except Exception as e:
print_error("Error retrieving %s: %s" % (key, str(e)))
return False
def _get(key, fn):
try:
value = fn()
doc[key] = value
return (value is not None)
except Exception as e:
print_error("Error retrieving %s: %s" % (key, str(e)))
return False
def run(command=None):
parser = LazyArgumentParser("rez")
parser.add_argument("-i", "--info", action=InfoAction,
help="print information about rez and exit")
parser.add_argument("-V", "--version", action="version",
version="Rez %s" % __version__)
# add args common to all subcommands... we add them both to the top parser,
# AND to the subparsers, for two reasons:
# 1) this allows us to do EITHER "rez --debug build" OR
# "rez build --debug"
# 2) this allows the flags to be used when using either "rez" or
# "rez-build" - ie, this will work: "rez-build --debug"
_add_common_args(parser)
# add lazy subparsers
subparser = parser.add_subparsers(dest='cmd', metavar='COMMAND')
for subcommand in subcommands:
module_name = "rez.cli.%s" % subcommand
subparser.add_parser(
subcommand,
help='', # required so that it can be setup later
setup_subparser=SetupRezSubParser(module_name))
# parse args, but split extras into groups separated by "--"
all_args = ([command] + sys.argv[1:]) if command else sys.argv[1:]
arg_groups = [[]]
for arg in all_args:
if arg == '--':
arg_groups.append([])
continue
arg_groups[-1].append(arg)
opts = parser.parse_args(arg_groups[0])
if opts.debug or _env_var_true("REZ_DEBUG"):
exc_type = None
else:
exc_type = RezError
def run_cmd():
return opts.func(opts, opts.parser, arg_groups[1:])
if opts.profile:
import cProfile
cProfile.runctx("run_cmd()", globals(), locals(), filename=opts.profile)
returncode = 0
else:
try:
returncode = run_cmd()
except (NotImplementedError, RezSystemError) as e:
raise
except exc_type as e:
print_error("%s: %s" % (e.__class__.__name__, str(e)))
#print >> sys.stderr, "rez: %s: %s" % (e.__class__.__name__, str(e))
sys.exit(1)
sys.exit(returncode or 0)
def _publish_message(host, amqp_settings, routing_key, data):
"""Publish an AMQP message.
Returns:
bool: True if message was sent successfully.
"""
if host == "stdout":
print("Published to %s: %s" % (routing_key, data))
return True
set_pika_log_level()
conn_kwargs = dict()
# name the conn like 'rez.publish.{host}'
conn_kwargs["client_properties"] = {
"connection_name": "rez.publish.%s" % socket.gethostname()
}
host, port = parse_host_and_port(url=host)
conn_kwargs["host"] = host
if port is not None:
conn_kwargs["port"] = port
if amqp_settings.get("userid"):
conn_kwargs["credentials"] = PlainCredentials(
username=amqp_settings.get("userid"),
password=amqp_settings.get("password"))
params = ConnectionParameters(
socket_timeout=amqp_settings.get("connect_timeout"), **conn_kwargs)
props = BasicProperties(
content_type="application/json",
content_encoding="utf-8",
delivery_mode=amqp_settings.get("message_delivery_mode"))
try:
conn = BlockingConnection(params)
except socket.error as e:
print_error("Cannot connect to the message broker: %s" % e)
return False
try:
channel = conn.channel()
channel.basic_publish(exchange=amqp_settings["exchange_name"],
routing_key=routing_key,
body=json.dumps(data),
properties=props)
except Exception as e:
print_error("Failed to publish message: %s" % (e))
return False
finally:
conn.close()
return True
def physical_cores(self):
"""Return the number of physical cpu cores on the system."""
try:
return self._physical_cores_base()
except Exception as e:
from rez.utils.logging_ import print_error
print_error("Error detecting physical core count, defaulting to 1: %s"
% str(e))
return 1
def physical_cores(self):
"""Return the number of physical cpu cores on the system."""
try:
return self._physical_cores_base()
except Exception as e:
from rez.utils.logging_ import print_error
print_error("Error detecting physical core count, defaulting to 1: %s"
% str(e))
return 1
def _get_global_level():
# load globally configured preprocess function
package_preprocess_function = self.config.package_preprocess_function
if not package_preprocess_function:
return None
elif isfunction(package_preprocess_function):
preprocess_func = package_preprocess_function
else:
if '.' not in package_preprocess_function:
print_error(
"Setting 'package_preprocess_function' must be of "
"form 'module[.module.module...].funcname'. Package "
"preprocessing has not been applied.")
return None
elif isinstance(package_preprocess_function, basestring):
if '.' not in package_preprocess_function:
print_error(
"Setting 'package_preprocess_function' must be of "
"form 'module[.module.module...].funcname'. "
"Package preprocessing has not been applied."
)
return None
name, funcname = package_preprocess_function.rsplit('.', 1)
try:
module = __import__(name=name, fromlist=[funcname])
except Exception as e:
print_error(
"Failed to load preprocessing function '%s': %s"
% (package_preprocess_function, str(e))
)
return None
setattr(module, "InvalidPackageError", InvalidPackageError)
preprocess_func = getattr(module, funcname)
else:
print_error(
"Invalid package_preprocess_function: %s" % package_preprocess_function
)
return None
if not preprocess_func or not isfunction(preprocess_func):
print_error("Function '%s' not found" % package_preprocess_function)
return None
return preprocess_func
def logical_cores(self):
"""Return the number of cpu cores as reported to the os.
May be different from physical_cores if, ie, intel's hyperthreading is
enabled.
"""
try:
return self._logical_cores()
except Exception as e:
from rez.utils.logging_ import print_error
print_error("Error detecting logical core count, defaulting to 1: %s"
% str(e))
return 1
def logical_cores(self):
"""Return the number of cpu cores as reported to the os.
May be different from physical_cores if, ie, intel's hyperthreading is
enabled.
"""
try:
return self._logical_cores()
except Exception as e:
from rez.utils.logging_ import print_error
print_error("Error detecting logical core count, defaulting to 1: %s"
% str(e))
return 1
def _publish_message(host, amqp_settings, routing_key, data):
"""Publish an AMQP message.
Returns:
bool: True if message was sent successfully.
"""
if host == "stdout":
print("Published to %s: %s" % (routing_key, data))
return True
creds = PlainCredentials(
username=amqp_settings.get("userid"),
password=amqp_settings.get("password")
)
params = ConnectionParameters(
host=host,
credentials=creds,
socket_timeout=amqp_settings.get("connect_timeout")
)
props = BasicProperties(
content_type="application/json",
content_encoding="utf-8",
delivery_mode=amqp_settings.get("message_delivery_mode")
)
try:
conn = BlockingConnection(params)
except socket.error as e:
print_error("Cannot connect to the message broker: %s" % e)
return False
try:
channel = conn.channel()
channel.basic_publish(
exchange=amqp_settings["exchange_name"],
routing_key=routing_key,
body=json.dumps(data),
properties=props
)
except Exception as e:
print_error("Failed to publish message: %s" % (e))
return False
finally:
conn.close()
return True
def publish_message(self, data):
if not self.settings.host:
print_error("Did not publish message, host is not specified")
return
routing_key = self.settings.exchange_routing_key
print("Publishing AMQP message on %s..." % routing_key)
publish_message(host=self.settings.host,
amqp_settings=self.settings,
routing_key=routing_key,
data=data)
if config.debug("package_release"):
print_debug("Published message: %s" % (data))
def test_print(self):
"""Test valid msg and nargs combinations for print_*."""
for msg in ("Hello", "Hello %s", "Hello %s %s"):
logging_.print_debug(msg)
logging_.print_info(msg)
logging_.print_warning(msg)
logging_.print_error(msg)
logging_.print_critical(msg)
for nargs in ([], ["foo"], ["foo", "bar"]):
logging_.print_debug(msg, *nargs)
logging_.print_info(msg, *nargs)
logging_.print_warning(msg, *nargs)
logging_.print_error(msg, *nargs)
logging_.print_critical(msg, *nargs)
def test_print(self):
"""Test valid msg and nargs combinations for print_*."""
for msg in ("Hello", "Hello %s", "Hello %s %s"):
logging_.print_debug(msg)
logging_.print_info(msg)
logging_.print_warning(msg)
logging_.print_error(msg)
logging_.print_critical(msg)
for nargs in ([], ["foo"], ["foo", "bar"]):
logging_.print_debug(msg, *nargs)
logging_.print_info(msg, *nargs)
logging_.print_warning(msg, *nargs)
logging_.print_error(msg, *nargs)
logging_.print_critical(msg, *nargs)
def _publish_message(host, amqp_settings, routing_key, data):
"""Publish an AMQP message.
Returns:
bool: True if message was sent successfully.
"""
if host == "stdout":
print("Published to %s: %s" % (routing_key, data))
return True
try:
conn = Connection(**remove_nones(
host=host,
userid=amqp_settings.get("userid"),
password=amqp_settings.get("password"),
connect_timeout=amqp_settings.get("connect_timeout")
))
except socket.error as e:
print_error("Cannot connect to the message broker: %s" % (e))
return False
channel = conn.channel()
# build the message
msg = basic_message.Message(**remove_nones(
body=json.dumps(data),
delivery_mode=amqp_settings.get("message_delivery_mode"),
content_type="application/json",
content_encoding="utf-8"
))
# publish the message
try:
channel.basic_publish(
msg,
amqp_settings["exchange_name"],
routing_key
)
except Exception as e:
print_error("Failed to publish message: %s" % (e))
return False
finally:
conn.close()
return True
def _publish_message(host, amqp_settings, routing_key, data):
"""Publish an AMQP message.
Returns:
bool: True if message was sent successfully.
"""
if host == "stdout":
print("Published to %s: %s" % (routing_key, data))
return True
try:
conn = Connection(**remove_nones(
host=host,
userid=amqp_settings.get("userid"),
password=amqp_settings.get("password"),
connect_timeout=amqp_settings.get("connect_timeout")
))
except socket.error as e:
print_error("Cannot connect to the message broker: %s" % (e))
return False
channel = conn.channel()
# build the message
msg = basic_message.Message(**remove_nones(
body=json.dumps(data),
delivery_mode=amqp_settings.get("message_delivery_mode"),
content_type="application/json",
content_encoding="utf-8"
))
# publish the message
try:
channel.basic_publish(
msg,
amqp_settings["exchange_name"],
routing_key
)
except Exception as e:
print_error("Failed to publish message: %s" % (e))
return False
finally:
conn.close()
return True
def get_mapping(rel_src):
# Remapping of other installed files according to manifest
for remap in config.pip_install_remaps:
path = remap['record_path']
if re.search(path, rel_src):
pip_subpath = re.sub(path, remap['pip_install'], rel_src)
rez_subpath = re.sub(path, remap['rez_install'], rel_src)
return (pip_subpath, rez_subpath)
tokenised_path = rel_src.replace(os.pardir, '{pardir}')
tokenised_path = tokenised_path.replace(os.sep, '{sep}')
dist_record = '{dist.name}-{dist.version}.dist-info{os.sep}RECORD'
dist_record = dist_record.format(dist=distribution, os=os)
try_this_message = r"""
Unknown source file in {0}! '{1}'
To resolve, try:
1. Manually install the pip package using 'pip install --target'
to a temporary location.
2. See where '{1}'
actually got installed to by pip, RELATIVE to --target location
3. Create a new rule to 'pip_install_remaps' configuration like:
{{
"record_path": r"{2}",
"pip_install": r"<RELATIVE path pip installed to in 2.>",
"rez_install": r"<DESTINATION sub-path in rez package>",
}}
4. Try rez-pip install again.
If path remapping is not enough, consider submitting a new issue
via https://github.com/nerdvegas/rez/issues/new
""".format(dist_record, rel_src, tokenised_path)
print_error(dedent(try_this_message).lstrip())
raise IOError(
89, # errno.EDESTADDRREQ : Destination address required
"Don't know what to do with relative path in {0}, see "
"above error message for".format(dist_record),
rel_src,
)
def remove_variant(pkgcache, uri, opts):
from rez.packages import get_variant_from_uri
from rez.utils.logging_ import print_info, print_warning, print_error
from rez.package_cache import PackageCache
print_info("Removing variant %r from package cache at %s:", uri, pkgcache.path)
variant = get_variant_from_uri(uri)
if variant is None:
print("No such variant: %s" % uri, file=sys.stderr)
sys.exit(1)
status = pkgcache.remove_variant(variant)
if status == PackageCache.VARIANT_NOT_FOUND:
print_error("No such variant found in cache")
elif status == PackageCache.VARIANT_COPYING:
print_warning("Another process is currently caching this variant")
else:
print_info("Variant successfully removed")
def command(opts, parser, extra_arg_groups=None):
from rez.utils.logging_ import print_error
from rez.bundle_context import bundle_context
from rez.resolved_context import ResolvedContext
rxt_filepath = os.path.abspath(os.path.expanduser(opts.RXT))
dest_dir = os.path.abspath(os.path.expanduser(opts.DEST_DIR))
# sanity checks
if not os.path.exists(rxt_filepath):
print_error("File does not exist: %s", rxt_filepath)
sys.exit(1)
context = ResolvedContext.load(rxt_filepath)
bundle_context(context=context,
dest_dir=dest_dir,
force=opts.force,
skip_non_relocatable=opts.skip_non_relocatable,
verbose=opts.verbose)
def get_recipients(self):
value = self.settings.recipients
if isinstance(value, list):
return value
if os.path.exists(value):
filepath = value
try:
return self.load_recipients(filepath)
except Exception as e:
print_error("failed to load recipients config: %s. Emails "
"not sent" % str(e))
elif '@' in value:
return [value] # assume it's an email address
else:
print_error("email recipient file does not exist: %s. Emails not "
"sent" % value)
return []
def get_recipients(self):
value = self.settings.recipients
if isinstance(value, list):
return value
if os.path.exists(value):
filepath = value
try:
return self.load_recipients(filepath)
except Exception as e:
print_error("failed to load recipients config: %s. Emails "
"not sent" % str(e))
elif '@' in value:
return [value] # assume it's an email address
else:
print_error("email recipient file does not exist: %s. Emails not "
"sent" % value)
return []
def publish_message(self, data):
if not self.settings.host:
print_error("Did not publish message, host is not specified")
return
try:
conn = Connection(host=self.settings.host,
port=self.settings.port,
userid=self.settings.userid,
password=self.settings.password,
connect_timeout=self.settings.connect_timeout)
except socket.error as e:
print_error("Cannot connect to the message broker: %s" % (e))
return
channel = conn.channel()
# Declare the exchange
try:
channel.exchange_declare(
self.settings.exchange_name,
self.settings.exchange_type,
durable=self.settings.exchange_durable,
auto_delete=self.settings.exchange_auto_delete)
except Exception as e:
print_error("Failed to declare an exchange: %s" % (e))
return
# build the message
msg = basic_message.Message(
body=json.dumps(data),
delivery_mode=self.settings.message_delivery_mode,
content_type="application/json",
content_encoding="utf-8")
routing_key = self.settings.exchange_routing_key
print "Publishing AMQP message on %s..." % routing_key
# publish the message
try:
channel.basic_publish(msg, self.settings.exchange_name,
routing_key)
except Exception as e:
print_error("Failed to publish message: %s" % (e))
return
if config.debug("package_release"):
print_debug("Published message: %s" % (data))
def publish_message(self, data):
if not self.settings.host:
print_error("Did not publish message, host is not specified")
return
try:
conn = Connection(host=self.settings.host,
port=self.settings.port,
userid=self.settings.userid,
password=self.settings.password,
connect_timeout=self.settings.connect_timeout)
except socket.error as e:
print_error("Cannot connect to the message broker: %s" % (e))
return
channel = conn.channel()
# Declare the exchange
try:
channel.exchange_declare(
self.settings.exchange_name,
self.settings.exchange_type,
durable=self.settings.exchange_durable,
auto_delete=self.settings.exchange_auto_delete)
except Exception as e:
print_error("Failed to declare an exchange: %s" % (e))
return
# build the message
msg = basic_message.Message(
body=json.dumps(data),
delivery_mode=self.settings.message_delivery_mode,
content_type="application/json",
content_encoding="utf-8")
routing_key = self.settings.exchange_routing_key
print "Publishing AMQP message on %s..." % routing_key
# publish the message
try:
channel.basic_publish(msg, self.settings.exchange_name, routing_key)
except Exception as e:
print_error("Failed to publish message: %s" % (e))
return
if config.debug("package_release"):
print_debug("Published message: %s" % (data))
def _get_preprocessed(self, data):
"""
Returns:
(DeveloperPackage, new_data) 2-tuple IF the preprocess function
changed the package; otherwise None.
"""
from rez.serialise import process_python_objects
from rez.utils.data_utils import get_dict_diff_str
from copy import deepcopy
def _get_package_level():
return getattr(self, "preprocess", None)
def _get_global_level():
# load globally configured preprocess function
package_preprocess_function = self.config.package_preprocess_function
if not package_preprocess_function:
return None
elif isfunction(package_preprocess_function):
preprocess_func = package_preprocess_function
else:
if '.' not in package_preprocess_function:
print_error(
"Setting 'package_preprocess_function' must be of "
"form 'module[.module.module...].funcname'. Package "
"preprocessing has not been applied.")
return None
elif isinstance(package_preprocess_function, basestring):
if '.' not in package_preprocess_function:
print_error(
"Setting 'package_preprocess_function' must be of "
"form 'module[.module.module...].funcname'. "
"Package preprocessing has not been applied.")
return None
name, funcname = package_preprocess_function.rsplit('.', 1)
try:
module = __import__(name=name, fromlist=[funcname])
except Exception as e:
print_error(
"Failed to load preprocessing function '%s': %s" %
(package_preprocess_function, str(e)))
return None
setattr(module, "InvalidPackageError", InvalidPackageError)
preprocess_func = getattr(module, funcname)
else:
print_error("Invalid package_preprocess_function: %s" %
package_preprocess_function)
return None
if not preprocess_func or not isfunction(preprocess_func):
print_error("Function '%s' not found" %
package_preprocess_function)
return None
return preprocess_func
with add_sys_paths(config.package_definition_build_python_paths):
preprocess_mode = PreprocessMode[
self.config.package_preprocess_mode]
package_preprocess = _get_package_level()
global_preprocess = _get_global_level()
if preprocess_mode == PreprocessMode.after:
preprocessors = [global_preprocess, package_preprocess]
elif preprocess_mode == PreprocessMode.before:
preprocessors = [package_preprocess, global_preprocess]
else:
preprocessors = [package_preprocess or global_preprocess]
preprocessed_data = deepcopy(data)
for preprocessor in preprocessors:
if not preprocessor:
continue
level = "global" if preprocessor == global_preprocess else "local"
print_info("Applying {0} preprocess function".format(level))
# apply preprocessing
try:
preprocessor(this=self, data=preprocessed_data)
except InvalidPackageError:
raise
except Exception as e:
print_error("Failed to apply preprocess: %s: %s" %
(e.__class__.__name__, str(e)))
return None
# if preprocess added functions, these may need to be converted to
# SourceCode instances
preprocessed_data = process_python_objects(preprocessed_data)
if preprocessed_data == data:
return None
# recreate package from modified package data
package = create_package(self.name,
preprocessed_data,
package_cls=self.__class__)
# print summary of changed package attributes
txt = get_dict_diff_str(
data,
preprocessed_data,
title="Package attributes were changed in preprocessing:")
print_info(txt)
return package, preprocessed_data
def run(command=None):
sys.dont_write_bytecode = True
parser = LazyArgumentParser("rez")
parser.add_argument("-i",
"--info",
action=InfoAction,
help="print information about rez and exit")
parser.add_argument("-V",
"--version",
action="version",
version="Rez %s" % __version__)
# add args common to all subcommands... we add them both to the top parser,
# AND to the subparsers, for two reasons:
# 1) this allows us to do EITHER "rez --debug build" OR
# "rez build --debug"
# 2) this allows the flags to be used when using either "rez" or
# "rez-build" - ie, this will work: "rez-build --debug"
_add_common_args(parser)
# add lazy subparsers
subparser = parser.add_subparsers(dest='cmd', metavar='COMMAND')
for subcommand in subcommands:
module_name = "rez.cli.%s" % subcommand
subparser.add_parser(
subcommand,
help='', # required so that it can be setup later
setup_subparser=SetupRezSubParser(module_name))
# construct args list. Note that commands like 'rez-env foo' and
# 'rez env foo' are equivalent
if command:
args = [command] + sys.argv[1:]
elif len(sys.argv) > 1 and sys.argv[1] in subcommands:
command = sys.argv[1]
args = sys.argv[1:]
else:
args = sys.argv[1:]
# parse args depending on subcommand behaviour
if command:
arg_mode = subcommands[command].get("arg_mode")
else:
arg_mode = None
if arg_mode == "grouped":
# args split into groups by '--'
arg_groups = [[]]
for arg in args:
if arg == '--':
arg_groups.append([])
continue
arg_groups[-1].append(arg)
opts = parser.parse_args(arg_groups[0])
extra_arg_groups = arg_groups[1:]
elif arg_mode == "passthrough":
# unknown args passed in first extra_arg_group
opts, extra_args = parser.parse_known_args(args)
extra_arg_groups = [extra_args]
else:
# native arg parsing
opts = parser.parse_args(args)
extra_arg_groups = []
if opts.debug or _env_var_true("REZ_DEBUG"):
exc_type = None
else:
exc_type = RezError
def run_cmd():
try:
# python3 will not automatically handle cases where no sub parser
# has been selected. In these cases func will not exist, and an
# AttributeError will be raised.
func = opts.func
except AttributeError:
parser.error("too few arguments.")
else:
return func(opts, opts.parser, extra_arg_groups)
if opts.profile:
import cProfile
cProfile.runctx("run_cmd()",
globals(),
locals(),
filename=opts.profile)
returncode = 0
else:
try:
returncode = run_cmd()
except (NotImplementedError, RezSystemError) as e:
raise
except exc_type as e:
print_error("%s: %s" % (e.__class__.__name__, str(e)))
sys.exit(1)
sys.exit(returncode or 0)
def _handle(e):
print_error(str(e))
def _handle(e):
print_error(str(e))
def run(command=None):
parser = LazyArgumentParser("rez")
parser.add_argument("-i",
"--info",
action=InfoAction,
help="print information about rez and exit")
parser.add_argument("-V",
"--version",
action="version",
version="Rez %s" % __version__)
# add args common to all subcommands... we add them both to the top parser,
# AND to the subparsers, for two reasons:
# 1) this allows us to do EITHER "rez --debug build" OR
# "rez build --debug"
# 2) this allows the flags to be used when using either "rez" or
# "rez-build" - ie, this will work: "rez-build --debug"
_add_common_args(parser)
# add lazy subparsers
subparser = parser.add_subparsers(dest='cmd', metavar='COMMAND')
for subcommand in subcommands:
module_name = "rez.cli.%s" % subcommand
subparser.add_parser(
subcommand,
help='', # required so that it can be setup later
setup_subparser=SetupRezSubParser(module_name))
# parse args, but split extras into groups separated by "--"
all_args = ([command] + sys.argv[1:]) if command else sys.argv[1:]
arg_groups = [[]]
for arg in all_args:
if arg == '--':
arg_groups.append([])
continue
arg_groups[-1].append(arg)
opts = parser.parse_args(arg_groups[0])
if opts.debug or _env_var_true("REZ_DEBUG"):
exc_type = None
else:
exc_type = RezError
def run_cmd():
return opts.func(opts, opts.parser, arg_groups[1:])
if opts.profile:
import cProfile
cProfile.runctx("run_cmd()",
globals(),
locals(),
filename=opts.profile)
returncode = 0
else:
try:
returncode = run_cmd()
except (NotImplementedError, RezSystemError) as e:
raise
except exc_type as e:
print_error("%s: %s" % (e.__class__.__name__, str(e)))
#print >> sys.stderr, "rez: %s: %s" % (e.__class__.__name__, str(e))
sys.exit(1)
sys.exit(returncode or 0)
def _get_preprocessed(self, data):
"""
Returns:
(DeveloperPackage, new_data) 2-tuple IFF the preprocess function
changed the package; otherwise None.
"""
from rez.serialise import process_python_objects
from rez.utils.data_utils import get_dict_diff
from copy import deepcopy
with add_sys_paths(config.package_definition_build_python_paths):
preprocess = getattr(self, "preprocess", None)
if preprocess:
preprocess_func = preprocess.func
print_info("Applying preprocess from package.py")
else:
# load globally configured preprocess function
dotted = self.config.package_preprocess_function
if not dotted:
return None
if '.' not in dotted:
print_error(
"Setting 'package_preprocess_function' must be of "
"form 'module[.module.module...].funcname'. Package "
"preprocessing has not been applied.")
return None
name, funcname = dotted.rsplit('.', 1)
try:
module = __import__(name=name, fromlist=[funcname])
except Exception as e:
print_error(
"Failed to load preprocessing function '%s': %s" %
(dotted, str(e)))
return None
setattr(module, "InvalidPackageError", InvalidPackageError)
preprocess_func = getattr(module, funcname)
if not preprocess_func or not isfunction(isfunction):
print_error("Function '%s' not found" % dotted)
return None
print_info("Applying preprocess function %s" % dotted)
preprocessed_data = deepcopy(data)
# apply preprocessing
try:
preprocess_func(this=self, data=preprocessed_data)
except InvalidPackageError:
raise
except Exception as e:
print_error("Failed to apply preprocess: %s: %s" %
(e.__class__.__name__, str(e)))
return None
# if preprocess added functions, these need to be converted to
# SourceCode instances
preprocessed_data = process_python_objects(preprocessed_data)
if preprocessed_data == data:
return None
# recreate package from modified package data
package = create_package(self.name,
preprocessed_data,
package_cls=self.__class__)
# print summary of changed package attributes
added, removed, changed = get_dict_diff(data, preprocessed_data)
lines = ["Package attributes were changed in preprocessing:"]
if added:
lines.append("Added attributes: %s" % ['.'.join(x) for x in added])
if removed:
lines.append("Removed attributes: %s" %
['.'.join(x) for x in removed])
if changed:
lines.append("Changed attributes: %s" %
['.'.join(x) for x in changed])
txt = '\n'.join(lines)
print_info(txt)
return package, preprocessed_data
def _get_preprocessed(self, data):
"""
Returns:
(DeveloperPackage, new_data) 2-tuple IFF the preprocess function
changed the package; otherwise None.
"""
from rez.serialise import process_python_objects
from rez.utils.data_utils import get_dict_diff_str
from copy import deepcopy
with add_sys_paths(config.package_definition_build_python_paths):
preprocess_func = getattr(self, "preprocess", None)
if preprocess_func:
print_info("Applying preprocess from package.py")
else:
# load globally configured preprocess function
dotted = self.config.package_preprocess_function
if not dotted:
return None
if '.' not in dotted:
print_error(
"Setting 'package_preprocess_function' must be of "
"form 'module[.module.module...].funcname'. Package "
"preprocessing has not been applied.")
return None
name, funcname = dotted.rsplit('.', 1)
try:
module = __import__(name=name, fromlist=[funcname])
except Exception as e:
print_error("Failed to load preprocessing function '%s': %s"
% (dotted, str(e)))
return None
setattr(module, "InvalidPackageError", InvalidPackageError)
preprocess_func = getattr(module, funcname)
if not preprocess_func or not isfunction(isfunction):
print_error("Function '%s' not found" % dotted)
return None
print_info("Applying preprocess function %s" % dotted)
preprocessed_data = deepcopy(data)
# apply preprocessing
try:
preprocess_func(this=self, data=preprocessed_data)
except InvalidPackageError:
raise
except Exception as e:
print_error("Failed to apply preprocess: %s: %s"
% (e.__class__.__name__, str(e)))
return None
# if preprocess added functions, these may need to be converted to
# SourceCode instances
preprocessed_data = process_python_objects(preprocessed_data)
if preprocessed_data == data:
return None
# recreate package from modified package data
package = create_package(self.name, preprocessed_data,
package_cls=self.__class__)
# print summary of changed package attributes
txt = get_dict_diff_str(
data,
preprocessed_data,
title="Package attributes were changed in preprocessing:"
)
print_info(txt)
return package, preprocessed_data
def run(command=None):
setup_logging()
parser = LazyArgumentParser("rez")
parser.add_argument("-i", "--info", action=InfoAction,
help="print information about rez and exit")
parser.add_argument("-V", "--version", action="version",
version="Rez %s" % __version__)
# add args common to all subcommands... we add them both to the top parser,
# AND to the subparsers, for two reasons:
# 1) this allows us to do EITHER "rez --debug build" OR
# "rez build --debug"
# 2) this allows the flags to be used when using either "rez" or
# "rez-build" - ie, this will work: "rez-build --debug"
_add_common_args(parser)
# add lazy subparsers
subparser = parser.add_subparsers(dest='cmd', metavar='COMMAND')
for subcommand in subcommands:
module_name = "rez.cli.%s" % subcommand
subparser.add_parser(
subcommand,
help='', # required so that it can be setup later
setup_subparser=SetupRezSubParser(module_name))
# construct args list. Note that commands like 'rez-env foo' and
# 'rez env foo' are equivalent
if command:
args = [command] + sys.argv[1:]
elif len(sys.argv) > 1 and sys.argv[1] in subcommands:
command = sys.argv[1]
args = sys.argv[1:]
else:
args = sys.argv[1:]
# parse args depending on subcommand behaviour
if command:
arg_mode = subcommands[command].get("arg_mode")
else:
arg_mode = None
if arg_mode == "grouped":
# args split into groups by '--'
arg_groups = [[]]
for arg in args:
if arg == '--':
arg_groups.append([])
continue
arg_groups[-1].append(arg)
opts = parser.parse_args(arg_groups[0])
extra_arg_groups = arg_groups[1:]
elif arg_mode == "passthrough":
# unknown args passed in first extra_arg_group
opts, extra_args = parser.parse_known_args(args)
extra_arg_groups = [extra_args]
else:
# native arg parsing
opts = parser.parse_args(args)
extra_arg_groups = []
if opts.debug or _env_var_true("REZ_DEBUG"):
exc_type = None
else:
exc_type = RezError
def run_cmd():
return opts.func(opts, opts.parser, extra_arg_groups)
if opts.profile:
import cProfile
cProfile.runctx("run_cmd()", globals(), locals(), filename=opts.profile)
returncode = 0
else:
try:
returncode = run_cmd()
except (NotImplementedError, RezSystemError) as e:
raise
except exc_type as e:
print_error("%s: %s" % (e.__class__.__name__, str(e)))
sys.exit(1)
sys.exit(returncode or 0)
def run(command=None):
global _hyphened_command
sys.dont_write_bytecode = True
# construct args list. Note that commands like 'rez-env foo' and
# 'rez env foo' are equivalent
#
if command:
# like 'rez-foo arg1 arg2'
args = [command] + sys.argv[1:]
_hyphened_command = True
elif len(sys.argv) > 1 and sys.argv[1] in subcommands:
# like 'rez foo arg1 arg2'
command = sys.argv[1]
args = sys.argv[1:]
else:
# like 'rez -i'
args = sys.argv[1:]
# parse args depending on subcommand behaviour
if command:
arg_mode = subcommands[command].get("arg_mode")
else:
arg_mode = None
parser = setup_parser()
if arg_mode == "grouped":
# args split into groups by '--'
arg_groups = [[]]
for arg in args:
if arg == '--':
arg_groups.append([])
continue
arg_groups[-1].append(arg)
opts = parser.parse_args(arg_groups[0])
extra_arg_groups = arg_groups[1:]
elif arg_mode == "passthrough":
# unknown args passed in first extra_arg_group
opts, extra_args = parser.parse_known_args(args)
extra_arg_groups = [extra_args]
else:
# native arg parsing
opts = parser.parse_args(args)
extra_arg_groups = []
if opts.debug or _env_var_true("REZ_DEBUG"):
exc_type = _NeverError
else:
exc_type = RezError
def run_cmd():
try:
# python3 will not automatically handle cases where no sub parser
# has been selected. In these cases func will not exist, and an
# AttributeError will be raised.
func = opts.func
except AttributeError:
parser.error("too few arguments.")
else:
return func(opts, opts.parser, extra_arg_groups)
if opts.profile:
import cProfile
cProfile.runctx("run_cmd()",
globals(),
locals(),
filename=opts.profile)
returncode = 0
else:
try:
returncode = run_cmd()
except (NotImplementedError, RezSystemError):
raise
except exc_type as e:
print_error("%s: %s" % (e.__class__.__name__, str(e)))
sys.exit(1)
sys.exit(returncode or 0)
def run_test(self, test_name):
"""Run a test.
Runs the test in its correct environment. Note that if tests share the
same requirements, the contexts will be reused.
Args:
test_name (str): Name of test to run.
Returns:
int: Exit code of first failed test, or 0 if none failed. If the first
test to fail did so because it was not able to run (eg its
environment could not be configured), -1 is returned.
"""
package = self.get_package()
exitcode = 0
if test_name not in self.get_test_names():
raise PackageTestError("Test '%s' not found in package %s" %
(test_name, package.uri))
if self.use_current_env:
if package is None:
self._add_test_result(
test_name, None, "skipped",
"The current environment does not contain a package "
"matching the request")
return
current_context = ResolvedContext.get_current()
current_variant = current_context.get_resolved_package(
package.name)
target_variants = [current_variant]
else:
target_variants = self._get_target_variants(test_name)
for variant in target_variants:
# get test info for this variant. If None, that just means that this
# variant doesn't provide this test. That's ok - 'tests' might be
# implemented as a late function attribute that provides some tests
# for some variants and not others
#
test_info = self._get_test_info(test_name, variant)
if not test_info:
self._add_test_result(
test_name, variant, "skipped",
"The test is not declared in this variant")
continue
command = test_info["command"]
requires = test_info["requires"]
on_variants = test_info["on_variants"]
# show progress
if self.verbose > 1:
self._print_header("\nRunning test: %s\nPackage: %s\n%s\n",
test_name, variant.uri, '-' * 80)
elif self.verbose:
self._print_header("\nRunning test: %s\n%s\n", test_name,
'-' * 80)
# apply variant selection filter if specified
if isinstance(on_variants, dict):
filter_type = on_variants["type"]
func = getattr(self, "_on_variant_" + filter_type)
do_test = func(variant, on_variants)
if not do_test:
reason = (
"Test skipped as specified by on_variants '%s' filter"
% filter_type)
print_info(reason)
self._add_test_result(test_name, variant, "skipped",
reason)
continue
# add requirements to force the current variant to be resolved.
# TODO this is not perfect, and will need to be updated when
# explicit variant selection is added to rez (this is a new
# feature). Until then, there's no guarantee that we'll resolve to
# the variant we want, so we take that into account here.
#
requires.extend(map(str, variant.variant_requires))
# create test runtime env
exc = None
try:
context = self._get_context(requires)
except RezError as e:
exc = e
fail_reason = None
if exc is not None:
fail_reason = "The test environment failed to resolve: %s" % exc
elif context is None:
fail_reason = "The current environment does not meet test requirements"
elif not context.success:
fail_reason = "The test environment failed to resolve"
if fail_reason:
self._add_test_result(test_name, variant, "failed",
fail_reason)
print_error(fail_reason)
if not exitcode:
exitcode = -1
if self.stop_on_fail:
self.stopped_on_fail = True
return exitcode
continue
# check that this has actually resolved the variant we want
resolved_variant = context.get_resolved_package(package.name)
assert resolved_variant
if resolved_variant.handle != variant.handle:
print_warning(
"Could not resolve environment for this variant (%s). This "
"is a known issue and will be fixed once 'explicit variant "
"selection' is added to rez.", variant.uri)
self._add_test_result(
test_name, variant, "skipped",
"Could not resolve to variant (known issue)")
continue
# expand refs like {root} in commands
if isinstance(command, basestring):
command = variant.format(command)
else:
command = map(variant.format, command)
# run the test in the context
if self.verbose:
if self.verbose > 1:
context.print_info(self.stdout)
print('')
if isinstance(command, basestring):
cmd_str = command
else:
cmd_str = ' '.join(map(quote, command))
self._print_header("Running test command: %s", cmd_str)
if self.dry_run:
self._add_test_result(test_name, variant, "skipped",
"Dry run mode")
continue
def _pre_test_commands(executor):
# run package.py:pre_test_commands() if present
pre_test_commands = getattr(variant, "pre_test_commands")
if not pre_test_commands:
return
test_ns = {"name": test_name}
with executor.reset_globals():
executor.bind("this", variant)
executor.bind("test", RO_AttrDictWrapper(test_ns))
executor.execute_code(pre_test_commands)
retcode, _, _ = context.execute_shell(
command=command,
actions_callback=_pre_test_commands,
stdout=self.stdout,
stderr=self.stderr,
block=True)
if retcode:
print_warning("Test command exited with code %d", retcode)
self._add_test_result(
test_name, variant, "failed",
"Test failed with exit code %d" % retcode)
if not exitcode:
exitcode = retcode
if self.stop_on_fail:
self.stopped_on_fail = True
return exitcode
continue
# test passed
self._add_test_result(test_name, variant, "success",
"Test succeeded")
# just test against one variant in this case
if on_variants is False:
break
return exitcode
def bind_package(name, path=None, version_range=None, no_deps=False,
bind_args=None, quiet=False):
"""Bind software available on the current system, as a rez package.
Note:
`bind_args` is provided when software is bound via the 'rez-bind'
command line tool. Bind modules can define their own command line
options, and they will be present in `bind_args` if applicable.
Args:
name (str): Package name.
path (str): Package path to install into; local packages path if None.
version_range (`VersionRange`): If provided, only bind the software if
it falls within this version range.
no_deps (bool): If True, don't bind dependencies.
bind_args (list of str): Command line options.
quiet (bool): If True, suppress superfluous output.
Returns:
List of `Variant`: The variant(s) that were installed as a result of
binding this package.
"""
pending = set([name])
installed_variants = []
installed_package_names = set()
primary = True
# bind package and possibly dependencies
while pending:
pending_ = pending
pending = set()
exc_type = RezUncatchableError
for name_ in pending_:
# turn error on binding of dependencies into a warning - we don't
# want to skip binding some dependencies because others failed
try:
variants_ = _bind_package(name_,
path=path,
version_range=version_range,
bind_args=bind_args,
quiet=quiet)
except exc_type as e:
print_error("Could not bind '%s': %s: %s"
% (name_, e.__class__.__name__, str(e)))
continue
installed_variants.extend(variants_)
for variant in variants_:
installed_package_names.add(variant.name)
# add dependencies
if not no_deps:
for variant in variants_:
for requirement in variant.requires:
if not requirement.conflict:
pending.add(requirement.name)
# non-primary packages are treated a little differently
primary = False
version_range = None
bind_args = None
exc_type = RezBindError
if installed_variants and not quiet:
print("The following packages were installed:")
print()
_print_package_list(installed_variants)
return installed_variants