from rez.utils.formatting import PackageRequest, indent, \
dict_to_attributes_code, as_block_string
from rez.utils.schema import Required
from rez.utils.yaml import dump_yaml
from pprint import pformat
# preferred order of keys in a package definition file
package_key_order = [
'name', 'version', 'description', 'authors', 'tools', 'has_plugins',
'plugin_for', 'requires', 'build_requires', 'private_build_requires',
'variants', 'commands', 'pre_commands', 'post_commands', 'help', 'config',
'uuid', 'timestamp', 'release_message', 'changelog', 'vcs', 'revision',
'previous_version', 'previous_revision'
]
version_schema = Or(basestring, And(Version, Use(str)))
package_request_schema = Or(basestring, And(PackageRequest, Use(str)))
source_code_schema = Or(SourceCode, And(basestring, Use(SourceCode)))
# package serialisation schema
package_serialise_schema = Schema({
Required("name"):
basestring,
Optional("version"):
version_schema,
Optional("description"):
basestring,
Optional("authors"): [basestring],
Optional("tools"): [basestring],
class CommandReleaseHook(ReleaseHook):
commands_schema = Schema({
"command": basestring,
Optional("args"): Or(
And(
basestring,
Use(lambda x: x.strip().split())
),
[basestring]
),
Optional("pretty_args"): bool,
Optional("user"): basestring,
Optional("env"): dict
})
schema_dict = {
"print_commands": bool,
"print_output": bool,
"print_error": bool,
"cancel_on_error": bool,
"stop_on_error": bool,
"pre_build_commands": [commands_schema],
"pre_release_commands": [commands_schema],
"post_release_commands": [commands_schema]
}
@classmethod
def name(cls):
return "command"
def __init__(self, source_path):
super(CommandReleaseHook, self).__init__(source_path)
def execute_command(self, cmd_name, cmd_arguments, user, errors, env=None):
def _err(msg):
errors.append(msg)
if self.settings.print_error:
print(msg, file=sys.stderr)
kwargs = {}
if env:
kwargs["env"] = env
def _execute(commands):
process = Popen(commands, stdout=PIPE, stderr=STDOUT, **kwargs)
stdout, _ = process.communicate()
if process.returncode != 0:
msg = "command failed:\n%s" % stdout
_err(msg)
return False
if self.settings.print_output:
print(stdout.strip())
return True
if not os.path.isfile(cmd_name):
cmd_full_path = which(cmd_name)
else:
cmd_full_path = cmd_name
if not cmd_full_path:
msg = "%s: command not found" % cmd_name
_err(msg)
return False
cmds = [cmd_full_path] + (cmd_arguments or [])
if user == 'root':
cmds = ['sudo'] + cmds
return _execute(cmds)
elif user and user != getpass.getuser():
raise NotImplementedError # TODO
else:
return _execute(cmds)
def pre_build(self, user, install_path, variants=None, **kwargs):
errors = []
self._execute_commands(self.settings.pre_build_commands,
install_path=install_path,
package=self.package,
errors=errors,
variants=variants)
if errors and self.settings.cancel_on_error:
raise ReleaseHookCancellingError(
"The following pre-build commands failed:\n%s"
% '\n\n'.join(errors))
def pre_release(self, user, install_path, variants=None, **kwargs):
errors = []
self._execute_commands(self.settings.pre_release_commands,
install_path=install_path,
package=self.package,
errors=errors,
variants=variants)
if errors and self.settings.cancel_on_error:
raise ReleaseHookCancellingError(
"The following pre-release commands failed:\n%s"
% '\n\n'.join(errors))
def post_release(self, user, install_path, variants, **kwargs):
# note that the package we use here is the *installed* package, not the
# developer package (self.package). Otherwise, attributes such as 'root'
# will be None
errors = []
if variants:
package = variants[0].parent
else:
package = self.package
self._execute_commands(self.settings.post_release_commands,
install_path=install_path,
package=package,
errors=errors,
variants=variants)
if errors:
print_debug(
"The following post-release commands failed:\n"
+ '\n\n'.join(errors)
)
def _execute_commands(self, commands, install_path, package, errors=None,
variants=None):
release_dict = dict(path=install_path)
variant_infos = []
if variants:
for variant in variants:
if isinstance(variant, six.integer_types):
variant_infos.append(variant)
else:
package = variant.parent
var_dict = dict(variant.resource.variables)
# using '%s' will preserve potential str/unicode nature
var_dict['variant_requires'] = ['%s' % x
for x in variant.resource.variant_requires]
variant_infos.append(var_dict)
formatter = scoped_formatter(system=system,
release=release_dict,
package=package,
variants=variant_infos,
num_variants=len(variant_infos))
for conf in commands:
program = conf["command"]
env_ = None
env = conf.get("env")
if env:
env_ = os.environ.copy()
env_.update(env)
# If we have, ie, a list, and format_pretty is True, it will be printed
# as "1 2 3" instead of "[1, 2, 3]"
formatter.__dict__['format_pretty'] = conf.get(
"pretty_args", True)
args = conf.get("args", [])
args = [formatter.format(x) for x in args]
args = [expandvars(x, environ=env_) for x in args]
if self.settings.print_commands or config.debug("package_release"):
from subprocess import list2cmdline
toks = [program] + args
msgs = []
msgs.append("running command: %s" % list2cmdline(toks))
if env:
for key, value in env.items():
msgs.append(" with: %s=%s" % (key, value))
if self.settings.print_commands:
print('\n'.join(msgs))
else:
for msg in msgs:
print_debug(msg)
if not self.execute_command(cmd_name=program,
cmd_arguments=args,
user=conf.get("user"),
errors=errors,
env=env_):
if self.settings.stop_on_error:
return
_function_schema, late_bound
from rez.package_repository import create_memory_package_repository
from rez.packages import Package
from rez.package_py_utils import expand_requirement
from rez.vendor.schema.schema import Schema, Optional, Or, Use, And
from rez.vendor.six import six
from rez.vendor.version.version import Version
from contextlib import contextmanager
import os
basestring = six.string_types[0]
# this schema will automatically harden request strings like 'python-*'; see
# the 'expand_requires' function for more info.
#
package_request_schema = Or(And(basestring, Use(expand_requirement)),
And(PackageRequest, Use(str)))
tests_schema = Schema({
Optional(basestring):
Or(
Or(basestring, [basestring]),
extensible_schema_dict({
"command":
Or(basestring, [basestring]),
Optional("requires"): [package_request_schema],
Optional("run_on"):
Or(basestring, [basestring]),
Optional("on_variants"):
Or(bool, {
"type": "requires",
class ContextSettingsWidget(QtGui.QWidget, ContextViewMixin):
titles = {
"packages_path": "Search path for Rez packages",
"implicit_packages":
"Packages that are implicitly added to the request",
"package_filter": "Package exclusion/inclusion rules",
"caching": "Enables resolve caching"
}
schema_dict = {
"packages_path": [basestring],
"implicit_packages": [basestring],
"package_filter":
Or(And(None, Use(lambda x: [])), And(dict, Use(lambda x: [x])),
[dict]),
"caching":
bool
}
def __init__(self, context_model=None, attributes=None, parent=None):
"""
Args:
attributes (list of str): Select only certain settings to expose. If
None, all settings are exposed.
"""
super(ContextSettingsWidget, self).__init__(parent)
ContextViewMixin.__init__(self, context_model)
self.schema_keys = set(self.schema_dict.keys())
if attributes:
self.schema_keys &= set(attributes)
assert self.schema_keys
schema_dict = dict((k, v) for k, v in self.schema_dict.items()
if k in self.schema_keys)
self.schema = Schema(schema_dict)
self.edit = QtGui.QTextEdit()
self.edit.setStyleSheet("font: 12pt 'Courier'")
self.default_btn = QtGui.QPushButton("Set To Defaults")
self.discard_btn = QtGui.QPushButton("Discard Changes...")
self.apply_btn = QtGui.QPushButton("Apply")
self.discard_btn.setEnabled(False)
self.apply_btn.setEnabled(False)
btn_pane = create_pane(
[None, self.default_btn, self.discard_btn, self.apply_btn], True)
layout = QtGui.QVBoxLayout()
layout.addWidget(self.edit)
layout.addWidget(btn_pane)
self.setLayout(layout)
self.apply_btn.clicked.connect(self.apply_changes)
self.default_btn.clicked.connect(self.set_defaults)
self.discard_btn.clicked.connect(partial(self.discard_changes, True))
self.edit.textChanged.connect(self._settingsChanged)
self._update_text()
def _contextChanged(self, flags=0):
if not (flags & ContextModel.CONTEXT_CHANGED):
return
self._update_text()
def apply_changes(self):
def _content_error(title, text):
ret = QtGui.QMessageBox.warning(self, title, text,
QtGui.QMessageBox.Discard,
QtGui.QMessageBox.Cancel)
if ret == QtGui.QMessageBox.Discard:
self.discard_changes()
# load new content
try:
txt = self.edit.toPlainText()
data = yaml.load(str(txt))
except YAMLError as e:
_content_error("Invalid syntax", str(e))
return
# check against schema
if self.schema:
try:
data = self.schema.validate(data)
except SchemaError as e:
_content_error("Settings validation failure", str(e))
return
# apply to context model
self.context_model.set_packages_path(data["packages_path"])
self.context_model.set_package_filter(data["package_filter"])
self.context_model.set_caching(data["caching"])
self._update_text()
def discard_changes(self, prompt=False):
if prompt:
ret = QtGui.QMessageBox.warning(
self, "The context settings have been modified.",
"Your changes will be lost. Are you sure?",
QtGui.QMessageBox.Ok, QtGui.QMessageBox.Cancel)
if ret != QtGui.QMessageBox.Ok:
return
self._update_text()
def set_defaults(self):
packages_path = config.packages_path
caching = config.caching
implicits = [str(x) for x in config.implicit_packages]
package_filter = config.package_filter
data = {
"packages_path": packages_path,
"implicit_packages": implicits,
"package_filter": package_filter,
"caching": caching
}
data = dict((k, v) for k, v in data.items() if k in self.schema_keys)
self._set_text(data)
self.discard_btn.setEnabled(True)
self.apply_btn.setEnabled(True)
def _update_text(self):
model = self.context_model
implicits = [str(x) for x in model.implicit_packages]
data = {
"packages_path": model.packages_path,
"implicit_packages": implicits,
"package_filter": model.package_filter,
"caching": model.caching
}
data = dict((k, v) for k, v in data.items() if k in self.schema_keys)
self._set_text(data)
self.discard_btn.setEnabled(False)
self.apply_btn.setEnabled(False)
def _set_text(self, data):
lines = []
for key, value in data.items():
lines.append('')
txt = yaml.dump({key: value}, default_flow_style=False)
title = self.titles.get(key)
if title:
lines.append("# %s" % title)
lines.append(txt.rstrip())
txt = '\n'.join(lines) + '\n'
txt = txt.lstrip()
self.edit.setPlainText(txt)
def _settingsChanged(self):
self.discard_btn.setEnabled(True)
self.apply_btn.setEnabled(True)
# ------------------------------------------------------------------------------
help_schema = Or(
basestring, # single help entry
[[basestring]]) # multiple help entries
_is_late = And(SourceCode, lambda x: hasattr(x, "_late"))
def late_bound(schema):
return Or(SourceCode, schema)
# used when 'requires' is late bound
late_requires_schema = Schema(
[Or(PackageRequest, And(basestring, Use(PackageRequest)))])
# ------------------------------------------------------------------------------
# schema dicts
# ------------------------------------------------------------------------------
# requirements of all package-related resources
#
base_resource_schema_dict = {Required("name"): basestring}
# package family
#
package_family_schema_dict = base_resource_schema_dict.copy()
from rez.utils.schema import Required, schema_keys
from rez.utils.formatting import PackageRequest
from rez.utils.data_utils import AttrDictWrapper
from rez.package_resources_ import help_schema, _commands_schema
from rez.package_repository import create_memory_package_repository
from rez.packages_ import Package
from rez.vendor.schema.schema import Schema, Optional, Or, Use, And
from rez.vendor.version.version import Version
from contextlib import contextmanager
import os
package_request_schema = Or(basestring, And(PackageRequest, Use(str)))
package_schema = Schema({
Required("name"):
basestring,
Optional("base"):
basestring,
Optional("version"):
Or(basestring, And(Version, Use(str))),
Optional('description'):
basestring,
Optional('authors'): [basestring],
Optional('requires'): [package_request_schema],
Optional('build_requires'): [package_request_schema],
Optional('private_build_requires'): [package_request_schema],
Optional('variants'): [[package_request_schema]],
Optional('uuid'):
basestring,
Optional('config'):
dict,
class OptionalDict(Dict):
schema = Or(And(None, Use(lambda x: {})), dict)
class OptionalDictOrDictList(Setting):
schema = Or(And(None, Use(lambda x: [])), And(dict, Use(lambda x: [x])),
[dict])
# here we are simulating a resource repository, in reality this might be a
# database or filesystem.
pets = dict(
kitten=dict(obi=dict(colors=["black", "white"], male=True, age=1.0),
scully=dict(colors=["tabby"], male=False, age=0.5),
mordor=dict(colors=["black"], male=True,
age="infinite")), # bad data
puppy=dict(taco=dict(colors=["brown"],
male=True,
age=0.6,
owner="joe.bloggs"),
ringo=dict(colors=["white", "grey"], male=True, age=0.8)))
pet_schema = Schema({
Required("name"): basestring,
Required("colors"): And([basestring], Use(set)),
Required("male"): bool,
Required("age"): float,
Optional("owner"): basestring
})
class BasePetResource(Resource):
schema_error = PetResourceError
def __init__(self, variables=None):
super(BasePetResource, self).__init__(variables)
self.validations = {}
# tracks validations
def _validate_key(self, key, attr, key_schema):
class OptionalStrList(StrList):
schema = Or(And(None, Use(lambda x: [])), [basestring])
class OptionalStrList(StrList):
schema = Or(And(None, Use(lambda x: [])),
[str_type])
class FileSystemCombinedPackageFamilyResource(PackageFamilyResource):
key = "filesystem.family.combined"
repository_type = "filesystem"
schema = Schema({
Optional("versions"): [
And(basestring, Use(Version))
],
Optional("version_overrides"): {
And(basestring, Use(VersionRange)): dict
}
})
@property
def ext(self):
return self.get("ext")
@property
def filepath(self):
filename = "%s.%s" % (self.name, self.ext)
return os.path.join(self.location, filename)
def _uri(self):
return self.filepath
def get_last_release_time(self):
try:
return os.path.getmtime(self.filepath)
except OSError:
return 0
def iter_packages(self):
# unversioned package
if config.allow_unversioned_packages and not self.versions:
package = self._repository.get_resource(
FileSystemCombinedPackageResource.key,
location=self.location,
name=self.name,
ext=self.ext)
yield package
return
# versioned packages
for version in self.versions:
package = self._repository.get_resource(
FileSystemCombinedPackageResource.key,
location=self.location,
name=self.name,
ext=self.ext,
version=str(version))
yield package
def _load(self):
format_ = FileFormat[self.ext]
data = load_from_file(
self.filepath,
format_,
disable_memcache=self._repository.disable_memcache
)
check_format_version(self.filepath, data)
return data
variant_schema = Schema(variant_schema_dict)
#------------------------------------------------------------------------------
# schemas for converting from POD datatypes
#------------------------------------------------------------------------------
_commands_schema = Or(SourceCode, # commands as converted function
callable, # commands as function
basestring, # commands in text block
[basestring]) # old-style (rez-1) commands
_package_request_schema = And(basestring, Use(PackageRequest))
package_pod_schema_dict = base_resource_schema_dict.copy()
large_string_dict = And(basestring, Use(lambda x: dedent(x).strip()))
package_pod_schema_dict.update({
Optional("base"): basestring,
Optional("version"): And(basestring, Use(Version)),
Optional('description'): large_string_dict,
Optional('authors'): [basestring],
Optional('requires'): [_package_request_schema],
class CommandReleaseHook(ReleaseHook):
commands_schema = Schema({
"command":
basestring,
Optional("args"):
Or(And(basestring, Use(lambda x: x.strip().split())), [basestring]),
Optional("user"):
basestring,
Optional("env"):
dict
})
schema_dict = {
"print_commands": bool,
"print_output": bool,
"print_error": bool,
"cancel_on_error": bool,
"stop_on_error": bool,
"pre_build_commands": [commands_schema],
"pre_release_commands": [commands_schema],
"post_release_commands": [commands_schema]
}
@classmethod
def name(cls):
return "command"
def __init__(self, source_path):
super(CommandReleaseHook, self).__init__(source_path)
def execute_command(self, cmd_name, cmd_arguments, user, errors, env=None):
def _err(msg):
errors.append(msg)
if self.settings.print_error:
print >> sys.stderr, msg
kwargs = {}
if env:
kwargs["_env"] = env
def _execute(cmd, arguments):
try:
result = cmd(*(arguments or []), **kwargs)
if self.settings.print_output:
print result.stdout.strip()
except ErrorReturnCode as e:
# `e` shows the command that was run
msg = "command failed:\n%s" % str(e)
_err(msg)
return False
return True
if not os.path.isfile(cmd_name):
cmd_full_path = which(cmd_name)
else:
cmd_full_path = cmd_name
if not cmd_full_path:
msg = "%s: command not found" % cmd_name
_err(msg)
return False
run_cmd = Command(cmd_full_path)
if user == 'root':
with sudo:
return _execute(run_cmd, cmd_arguments)
elif user and user != getpass.getuser():
raise NotImplementedError # TODO
else:
return _execute(run_cmd, cmd_arguments)
def pre_build(self, user, install_path, **kwargs):
errors = []
self._execute_commands(self.settings.pre_build_commands,
install_path=install_path,
package=self.package,
errors=errors)
if errors and self.settings.cancel_on_error:
raise ReleaseHookCancellingError(
"The following pre-build commands failed:\n%s" %
'\n\n'.join(errors))
def pre_release(self, user, install_path, **kwargs):
errors = []
self._execute_commands(self.settings.pre_release_commands,
install_path=install_path,
package=self.package,
errors=errors)
if errors and self.settings.cancel_on_error:
raise ReleaseHookCancellingError(
"The following pre-release commands failed:\n%s" %
'\n\n'.join(errors))
def post_release(self, user, install_path, variants, **kwargs):
# note that the package we use here is the *installed* package, not the
# developer package (self.package). Otherwise, attributes such as 'root'
# will be None
errors = []
if variants:
package = variants[0].parent
else:
package = self.package
self._execute_commands(self.settings.post_release_commands,
install_path=install_path,
package=package,
errors=errors)
if errors:
print_debug("The following post-release commands failed:\n" +
'\n\n'.join(errors))
def _execute_commands(self, commands, install_path, package, errors=None):
release_dict = dict(path=install_path)
formatter = scoped_formatter(system=system,
release=release_dict,
package=package)
for conf in commands:
program = conf["command"]
env_ = None
env = conf.get("env")
if env:
env_ = os.environ.copy()
env_.update(env)
args = conf.get("args", [])
args = [formatter.format(x) for x in args]
args = [expandvars(x, environ=env_) for x in args]
if self.settings.print_commands or config.debug("package_release"):
from subprocess import list2cmdline
toks = [program] + args
msgs = []
msgs.append("running command: %s" % list2cmdline(toks))
if env:
for key, value in env.iteritems():
msgs.append(" with: %s=%s" % (key, value))
if self.settings.print_commands:
print '\n'.join(msgs)
else:
for msg in msgs:
print_debug(msg)
if not self.execute_command(cmd_name=program,
cmd_arguments=args,
user=conf.get("user"),
errors=errors,
env=env_):
if self.settings.stop_on_error:
return
from rez.exceptions import PackageMetadataError
from rez.package_resources_ import help_schema, _commands_schema, \
_function_schema, late_bound
from rez.package_repository import create_memory_package_repository
from rez.packages_ import Package
from rez.package_py_utils import expand_requirement
from rez.vendor.schema.schema import Schema, Optional, Or, Use, And
from rez.vendor.version.version import Version
from contextlib import contextmanager
import os
# this schema will automatically harden request strings like 'python-*'; see
# the 'expand_requires' function for more info.
#
package_request_schema = Or(And(basestring, Use(expand_requirement)),
And(PackageRequest, Use(str)))
tests_schema = Schema({
Optional(basestring): Or(
Or(basestring, [basestring]),
{
"command": Or(basestring, [basestring]),
Optional("requires"): [package_request_schema]
}
)
})
package_schema = Schema({
Optional("requires_rez_version"): And(basestring, Use(Version)),
