class FileSystemCombinedPackageFamilyResource(PackageFamilyResource): key = "filesystem.family.combined" repository_type = "filesystem" schema = Schema({ Optional("versions"): [ And(basestring, Use(Version)) ], Optional("version_overrides"): { And(basestring, Use(VersionRange)): dict } }) @property def ext(self): return self.get("ext") @property def filepath(self): filename = "%s.%s" % (self.name, self.ext) return os.path.join(self.location, filename) def _uri(self): return self.filepath def get_last_release_time(self): try: return os.path.getmtime(self.filepath) except OSError: return 0 def iter_packages(self): # unversioned package if config.allow_unversioned_packages and not self.versions: package = self._repository.get_resource( FileSystemCombinedPackageResource.key, location=self.location, name=self.name, ext=self.ext) yield package return # versioned packages for version in self.versions: package = self._repository.get_resource( FileSystemCombinedPackageResource.key, location=self.location, name=self.name, ext=self.ext, version=str(version)) yield package def _load(self): format_ = FileFormat[self.ext] data = load_from_file(self.filepath, format_) check_format_version(self.filepath, data) return data
def schema(cls): from rez.utils.platform_ import platform_ # Note that this bakes the physical / logical cores at the time the # config is read... which should be fine return Or( And(int, lambda x: x > 0), And("physical_cores", Use(lambda x: platform_.physical_cores)), And("logical_cores", Use(lambda x: platform_.logical_cores)), )
def _to(value): if isinstance(value, dict): d = {} for k, v in value.iteritems(): if isinstance(k, basestring): k = Required(k) if required else Optional(k) d[k] = _to(v) if allow_custom_keys: d[Optional(basestring)] = modifier or object schema = Schema(d) elif modifier: schema = And(value, modifier) else: schema = value return schema
class PipInstallRemaps(Setting): """Ordered, pip install remappings.""" PARDIR, SEP = map(re.escape, (os.pardir, os.sep)) RE_TOKENS = {'sep': SEP, 's': SEP, 'pardir': PARDIR, 'p': PARDIR} TOKENS = {'sep': os.sep, 's': os.sep, 'pardir': os.pardir, 'p': os.pardir} KEYS = ["record_path", "pip_install", "rez_install"] schema = Schema([{key: And(str, len) for key in KEYS}]) def validate(self, data): """Extended to substitute regex-escaped path tokens.""" return [{ key: expression.format( **(self.RE_TOKENS if key == "record_path" else self.TOKENS)) for key, expression in remap.items() } for remap in super(PipInstallRemaps, self).validate(data)]
class OptionalStrList(StrList): schema = Or(And(None, Use(lambda x: [])), [str_type])
class OptionalDictOrDictList(Setting): schema = Or(And(None, Use(lambda x: [])), And(dict, Use(lambda x: [x])), [dict])
class OptionalDict(Dict): schema = Or(And(None, Use(lambda x: {})), dict)
class OptionalStrList(StrList): schema = Or(And(None, Use(lambda x: [])), [basestring])
# here we are simulating a resource repository, in reality this might be a # database or filesystem. pets = dict( kitten=dict(obi=dict(colors=["black", "white"], male=True, age=1.0), scully=dict(colors=["tabby"], male=False, age=0.5), mordor=dict(colors=["black"], male=True, age="infinite")), # bad data puppy=dict(taco=dict(colors=["brown"], male=True, age=0.6, owner="joe.bloggs"), ringo=dict(colors=["white", "grey"], male=True, age=0.8))) pet_schema = Schema({ Required("name"): basestring, Required("colors"): And([basestring], Use(set)), Required("male"): bool, Required("age"): float, Optional("owner"): basestring }) class BasePetResource(Resource): schema_error = PetResourceError def __init__(self, variables=None): super(BasePetResource, self).__init__(variables) self.validations = {} # tracks validations def _validate_key(self, key, attr, key_schema):
from rez.utils.schema import Required, schema_keys from rez.utils.formatting import PackageRequest from rez.utils.data_utils import AttrDictWrapper from rez.package_resources_ import help_schema, _commands_schema from rez.package_repository import create_memory_package_repository from rez.packages_ import Package from rez.vendor.schema.schema import Schema, Optional, Or, Use, And from rez.vendor.version.version import Version from contextlib import contextmanager import os package_request_schema = Or(basestring, And(PackageRequest, Use(str))) package_schema = Schema({ Required("name"): basestring, Optional("base"): basestring, Optional("version"): Or(basestring, And(Version, Use(str))), Optional('description'): basestring, Optional('authors'): [basestring], Optional('requires'): [package_request_schema], Optional('build_requires'): [package_request_schema], Optional('private_build_requires'): [package_request_schema], Optional('variants'): [[package_request_schema]], Optional('uuid'): basestring, Optional('config'): dict,
dict_to_attributes_code, as_block_string from rez.utils.schema import Required from rez.utils.yaml import dump_yaml from pprint import pformat # preferred order of keys in a package definition file package_key_order = [ 'name', 'nice_name', 'version', 'description', 'authors', 'tools', 'tools_info', 'excluded_tools', 'has_plugins', 'plugin_for', 'requires', 'build_requires', 'private_build_requires', 'variants', 'commands', 'pre_commands', 'post_commands', 'help', 'config', 'uuid', 'timestamp', 'release_message', 'changelog', 'vcs', 'revision', 'previous_version', 'previous_revision', 'has_plugins', 'plugin_for', 'plugin_launch_commands' ] version_schema = Or(basestring, And(Version, Use(str))) package_request_schema = Or(basestring, And(PackageRequest, Use(str))) source_code_schema = Or(SourceCode, And(basestring, Use(SourceCode))) # package serialisation schema package_serialise_schema = Schema({ Required("name"): basestring, Optional("nice_name"): basestring, Optional("version"): version_schema, Optional("description"): basestring,
class CommandReleaseHook(ReleaseHook): commands_schema = Schema({ "command": basestring, Optional("args"): Or(And(basestring, Use(lambda x: x.strip().split())), [basestring]), Optional("user"): basestring, Optional("env"): dict }) schema_dict = { "print_commands": bool, "print_output": bool, "print_error": bool, "cancel_on_error": bool, "stop_on_error": bool, "pre_build_commands": [commands_schema], "pre_release_commands": [commands_schema], "post_release_commands": [commands_schema] } @classmethod def name(cls): return "command" def __init__(self, source_path): super(CommandReleaseHook, self).__init__(source_path) def execute_command(self, cmd_name, cmd_arguments, user, errors, env=None): def _err(msg): errors.append(msg) if self.settings.print_error: print >> sys.stderr, msg kwargs = {} if env: kwargs["_env"] = env def _execute(cmd, arguments): try: result = cmd(*(arguments or []), **kwargs) if self.settings.print_output: print result.stdout.strip() except ErrorReturnCode as e: # `e` shows the command that was run msg = "command failed:\n%s" % str(e) _err(msg) return False return True if not os.path.isfile(cmd_name): cmd_full_path = which(cmd_name) else: cmd_full_path = cmd_name if not cmd_full_path: msg = "%s: command not found" % cmd_name _err(msg) return False run_cmd = Command(cmd_full_path) if user == 'root': with sudo: return _execute(run_cmd, cmd_arguments) elif user and user != getpass.getuser(): raise NotImplementedError # TODO else: return _execute(run_cmd, cmd_arguments) def pre_build(self, user, install_path, **kwargs): errors = [] self._execute_commands(self.settings.pre_build_commands, install_path=install_path, package=self.package, errors=errors) if errors and self.settings.cancel_on_error: raise ReleaseHookCancellingError( "The following pre-build commands failed:\n%s" % '\n\n'.join(errors)) def pre_release(self, user, install_path, **kwargs): errors = [] self._execute_commands(self.settings.pre_release_commands, install_path=install_path, package=self.package, errors=errors) if errors and self.settings.cancel_on_error: raise ReleaseHookCancellingError( "The following pre-release commands failed:\n%s" % '\n\n'.join(errors)) def post_release(self, user, install_path, variants, **kwargs): # note that the package we use here is the *installed* package, not the # developer package (self.package). Otherwise, attributes such as 'root' # will be None errors = [] if variants: package = variants[0].parent else: package = self.package self._execute_commands(self.settings.post_release_commands, install_path=install_path, package=package, errors=errors) if errors: print_debug("The following post-release commands failed:\n" + '\n\n'.join(errors)) def _execute_commands(self, commands, install_path, package, errors=None): release_dict = dict(path=install_path) formatter = scoped_formatter(system=system, release=release_dict, package=package) for conf in commands: program = conf["command"] env_ = None env = conf.get("env") if env: env_ = os.environ.copy() env_.update(env) args = conf.get("args", []) args = [formatter.format(x) for x in args] args = [expandvars(x, environ=env_) for x in args] if self.settings.print_commands or config.debug("package_release"): from subprocess import list2cmdline toks = [program] + args msgs = [] msgs.append("running command: %s" % list2cmdline(toks)) if env: for key, value in env.iteritems(): msgs.append(" with: %s=%s" % (key, value)) if self.settings.print_commands: print '\n'.join(msgs) else: for msg in msgs: print_debug(msg) if not self.execute_command(cmd_name=program, cmd_arguments=args, user=conf.get("user"), errors=errors, env=env_): if self.settings.stop_on_error: return
from rez.utils.data_utils import AttrDictWrapper from rez.utils.logging_ import print_warning from rez.package_resources_ import help_schema, _commands_schema, \ _function_schema, late_bound from rez.package_repository import create_memory_package_repository from rez.packages_ import Package from rez.package_py_utils import expand_requirement from rez.vendor.schema.schema import Schema, Optional, Or, Use, And from rez.vendor.version.version import Version from contextlib import contextmanager import os # this schema will automatically harden request strings like 'python-*'; see # the 'expand_requires' function for more info. # package_request_schema = Or(And(basestring, Use(expand_requirement)), And(PackageRequest, Use(str))) tests_schema = Schema({ Optional(basestring): Or( Or(basestring, [basestring]), { "command": Or(basestring, [basestring]), Optional("requires"): [package_request_schema] }) }) package_schema = Schema({ Required("name"): basestring, Optional("base"):
class CommandReleaseHook(ReleaseHook): commands_schema = Schema({ "command": basestring, Optional("args"): Or(And(basestring, Use(lambda x: x.strip().split())), [basestring]), Optional("user"): basestring }) schema_dict = { "print_commands": bool, "print_output": bool, "print_error": bool, "cancel_on_error": bool, "stop_on_error": bool, "pre_build_commands": [commands_schema], "pre_release_commands": [commands_schema], "post_release_commands": [commands_schema] } @classmethod def name(cls): return "command" def __init__(self, source_path): super(CommandReleaseHook, self).__init__(source_path) def execute_command(self, cmd_name, cmd_arguments, user, errors): def _err(msg): errors.append(msg) if self.settings.print_error: print >> sys.stderr, msg def _execute(cmd, arguments): try: result = cmd(*(arguments or [])) if self.settings.print_output: print result.stdout.strip() except ErrorReturnCode as e: # `e` shows the command that was run msg = "command failed:\n%s" % str(e) _err(msg) return False return True if not os.path.isfile(cmd_name): cmd_full_path = which(cmd_name) else: cmd_full_path = cmd_name if not cmd_full_path: msg = "%s: command not found" % cmd_name _err(msg) return False run_cmd = Command(cmd_full_path) if user == 'root': with sudo: return _execute(run_cmd, cmd_arguments) elif user and user != getpass.getuser(): raise NotImplementedError # TODO else: return _execute(run_cmd, cmd_arguments) def _release(self, commands, errors=None): for conf in commands: if self.settings.print_commands or config.debug("package_release"): from subprocess import list2cmdline toks = [conf["command"]] + conf.get("args", []) msg = "running command: %s" % list2cmdline(toks) if self.settings.print_commands: print msg else: print_debug(msg) if not self.execute_command(cmd_name=conf.get("command"), cmd_arguments=conf.get("args"), user=conf.get("user"), errors=errors): if self.settings.stop_on_error: return def pre_build(self, user, install_path, **kwargs): errors = [] self._release(self.settings.pre_build_commands, errors=errors) if errors and self.settings.cancel_on_error: raise ReleaseHookCancellingError( "The following pre-build commands failed:\n%s" % '\n\n'.join(errors)) def pre_release(self, user, install_path, **kwargs): errors = [] self._release(self.settings.pre_release_commands, errors=errors) if errors and self.settings.cancel_on_error: raise ReleaseHookCancellingError( "The following pre-release commands failed:\n%s" % '\n\n'.join(errors)) def post_release(self, user, install_path, variants, **kwargs): self._release(self.settings.post_release_commands)
class CommandReleaseHook(ReleaseHook): commands_schema = Schema({ "command": basestring, Optional("args"): Or(And(basestring, Use(lambda x: x.strip().split())), [basestring]), Optional("pretty_args"): bool, Optional("user"): basestring, Optional("env"): dict }) schema_dict = { "print_commands": bool, "print_output": bool, "print_error": bool, "cancel_on_error": bool, "stop_on_error": bool, "pre_build_commands": [commands_schema], "pre_release_commands": [commands_schema], "post_release_commands": [commands_schema] } @classmethod def name(cls): return "command" def __init__(self, source_path): super(CommandReleaseHook, self).__init__(source_path) def execute_command(self, cmd_name, cmd_arguments, user, errors, env=None): def _err(msg): errors.append(msg) if self.settings.print_error: print(msg, file=sys.stderr) kwargs = {} if env: kwargs["env"] = env def _execute(commands): process = Popen(commands, stdout=PIPE, stderr=STDOUT, **kwargs) stdout, _ = process.communicate() if process.returncode != 0: msg = "command failed:\n%s" % stdout _err(msg) return False if self.settings.print_output: print(stdout.strip()) return True if not os.path.isfile(cmd_name): cmd_full_path = which(cmd_name) else: cmd_full_path = cmd_name if not cmd_full_path: msg = "%s: command not found" % cmd_name _err(msg) return False cmds = [cmd_full_path] + (cmd_arguments or []) if user == 'root': cmds = ['sudo'] + cmds return _execute(cmds) elif user and user != getpass.getuser(): raise NotImplementedError # TODO else: return _execute(cmds) def pre_build(self, user, install_path, variants=None, **kwargs): errors = [] self._execute_commands(self.settings.pre_build_commands, install_path=install_path, package=self.package, errors=errors, variants=variants) if errors and self.settings.cancel_on_error: raise ReleaseHookCancellingError( "The following pre-build commands failed:\n%s" % '\n\n'.join(errors)) def pre_release(self, user, install_path, variants=None, **kwargs): errors = [] self._execute_commands(self.settings.pre_release_commands, install_path=install_path, package=self.package, errors=errors, variants=variants) if errors and self.settings.cancel_on_error: raise ReleaseHookCancellingError( "The following pre-release commands failed:\n%s" % '\n\n'.join(errors)) def post_release(self, user, install_path, variants, **kwargs): # note that the package we use here is the *installed* package, not the # developer package (self.package). Otherwise, attributes such as 'root' # will be None errors = [] if variants: package = variants[0].parent else: package = self.package self._execute_commands(self.settings.post_release_commands, install_path=install_path, package=package, errors=errors, variants=variants) if errors: print_debug("The following post-release commands failed:\n" + '\n\n'.join(errors)) def _execute_commands(self, commands, install_path, package, errors=None, variants=None): release_dict = dict(path=install_path) variant_infos = [] if variants: for variant in variants: if isinstance(variant, (int, long)): variant_infos.append(variant) else: package = variant.parent var_dict = dict(variant.resource.variables) # using '%s' will preserve potential str/unicode nature var_dict['variant_requires'] = [ '%s' % x for x in variant.resource.variant_requires ] variant_infos.append(var_dict) formatter = scoped_formatter(system=system, release=release_dict, package=package, variants=variant_infos, num_variants=len(variant_infos)) for conf in commands: program = conf["command"] env_ = None env = conf.get("env") if env: env_ = os.environ.copy() env_.update(env) # If we have, ie, a list, and format_pretty is True, it will be printed # as "1 2 3" instead of "[1, 2, 3]" formatter.__dict__['format_pretty'] = conf.get("pretty_args", True) args = conf.get("args", []) args = [formatter.format(x) for x in args] args = [expandvars(x, environ=env_) for x in args] if self.settings.print_commands or config.debug("package_release"): from subprocess import list2cmdline toks = [program] + args msgs = [] msgs.append("running command: %s" % list2cmdline(toks)) if env: for key, value in env.iteritems(): msgs.append(" with: %s=%s" % (key, value)) if self.settings.print_commands: print('\n'.join(msgs)) else: for msg in msgs: print_debug(msg) if not self.execute_command(cmd_name=program, cmd_arguments=args, user=conf.get("user"), errors=errors, env=env_): if self.settings.stop_on_error: return
variant_schema = Schema(variant_schema_dict) #------------------------------------------------------------------------------ # schemas for converting from POD datatypes #------------------------------------------------------------------------------ _commands_schema = Or(SourceCode, # commands as converted function callable, # commands as function basestring, # commands in text block [basestring]) # old-style (rez-1) commands _package_request_schema = And(basestring, Use(PackageRequest)) package_pod_schema_dict = base_resource_schema_dict.copy() large_string_dict = And(basestring, Use(lambda x: dedent(x).strip())) package_pod_schema_dict.update({ Optional("base"): basestring, Optional("version"): And(basestring, Use(Version)), Optional('description'): large_string_dict, Optional('authors'): [basestring], Optional('requires'): [_package_request_schema],
class ContextSettingsWidget(QtGui.QWidget, ContextViewMixin): titles = { "packages_path": "Search path for Rez packages", "implicit_packages": "Packages that are implicitly added to the request", "package_filter": "Package exclusion/inclusion rules" } schema_dict = { "packages_path": [basestring], "implicit_packages": [basestring], "package_filter": Or(And(None, Use(lambda x: [])), And(dict, Use(lambda x: [x])), [dict]) } def __init__(self, context_model=None, attributes=None, parent=None): """ Args: attributes (list of str): Select only certain settings to expose. If None, all settings are exposed. """ super(ContextSettingsWidget, self).__init__(parent) ContextViewMixin.__init__(self, context_model) self.schema_keys = set(self.schema_dict.iterkeys()) if attributes: self.schema_keys &= set(attributes) assert self.schema_keys schema_dict = dict((k, v) for k, v in self.schema_dict.iteritems() if k in self.schema_keys) self.schema = Schema(schema_dict) self.edit = QtGui.QTextEdit() self.edit.setStyleSheet("font: 12pt 'Courier'") self.default_btn = QtGui.QPushButton("Set To Defaults") self.discard_btn = QtGui.QPushButton("Discard Changes...") self.apply_btn = QtGui.QPushButton("Apply") self.discard_btn.setEnabled(False) self.apply_btn.setEnabled(False) btn_pane = create_pane([None, self.default_btn, self.discard_btn, self.apply_btn], True) layout = QtGui.QVBoxLayout() layout.addWidget(self.edit) layout.addWidget(btn_pane) self.setLayout(layout) self.apply_btn.clicked.connect(self.apply_changes) self.default_btn.clicked.connect(self.set_defaults) self.discard_btn.clicked.connect(partial(self.discard_changes, True)) self.edit.textChanged.connect(self._settingsChanged) self._update_text() def _contextChanged(self, flags=0): if not (flags & ContextModel.CONTEXT_CHANGED): return self._update_text() def apply_changes(self): def _content_error(title, text): ret = QtGui.QMessageBox.warning(self, title, text, QtGui.QMessageBox.Discard, QtGui.QMessageBox.Cancel) if ret == QtGui.QMessageBox.Discard: self.discard_changes() # load new content try: txt = self.edit.toPlainText() data = yaml.load(str(txt)) except YAMLError as e: _content_error("Invalid syntax", str(e)) return # check against schema if self.schema: try: data = self.schema.validate(data) except SchemaError as e: _content_error("Settings validation failure", str(e)) return # apply to context model self.context_model.set_packages_path(data["packages_path"]) self.context_model.set_package_filter(data["package_filter"]) self._update_text() def discard_changes(self, prompt=False): if prompt: ret = QtGui.QMessageBox.warning( self, "The context settings have been modified.", "Your changes will be lost. Are you sure?", QtGui.QMessageBox.Ok, QtGui.QMessageBox.Cancel) if ret != QtGui.QMessageBox.Ok: return self._update_text() def set_defaults(self): packages_path = config.packages_path implicits = [str(x) for x in config.implicit_packages] package_filter = config.package_filter data = {"packages_path": packages_path, "implicit_packages": implicits, "package_filter": package_filter} data = dict((k, v) for k, v in data.iteritems() if k in self.schema_keys) self._set_text(data) self.discard_btn.setEnabled(True) self.apply_btn.setEnabled(True) def _update_text(self): model = self.context_model implicits = [str(x) for x in model.implicit_packages] data = {"packages_path": model.packages_path, "implicit_packages": implicits, "package_filter": model.package_filter} data = dict((k, v) for k, v in data.iteritems() if k in self.schema_keys) self._set_text(data) self.discard_btn.setEnabled(False) self.apply_btn.setEnabled(False) def _set_text(self, data): lines = [] for key, value in data.iteritems(): lines.append('') txt = yaml.dump({key: value}, default_flow_style=False) title = self.titles.get(key) if title: lines.append("# %s" % title) lines.append(txt.rstrip()) txt = '\n'.join(lines) + '\n' txt = txt.lstrip() self.edit.setPlainText(txt) def _settingsChanged(self): self.discard_btn.setEnabled(True) self.apply_btn.setEnabled(True)
# package attributes that are rex-based functions package_rex_keys = ( "pre_commands", "commands", "post_commands" ) #------------------------------------------------------------------------------ # utility schemas #------------------------------------------------------------------------------ help_schema = Or(basestring, # single help entry [[basestring]]) # multiple help entries _is_late = And(SourceCode, lambda x: hasattr(x, "_late")) def late_bound(schema): return Or(SourceCode, schema) # used when 'requires' is late bound late_requires_schema = Schema([ Or(PackageRequest, And(basestring, Use(PackageRequest))) ]) #------------------------------------------------------------------------------ # schema dicts #------------------------------------------------------------------------------ # requirements of all package-related resources
"build_command", "preprocess", ) # package attributes that are rex-based functions package_rex_keys = ("pre_commands", "commands", "post_commands") # ------------------------------------------------------------------------------ # utility schemas # ------------------------------------------------------------------------------ help_schema = Or( basestring, # single help entry [[basestring]]) # multiple help entries _is_late = And(SourceCode, lambda x: hasattr(x, "_late")) def late_bound(schema): return Or(SourceCode, schema) # used when 'requires' is late bound late_requires_schema = Schema( [Or(PackageRequest, And(basestring, Use(PackageRequest)))]) # ------------------------------------------------------------------------------ # schema dicts # ------------------------------------------------------------------------------ # requirements of all package-related resources
from rez.exceptions import PackageMetadataError from rez.package_resources_ import help_schema, _commands_schema, \ _function_schema, late_bound from rez.package_repository import create_memory_package_repository from rez.packages_ import Package from rez.package_py_utils import expand_requirement from rez.vendor.schema.schema import Schema, Optional, Or, Use, And from rez.vendor.version.version import Version from contextlib import contextmanager import os # this schema will automatically harden request strings like 'python-*'; see # the 'expand_requires' function for more info. # package_request_schema = Or(And(basestring, Use(expand_requirement)), And(PackageRequest, Use(str))) tests_schema = Schema({ Optional(basestring): Or( Or(basestring, [basestring]), { "command": Or(basestring, [basestring]), Optional("requires"): [package_request_schema] } ) }) package_schema = Schema({ Optional("requires_rez_version"): And(basestring, Use(Version)),
package_schema = Schema(package_schema_dict) variant_schema = Schema(variant_schema_dict) #------------------------------------------------------------------------------ # schemas for converting from POD datatypes #------------------------------------------------------------------------------ _commands_schema = Or( SourceCode, # commands as converted function callable, # commands as function basestring, # commands in text block [basestring]) # old-style (rez-1) commands _package_request_schema = And(basestring, Use(PackageRequest)) package_pod_schema_dict = base_resource_schema_dict.copy() large_string_dict = And(basestring, Use(lambda x: dedent(x).strip())) package_pod_schema_dict.update({ Optional("base"): basestring, Optional("version"): And(basestring, Use(Version)), Optional('description'): large_string_dict, Optional('authors'): [basestring], Optional('requires'): [_package_request_schema], Optional('build_requires'): [_package_request_schema],