Пример #1
0
    def schema(cls):
        from rez.utils.platform_ import platform_

        # Note that this bakes the physical / logical cores at the time the
        # config is read... which should be fine
        return Or(
            And(int, lambda x: x > 0),
            And("physical_cores", Use(lambda x: platform_.physical_cores)),
            And("logical_cores", Use(lambda x: platform_.logical_cores)),
        )
Пример #2
0
class ForceOrBool(Bool):
    FORCE_STR = "force"

    # need force first, or Bool.schema will coerce "force" to True
    schema = Or(FORCE_STR, Bool.schema)
    all_words = Bool.all_words | frozenset([FORCE_STR])

    def _parse_env_var(self, value):
        if value == self.FORCE_STR:
            return value
        return super(ForceOrBool, self)._parse_env_var(value)
Пример #3
0
class EmailReleaseHook(ReleaseHook):

    schema_dict = {
        "subject":          basestring,
        "body":             basestring,
        "smtp_host":        basestring,
        "smtp_port":        int,
        "sender":           basestring,
        "recipients":       Or(basestring, [basestring])}

    @classmethod
    def name(cls):
        return "emailer"

    def __init__(self, source_path):
        super(EmailReleaseHook, self).__init__(source_path)

    def post_release(self, user, install_path, variants, release_message=None,
                     changelog=None, previous_version=None, **kwargs):
        if not variants:
            return  # nothing was released

        # construct email body
        release_dict = dict(path=install_path,
                            previous_version=previous_version or "None.",
                            message=release_message or "No release message.",
                            changelog=changelog or "No changelog.")

        paths_str = '\n'.join(x.root for x in variants)
        variants_dict = dict(count=len(variants),
                             paths=paths_str)

        formatter = scoped_formatter(release=release_dict,
                                     variants=variants_dict,
                                     system=system,
                                     package=self.package)

        body = formatter.format(self.settings.body)
        body = body.strip()
        body = body.replace("\n\n\n", "\n\n")

        # construct subject line, send email
        subject = formatter.format(self.settings.subject)
        self.send_email(subject, body)

    def send_email(self, subject, body):
        if not self.settings.recipients:
            return  # nothing to do, sending email to nobody

        if not self.settings.smtp_host:
            print_warning("did not send release email: "
                          "SMTP host is not specified")
            return

        recipients = self.get_recipients()
        if not recipients:
            return

        print("Sending release email to:")
        print('\n'.join("- %s" % x for x in recipients))

        msg = MIMEText(body)
        msg["Subject"] = subject
        msg["From"] = self.settings.sender
        msg["To"] = str(',').join(recipients)

        try:
            s = smtplib.SMTP(self.settings.smtp_host, self.settings.smtp_port)
            s.sendmail(from_addr=self.settings.sender,
                       to_addrs=recipients,
                       msg=msg.as_string())
            print('Email(s) sent.')
        except Exception as e:
            print_error("release email delivery failed: %s" % str(e))

    def get_recipients(self):
        value = self.settings.recipients

        if isinstance(value, list):
            return value

        if os.path.exists(value):
            filepath = value

            try:
                return self.load_recipients(filepath)
            except Exception as e:
                print_error("failed to load recipients config: %s. Emails "
                            "not sent" % str(e))
        elif '@' in value:
            return [value]  # assume it's an email address
        else:
            print_error("email recipient file does not exist: %s. Emails not "
                        "sent" % value)

        return []

    def load_recipients(self, filepath):
        def test(value, type_):
            if not isinstance(value, type_):
                raise TypeError("Expected %s, not %s" % type_, value)
            return value

        conf = load_yaml(filepath)
        recipients = set()

        for rule in test(conf.get("rules", []), list):
            filters = rule.get("filters")
            match = True

            if filters:
                for attr, test_value in test(filters, dict).items():

                    missing = object()
                    value = getattr(self.package, attr, missing)

                    if value is missing:
                        match = False
                    elif test_value is None:
                        match = True
                    elif isinstance(test_value, list):
                        match = (value in test_value)
                    else:
                        match = (value == test_value)

                    if not match:
                        break

            if match:
                rule_recipients = rule.get("recipients")
                recipients.update(test(rule_recipients, list))

        return sorted(recipients)
Пример #4
0
    dict_to_attributes_code, as_block_string
from rez.utils.schema import Required
from rez.utils.yaml import dump_yaml
from pprint import pformat

# preferred order of keys in a package definition file
package_key_order = [
    'name', 'nice_name', 'version', 'description', 'authors', 'tools',
    'tools_info', 'excluded_tools', 'has_plugins', 'plugin_for', 'requires',
    'build_requires', 'private_build_requires', 'variants', 'commands',
    'pre_commands', 'post_commands', 'help', 'config', 'uuid', 'timestamp',
    'release_message', 'changelog', 'vcs', 'revision', 'previous_version',
    'previous_revision', 'has_plugins', 'plugin_for', 'plugin_launch_commands'
]

version_schema = Or(basestring, And(Version, Use(str)))

package_request_schema = Or(basestring, And(PackageRequest, Use(str)))

source_code_schema = Or(SourceCode, And(basestring, Use(SourceCode)))

# package serialisation schema
package_serialise_schema = Schema({
    Required("name"):
    basestring,
    Optional("nice_name"):
    basestring,
    Optional("version"):
    version_schema,
    Optional("description"):
    basestring,
Пример #5
0
    "preprocess",
)

# package attributes that are rex-based functions
package_rex_keys = (
    "pre_commands",
    "commands",
    "post_commands"
)


#------------------------------------------------------------------------------
# utility schemas
#------------------------------------------------------------------------------

help_schema = Or(basestring,  # single help entry
                 [[basestring]])  # multiple help entries

_is_late = And(SourceCode, lambda x: hasattr(x, "_late"))

def late_bound(schema):
    return Or(SourceCode, schema)

# used when 'requires' is late bound
late_requires_schema = Schema([
    Or(PackageRequest, And(basestring, Use(PackageRequest)))
])


#------------------------------------------------------------------------------
# schema dicts
#------------------------------------------------------------------------------
Пример #6
0
class OptionalStrOrFunction(Setting):
    schema = Or(None, basestring, callable)

    def _parse_env_var(self, value):
        # note: env-var override only supports string, eg 'mymodule.preprocess_func'
        return value
Пример #7
0
 def schema(cls):
     from rez.resolved_context import RezToolsVisibility
     return Or(*(x.name for x in RezToolsVisibility))
Пример #8
0
 def schema(cls):
     from rez.resolved_context import SuiteVisibility
     return Or(*(x.name for x in SuiteVisibility))
Пример #9
0
Файл: config.py Проект: opcg/rez
class OptionalStr(Str):
    schema = Or(None, str_type)
Пример #10
0
Файл: config.py Проект: opcg/rez
    "package_filter": OptionalDictOrDictList,
    "new_session_popen_args": OptionalDict,
    "context_tracking_amqp": OptionalDict,
    "context_tracking_extra_fields": OptionalDict,

    # GUI settings
    "use_pyside": Bool,
    "use_pyqt": Bool,
    "gui_threads": Bool
})

# settings common to each plugin type
_plugin_config_dict = {
    "release_vcs": {
        "tag_name": str_type,
        "releasable_branches": Or(None, [str_type]),
        "check_tag": bool
    }
}

# -----------------------------------------------------------------------------
# Config
# -----------------------------------------------------------------------------


class Config(six.with_metaclass(LazyAttributeMeta, object)):
    """Rez configuration settings.

    You should call the `create_config` function, rather than constructing a
    `Config` object directly.
Пример #11
0
    'commands',
    'pre_commands',
    'post_commands',
    'help',
    'config',
    'uuid',
    'timestamp',
    'release_message',
    'changelog',
    'vcs',
    'revision',
    'previous_version',
    'previous_revision']


version_schema = Or(basestring, And(Version, Use(str)))


package_request_schema = Or(basestring, And(PackageRequest, Use(str)))


source_code_schema = Or(SourceCode, And(basestring, Use(SourceCode)))


# package serialisation schema
package_serialise_schema = Schema({
    Required("name"):                   basestring,
    Optional("version"):                version_schema,
    Optional("description"):            basestring,
    Optional("authors"):                [basestring],
    Optional("tools"):                  [basestring],
Пример #12
0
class CommandReleaseHook(ReleaseHook):

    commands_schema = Schema({
        "command":
        basestring,
        Optional("args"):
        Or(And(basestring, Use(lambda x: x.strip().split())), [basestring]),
        Optional("user"):
        basestring,
        Optional("env"):
        dict
    })

    schema_dict = {
        "print_commands": bool,
        "print_output": bool,
        "print_error": bool,
        "cancel_on_error": bool,
        "stop_on_error": bool,
        "pre_build_commands": [commands_schema],
        "pre_release_commands": [commands_schema],
        "post_release_commands": [commands_schema]
    }

    @classmethod
    def name(cls):
        return "command"

    def __init__(self, source_path):
        super(CommandReleaseHook, self).__init__(source_path)

    def execute_command(self, cmd_name, cmd_arguments, user, errors, env=None):
        def _err(msg):
            errors.append(msg)
            if self.settings.print_error:
                print >> sys.stderr, msg

        kwargs = {}
        if env:
            kwargs["_env"] = env

        def _execute(cmd, arguments):
            try:
                result = cmd(*(arguments or []), **kwargs)
                if self.settings.print_output:
                    print result.stdout.strip()
            except ErrorReturnCode as e:
                # `e` shows the command that was run
                msg = "command failed:\n%s" % str(e)
                _err(msg)
                return False
            return True

        if not os.path.isfile(cmd_name):
            cmd_full_path = which(cmd_name)
        else:
            cmd_full_path = cmd_name
        if not cmd_full_path:
            msg = "%s: command not found" % cmd_name
            _err(msg)
            return False

        run_cmd = Command(cmd_full_path)
        if user == 'root':
            with sudo:
                return _execute(run_cmd, cmd_arguments)
        elif user and user != getpass.getuser():
            raise NotImplementedError  # TODO
        else:
            return _execute(run_cmd, cmd_arguments)

    def pre_build(self, user, install_path, **kwargs):
        errors = []
        self._execute_commands(self.settings.pre_build_commands,
                               install_path=install_path,
                               package=self.package,
                               errors=errors)

        if errors and self.settings.cancel_on_error:
            raise ReleaseHookCancellingError(
                "The following pre-build commands failed:\n%s" %
                '\n\n'.join(errors))

    def pre_release(self, user, install_path, **kwargs):
        errors = []
        self._execute_commands(self.settings.pre_release_commands,
                               install_path=install_path,
                               package=self.package,
                               errors=errors)

        if errors and self.settings.cancel_on_error:
            raise ReleaseHookCancellingError(
                "The following pre-release commands failed:\n%s" %
                '\n\n'.join(errors))

    def post_release(self, user, install_path, variants, **kwargs):
        # note that the package we use here is the *installed* package, not the
        # developer package (self.package). Otherwise, attributes such as 'root'
        # will be None
        errors = []
        if variants:
            package = variants[0].parent
        else:
            package = self.package

        self._execute_commands(self.settings.post_release_commands,
                               install_path=install_path,
                               package=package,
                               errors=errors)
        if errors:
            print_debug("The following post-release commands failed:\n" +
                        '\n\n'.join(errors))

    def _execute_commands(self, commands, install_path, package, errors=None):
        release_dict = dict(path=install_path)
        formatter = scoped_formatter(system=system,
                                     release=release_dict,
                                     package=package)

        for conf in commands:
            program = conf["command"]

            env_ = None
            env = conf.get("env")
            if env:
                env_ = os.environ.copy()
                env_.update(env)

            args = conf.get("args", [])
            args = [formatter.format(x) for x in args]
            args = [expandvars(x, environ=env_) for x in args]

            if self.settings.print_commands or config.debug("package_release"):
                from subprocess import list2cmdline
                toks = [program] + args

                msgs = []
                msgs.append("running command: %s" % list2cmdline(toks))
                if env:
                    for key, value in env.iteritems():
                        msgs.append("    with: %s=%s" % (key, value))

                if self.settings.print_commands:
                    print '\n'.join(msgs)
                else:
                    for msg in msgs:
                        print_debug(msg)

            if not self.execute_command(cmd_name=program,
                                        cmd_arguments=args,
                                        user=conf.get("user"),
                                        errors=errors,
                                        env=env_):
                if self.settings.stop_on_error:
                    return
Пример #13
0
from rez.utils.data_utils import AttrDictWrapper
from rez.utils.logging_ import print_warning
from rez.package_resources_ import help_schema, _commands_schema, \
    _function_schema, late_bound
from rez.package_repository import create_memory_package_repository
from rez.packages_ import Package
from rez.package_py_utils import expand_requirement
from rez.vendor.schema.schema import Schema, Optional, Or, Use, And
from rez.vendor.version.version import Version
from contextlib import contextmanager
import os

# this schema will automatically harden request strings like 'python-*'; see
# the 'expand_requires' function for more info.
#
package_request_schema = Or(And(basestring, Use(expand_requirement)),
                            And(PackageRequest, Use(str)))

tests_schema = Schema({
    Optional(basestring):
    Or(
        Or(basestring, [basestring]), {
            "command": Or(basestring, [basestring]),
            Optional("requires"): [package_request_schema]
        })
})

package_schema = Schema({
    Required("name"):
    basestring,
    Optional("base"):
    basestring,
Пример #14
0
class CMakeBuildSystem(BuildSystem):
    """The CMake build system.

    The 'cmake' executable is run within the build environment. Rez supplies a
    library of cmake macros in the 'cmake_files' directory; these are added to
    cmake's searchpath and are available to use in your own CMakeLists.txt
    file.

    The following CMake variables are available:
    - REZ_BUILD_TYPE: One of 'local', 'central'. Describes whether an install
      is going to the local packages path, or the release packages path.
    - REZ_BUILD_INSTALL: One of 0 or 1. If 1, an installation is taking place;
      if 0, just a build is occurring.
    """

    build_systems = {'eclipse':     "Eclipse CDT4 - Unix Makefiles",
                     'codeblocks':  "CodeBlocks - Unix Makefiles",
                     'make':        "Unix Makefiles",
                     'nmake':       "NMake Makefiles",
                     'xcode':       "Xcode"}

    build_targets = ["Debug", "Release", "RelWithDebInfo"]

    schema_dict = {
        "build_target":     Or(*build_targets),
        "build_system":     Or(*build_systems.keys()),
        "cmake_args":       [basestring],
        "cmake_binary":     Or(None, basestring),
        "make_binary":     Or(None, basestring)}

    @classmethod
    def name(cls):
        return "cmake"

    @classmethod
    def child_build_system(cls):
        return "make"

    @classmethod
    def is_valid_root(cls, path):
        return os.path.isfile(os.path.join(path, "CMakeLists.txt"))

    @classmethod
    def bind_cli(cls, parser):
        settings = config.plugins.build_system.cmake
        parser.add_argument("--bt", "--build-target", dest="build_target",
                            type=str, choices=cls.build_targets,
                            default=settings.build_target,
                            help="set the build target (default: %(default)s).")
        parser.add_argument("--bs", "--build-system", dest="build_system",
                            type=str, choices=cls.build_systems.keys(),
                            default=settings.build_system,
                            help="set the cmake build system (default: %(default)s).")

    def __init__(self, working_dir, opts=None, write_build_scripts=False,
                 verbose=False, build_args=[], child_build_args=[]):
        super(CMakeBuildSystem, self).__init__(
            working_dir,
            opts=opts,
            write_build_scripts=write_build_scripts,
            verbose=verbose,
            build_args=build_args,
            child_build_args=child_build_args)

        self.settings = self.package.config.plugins.build_system.cmake
        self.build_target = (opts and opts.build_target) or \
                            self.settings.build_target
        self.cmake_build_system = (opts and opts.build_system) or \
                                  self.settings.build_system
        if self.cmake_build_system == 'xcode' and platform_.name != 'osx':
            raise RezCMakeError("Generation of Xcode project only available "
                                "on the OSX platform")

    def build(self, context, variant, build_path, install_path, install=False,
              build_type=BuildType.local):
        def _pr(s):
            if self.verbose:
                print s

        # find cmake binary
        if self.settings.cmake_binary:
            exe = self.settings.cmake_binary
        else:
            exe = context.which("cmake", fallback=True)
        if not exe:
            raise RezCMakeError("could not find cmake binary")
        found_exe = which(exe)
        if not found_exe:
            raise RezCMakeError("cmake binary does not exist: %s" % exe)

        sh = create_shell()

        # assemble cmake command
        cmd = [found_exe, "-d", self.working_dir]
        cmd += (self.settings.cmake_args or [])
        cmd += (self.build_args or [])

        cmd.append("-DCMAKE_INSTALL_PREFIX=%s" % install_path)
        cmd.append("-DCMAKE_MODULE_PATH=%s" % sh.get_key_token("CMAKE_MODULE_PATH"))
        cmd.append("-DCMAKE_BUILD_TYPE=%s" % self.build_target)
        cmd.append("-DREZ_BUILD_TYPE=%s" % build_type.name)
        cmd.append("-DREZ_BUILD_INSTALL=%d" % (1 if install else 0))
        cmd.extend(["-G", self.build_systems[self.cmake_build_system]])

        if config.rez_1_cmake_variables and \
                not config.disable_rez_1_compatibility and \
                build_type == BuildType.central:
            cmd.append("-DCENTRAL=1")

        # execute cmake within the build env
        _pr("Executing: %s" % ' '.join(cmd))
        if not os.path.abspath(build_path):
            build_path = os.path.join(self.working_dir, build_path)
            build_path = os.path.realpath(build_path)

        callback = functools.partial(self._add_build_actions,
                                     context=context,
                                     package=self.package,
                                     variant=variant,
                                     build_type=build_type)

        # run the build command and capture/print stderr at the same time
        retcode, _, _ = context.execute_shell(command=cmd,
                                              block=True,
                                              cwd=build_path,
                                              actions_callback=callback)
        ret = {}
        if retcode:
            ret["success"] = False
            return ret

        if self.write_build_scripts:
            # write out the script that places the user in a build env, where
            # they can run make directly themselves.
            build_env_script = os.path.join(build_path, "build-env")
            create_forwarding_script(build_env_script,
                                     module=("build_system", "cmake"),
                                     func_name="_FWD__spawn_build_shell",
                                     working_dir=self.working_dir,
                                     build_dir=build_path,
                                     variant_index=variant.index)
            ret["success"] = True
            ret["build_env_script"] = build_env_script
            return ret

        # assemble make command
        if self.settings.make_binary:
            cmd = [self.settings.make_binary]
        else:
            cmd = ["make"]
        cmd += (self.child_build_args or [])

        if not any(x.startswith("-j") for x in (self.child_build_args or [])):
            n = variant.config.build_thread_count or cpu_count()
            cmd.append("-j%d" % n)

        # execute make within the build env
        _pr("\nExecuting: %s" % ' '.join(cmd))
        retcode, _, _ = context.execute_shell(command=cmd,
                                              block=True,
                                              cwd=build_path,
                                              actions_callback=callback)

        if not retcode and install and "install" not in cmd:
            cmd.append("install")

            # execute make install within the build env
            _pr("\nExecuting: %s" % ' '.join(cmd))
            retcode, _, _ = context.execute_shell(command=cmd,
                                                  block=True,
                                                  cwd=build_path,
                                                  actions_callback=callback)

        ret["success"] = (not retcode)
        return ret

    @staticmethod
    def _add_build_actions(executor, context, package, variant, build_type):
        settings = package.config.plugins.build_system.cmake
        cmake_path = os.path.join(os.path.dirname(__file__), "cmake_files")
        template_path = os.path.join(os.path.dirname(__file__), "template_files")

        executor.env.CMAKE_MODULE_PATH.append(cmake_path)
        executor.env.REZ_BUILD_DOXYFILE = os.path.join(template_path, 'Doxyfile')
        executor.env.REZ_BUILD_VARIANT_INDEX = variant.index or 0
        executor.env.REZ_BUILD_THREAD_COUNT = package.config.build_thread_count or cpu_count()
        # build always occurs on a filesystem package, thus 'filepath' attribute
        # exists. This is not the case for packages in general.
        executor.env.REZ_BUILD_PROJECT_FILE = package.filepath
        executor.env.REZ_BUILD_PROJECT_VERSION = str(package.version)
        executor.env.REZ_BUILD_PROJECT_NAME = package.name
        executor.env.REZ_BUILD_PROJECT_DESCRIPTION = \
            (package.description or '').strip()
        executor.env.REZ_BUILD_REQUIRES_UNVERSIONED = \
            ' '.join(x.name for x in context.requested_packages(True))
        executor.env.REZ_BUILD_INSTALL_PYC = '1' if settings.install_pyc else '0'

        if config.rez_1_environment_variables and \
                not config.disable_rez_1_compatibility and \
                build_type == BuildType.central:
            executor.env.REZ_IN_REZ_RELEASE = 1
    _function_schema, late_bound
from rez.package_repository import create_memory_package_repository
from rez.packages_ import Package
from rez.package_py_utils import expand_requirement
from rez.vendor.schema.schema import Schema, Optional, Or, Use, And
from rez.vendor.six import six
from rez.vendor.version.version import Version
from contextlib import contextmanager
import os

basestring = six.string_types[0]

# this schema will automatically harden request strings like 'python-*'; see
# the 'expand_requires' function for more info.
#
package_request_schema = Or(And(basestring, Use(expand_requirement)),
                            And(PackageRequest, Use(str)))

tests_schema = Schema({
    Optional(basestring):
    Or(
        Or(basestring, [basestring]),
        extensible_schema_dict({
            "command":
            Or(basestring, [basestring]),
            Optional("requires"): [package_request_schema],
            Optional("run_on"):
            Or(basestring, [basestring]),
            Optional("on_variants"):
            Or(bool, {
                "type": "requires",
                "value": [package_request_schema]
Пример #16
0
class OptionalDict(Dict):
    schema = Or(And(None, Use(lambda x: {})), dict)
Пример #17
0
class OptionalDictOrDictList(Setting):
    schema = Or(And(None, Use(lambda x: [])), And(dict, Use(lambda x: [x])),
                [dict])
Пример #18
0
Файл: config.py Проект: opcg/rez
class OptionalStrList(StrList):
    schema = Or(And(None, Use(lambda x: [])), [str_type])
Пример #19
0
 def schema(cls):
     from rez.solver import VariantSelectMode
     return Or(*(x.name for x in VariantSelectMode))
Пример #20
0
class FileSystemPackageRepository(PackageRepository):
    """A filesystem-based package repository.

    Packages are stored on disk, in either 'package.yaml' or 'package.py' files.
    These files are stored into an organised directory structure like so:

        /LOCATION/pkgA/1.0.0/package.py
                      /1.0.1/package.py
                 /pkgB/2.1/package.py
                      /2.2/package.py

    Another supported storage format is to store all package versions within a
    single package family in one file, like so:

        /LOCATION/pkgC.yaml
        /LOCATION/pkgD.py

    These 'combined' package files allow for differences between package
    versions via a 'package_overrides' section:

        name: pkgC

        versions:
        - '1.0'
        - '1.1'
        - '1.2'

        version_overrides:
            '1.0':
                requires:
                - python-2.5
            '1.1+':
                requires:
                - python-2.6
    """
    schema_dict = {"file_lock_timeout": int,
                   "file_lock_dir": Or(None, str),
                   "package_filenames": [basestring]}

    building_prefix = ".building"
    package_file_mode = (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)

    @classmethod
    def name(cls):
        return "filesystem"

    def __init__(self, location, resource_pool):
        """Create a filesystem package repository.

        Args:
            location (str): Path containing the package repository.
        """
        super(FileSystemPackageRepository, self).__init__(location, resource_pool)

        global _settings
        _settings = config.plugins.package_repository.filesystem

        self.register_resource(FileSystemPackageFamilyResource)
        self.register_resource(FileSystemPackageResource)
        self.register_resource(FileSystemVariantResource)

        self.register_resource(FileSystemCombinedPackageFamilyResource)
        self.register_resource(FileSystemCombinedPackageResource)
        self.register_resource(FileSystemCombinedVariantResource)

        self.get_families = lru_cache(maxsize=None)(self._get_families)
        self.get_family = lru_cache(maxsize=None)(self._get_family)
        self.get_packages = lru_cache(maxsize=None)(self._get_packages)
        self.get_variants = lru_cache(maxsize=None)(self._get_variants)
        self.get_file = lru_cache(maxsize=None)(self._get_file)

    def _uid(self):
        t = ["filesystem", self.location]
        if os.path.exists(self.location):
            st = os.stat(self.location)
            t.append(st.st_ino)
        return tuple(t)

    def get_package_family(self, name):
        return self.get_family(name)

    @pool_memcached_connections
    def iter_package_families(self):
        for family in self.get_families():
            yield family

    @pool_memcached_connections
    def iter_packages(self, package_family_resource):
        for package in self.get_packages(package_family_resource):
            yield package

    def iter_variants(self, package_resource):
        for variant in self.get_variants(package_resource):
            yield variant

    def get_parent_package_family(self, package_resource):
        return package_resource.parent

    def get_parent_package(self, variant_resource):
        return variant_resource.parent

    def get_variant_state_handle(self, variant_resource):
        package_resource = variant_resource.parent
        return package_resource.state_handle

    def get_last_release_time(self, package_family_resource):
        return package_family_resource.get_last_release_time()

    @cached_property
    def file_lock_dir(self):
        dirname = _settings.file_lock_dir
        if not dirname:
            return None

        # sanity check
        if os.path.isabs(dirname) or os.path.basename(dirname) != dirname:
            raise ConfigurationError(
                "filesystem package repository setting 'file_lock_dir' must be "
                "a single relative directory such as '.lock'")

        # fall back to location path if lock dir doesn't exist.
        path = os.path.join(self.location, dirname)
        if not os.path.exists(path):
            return None

        return dirname

    def pre_variant_install(self, variant_resource):
        if not variant_resource.version:
            return

        # create 'building' tagfile, this makes sure that a resolve doesn't
        # pick up this package if it doesn't yet have a package.py created.
        path = self.location

        family_path = os.path.join(path, variant_resource.name)
        if not os.path.isdir(family_path):
            os.makedirs(family_path)

        filename = self.building_prefix + str(variant_resource.version)
        filepath = os.path.join(family_path, filename)

        with open(filepath, 'w'):  # create empty file
            pass

    def install_variant(self, variant_resource, dry_run=False, overrides=None):
        overrides = overrides or {}

        # Name and version overrides are a special case - they change the
        # destination variant to be created/replaced.
        #
        variant_name = variant_resource.name
        variant_version = variant_resource.version

        if "name" in overrides:
            variant_name = overrides["name"]
            if variant_name is self.remove:
                raise PackageRepositoryError(
                    "Cannot remove package attribute 'name'")

        if "version" in overrides:
            ver = overrides["version"]
            if ver is self.remove:
                raise PackageRepositoryError(
                    "Cannot remove package attribute 'version'")

            if isinstance(ver, basestring):
                ver = Version(ver)
                overrides = overrides.copy()
                overrides["version"] = ver

            variant_version = ver

        # cannot install over one's self, just return existing variant
        if variant_resource._repository is self and \
                variant_name == variant_resource.name and \
                variant_version == variant_resource.version:
            return variant_resource

        # check repo exists on disk
        path = self.location
        if not os.path.exists(path):
            raise PackageRepositoryError(
                "Package repository path does not exist: %s" % path)

        # install the variant
        def _create_variant():
            return self._create_variant(
                variant_resource,
                dry_run=dry_run,
                overrides=overrides
            )

        if dry_run:
            variant = _create_variant()
        else:
            with self._lock_package(variant_name, variant_version):
                variant = _create_variant()

        return variant

    @contextmanager
    def _lock_package(self, package_name, package_version=None):
        from rez.vendor.lockfile import LockFile

        path = self.location

        if self.file_lock_dir:
            path = os.path.join(path, self.file_lock_dir)

        if not os.path.exists(path):
            raise PackageRepositoryError(
                "Lockfile directory %s does not exist - please create and try "
                "again" % path)

        filename = ".lock.%s" % package_name
        if package_version:
            filename += "-%s" % str(package_version)

        lock_file = os.path.join(path, filename)
        lock = LockFile(lock_file)

        try:
            lock.acquire(timeout=_settings.file_lock_timeout)
            yield

        finally:
            if lock.is_locked():
                lock.release()

    def clear_caches(self):
        super(FileSystemPackageRepository, self).clear_caches()
        self.get_families.cache_clear()
        self.get_family.cache_clear()
        self.get_packages.cache_clear()
        self.get_variants.cache_clear()
        self.get_file.cache_clear()
        self._get_family_dirs.forget()
        self._get_version_dirs.forget()
        # unfortunately we need to clear file cache across the board
        clear_file_caches()

    def get_package_payload_path(self, package_name, package_version=None):
        path = os.path.join(self.location, package_name)

        if package_version:
            path = os.path.join(path, str(package_version))

        return path

    # -- internal

    def _get_family_dirs__key(self):
        if os.path.isdir(self.location):
            st = os.stat(self.location)
            return str(("listdir", self.location, st.st_ino, st.st_mtime))
        else:
            return str(("listdir", self.location))

    @memcached(servers=config.memcached_uri if config.cache_listdir else None,
               min_compress_len=config.memcached_listdir_min_compress_len,
               key=_get_family_dirs__key,
               debug=config.debug_memcache)
    def _get_family_dirs(self):
        dirs = []
        if not os.path.isdir(self.location):
            return dirs

        for name in os.listdir(self.location):
            path = os.path.join(self.location, name)
            if os.path.isdir(path):
                if is_valid_package_name(name) and name != self.file_lock_dir:
                    dirs.append((name, None))
            else:
                name_, ext_ = os.path.splitext(name)
                if ext_ in (".py", ".yaml") and is_valid_package_name(name_):
                    dirs.append((name_, ext_[1:]))

        return dirs

    def _get_version_dirs__key(self, root):
        st = os.stat(root)
        return str(("listdir", root, st.st_ino, st.st_mtime))

    @memcached(servers=config.memcached_uri if config.cache_listdir else None,
               min_compress_len=config.memcached_listdir_min_compress_len,
               key=_get_version_dirs__key,
               debug=config.debug_memcache)
    def _get_version_dirs(self, root):

        # simpler case if this test is on
        #
        if _settings.check_package_definition_files:
            dirs = []

            for name in os.listdir(root):
                if name.startswith('.'):
                    continue

                path = os.path.join(root, name)
                if os.path.isdir(path):
                    if not self._is_valid_package_directory(path):
                        continue

                dirs.append(name)
            return dirs

        # with test off, we have to check for 'building' dirs, these have to be
        # tested regardless. Failed releases may cause 'building files' to be
        # left behind, so we need to clear these out also
        #
        dirs = set()
        building_dirs = set()

        # find dirs and dirs marked as 'building'
        for name in os.listdir(root):
            if name.startswith('.'):
                if not name.startswith(self.building_prefix):
                    continue

                ver_str = name[len(self.building_prefix):]
                building_dirs.add(ver_str)

            path = os.path.join(root, name)
            if os.path.isdir(path):
                dirs.add(name)

        # check 'building' dirs for validity
        for name in building_dirs:
            if name not in dirs:
                continue

            path = os.path.join(root, name)
            if not self._is_valid_package_directory(path):
                # package probably still being built
                dirs.remove(name)

        return list(dirs)

    # True if `path` contains package.py or similar
    def _is_valid_package_directory(self, path):
        return bool(self._get_file(path, "package")[0])

    def _get_families(self):
        families = []
        for name, ext in self._get_family_dirs():
            if ext is None:  # is a directory
                family = self.get_resource(
                    FileSystemPackageFamilyResource.key,
                    location=self.location,
                    name=name)
            else:
                family = self.get_resource(
                    FileSystemCombinedPackageFamilyResource.key,
                    location=self.location,
                    name=name,
                    ext=ext)
            families.append(family)

        return families

    def _get_family(self, name):
        is_valid_package_name(name, raise_error=True)
        if os.path.isdir(os.path.join(self.location, name)):
            family = self.get_resource(
                FileSystemPackageFamilyResource.key,
                location=self.location,
                name=name)
            return family
        else:
            filepath, format_ = self.get_file(self.location, package_filename=name)
            if filepath:
                family = self.get_resource(
                    FileSystemCombinedPackageFamilyResource.key,
                    location=self.location,
                    name=name,
                    ext=format_.extension)
                return family
        return None

    def _get_packages(self, package_family_resource):
        return [x for x in package_family_resource.iter_packages()]

    def _get_variants(self, package_resource):
        return [x for x in package_resource.iter_variants()]

    def _get_file(self, path, package_filename=None):
        if package_filename:
            package_filenames = [package_filename]
        else:
            package_filenames = _settings.package_filenames

        for name in package_filenames:
            for format_ in (FileFormat.py, FileFormat.yaml):
                filename = "%s.%s" % (name, format_.extension)
                filepath = os.path.join(path, filename)
                if os.path.isfile(filepath):
                    return filepath, format_
        return None, None

    def _create_family(self, name):
        path = os.path.join(self.location, name)
        if not os.path.exists(path):
            os.makedirs(path)
        self.clear_caches()
        return self.get_package_family(name)

    def _create_variant(self, variant, dry_run=False, overrides=None):
        # special case overrides
        variant_name = overrides.get("name") or variant.name
        variant_version = overrides.get("version") or variant.version

        overrides = (overrides or {}).copy()
        overrides.pop("name", None)
        overrides.pop("version", None)

        # find or create the package family
        family = self.get_package_family(variant_name)
        if not family:
            family = self._create_family(variant_name)

        if isinstance(family, FileSystemCombinedPackageFamilyResource):
            raise NotImplementedError(
                "Cannot install variant into combined-style package file %r."
                % family.filepath)

        # find the package if it already exists
        existing_package = None

        for package in self.iter_packages(family):
            if package.version == variant_version:
                # during a build, the family/version dirs get created ahead of
                # time, which causes a 'missing package definition file' error.
                # This is fine, we can just ignore it and write the new file.
                try:
                    package.validate_data()
                except PackageDefinitionFileMissing:
                    break

                uuids = set([variant.uuid, package.uuid])
                if len(uuids) > 1 and None not in uuids:
                    raise ResourceError(
                        "Cannot install variant %r into package %r - the "
                        "packages are not the same (UUID mismatch)"
                        % (variant, package))

                existing_package = package

                if variant.index is None:
                    if package.variants:
                        raise ResourceError(
                            "Attempting to install a package without variants "
                            "(%r) into an existing package with variants (%r)"
                            % (variant, package))
                elif not package.variants:
                    raise ResourceError(
                        "Attempting to install a variant (%r) into an existing "
                        "package without variants (%r)" % (variant, package))

        existing_package_data = None
        release_data = {}

        # Need to treat 'config' as special case. In validated data, this is
        # converted to a Config object. We need it as the raw dict that you'd
        # see in a package.py.
        #
        def _get_package_data(pkg):
            data = pkg.validated_data()
            if hasattr(pkg, "_data"):
                raw_data = pkg._data
            else:
                raw_data = pkg.resource._data

            raw_config_data = raw_data.get('config')
            data.pop("config", None)

            if raw_config_data:
                data["config"] = raw_config_data

            return data

        def _remove_build_keys(obj):
            for key in package_build_only_keys:
                obj.pop(key, None)

        new_package_data = _get_package_data(variant.parent)
        new_package_data.pop("variants", None)
        new_package_data["name"] = variant_name
        if variant_version:
            new_package_data["version"] = variant_version
        package_changed = False

        _remove_build_keys(new_package_data)

        if existing_package:
            debug_print(
                "Found existing package for installation of variant %s: %s",
                variant.uri, existing_package.uri
            )

            existing_package_data = _get_package_data(existing_package)
            _remove_build_keys(existing_package_data)

            # detect case where new variant introduces package changes outside of variant
            data_1 = existing_package_data.copy()
            data_2 = new_package_data.copy()

            for key in package_release_keys:
                data_2.pop(key, None)
                value = data_1.pop(key, None)
                if value is not None:
                    release_data[key] = value

            for key in ("format_version", "base", "variants"):
                data_1.pop(key, None)
                data_2.pop(key, None)

            package_changed = (data_1 != data_2)

            if debug_print:
                if package_changed:
                    from rez.utils.data_utils import get_dict_diff_str

                    debug_print("Variant %s package data differs from package %s",
                                variant.uri, existing_package.uri)

                    txt = get_dict_diff_str(data_1, data_2, "Changes:")
                    debug_print(txt)
                else:
                    debug_print("Variant %s package data matches package %s",
                                variant.uri, existing_package.uri)

        # check for existing installed variant
        existing_installed_variant = None
        installed_variant_index = None

        if existing_package:
            if variant.index is None:
                existing_installed_variant = \
                    self.iter_variants(existing_package).next()
            else:
                variant_requires = variant.variant_requires

                for variant_ in self.iter_variants(existing_package):
                    variant_requires_ = existing_package.variants[variant_.index]
                    if variant_requires_ == variant_requires:
                        installed_variant_index = variant_.index
                        existing_installed_variant = variant_

        if existing_installed_variant:
            debug_print(
                "Variant %s already has installed equivalent: %s",
                variant.uri, existing_installed_variant.uri
            )

        if dry_run:
            if not package_changed:
                return existing_installed_variant
            else:
                return None

        # construct package data for new installed package definition
        if existing_package:
            _, file_ = os.path.split(existing_package.filepath)
            package_filename, package_extension = os.path.splitext(file_)
            package_extension = package_extension[1:]
            package_format = FileFormat[package_extension]

            if package_changed:
                # graft together new package data, with existing package variants,
                # and other data that needs to stay unchanged (eg timestamp)
                package_data = new_package_data

                if variant.index is not None:
                    package_data["variants"] = existing_package_data.get("variants", [])
            else:
                package_data = existing_package_data
        else:
            package_data = new_package_data
            package_filename = _settings.package_filenames[0]
            package_extension = "py"
            package_format = FileFormat.py

        # merge existing release data (if any) into the package. Note that when
        # this data becomes variant-specific, this step will no longer be needed
        package_data.update(release_data)

        # merge the new variant into the package
        if installed_variant_index is None and variant.index is not None:
            variant_requires = variant.variant_requires
            if not package_data.get("variants"):
                package_data["variants"] = []
            package_data["variants"].append(variant_requires)
            installed_variant_index = len(package_data["variants"]) - 1

        # a little data massaging is needed
        package_data.pop("base", None)

        # create version dir if it doesn't already exist
        family_path = os.path.join(self.location, variant_name)
        if variant_version:
            pkg_base_path = os.path.join(family_path, str(variant_version))
        else:
            pkg_base_path = family_path
        if not os.path.exists(pkg_base_path):
            os.makedirs(pkg_base_path)

        # Apply overrides.
        #
        # If we're installing into an existing package, then existing attributes
        # in that package take precedence over `overrides`. If we're installing
        # to a new package, then `overrides` takes precedence always.
        #
        # This is done so that variants added to an existing package don't change
        # attributes such as 'timestamp' or release-related fields like 'revision'.
        #
        for key, value in overrides.iteritems():
            if existing_package:
                if key not in package_data:
                    package_data[key] = value
            else:
                if value is self.remove:
                    package_data.pop(key, None)
                else:
                    package_data[key] = value

        # timestamp defaults to now if not specified
        if not package_data.get("timestamp"):
            package_data["timestamp"] = int(time.time())

        # format version is always set
        package_data["format_version"] = format_version

        # write out new package definition file
        package_file = ".".join([package_filename, package_extension])
        filepath = os.path.join(pkg_base_path, package_file)

        with make_path_writable(pkg_base_path):
            with open_file_for_write(filepath, mode=self.package_file_mode) as f:
                dump_package_data(package_data, buf=f, format_=package_format)

        # delete the tmp 'building' file.
        if variant_version:
            filename = self.building_prefix + str(variant_version)
            filepath = os.path.join(family_path, filename)
            if os.path.exists(filepath):
                try:
                    os.remove(filepath)
                except:
                    pass

        # delete other stale building files; previous failed releases may have
        # left some around
        try:
            self._delete_stale_build_tagfiles(family_path)
        except:
            pass

        # touch the family dir, this keeps memcached resolves updated properly
        os.utime(family_path, None)

        # load new variant
        new_variant = None
        self.clear_caches()
        family = self.get_package_family(variant_name)

        if family:
            for package in self.iter_packages(family):
                if package.version == variant_version:
                    for variant_ in self.iter_variants(package):
                        if variant_.index == installed_variant_index:
                            new_variant = variant_
                            break
                elif new_variant:
                    break

        if not new_variant:
            raise RezSystemError("Internal failure - expected installed variant")
        return new_variant

    def _delete_stale_build_tagfiles(self, family_path):
        now = time.time()

        for name in os.listdir(family_path):
            if not name.startswith(self.building_prefix):
                continue

            tagfilepath = os.path.join(family_path, name)
            ver_str = name[len(self.building_prefix):]
            pkg_path = os.path.join(family_path, ver_str)

            if os.path.exists(pkg_path):
                # build tagfile not needed if package is valid
                if self._is_valid_package_directory(pkg_path):
                    os.remove(tagfilepath)
                    continue
            else:
                # remove tagfile if pkg is gone. Delete only tagfiles over a certain
                # age, otherwise might delete a tagfile another process has created
                # just before it created the package directory.
                st = os.stat(tagfilepath)
                age = now - st.st_mtime

                if age > 3600:
                    os.remove(tagfilepath)
Пример #21
0
 def schema(cls):
     from rez.utils.execution import ExecutableScriptMode
     return Or(*(x.name for x in ExecutableScriptMode))
Пример #22
0
    "package_orderers": OptionalDictOrDictList,
    "new_session_popen_args": OptionalDict,
    "context_tracking_amqp": OptionalDict,
    "context_tracking_extra_fields": OptionalDict,

    # GUI settings
    "use_pyside": Bool,
    "use_pyqt": Bool,
    "gui_threads": Bool
})

# settings common to each plugin type
_plugin_config_dict = {
    "release_vcs": {
        "tag_name": basestring,
        "releasable_branches": Or(None, [basestring]),
        "check_tag": bool
    }
}

# -----------------------------------------------------------------------------
# Config
# -----------------------------------------------------------------------------


class Config(six.with_metaclass(LazyAttributeMeta, object)):
    """Rez configuration settings.

    You should call the `create_config` function, rather than constructing a
    `Config` object directly.
Пример #23
0
 def schema(cls):
     from rez.developer_package import PreprocessMode
     return Or(*(x.name for x in PreprocessMode))
Пример #24
0
class OptionalStr(Str):
    schema = Or(None, basestring)
Пример #25
0
from rez.utils.schema import Required, schema_keys
from rez.utils.formatting import PackageRequest
from rez.utils.data_utils import AttrDictWrapper
from rez.package_resources_ import help_schema, _commands_schema
from rez.package_repository import create_memory_package_repository
from rez.packages_ import Package
from rez.vendor.schema.schema import Schema, Optional, Or, Use, And
from rez.vendor.version.version import Version
from contextlib import contextmanager
import os


package_request_schema = Or(basestring,
                            And(PackageRequest, Use(str)))


package_schema = Schema({
    Required("name"):                   basestring,
    Optional("base"):                   basestring,
    Optional("version"):                Or(basestring,
                                           And(Version, Use(str))),
    Optional('description'):            basestring,
    Optional('authors'):                [basestring],

    Optional('requires'):               [package_request_schema],
    Optional('build_requires'):         [package_request_schema],
    Optional('private_build_requires'): [package_request_schema],
    Optional('variants'):               [[package_request_schema]],

    Optional('uuid'):                   basestring,
    Optional('config'):                 dict,
Пример #26
0
class OptionalStrList(StrList):
    schema = Or(And(None, Use(lambda x: [])), [basestring])
Пример #27
0
def late_bound(schema):
    return Or(SourceCode, schema)
Пример #28
0
class OptionalBool(Bool):
    # need None first, or Bool.schema will coerce None to False
    schema = Or(None, Bool.schema)
Пример #29
0
from rez.exceptions import PackageMetadataError
from rez.package_resources_ import help_schema, _commands_schema, \
    _function_schema, late_bound
from rez.package_repository import create_memory_package_repository
from rez.packages_ import Package
from rez.package_py_utils import expand_requirement
from rez.vendor.schema.schema import Schema, Optional, Or, Use, And
from rez.vendor.version.version import Version
from contextlib import contextmanager
import os


# this schema will automatically harden request strings like 'python-*'; see
# the 'expand_requires' function for more info.
#
package_request_schema = Or(And(basestring, Use(expand_requirement)),
                            And(PackageRequest, Use(str)))

tests_schema = Schema({
    Optional(basestring): Or(
        Or(basestring, [basestring]),
        {
            "command": Or(basestring, [basestring]),
            Optional("requires"): [package_request_schema]
        }
    )
})


package_schema = Schema({
    Optional("requires_rez_version"):   And(basestring, Use(Version)),
Пример #30
0
from rez.utils.formatting import PackageRequest, indent, \
    dict_to_attributes_code, as_block_string
from rez.utils.schema import Required
from rez.utils.yaml import dump_yaml
from pprint import pformat

# preferred order of keys in a package definition file
package_key_order = [
    'name', 'version', 'description', 'authors', 'tools', 'has_plugins',
    'plugin_for', 'requires', 'build_requires', 'private_build_requires',
    'variants', 'commands', 'pre_commands', 'post_commands', 'help', 'config',
    'uuid', 'timestamp', 'release_message', 'changelog', 'vcs', 'revision',
    'previous_version', 'previous_revision'
]

version_schema = Or(basestring, And(Version, Use(str)))

package_request_schema = Or(basestring, And(PackageRequest, Use(str)))

source_code_schema = Or(SourceCode, And(basestring, Use(SourceCode)))

tests_schema = Schema({
    Optional(basestring):
    Or(
        Or(basestring, [basestring]), {
            "command": Or(basestring, [basestring]),
            Optional("requires"): [package_request_schema]
        })
})

# package serialisation schema