Ejemplo n.º 1
0
    def load_plugins(self):
        import pkgutil
        from rez.backport.importlib import import_module
        type_module_name = 'rezplugins.' + self.type_name
        package = import_module(type_module_name)

        # on import, the `__path__` variable of the imported package is extended
        # to include existing directories on the plugin search path (via
        # extend_path, above). this means that `walk_packages` will walk over all
        # modules on the search path at the same level (.e.g in a
        # 'rezplugins/type_name' sub-directory).
        paths = [package.__path__] if isinstance(package.__path__, basestring) \
            else package.__path__
        for path in paths:
            for loader, modname, ispkg in pkgutil.walk_packages(
                    [path], package.__name__ + '.'):
                if loader is not None:
                    plugin_name = modname.split('.')[-1]
                    if plugin_name.startswith('_'):
                        continue
                    if config.debug("plugins"):
                        print_debug("loading %s plugin at %s: %s..."
                                    % (self.type_name, path, modname))
                    try:
                        # load_module will force reload the module if it's
                        # already loaded, so check for that
                        module = sys.modules.get(modname)
                        if module is None:
                            module = loader.find_module(modname).load_module(modname)
                        if hasattr(module, 'register_plugin') and \
                                hasattr(module.register_plugin, '__call__'):
                            plugin_class = module.register_plugin()
                            if plugin_class != None:
                                self.register_plugin(plugin_name, plugin_class, module)
                            else:
                                if config.debug("plugins"):
                                    print_warning(
                                        "'register_plugin' function at %s: %s did not return a class."
                                        % (path, modname))
                        else:
                            if config.debug("plugins"):
                                print_warning(
                                    "no 'register_plugin' function at %s: %s"
                                    % (path, modname))

                            # delete from sys.modules?

                    except Exception as e:
                        nameish = modname.split('.')[-1]
                        self.failed_plugins[nameish] = str(e)
                        if config.debug("plugins"):
                            import traceback
                            from StringIO import StringIO
                            out = StringIO()
                            traceback.print_exc(file=out)
                            print_debug(out.getvalue())

            # load config
            data, _ = _load_config_from_filepaths([os.path.join(path, "rezconfig")])
            deep_update(self.config_data, data)
Ejemplo n.º 2
0
    def load_plugins(self):
        import pkgutil
        from rez.backport.importlib import import_module
        type_module_name = 'rezplugins.' + self.type_name
        package = import_module(type_module_name)

        # on import, the `__path__` variable of the imported package is extended
        # to include existing directories on the plugin search path (via
        # extend_path, above). this means that `walk_packages` will walk over all
        # modules on the search path at the same level (.e.g in a
        # 'rezplugins/type_name' sub-directory).
        paths = [package.__path__] if isinstance(package.__path__, basestring) \
            else package.__path__
        for path in paths:
            for loader, modname, ispkg in pkgutil.walk_packages(
                    [path], package.__name__ + '.'):
                if loader is not None:
                    plugin_name = modname.split('.')[-1]
                    if plugin_name.startswith('_'):
                        continue
                    if config.debug("plugins"):
                        print_debug("loading %s plugin at %s: %s..."
                                    % (self.type_name, path, modname))
                    try:
                        # load_module will force reload the module if it's
                        # already loaded, so check for that
                        module = sys.modules.get(modname)
                        if module is None:
                            module = loader.find_module(modname).load_module(modname)
                        if hasattr(module, 'register_plugin') and \
                                hasattr(module.register_plugin, '__call__'):
                            plugin_class = module.register_plugin()
                            if plugin_class != None:
                                self.register_plugin(plugin_name, plugin_class, module)
                            else:
                                if config.debug("plugins"):
                                    print_warning(
                                        "'register_plugin' function at %s: %s did not return a class."
                                        % (path, modname))
                        else:
                            if config.debug("plugins"):
                                print_warning(
                                    "no 'register_plugin' function at %s: %s"
                                    % (path, modname))

                            # delete from sys.modules?

                    except Exception as e:
                        nameish = modname.split('.')[-1]
                        self.failed_plugins[nameish] = str(e)
                        if config.debug("plugins"):
                            import traceback
                            from StringIO import StringIO
                            out = StringIO()
                            traceback.print_exc(file=out)
                            print_debug(out.getvalue())

            # load config
            data, _ = _load_config_from_filepaths([os.path.join(path, "rezconfig")])
            deep_update(self.config_data, data)
Ejemplo n.º 3
0
def set_pika_log_level():
    mod_name = "rez.vendor.pika"

    if config.debug("context_tracking"):
        logging.getLogger(mod_name).setLevel(logging.DEBUG)
    else:
        logging.getLogger(mod_name).setLevel(logging.WARNING)
Ejemplo n.º 4
0
    def _data(self):
        if not self.schema:
            return None

        data = self._load()
        if config.debug("resources"):
            print_debug("Loaded resource: %s" % str(self))
        return data
Ejemplo n.º 5
0
    def _data(self):
        if not self.schema:
            return None

        data = self._load()
        if config.debug("resources"):
            print_debug("Loaded resource: %s" % str(self))
        return data
Ejemplo n.º 6
0
def load_plugin_cmd():
    """Load subcommand from command type plugin

    The command type plugin module should have attribute `command_behavior`,
    and the value must be a dict if provided. For example:

        # in your command plugin module
        command_behavior = {
            "hidden": False,   # (bool): default False
            "arg_mode": None,  #  (str): "passthrough", "grouped", default None
        }

    If the attribute not present, default behavior will be given.

    """
    from rez.config import config
    from rez.utils.logging_ import print_debug
    from rez.plugin_managers import plugin_manager

    ext_plugins = dict()

    for plugin_name in plugin_manager.get_plugins("command"):
        module = plugin_manager.get_plugin_module("command", plugin_name)

        behavior = getattr(module, "command_behavior", None)
        if behavior is None:
            behavior = dict()

            if config.debug("plugins"):
                print_debug("Attribute 'command_behavior' not found in plugin "
                            "module %s, registering with default behavior." %
                            module.__name__)
        try:
            data = behavior.copy()
            data.update({"module_name": module.__name__})
            ext_plugins[plugin_name] = data

        except Exception:
            if config.debug("plugins"):
                import traceback
                from rez.vendor.six.six import StringIO
                out = StringIO()
                traceback.print_exc(file=out)
                print_debug(out.getvalue())

    return ext_plugins
Ejemplo n.º 7
0
    def load_module(self, name, package):
        from hashlib import sha1
        from rez.config import config  # avoiding circular import
        from rez.developer_package import DeveloperPackage

        # in rare cases, a @late bound function may get called before the
        # package is built. An example is 'requires' and the other requires-like
        # functions. These need to be evaluated before a build, but it does also
        # make sense to sometimes implement these as late-bound functions. We
        # detect this case here, and load the modules from the original (pre-
        # copied into package payload) location.
        #
        if isinstance(package, DeveloperPackage):
            # load sourcefile from original location
            path = config.package_definition_python_path
            filepath = os.path.join(path, "%s.py" % name)

            if not os.path.exists(filepath):
                return None

            with open(filepath, "rb") as f:
                hash_str = sha1(f.read().strip()).hexdigest()

        else:
            # load sourcefile that's been copied into package install payload
            path = os.path.join(package.base, self.include_modules_subpath)
            pathname = os.path.join(path, "%s.py" % name)
            hashname = os.path.join(path, "%s.sha1" % name)

            if os.path.isfile(pathname) and os.path.isfile(hashname):
                with open(hashname, "r") as f:
                    hash_str = f.readline()
                filepath = pathname

            else:
                # Fallback for backward compat
                pathname = os.path.join(path, "%s-*.py" % name)
                hashnames = glob(pathname)
                if not hashnames:
                    return None

                filepath = hashnames[0]
                hash_str = filepath.rsplit('-', 1)[-1].split('.', 1)[0]
                # End, for details of backward compat,
                # see https://github.com/nerdvegas/rez/issues/934
                # and https://github.com/nerdvegas/rez/pull/935

        module = self.modules.get(hash_str)
        if module is not None:
            return module

        if config.debug("file_loads"):
            print_debug("Loading include sourcefile: %s" % filepath)

        module = py23.load_module_from_file(name, filepath)
        self.modules[hash_str] = module
        return module
Ejemplo n.º 8
0
    def append_if_valid(dir_):
        if os.path.isdir(dir_):
            subdir = os.path.normcase(os.path.join(dir_, pname))
            initfile = os.path.join(subdir, init_py)
            if subdir not in path and os.path.isfile(initfile):
                path.append(subdir)

        elif config.debug("plugins"):
            print_debug("skipped nonexistant rez plugin path: %s" % dir_)
Ejemplo n.º 9
0
def _load_file(filepath, format_, update_data_callback):
    load_func = load_functions[format_]

    if config.debug("file_loads"):
        print_debug("Loading file: %s" % filepath)

    with open(filepath) as f:
        result = load_func(f, filepath=filepath)

    if update_data_callback:
        result = update_data_callback(format_, result)
    return result
Ejemplo n.º 10
0
def _load_file(filepath, format_, update_data_callback):
    load_func = load_functions[format_]

    if config.debug("file_loads"):
        print_debug("Loading file: %s" % filepath)

    with open(filepath) as f:
        result = load_func(f, filepath=filepath)

    if update_data_callback:
        result = update_data_callback(format_, result)
    return result
Ejemplo n.º 11
0
    def load_module(self, name, package):
        from rez.config import config  # avoiding circular import
        from rez.developer_package import DeveloperPackage

        # in rare cases, a @late bound function may get called before the
        # package is built. An example is 'requires' and the other requires-like
        # functions. These need to be evaluated before a build, but it does also
        # make sense to sometimes implement these as late-bound functions. We
        # detect this case here, and load the modules from the original (pre-
        # copied into package payload) location.
        #
        if isinstance(package, DeveloperPackage):
            from hashlib import sha1

            # load sourcefile from original location
            path = config.package_definition_python_path
            filepath = os.path.join(path, "%s.py" % name)

            if not os.path.exists(filepath):
                return None

            with open(filepath) as f:
                txt = f.read().strip()

            hash_str = sha1(txt).hexdigest()
        else:
            # load sourcefile that's been copied into package install payload
            path = os.path.join(package.base, self.include_modules_subpath)
            pathname = os.path.join(path, "%s-*.py" % name)

            pathnames = glob(pathname)
            if not pathnames:
                return None

            filepath = pathnames[0]
            hash_str = filepath.rsplit('-', 1)[-1].split('.', 1)[0]

        module = self.modules.get(hash_str)
        if module is not None:
            return module

        if config.debug("file_loads"):
            print_debug("Loading include sourcefile: %s" % filepath)

        with open(filepath) as f:
            module = imp.load_source(name, filepath, f)

        self.modules[hash_str] = module
        return module
Ejemplo n.º 12
0
    def load_module(self, name, package):
        from rez.config import config  # avoiding circular import
        from rez.developer_package import DeveloperPackage

        # in rare cases, a @late bound function may get called before the
        # package is built. An example is 'requires' and the other requires-like
        # functions. These need to be evaluated before a build, but it does also
        # make sense to sometimes implement these as late-bound functions. We
        # detect this case here, and load the modules from the original (pre-
        # copied into package payload) location.
        #
        if isinstance(package, DeveloperPackage):
            from hashlib import sha1

            # load sourcefile from original location
            path = config.package_definition_python_path
            filepath = os.path.join(path, "%s.py" % name)

            if not os.path.exists(filepath):
                return None

            with open(filepath) as f:
                txt = f.read().strip()

            hash_str = sha1(txt).hexdigest()
        else:
            # load sourcefile that's been copied into package install payload
            path = os.path.join(package.base, self.include_modules_subpath)
            pathname = os.path.join(path, "%s-*.py" % name)

            pathnames = glob(pathname)
            if not pathnames:
                return None

            filepath = pathnames[0]
            hash_str = filepath.rsplit('-', 1)[-1].split('.', 1)[0]

        module = self.modules.get(hash_str)
        if module is not None:
            return module

        if config.debug("file_loads"):
            print_debug("Loading include sourcefile: %s" % filepath)

        with open(filepath) as f:
            module = imp.load_source(name, filepath, f)

        self.modules[hash_str] = module
        return module
Ejemplo n.º 13
0
Archivo: amqp.py Proyecto: opcg/rez
    def publish_message(self, data):
        if not self.settings.host:
            print_error("Did not publish message, host is not specified")
            return

        routing_key = self.settings.exchange_routing_key
        print("Publishing AMQP message on %s..." % routing_key)

        publish_message(host=self.settings.host,
                        amqp_settings=self.settings,
                        routing_key=routing_key,
                        data=data)

        if config.debug("package_release"):
            print_debug("Published message: %s" % (data))
Ejemplo n.º 14
0
    def publish_message(self, data):
        if not self.settings.host:
            print_error("Did not publish message, host is not specified")
            return

        try:
            conn = Connection(host=self.settings.host,
                              port=self.settings.port,
                              userid=self.settings.userid,
                              password=self.settings.password,
                              connect_timeout=self.settings.connect_timeout)
        except socket.error as e:
            print_error("Cannot connect to the message broker: %s" % (e))
            return

        channel = conn.channel()

        # Declare the exchange
        try:
            channel.exchange_declare(
                self.settings.exchange_name,
                self.settings.exchange_type,
                durable=self.settings.exchange_durable,
                auto_delete=self.settings.exchange_auto_delete)
        except Exception as e:
            print_error("Failed to declare an exchange: %s" % (e))
            return

        # build the message
        msg = basic_message.Message(
            body=json.dumps(data),
            delivery_mode=self.settings.message_delivery_mode,
            content_type="application/json",
            content_encoding="utf-8")

        routing_key = self.settings.exchange_routing_key
        print "Publishing AMQP message on %s..." % routing_key

        # publish the message
        try:
            channel.basic_publish(msg, self.settings.exchange_name,
                                  routing_key)
        except Exception as e:
            print_error("Failed to publish message: %s" % (e))
            return

        if config.debug("package_release"):
            print_debug("Published message: %s" % (data))
Ejemplo n.º 15
0
def extend_path(path, name):
    """Extend a package's path.

    Intended use is to place the following code in a package's __init__.py:

        from pkgutil import extend_path
        __path__ = extend_path(__path__, __name__)

    This will add to the package's __path__ all subdirectories of
    directories on sys.path named after the package.  This is useful
    if one wants to distribute different parts of a single logical
    package as multiple directories.

    If the input path is not a list (as is the case for frozen
    packages) it is returned unchanged.  The input path is not
    modified; an extended copy is returned.  Items are only appended
    to the copy at the end.

    It is assumed that 'plugin_path' is a sequence.  Items of 'plugin_path'
    that are not (unicode or 8-bit) strings referring to existing
    directories are ignored.  Unicode items of sys.path that cause
    errors when used as filenames may cause this function to raise an
    exception (in line with os.path.isdir() behavior).
    """
    if not isinstance(path, list):
        # This could happen e.g. when this is called from inside a
        # frozen package.  Return the path unchanged in that case.
        return path

    pname = os.path.join(*name.split('.'))  # Reconstitute as relative path
    # Just in case os.extsep != '.'
    init_py = "__init__" + os.extsep + "py"
    path = path[:]

    for dir in config.plugin_path:
        if not os.path.isdir(dir):
            if config.debug("plugins"):
                print_debug("skipped nonexistant rez plugin path: %s" % dir)
            continue

        subdir = os.path.join(dir, pname)
        # XXX This may still add duplicate entries to path on
        # case-insensitive filesystems
        initfile = os.path.join(subdir, init_py)
        if subdir not in path and os.path.isfile(initfile):
            path.append(subdir)

    return path
Ejemplo n.º 16
0
def extend_path(path, name):
    """Extend a package's path.

    Intended use is to place the following code in a package's __init__.py:

        from pkgutil import extend_path
        __path__ = extend_path(__path__, __name__)

    This will add to the package's __path__ all subdirectories of
    directories on sys.path named after the package.  This is useful
    if one wants to distribute different parts of a single logical
    package as multiple directories.

    If the input path is not a list (as is the case for frozen
    packages) it is returned unchanged.  The input path is not
    modified; an extended copy is returned.  Items are only appended
    to the copy at the end.

    It is assumed that 'plugin_path' is a sequence.  Items of 'plugin_path'
    that are not (unicode or 8-bit) strings referring to existing
    directories are ignored.  Unicode items of sys.path that cause
    errors when used as filenames may cause this function to raise an
    exception (in line with os.path.isdir() behavior).
    """
    if not isinstance(path, list):
        # This could happen e.g. when this is called from inside a
        # frozen package.  Return the path unchanged in that case.
        return path

    pname = os.path.join(*name.split('.'))  # Reconstitute as relative path
    # Just in case os.extsep != '.'
    init_py = "__init__" + os.extsep + "py"
    path = path[:]

    for dir in config.plugin_path:
        if not os.path.isdir(dir):
            if config.debug("plugins"):
                print_debug("skipped nonexistant rez plugin path: %s" % dir)
            continue

        subdir = os.path.join(dir, pname)
        # XXX This may still add duplicate entries to path on
        # case-insensitive filesystems
        initfile = os.path.join(subdir, init_py)
        if subdir not in path and os.path.isfile(initfile):
            path.append(subdir)

    return path
Ejemplo n.º 17
0
    def publish_message(self, data):
        if not self.settings.host:
            print_error("Did not publish message, host is not specified")
            return

        try:
            conn = Connection(host=self.settings.host,
                              port=self.settings.port,
                              userid=self.settings.userid,
                              password=self.settings.password,
                              connect_timeout=self.settings.connect_timeout)
        except socket.error as e:
            print_error("Cannot connect to the message broker: %s" % (e))
            return

        channel = conn.channel()

        # Declare the exchange
        try:
            channel.exchange_declare(
                self.settings.exchange_name,
                self.settings.exchange_type,
                durable=self.settings.exchange_durable,
                auto_delete=self.settings.exchange_auto_delete)
        except Exception as e:
            print_error("Failed to declare an exchange: %s" % (e))
            return

        # build the message
        msg = basic_message.Message(
            body=json.dumps(data),
            delivery_mode=self.settings.message_delivery_mode,
            content_type="application/json",
            content_encoding="utf-8")

        routing_key = self.settings.exchange_routing_key
        print "Publishing AMQP message on %s..." % routing_key

        # publish the message
        try:
            channel.basic_publish(msg, self.settings.exchange_name, routing_key)
        except Exception as e:
            print_error("Failed to publish message: %s" % (e))
            return

        if config.debug("package_release"):
            print_debug("Published message: %s" % (data))
Ejemplo n.º 18
0
    def _release(self, commands, errors=None):
        for conf in commands:
            if self.settings.print_commands or config.debug("package_release"):
                from subprocess import list2cmdline
                toks = [conf["command"]] + conf.get("args", [])
                msg = "running command: %s" % list2cmdline(toks)
                if self.settings.print_commands:
                    print msg
                else:
                    print_debug(msg)

            if not self.execute_command(cmd_name=conf.get("command"),
                                        cmd_arguments=conf.get("args"),
                                        user=conf.get("user"),
                                        errors=errors):
                if self.settings.stop_on_error:
                    return
Ejemplo n.º 19
0
    def _execute_commands(self, commands, install_path, package, errors=None):
        release_dict = dict(path=install_path)
        formatter = scoped_formatter(system=system,
                                     release=release_dict,
                                     package=package)

        for conf in commands:
            program = conf["command"]

            env_ = None
            env = conf.get("env")
            if env:
                env_ = os.environ.copy()
                env_.update(env)

            args = conf.get("args", [])
            args = [formatter.format(x) for x in args]
            args = [expandvars(x, environ=env_) for x in args]

            if self.settings.print_commands or config.debug("package_release"):
                from subprocess import list2cmdline
                toks = [program] + args

                msgs = []
                msgs.append("running command: %s" % list2cmdline(toks))
                if env:
                    for key, value in env.iteritems():
                        msgs.append("    with: %s=%s" % (key, value))

                if self.settings.print_commands:
                    print '\n'.join(msgs)
                else:
                    for msg in msgs:
                        print_debug(msg)

            if not self.execute_command(cmd_name=program,
                                        cmd_arguments=args,
                                        user=conf.get("user"),
                                        errors=errors,
                                        env=env_):
                if self.settings.stop_on_error:
                    return
Ejemplo n.º 20
0
    def _execute_commands(self, commands, install_path, package, errors=None):
        release_dict = dict(path=install_path)
        formatter = scoped_formatter(system=system,
                                     release=release_dict,
                                     package=package)

        for conf in commands:
            program = conf["command"]

            env_ = None
            env = conf.get("env")
            if env:
                env_ = os.environ.copy()
                env_.update(env)

            args = conf.get("args", [])
            args = [formatter.format(x) for x in args]
            args = [expandvars(x, environ=env_) for x in args]

            if self.settings.print_commands or config.debug("package_release"):
                from subprocess import list2cmdline
                toks = [program] + args

                msgs = []
                msgs.append("running command: %s" % list2cmdline(toks))
                if env:
                    for key, value in env.iteritems():
                        msgs.append("    with: %s=%s" % (key, value))

                if self.settings.print_commands:
                    print '\n'.join(msgs)
                else:
                    for msg in msgs:
                        print_debug(msg)

            if not self.execute_command(cmd_name=program,
                                        cmd_arguments=args,
                                        user=conf.get("user"),
                                        errors=errors,
                                        env=env_):
                if self.settings.stop_on_error:
                    return
Ejemplo n.º 21
0
    def _execute_commands(self, commands, install_path, package, errors=None,
                          variants=None):
        release_dict = dict(path=install_path)
        variant_infos = []
        if variants:
            for variant in variants:
                if isinstance(variant, (int, long)):
                    variant_infos.append(variant)
                else:
                    package = variant.parent
                    var_dict = dict(variant.resource.variables)
                    # using '%s' will preserve potential str/unicode nature
                    var_dict['variant_requires'] = ['%s' % x
                                                    for x in variant.resource.variant_requires]
                    variant_infos.append(var_dict)
        formatter = scoped_formatter(system=system,
                                     release=release_dict,
                                     package=package,
                                     variants=variant_infos,
                                     num_variants=len(variant_infos))

        for conf in commands:
            program = conf["command"]

            env_ = None
            env = conf.get("env")
            if env:
                env_ = os.environ.copy()
                env_.update(env)

            # If we have, ie, a list, and format_pretty is True, it will be printed
            # as "1 2 3" instead of "[1, 2, 3]"
            formatter.__dict__['format_pretty'] = conf.get(
                "pretty_args", True)

            args = conf.get("args", [])
            args = [formatter.format(x) for x in args]
            args = [expandvars(x, environ=env_) for x in args]

            if self.settings.print_commands or config.debug("package_release"):
                from subprocess import list2cmdline
                toks = [program] + args

                msgs = []
                msgs.append("running command: %s" % list2cmdline(toks))
                if env:
                    for key, value in env.iteritems():
                        msgs.append("    with: %s=%s" % (key, value))

                if self.settings.print_commands:
                    print '\n'.join(msgs)
                else:
                    for msg in msgs:
                        print_debug(msg)

            if not self.execute_command(cmd_name=program,
                                        cmd_arguments=args,
                                        user=conf.get("user"),
                                        errors=errors,
                                        env=env_):
                if self.settings.stop_on_error:
                    return
Ejemplo n.º 22
0
    def load_plugins(self):
        import pkgutil
        from rez.backport.importlib import import_module
        type_module_name = 'rezplugins.' + self.type_name
        package = import_module(type_module_name)

        # on import, the `__path__` variable of the imported package is extended
        # to include existing directories on the plugin search path (via
        # extend_path, above). this means that `walk_packages` will walk over all
        # modules on the search path at the same level (.e.g in a
        # 'rezplugins/type_name' sub-directory).
        paths = [package.__path__] if isinstance(package.__path__, basestring) \
            else package.__path__

        # reverse plugin path order, so that custom plugins have a chance to
        # be found before the builtin plugins (from /rezplugins).
        paths = reversed(paths)

        for path in paths:
            if config.debug("plugins"):
                print_debug("searching plugin path %s...", path)

            for importer, modname, ispkg in pkgutil.iter_modules(
                [path], package.__name__ + '.'):

                if importer is None:
                    continue

                plugin_name = modname.split('.')[-1]
                if plugin_name.startswith('_') or plugin_name == 'rezconfig':
                    continue

                if plugin_name in self.plugin_modules:
                    # same named plugins will have identical module name,
                    # which will just reuse previous imported module from
                    # `sys.modules` below. skipping the rest of the process
                    # for good.
                    if config.debug("plugins"):
                        print_warning(
                            "skipped same named %s plugin at %s: %s" %
                            (self.type_name, path, modname))
                    continue

                if config.debug("plugins"):
                    print_debug("loading %s plugin at %s: %s..." %
                                (self.type_name, path, modname))
                try:
                    # nerdvegas/rez#218
                    # load_module will force reload the module if it's
                    # already loaded, so check for that
                    plugin_module = sys.modules.get(modname)
                    if plugin_module is None:
                        loader = importer.find_module(modname)
                        plugin_module = loader.load_module(modname)

                    elif os.path.dirname(plugin_module.__file__) != path:
                        if config.debug("plugins"):
                            # this should not happen but if it does, tell why.
                            print_warning(
                                "plugin module %s is not loaded from current "
                                "load path but reused from previous imported "
                                "path: %s" % (modname, plugin_module.__file__))

                    if (hasattr(plugin_module, "register_plugin")
                            and callable(plugin_module.register_plugin)):

                        plugin_class = plugin_module.register_plugin()
                        if plugin_class is not None:
                            self.register_plugin(plugin_name, plugin_class,
                                                 plugin_module)
                        else:
                            if config.debug("plugins"):
                                print_warning(
                                    "'register_plugin' function at %s: %s did "
                                    "not return a class." % (path, modname))
                    else:
                        if config.debug("plugins"):
                            print_warning(
                                "no 'register_plugin' function at %s: %s" %
                                (path, modname))

                        # delete from sys.modules?

                except Exception as e:
                    nameish = modname.split('.')[-1]
                    self.failed_plugins[nameish] = str(e)
                    if config.debug("plugins"):
                        import traceback
                        from rez.vendor.six.six import StringIO
                        out = StringIO()
                        traceback.print_exc(file=out)
                        print_debug(out.getvalue())

            # load config
            data, _ = _load_config_from_filepaths(
                [os.path.join(path, "rezconfig")])
            deep_update(self.config_data, data)
Ejemplo n.º 23
0
Archivo: pip.py Proyecto: shiyi9/rez
def pip_install_package(source_name,
                        pip_version=None,
                        python_version=None,
                        mode=InstallMode.min_deps,
                        release=False):
    """Install a pip-compatible python package as a rez package.
    Args:
        source_name (str): Name of package or archive/url containing the pip
            package source. This is the same as the arg you would pass to
            the 'pip install' command.
        pip_version (str or `Version`): Version of pip to use to perform the
            install, uses latest if None.
        python_version (str or `Version`): Python version to use to perform the
            install, and subsequently have the resulting rez package depend on.
        mode (`InstallMode`): Installation mode, determines how dependencies are
            managed.
        release (bool): If True, install as a released package; otherwise, it
            will be installed as a local package.

    Returns:
        2-tuple:
            List of `Variant`: Installed variants;
            List of `Variant`: Skipped variants (already installed).
    """
    installed_variants = []
    skipped_variants = []

    py_exe, context = find_pip(pip_version, python_version)

    # TODO: should check if packages_path is writable before continuing with pip
    #
    packages_path = (config.release_packages_path
                     if release else config.local_packages_path)

    tmpdir = mkdtemp(suffix="-rez", prefix="pip-")
    stagingdir = os.path.join(tmpdir, "rez_staging")
    stagingsep = "".join([os.path.sep, "rez_staging", os.path.sep])

    destpath = os.path.join(stagingdir, "python")
    # TODO use binpath once https://github.com/pypa/pip/pull/3934 is approved
    binpath = os.path.join(stagingdir, "bin")

    if context and config.debug("package_release"):
        buf = StringIO()
        print("\n\npackage download environment:", file=buf)
        context.print_info(buf)
        _log(buf.getvalue())

    # Build pip commandline
    cmd = [
        py_exe, "-m", "pip", "install", "--use-pep517",
        "--target=%s" % destpath
    ]

    if mode == InstallMode.no_deps:
        cmd.append("--no-deps")
    cmd.append(source_name)

    _cmd(context=context, command=cmd)
    _system = System()

    # determine version of python in use
    if context is None:
        # since we had to use system pip, we have to assume system python version
        py_ver_str = '.'.join(map(str, sys.version_info))
        py_ver = Version(py_ver_str)
    else:
        python_variant = context.get_resolved_package("python")
        py_ver = python_variant.version

    # moving bin folder to expected relative location as per wheel RECORD files
    staged_binpath = os.path.join(destpath, "bin")
    if os.path.isdir(staged_binpath):
        shutil.move(os.path.join(destpath, "bin"), binpath)

    # Collect resulting python packages using distlib
    distribution_path = DistributionPath([destpath])
    distributions = list(distribution_path.get_distributions())
    dist_names = [x.name for x in distributions]

    # get list of package and dependencies
    for distribution in distributions:
        # convert pip requirements into rez requirements
        rez_requires = get_rez_requirements(installed_dist=distribution,
                                            python_version=py_ver,
                                            name_casings=dist_names)

        # log the pip -> rez translation, for debugging
        _log("Pip to rez translation information for " +
             distribution.name_and_version + ":\n" +
             pformat({
                 "pip": {
                     "run_requires": map(str, distribution.run_requires)
                 },
                 "rez": rez_requires
             }))

        # iterate over installed files and determine dest filepaths
        tools = []
        src_dst_lut = {}

        for installed_file in distribution.list_installed_files():
            # distlib expects the script files to be located in ../../bin/
            # when in fact ../bin seems to be the resulting path after the
            # installation as such we need to point the bin files to the
            # expected location to match wheel RECORD files
            installed_filepath = os.path.normpath(installed_file[0])
            bin_prefix = os.path.join('..', '..', 'bin') + os.sep

            if installed_filepath.startswith(bin_prefix):
                # account for extra parentdir as explained above
                installed = os.path.join(destpath, '_', installed_filepath)
            else:
                installed = os.path.join(destpath, installed_filepath)

            source_file = os.path.normpath(installed)

            if os.path.exists(source_file):
                destination_file = os.path.relpath(source_file, stagingdir)
                exe = False

                if is_exe(source_file) and destination_file.startswith("bin" +
                                                                       os.sep):
                    _file = os.path.basename(destination_file)
                    tools.append(_file)
                    exe = True

                src_dst_lut[source_file] = [destination_file, exe]
            else:
                _log("Source file does not exist: " + source_file + "!")

        def make_root(variant, path):
            """Using distlib to iterate over all installed files of the current
            distribution to copy files to the target directory of the rez package
            variant
            """
            for source_file, data in src_dst_lut.items():
                destination_file, exe = data
                destination_file = os.path.normpath(
                    os.path.join(path, destination_file))

                if not os.path.exists(os.path.dirname(destination_file)):
                    os.makedirs(os.path.dirname(destination_file))

                shutil.copyfile(source_file, destination_file)
                if exe:
                    shutil.copystat(source_file, destination_file)

        # create the rez package
        name = pip_to_rez_package_name(distribution.name)
        version = pip_to_rez_version(distribution.version)
        requires = rez_requires["requires"]
        variant_requires = rez_requires["variant_requires"]
        metadata = rez_requires["metadata"]

        with make_package(name, packages_path, make_root=make_root) as pkg:
            # basics (version etc)
            pkg.version = version

            if distribution.metadata.summary:
                pkg.description = distribution.metadata.summary

            # requirements and variants
            if requires:
                pkg.requires = requires

            if variant_requires:
                pkg.variants = [variant_requires]

            # commands
            commands = []
            commands.append("env.PYTHONPATH.append('{root}/python')")

            if tools:
                pkg.tools = tools
                commands.append("env.PATH.append('{root}/bin')")

            pkg.commands = '\n'.join(commands)

            # Make the package use hashed variants. This is required because we
            # can't control what ends up in its variants, and that can easily
            # include problematic chars (>, +, ! etc).
            # TODO: https://github.com/nerdvegas/rez/issues/672
            #
            pkg.hashed_variants = True

            # add some custom attributes to retain pip-related info
            pkg.pip_name = distribution.name_and_version
            pkg.from_pip = True
            pkg.is_pure_python = metadata["is_pure_python"]

        installed_variants.extend(pkg.installed_variants or [])
        skipped_variants.extend(pkg.skipped_variants or [])

    # cleanup
    shutil.rmtree(tmpdir)

    return installed_variants, skipped_variants
Ejemplo n.º 24
0
def pip_install_package(source_name,
                        pip_version=None,
                        python_version=None,
                        mode=InstallMode.min_deps,
                        release=False,
                        prefix=None,
                        extra_args=None):
    """Install a pip-compatible python package as a rez package.
    Args:
        source_name (str): Name of package or archive/url containing the pip
            package source. This is the same as the arg you would pass to
            the 'pip install' command.
        pip_version (str or `Version`): Version of pip to use to perform the
            install, uses latest if None.
        python_version (str or `Version`): Python version to use to perform the
            install, and subsequently have the resulting rez package depend on.
        mode (`InstallMode`): Installation mode, determines how dependencies are
            managed.
        release (bool): If True, install as a released package; otherwise, it
            will be installed as a local package.
        extra_args (List[str]): Additional options to the pip install command.

    Returns:
        2-tuple:
            List of `Variant`: Installed variants;
            List of `Variant`: Skipped variants (already installed).
    """
    installed_variants = []
    skipped_variants = []

    py_exe, context = find_pip(pip_version, python_version)
    print_info("Installing %r with pip taken from %r", source_name, py_exe)

    # TODO: should check if packages_path is writable before continuing with pip
    #
    if prefix is not None:
        packages_path = prefix
    else:
        packages_path = (config.release_packages_path
                         if release else config.local_packages_path)

    targetpath = mkdtemp(suffix="-rez", prefix="pip-")

    if context and config.debug("package_release"):
        buf = StringIO()
        print("\n\npackage download environment:", file=buf)
        context.print_info(buf)
        _log(buf.getvalue())

    # Build pip commandline
    cmd = [py_exe, "-m", "pip", "install"]

    _extra_args = extra_args or config.pip_extra_args or []

    if "--no-use-pep517" not in _extra_args:
        cmd.append("--use-pep517")

    if not _option_present(_extra_args, "-t", "--target"):
        cmd.append("--target=%s" % targetpath)

    if mode == InstallMode.no_deps and "--no-deps" not in _extra_args:
        cmd.append("--no-deps")

    cmd.extend(_extra_args)
    cmd.append(source_name)

    # run pip
    #
    # Note: https://github.com/pypa/pip/pull/3934. If/when this PR is merged,
    # it will allow explicit control of where to put bin files.
    #
    _cmd(context=context, command=cmd)

    # determine version of python in use
    if context is None:
        # since we had to use system pip, we have to assume system python version
        py_ver_str = '.'.join(map(str, sys.version_info))
        py_ver = Version(py_ver_str)
    else:
        python_variant = context.get_resolved_package("python")
        py_ver = python_variant.version

    # Collect resulting python packages using distlib
    distribution_path = DistributionPath([targetpath])
    distributions = list(distribution_path.get_distributions())
    dist_names = [x.name for x in distributions]

    def log_append_pkg_variants(pkg_maker):
        template = '{action} [{package.qualified_name}] {package.uri}{suffix}'
        actions_variants = [
            (
                print_info,
                'Installed',
                installed_variants,
                pkg_maker.installed_variants or [],
            ),
            (
                print_debug,
                'Skipped',
                skipped_variants,
                pkg_maker.skipped_variants or [],
            ),
        ]
        for print_, action, variants, pkg_variants in actions_variants:
            for variant in pkg_variants:
                variants.append(variant)
                package = variant.parent
                suffix = (' (%s)' % variant.subpath) if variant.subpath else ''
                print_(template.format(**locals()))

    # get list of package and dependencies
    for distribution in distributions:
        # convert pip requirements into rez requirements
        rez_requires = get_rez_requirements(installed_dist=distribution,
                                            python_version=py_ver,
                                            name_casings=dist_names)

        # log the pip -> rez requirements translation, for debugging
        _log("Pip to rez requirements translation information for " +
             distribution.name_and_version + ":\n" +
             pformat({
                 "pip": {
                     "run_requires": map(str, distribution.run_requires)
                 },
                 "rez": rez_requires
             }))

        # determine where pip files need to be copied into rez package
        src_dst_lut = _get_distribution_files_mapping(distribution, targetpath)

        # build tools list
        tools = []
        for relpath in src_dst_lut.values():
            dir_, filename = os.path.split(relpath)
            if dir_ == "bin":
                tools.append(filename)

        # Sanity warning to see if any files will be copied
        if not src_dst_lut:
            message = 'No source files exist for {}!'
            if not _verbose:
                message += '\nTry again with rez-pip --verbose ...'
            print_warning(message.format(distribution.name_and_version))

        def make_root(variant, path):
            """Using distlib to iterate over all installed files of the current
            distribution to copy files to the target directory of the rez package
            variant
            """
            for rel_src, rel_dest in src_dst_lut.items():
                src = os.path.join(targetpath, rel_src)
                dest = os.path.join(path, rel_dest)

                if not os.path.exists(os.path.dirname(dest)):
                    os.makedirs(os.path.dirname(dest))

                shutil.copyfile(src, dest)

                if _is_exe(src):
                    shutil.copystat(src, dest)

        # create the rez package
        name = pip_to_rez_package_name(distribution.name)
        version = pip_to_rez_version(distribution.version)
        requires = rez_requires["requires"]
        variant_requires = rez_requires["variant_requires"]
        metadata = rez_requires["metadata"]

        with make_package(name, packages_path, make_root=make_root) as pkg:
            # basics (version etc)
            pkg.version = version

            if distribution.metadata.summary:
                pkg.description = distribution.metadata.summary

            # requirements and variants
            if requires:
                pkg.requires = requires

            if variant_requires:
                pkg.variants = [variant_requires]

            # commands
            commands = []
            commands.append("env.PYTHONPATH.append('{root}/python')")

            if tools:
                pkg.tools = tools
                commands.append("env.PATH.append('{root}/bin')")

            pkg.commands = '\n'.join(commands)

            # Make the package use hashed variants. This is required because we
            # can't control what ends up in its variants, and that can easily
            # include problematic chars (>, +, ! etc).
            # TODO: https://github.com/nerdvegas/rez/issues/672
            #
            pkg.hashed_variants = True

            # add some custom attributes to retain pip-related info
            pkg.pip_name = distribution.name_and_version
            pkg.from_pip = True
            pkg.is_pure_python = metadata["is_pure_python"]

            distribution_metadata = distribution.metadata.todict()

            help_ = []

            if "home_page" in distribution_metadata:
                help_.append(["Home Page", distribution_metadata["home_page"]])

            if "download_url" in distribution_metadata:
                help_.append(
                    ["Source Code", distribution_metadata["download_url"]])

            if help_:
                pkg.help = help_

            if "author" in distribution_metadata:
                author = distribution_metadata["author"]

                if "author_email" in distribution_metadata:
                    author += ' ' + distribution_metadata["author_email"]

                pkg.authors = [author]

        log_append_pkg_variants(pkg)

    # cleanup
    shutil.rmtree(targetpath)

    # print summary
    #
    if installed_variants:
        print_info("%d packages were installed.", len(installed_variants))
    else:
        print_warning("NO packages were installed.")
    if skipped_variants:
        print_warning(
            "%d packages were already installed.",
            len(skipped_variants),
        )

    return installed_variants, skipped_variants
Ejemplo n.º 25
0
    cmd_str = ' '.join(quote(x) for x in command)
    _log("running: %s" % cmd_str)

    if context is None:
        p = Popen(command)
    else:
        p = context.execute_shell(command=command, block=False)

    with p:
        p.wait()

    if p.returncode:
        raise BuildError("Failed to download source with pip: %s" % cmd_str)


_verbose = config.debug("package_release")


def _log(msg):
    if _verbose:
        print_debug(msg)


# Copyright 2013-2016 Allan Johns.
#
# This library is free software: you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
Ejemplo n.º 26
0
Archivo: pip.py Proyecto: oktomus/rez3
def pip_install_package(source_name, pip_version=None, python_version=None,
                        mode=InstallMode.min_deps, release=False):
    """Install a pip-compatible python package as a rez package.
    Args:
        source_name (str): Name of package or archive/url containing the pip
            package source. This is the same as the arg you would pass to
            the 'pip install' command.
        pip_version (str or `Version`): Version of pip to use to perform the
            install, uses latest if None.
        python_version (str or `Version`): Python version to use to perform the
            install, and subsequently have the resulting rez package depend on.
        mode (`InstallMode`): Installation mode, determines how dependencies are
            managed.
        release (bool): If True, install as a released package; otherwise, it
            will be installed as a local package.

    Returns:
        2-tuple:
            List of `Variant`: Installed variants;
            List of `Variant`: Skipped variants (already installed).
    """
    installed_variants = []
    skipped_variants = []

    pip_exe, context = find_pip(pip_version, python_version)

    # TODO: should check if packages_path is writable before continuing with pip
    #
    packages_path = (config.release_packages_path if release
                     else config.local_packages_path)

    tmpdir = mkdtemp(suffix="-rez", prefix="pip-")
    stagingdir = os.path.join(tmpdir, "rez_staging")
    stagingsep = "".join([os.path.sep, "rez_staging", os.path.sep])

    destpath = os.path.join(stagingdir, "python")
    binpath = os.path.join(stagingdir, "bin")
    incpath = os.path.join(stagingdir, "include")
    datapath = stagingdir

    if context and config.debug("package_release"):
        buf = StringIO()
        print >> buf, "\n\npackage download environment:"
        context.print_info(buf)
        _log(buf.getvalue())

    # Build pip commandline
    cmd = [pip_exe, "install",
           "--install-option=--install-lib=%s" % destpath,
           "--install-option=--install-scripts=%s" % binpath,
           "--install-option=--install-headers=%s" % incpath,
           "--install-option=--install-data=%s" % datapath]

    if mode == InstallMode.no_deps:
        cmd.append("--no-deps")
    cmd.append(source_name)

    _cmd(context=context, command=cmd)
    _system = System()

    # Collect resulting python packages using distlib
    distribution_path = DistributionPath([destpath], include_egg=True)
    distributions = [d for d in distribution_path.get_distributions()]

    for distribution in distribution_path.get_distributions():
        requirements = []
        if distribution.metadata.run_requires:
            # Handle requirements. Currently handles conditional environment based
            # requirements and normal requirements
            # TODO: Handle optional requirements?
            for requirement in distribution.metadata.run_requires:
                if "environment" in requirement:
                    if interpret(requirement["environment"]):
                        requirements.extend(_get_dependencies(requirement, distributions))
                elif "extra" in requirement:
                    # Currently ignoring optional requirements
                    pass
                else:
                    requirements.extend(_get_dependencies(requirement, distributions))

        tools = []
        src_dst_lut = {}

        for installed_file in distribution.list_installed_files(allow_fail=True):
            source_file = os.path.normpath(os.path.join(destpath, installed_file[0]))

            if os.path.exists(source_file):
                destination_file = installed_file[0].split(stagingsep)[1]
                exe = False

                if is_exe(source_file) and \
                        destination_file.startswith("%s%s" % ("bin", os.path.sep)):
                    _, _file = os.path.split(destination_file)
                    tools.append(_file)
                    exe = True

                data = [destination_file, exe]
                src_dst_lut[source_file] = data
            else:
                _log("Source file does not exist: " + source_file + "!")

        def make_root(variant, path):
            """Using distlib to iterate over all installed files of the current
            distribution to copy files to the target directory of the rez package
            variant
            """
            for source_file, data in src_dst_lut.items():
                destination_file, exe = data
                destination_file = os.path.normpath(os.path.join(path, destination_file))

                if not os.path.exists(os.path.dirname(destination_file)):
                    os.makedirs(os.path.dirname(destination_file))

                shutil.copyfile(source_file, destination_file)
                if exe:
                    shutil.copystat(source_file, destination_file)

        # determine variant requirements
        # TODO detect if platform/arch/os necessary, no if pure python
        variant_reqs = []
        variant_reqs.append("platform-%s" % _system.platform)
        variant_reqs.append("arch-%s" % _system.arch)
        variant_reqs.append("os-%s" % _system.os)

        if context is None:
            # since we had to use system pip, we have to assume system python version
            py_ver = '.'.join(map(str, sys.version_info[:2]))
        else:
            python_variant = context.get_resolved_package("python")
            py_ver = python_variant.version.trim(2)

        variant_reqs.append("python-%s" % py_ver)

        name, _ = parse_name_and_version(distribution.name_and_version)
        name = distribution.name[0:len(name)].replace("-", "_")

        with make_package(name, packages_path, make_root=make_root) as pkg:
            pkg.version = distribution.version
            if distribution.metadata.summary:
                pkg.description = distribution.metadata.summary

            pkg.variants = [variant_reqs]
            if requirements:
                pkg.requires = requirements

            commands = []
            commands.append("env.PYTHONPATH.append('{root}/python')")

            if tools:
                pkg.tools = tools
                commands.append("env.PATH.append('{root}/bin')")

            pkg.commands = '\n'.join(commands)

        installed_variants.extend(pkg.installed_variants or [])
        skipped_variants.extend(pkg.skipped_variants or [])

    # cleanup
    shutil.rmtree(tmpdir)

    return installed_variants, skipped_variants
Ejemplo n.º 27
0
def log(msg):
    if config.debug("bind_modules"):
        print_debug(msg)
Ejemplo n.º 28
0
    def _execute_commands(self,
                          commands,
                          install_path,
                          package,
                          errors=None,
                          variants=None):
        release_dict = dict(path=install_path)
        variant_infos = []
        if variants:
            for variant in variants:
                if isinstance(variant, (int, long)):
                    variant_infos.append(variant)
                else:
                    package = variant.parent
                    var_dict = dict(variant.resource.variables)
                    # using '%s' will preserve potential str/unicode nature
                    var_dict['variant_requires'] = [
                        '%s' % x for x in variant.resource.variant_requires
                    ]
                    variant_infos.append(var_dict)
        formatter = scoped_formatter(system=system,
                                     release=release_dict,
                                     package=package,
                                     variants=variant_infos,
                                     num_variants=len(variant_infos))

        for conf in commands:
            program = conf["command"]

            env_ = None
            env = conf.get("env")
            if env:
                env_ = os.environ.copy()
                env_.update(env)

            # If we have, ie, a list, and format_pretty is True, it will be printed
            # as "1 2 3" instead of "[1, 2, 3]"
            formatter.__dict__['format_pretty'] = conf.get("pretty_args", True)

            args = conf.get("args", [])
            args = [formatter.format(x) for x in args]
            args = [expandvars(x, environ=env_) for x in args]

            if self.settings.print_commands or config.debug("package_release"):
                from subprocess import list2cmdline
                toks = [program] + args

                msgs = []
                msgs.append("running command: %s" % list2cmdline(toks))
                if env:
                    for key, value in env.iteritems():
                        msgs.append("    with: %s=%s" % (key, value))

                if self.settings.print_commands:
                    print('\n'.join(msgs))
                else:
                    for msg in msgs:
                        print_debug(msg)

            if not self.execute_command(cmd_name=program,
                                        cmd_arguments=args,
                                        user=conf.get("user"),
                                        errors=errors,
                                        env=env_):
                if self.settings.stop_on_error:
                    return
Ejemplo n.º 29
0
def convert_old_commands(commands, annotate=True):
    """Converts old-style package commands into equivalent Rex code."""
    from rez.config import config
    from rez.utils.logging_ import print_debug

    def _repl(s):
        return s.replace('\\"', '"')

    def _encode(s):
        # this replaces all occurrances of '\"' with '"', *except* for those
        # occurrances of '\"' that are within double quotes, which themselves
        # are not escaped. In other words, the string:
        # 'hey "there \"you\" " \"dude\" '
        # ..will convert to:
        # 'hey "there \"you\" " "dude" '
        s_new = ''
        prev_i = 0

        for m in within_unescaped_quotes_regex.finditer(s):
            s_ = s[prev_i:m.start()]
            s_new += _repl(s_)

            s_new += s[m.start():m.end()]
            prev_i = m.end()

        s_ = s[prev_i:]
        s_new += _repl(s_)
        return repr(s_new)

    loc = []

    for cmd in commands:
        if annotate:
            txt = "OLD COMMAND: %s" % cmd
            line = "comment(%s)" % _encode(txt)
            loc.append(line)

        cmd = convert_old_command_expansions(cmd)
        toks = cmd.strip().split()

        try:
            if toks[0] == "export":
                var, value = cmd.split(' ', 1)[1].split('=', 1)
                for bookend in ('"', "'"):
                    if value.startswith(bookend) and value.endswith(bookend):
                        value = value[1:-1]
                        break

                # As the only old-style commands were Linux/Bash based,
                # we assume using the default separator ":" is ok - we don't
                # need to use os.pathsep as we don't expected to see a
                # Windows path here.
                separator = config.env_var_separators.get(var, ":")

                # This is a special case.  We don't want to include "';'" in
                # our env var separators map as it's not really the correct
                # behaviour/something we want to promote.  It's included here for
                # backwards compatibility only, and to not propogate elsewhere.
                if var == "CMAKE_MODULE_PATH":
                    value = value.replace("'%s'" % separator, separator)
                    value = value.replace('"%s"' % separator, separator)
                    value = value.replace(":", separator)

                parts = value.split(separator)
                parts = [x for x in parts if x]
                if len(parts) > 1:
                    idx = None
                    var1 = "$%s" % var
                    var2 = "${%s}" % var
                    if var1 in parts:
                        idx = parts.index(var1)
                    elif var2 in parts:
                        idx = parts.index(var2)
                    if idx in (0, len(parts) - 1):
                        func = "appendenv" if idx == 0 else "prependenv"
                        parts = parts[1:] if idx == 0 else parts[:-1]
                        val = separator.join(parts)
                        loc.append("%s('%s', %s)" % (func, var, _encode(val)))
                        continue

                loc.append("setenv('%s', %s)" % (var, _encode(value)))
            elif toks[0].startswith('#'):
                loc.append("comment(%s)" % _encode(' '.join(toks[1:])))
            elif toks[0] == "alias":
                match = re.search("alias (?P<key>.*?)=(?P<value>.*)", cmd)
                key = match.groupdict()['key'].strip()
                value = match.groupdict()['value'].strip()
                if (value.startswith('"') and value.endswith('"')) or \
                        (value.startswith("'") and value.endswith("'")):
                    value = value[1:-1]
                loc.append("alias('%s', %s)" % (key, _encode(value)))
            else:
                # assume we can execute this as a straight command
                loc.append("command(%s)" % _encode(cmd))
        except:
            # if anything goes wrong, just fall back to bash command
            loc.append("command(%s)" % _encode(cmd))

    rex_code = '\n'.join(loc)
    if config.debug("old_commands"):
        br = '-' * 80
        msg = textwrap.dedent("""
            %s
            OLD COMMANDS:
            %s

            NEW COMMANDS:
            %s
            %s
            """) % (br, '\n'.join(commands), rex_code, br)
        print_debug(msg)
    return rex_code
Ejemplo n.º 30
0
def pip_install_package(source_name,
                        pip_version=None,
                        python_version=None,
                        mode=InstallMode.min_deps,
                        release=False):
    """Install a pip-compatible python package as a rez package.
    Args:
        source_name (str): Name of package or archive/url containing the pip
            package source. This is the same as the arg you would pass to
            the 'pip install' command.
        pip_version (str or `Version`): Version of pip to use to perform the
            install, uses latest if None.
        python_version (str or `Version`): Python version to use to perform the
            install, and subsequently have the resulting rez package depend on.
        mode (`InstallMode`): Installation mode, determines how dependencies are
            managed.
        release (bool): If True, install as a released package; otherwise, it
            will be installed as a local package.

    Returns:
        2-tuple:
            List of `Variant`: Installed variants;
            List of `Variant`: Skipped variants (already installed).
    """
    installed_variants = []
    skipped_variants = []

    pip_exe, context = find_pip(pip_version, python_version)

    # TODO: should check if packages_path is writable before continuing with pip
    #
    packages_path = (config.release_packages_path
                     if release else config.local_packages_path)

    tmpdir = mkdtemp(suffix="-rez", prefix="pip-")
    stagingdir = os.path.join(tmpdir, "rez_staging")
    stagingsep = "".join([os.path.sep, "rez_staging", os.path.sep])

    destpath = os.path.join(stagingdir, "python")
    # TODO use binpath once https://github.com/pypa/pip/pull/3934 is approved
    binpath = os.path.join(stagingdir, "bin")

    if context and config.debug("package_release"):
        buf = StringIO()
        print("\n\npackage download environment:", file=buf)
        context.print_info(buf)
        _log(buf.getvalue())

    # Build pip commandline
    cmd = [pip_exe, "install", "--use-pep517", "--target=%s" % destpath]

    if mode == InstallMode.no_deps:
        cmd.append("--no-deps")
    cmd.append(source_name)

    _cmd(context=context, command=cmd)
    _system = System()

    def pure_python_package(installed_dist):

        true_table = {"true": True, "false": False}

        packages = pkg_resources.find_distributions(destpath)
        dist = next(
            (package
             for package in packages if package.key == installed_dist.key),
            None)
        wheel_data = dist.get_metadata('WHEEL')
        # see https://www.python.org/dev/peps/pep-0566/#json-compatible-metadata
        wheel_data = Parser().parsestr(wheel_data)

        # see https://www.python.org/dev/peps/pep-0427/#what-s-the-deal-with-purelib-vs-platlib
        return true_table[wheel_data["Root-Is-Purelib"]]

    # Collect resulting python packages using distlib
    distribution_path = DistributionPath([destpath])
    distributions = [d for d in distribution_path.get_distributions()]

    # moving bin folder to expected relative location as per wheel RECORD files
    staged_binpath = os.path.join(destpath, "bin")
    if os.path.isdir(staged_binpath):
        shutil.move(os.path.join(destpath, "bin"), binpath)

    for distribution in distribution_path.get_distributions():
        requirements = []
        if distribution.metadata.run_requires:
            # Handle requirements. Currently handles conditional environment based
            # requirements and normal requirements
            # TODO: Handle optional requirements?
            for requirement in distribution.metadata.run_requires:
                if "environment" in requirement:
                    if interpret(requirement["environment"]):
                        requirements.extend(
                            _get_dependencies(requirement, distributions))
                elif "extra" in requirement:
                    # Currently ignoring optional requirements
                    pass
                else:
                    requirements.extend(
                        _get_dependencies(requirement, distributions))

        tools = []
        src_dst_lut = {}

        for installed_file in distribution.list_installed_files():
            # distlib expects the script files to be located in ../../bin/
            # when in fact ../bin seems to be the resulting path after the
            # installation as such we need to point the bin files to the
            # expected location to match wheel RECORD files
            installed_filepath = os.path.normpath(installed_file[0])
            bin_prefix = os.path.join('..', '..', 'bin') + os.sep

            if installed_filepath.startswith(bin_prefix):
                # account for extra parentdir as explained above
                installed = os.path.join(destpath, '_', installed_filepath)
            else:
                installed = os.path.join(destpath, installed_filepath)

            source_file = os.path.normpath(installed)

            if os.path.exists(source_file):
                destination_file = os.path.relpath(source_file, stagingdir)
                exe = False

                if is_exe(source_file) and destination_file.startswith("bin" +
                                                                       os.sep):
                    _file = os.path.basename(destination_file)
                    tools.append(_file)
                    exe = True

                data = [destination_file, exe]
                src_dst_lut[source_file] = data
            else:
                _log("Source file does not exist: " + source_file + "!")

        def make_root(variant, path):
            """Using distlib to iterate over all installed files of the current
            distribution to copy files to the target directory of the rez package
            variant
            """
            for source_file, data in src_dst_lut.items():
                destination_file, exe = data
                destination_file = os.path.normpath(
                    os.path.join(path, destination_file))

                if not os.path.exists(os.path.dirname(destination_file)):
                    os.makedirs(os.path.dirname(destination_file))

                shutil.copyfile(source_file, destination_file)
                if exe:
                    shutil.copystat(source_file, destination_file)

        # determine variant requirements
        variant_reqs = []

        pure = pure_python_package(distribution)

        if not pure:
            variant_reqs.append("platform-%s" % _system.platform)
            variant_reqs.append("arch-%s" % _system.arch)

        # Add the python version requirement. Note that we specify python to
        # minor version because of environment markers - these often mean that
        # you cannot use a loose python requirement (ie major version only)
        # because then the package requirements would not be correct for all
        # versions of python within that major version.
        #
        # This is not perfect. It means that often we will overspecify the required
        # python version; and theoretically there could be markers that specify
        # python down to the patch version. However, accurately varianting on
        # python based on markers may be overly complicated, and may also
        # result in odd varianting cases.
        #
        # https://www.python.org/dev/peps/pep-0508/#environment-markers
        #
        if context is None:
            # since we had to use system pip, we have to assume system python version
            py_ver = '.'.join(map(str, sys.version_info[:2]))
        else:
            python_variant = context.get_resolved_package("python")
            py_ver = python_variant.version.trim(2)

        variant_reqs.append("python-%s" % py_ver)

        name = pip_to_rez_package_name(distribution)

        with make_package(name, packages_path, make_root=make_root) as pkg:
            pkg.version = pip_to_rez_version(distribution.version)
            if distribution.metadata.summary:
                pkg.description = distribution.metadata.summary

            pkg.variants = [variant_reqs]
            if requirements:
                pkg.requires = requirements

            commands = []
            commands.append("env.PYTHONPATH.append('{root}/python')")

            if tools:
                pkg.tools = tools
                commands.append("env.PATH.append('{root}/bin')")

            pkg.commands = '\n'.join(commands)

        installed_variants.extend(pkg.installed_variants or [])
        skipped_variants.extend(pkg.skipped_variants or [])

    # cleanup
    shutil.rmtree(tmpdir)

    return installed_variants, skipped_variants
Ejemplo n.º 31
0
def convert_old_commands(commands, annotate=True):
    """Converts old-style package commands into equivalent Rex code."""
    from rez.config import config
    from rez.utils.logging_ import print_debug

    def _repl(s):
        return s.replace('\\"', '"')

    def _encode(s):
        # this replaces all occurrances of '\"' with '"', *except* for those
        # occurrances of '\"' that are within double quotes, which themselves
        # are not escaped. In other words, the string:
        # 'hey "there \"you\" " \"dude\" '
        # ..will convert to:
        # 'hey "there \"you\" " "dude" '
        s_new = ''
        prev_i = 0

        for m in within_unescaped_quotes_regex.finditer(s):
            s_ = s[prev_i:m.start()]
            s_new += _repl(s_)

            s_new += s[m.start():m.end()]
            prev_i = m.end()

        s_ = s[prev_i:]
        s_new += _repl(s_)
        return repr(s_new)

    loc = []

    for cmd in commands:
        if annotate:
            txt = "OLD COMMAND: %s" % cmd
            line = "comment(%s)" % _encode(txt)
            loc.append(line)

        cmd = convert_old_command_expansions(cmd)
        toks = cmd.strip().split()

        if toks[0] == "export":
            var, value = cmd.split(' ', 1)[1].split('=', 1)
            for bookend in ('"', "'"):
                if value.startswith(bookend) and value.endswith(bookend):
                    value = value[1:-1]
                    break

            # As the only old-style commands were Linux/Bash based,
            # we assume using the default separator ":" is ok - we don't
            # need to use os.pathsep as we don't expected to see a
            # Windows path here.
            separator = config.env_var_separators.get(var, ":")

            # This is a special case.  We don't want to include "';'" in
            # our env var separators map as it's not really the correct
            # behaviour/something we want to promote.  It's included here for
            # backwards compatibility only, and to not propogate elsewhere.
            if var == "CMAKE_MODULE_PATH":
                value = value.replace("'%s'" % separator, separator)
                value = value.replace('"%s"' % separator, separator)
                value = value.replace(":", separator)

            parts = value.split(separator)
            parts = [x for x in parts if x]
            if len(parts) > 1:
                idx = None
                var1 = "$%s" % var
                var2 = "${%s}" % var
                if var1 in parts:
                    idx = parts.index(var1)
                elif var2 in parts:
                    idx = parts.index(var2)
                if idx in (0, len(parts) - 1):
                    func = "appendenv" if idx == 0 else "prependenv"
                    parts = parts[1:] if idx == 0 else parts[:-1]
                    val = separator.join(parts)
                    loc.append("%s('%s', %s)" % (func, var, _encode(val)))
                    continue

            loc.append("setenv('%s', %s)" % (var, _encode(value)))
        elif toks[0].startswith('#'):
            loc.append("comment(%s)" % _encode(' '.join(toks[1:])))
        elif toks[0] == "alias":
            match = re.search("alias (?P<key>.*?)=(?P<value>.*)", cmd)
            key = match.groupdict()['key'].strip()
            value = match.groupdict()['value'].strip()
            if (value.startswith('"') and value.endswith('"')) or \
                    (value.startswith("'") and value.endswith("'")):
                value = value[1:-1]
            loc.append("alias('%s', %s)" % (key, _encode(value)))
        else:
            # assume we can execute this as a straight command
            loc.append("command(%s)" % _encode(cmd))

    rex_code = '\n'.join(loc)
    if config.debug("old_commands"):
        br = '-' * 80
        msg = textwrap.dedent(
            """
            %s
            OLD COMMANDS:
            %s

            NEW COMMANDS:
            %s
            %s
            """) % (br, '\n'.join(commands), rex_code, br)
        print_debug(msg)
    return rex_code
Ejemplo n.º 32
0
def log(msg):
    if config.debug("bind_modules"):
        print_debug(msg)
Ejemplo n.º 33
0
Archivo: pip.py Proyecto: mottosso/rez
def pip_install_package(source_name, python_version=None,
                        release=False, no_deps=False,
                        prefix=None, auto_variants=True,
                        variants=None):
    """Install a pip-compatible python package as a rez package.
    Args:
        source_name (str): Name of package or archive/url containing the pip
            package source. This is the same as the arg you would pass to
            the 'pip install' command.
        python_version (str or `Version`): Python version to use to perform the
            install, and subsequently have the resulting rez package depend on.
        prefix (str, optional): Override install location to here,
            similar to `rez build --prefix`
        no_deps (bool, optional): The `pip --no-deps` argument
        auto_variants (bool, optional): Compute variants from the PyPI
            classifiers portion of setup()
        release (bool): If True, install as a released package; otherwise, it
            will be installed as a local package.
        prefix (str, optional): Override release path with this absolute path

    Returns:
        2-tuple:
            List of `Variant`: Installed variants;
            List of `Variant`: Skipped variants (already installed).
    """

    installed_variants = []
    skipped_variants = []

    if prefix is not None:
        config.release_packages_path = prefix

    # TODO: should check if packages_path is writable
    # before continuing with pip
    #
    packages_path = (config.release_packages_path if release
                     else config.local_packages_path)

    tmpdir = mkdtemp(suffix="-rez", prefix="pip-")
    stagingdir = os.path.join(tmpdir, "rez_staging")
    stagingsep = "".join([os.path.sep, "rez_staging", os.path.sep])

    destpath = os.path.join(stagingdir, "python")

    python_exe, context = find_python(python_version)
    if context and config.debug("package_release"):
        buf = StringIO()
        print >> buf, "\n\npackage download environment:"
        context.print_info(buf)
        _log(buf.getvalue())

    # Build pip commandline
    cmd = [
        python_exe, "-m", "pip", "install",
        "--target", destpath,

        # Only ever consider wheels, anything else is ancient
        "--use-pep517",

        # Handle case where the Python distribution used alongside
        # pip already has a package installed in its `site-packages/` dir.
        "--ignore-installed",
    ]

    if no_deps:
        # Delegate the installation of dependencies to the user
        # This is important, as each dependency may have different
        # requirements of its own, and variants to go with it.
        cmd.append("--no-deps")

    cmd.append(source_name)

    _cmd(context=context, command=cmd)

    # Collect resulting python packages using distlib
    distribution_path = DistributionPath([destpath])
    distributions = [d for d in distribution_path.get_distributions()]

    for distribution in distribution_path.get_distributions():

        requirements = []
        if distribution.metadata.run_requires:
            # Handle requirements. Currently handles
            # conditional environment based
            # requirements and normal requirements
            # TODO: Handle optional requirements?
            for requirement in distribution.metadata.run_requires:
                if "environment" in requirement:
                    if interpret(requirement["environment"]):
                        requirements.extend(_get_dependencies(
                            requirement, distributions))
                elif "extra" in requirement:
                    # Currently ignoring optional requirements
                    pass
                else:
                    requirements.extend(_get_dependencies(
                        requirement, distributions))

        tools = []
        src_dst_lut = {}
        files = distribution.list_installed_files()

        for installed_file in files:
            source_file = os.path.join(destpath, installed_file[0])
            source_file = os.path.normpath(source_file)

            if os.path.exists(source_file):
                destination_file = source_file.split(stagingsep)[1]
                exe = False

                starts_with_bin = destination_file.startswith(
                    "%s%s" % ("bin", os.path.sep)
                )

                if is_exe(source_file) and starts_with_bin:
                    _, _file = os.path.split(destination_file)
                    tools.append(_file)
                    exe = True

                data = [destination_file, exe]
                src_dst_lut[source_file] = data
            else:
                _log("Source file does not exist: " + source_file + "!")

        def make_root(variant, path):
            """Using distlib to iterate over all installed files of the current
            distribution to copy files to the target directory of the rez
            package variant

            """

            for source_file, data in src_dst_lut.items():
                destination_file, exe = data
                destination_file = os.path.join(path, destination_file)
                destination_file = os.path.normpath(destination_file)

                if not os.path.exists(os.path.dirname(destination_file)):
                    os.makedirs(os.path.dirname(destination_file))

                shutil.copyfile(source_file, destination_file)
                if exe:
                    shutil.copystat(source_file, destination_file)

        name, _ = parse_name_and_version(distribution.name_and_version)
        name = distribution.name[0:len(name)].replace("-", "_")

        # determine variant requirements
        variants_ = variants or []

        if (not variants_) and auto_variants:
            variants_.extend(wheel_to_variants(distribution))

            if variants_:
                print_info("'%s' - Automatically detected variants: %s" % (
                    name, ", ".join(variants_))
                )

        with make_package(name, packages_path, make_root=make_root) as pkg:
            pkg.version = distribution.version
            if distribution.metadata.summary:
                pkg.description = distribution.metadata.summary

            if variants_:
                pkg.variants = [variants_]

            if requirements:
                pkg.requires = requirements

            commands = []
            commands.append("env.PYTHONPATH.append('{root}/python')")

            if tools:
                pkg.tools = tools
                commands.append("env.PATH.append('{root}/bin')")

            pkg.commands = '\n'.join(commands)

        installed_variants.extend(pkg.installed_variants or [])
        skipped_variants.extend(pkg.skipped_variants or [])

    # cleanup
    shutil.rmtree(tmpdir)

    return installed_variants, skipped_variants
Ejemplo n.º 34
0
Archivo: pip.py Proyecto: mottosso/rez
def _cmd(context, command):
    cmd_str = ' '.join(quote(x) for x in command)
    _log("running: %s" % cmd_str)

    if context is None:
        p = popen(command)
    else:
        p = context.execute_shell(command=command, block=False)

    p.wait()

    if p.returncode:
        raise BuildError("Failed to download source with pip: %s" % cmd_str)


_verbose = config.debug("package_release")


def _log(msg):
    if _verbose:
        print_debug(msg)


# Copyright 2013-2016 Allan Johns.
#
# This library is free software: you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,