コード例 #1
0
ファイル: caching.py プロジェクト: mikrosimage/tk-core
def cache_apps(sg_connection, cfg_descriptor, bundle_cache_root):
    """
    Iterates over all environments within the given configuration descriptor
    and caches all items into the bundle cache root.

    :param sg_connection: Shotgun connection
    :param cfg_descriptor: Config descriptor
    :param bundle_cache_root: Root where to cache payload
    """
    # introspect the config and cache everything
    logger.info("Introspecting environments...")
    env_path = os.path.join(cfg_descriptor.get_path(), "env")

    # find all environment files
    env_filenames = []
    for filename in os.listdir(env_path):
        if filename.endswith(".yml"):
            # matching the env filter (or no filter set)
            logger.info("> found %s" % filename)
            env_filenames.append(os.path.join(env_path, filename))

    # traverse and cache
    for env_path in env_filenames:
        logger.info("Processing %s..." % env_path)
        env = environment.Environment(env_path)

        for eng in env.get_engines():
            # resolve descriptor and clone cache into bundle cache
            _cache_descriptor(
                sg_connection,
                Descriptor.ENGINE,
                env.get_engine_descriptor_dict(eng),
                bundle_cache_root
            )

            for app in env.get_apps(eng):
                # resolve descriptor and clone cache into bundle cache
                _cache_descriptor(
                    sg_connection,
                    Descriptor.APP,
                    env.get_app_descriptor_dict(eng, app),
                    bundle_cache_root
                )

        for framework in env.get_frameworks():
            _cache_descriptor(
                sg_connection,
                Descriptor.FRAMEWORK,
                env.get_framework_descriptor_dict(framework),
                bundle_cache_root
            )

    logger.info("Total size of bundle cache: %d KiB" % (filesystem.compute_folder_size(bundle_cache_root) / 1024))
コード例 #2
0
def _cache_apps(sg_connection, cfg_descriptor, bundle_cache_root):
    """
    Iterates over all environments within the given configuration descriptor
    and caches all items into the bundle cache root.

    :param sg_connection: Shotgun connection
    :param cfg_descriptor: Config descriptor
    :param bundle_cache_root: Root where to cache payload
    """
    # introspect the config and cache everything
    logger.info("Introspecting environments...")
    env_path = os.path.join(cfg_descriptor.get_path(), "env")

    # find all environment files
    env_filenames = []
    for filename in os.listdir(env_path):
        if filename.endswith(".yml"):
            # matching the env filter (or no filter set)
            logger.info("> found %s" % filename)
            env_filenames.append(os.path.join(env_path, filename))

    # traverse and cache
    for env_path in env_filenames:
        logger.info("Processing %s..." % env_path)
        env = environment.Environment(env_path)

        for eng in env.get_engines():
            # resolve descriptor and clone cache into bundle cache
            _cache_descriptor(
                sg_connection,
                Descriptor.ENGINE,
                env.get_engine_descriptor_dict(eng),
                bundle_cache_root
            )

            for app in env.get_apps(eng):
                # resolve descriptor and clone cache into bundle cache
                _cache_descriptor(
                    sg_connection,
                    Descriptor.APP,
                    env.get_app_descriptor_dict(eng, app),
                    bundle_cache_root
                )

        for framework in env.get_frameworks():
            _cache_descriptor(
                sg_connection,
                Descriptor.FRAMEWORK,
                env.get_framework_descriptor_dict(framework),
                bundle_cache_root
            )

    logger.info("Total size of bundle cache: %d KiB" % (filesystem.compute_folder_size(bundle_cache_root) / 1024))
コード例 #3
0
ファイル: caching.py プロジェクト: mikrosimage/tk-core
def _cache_descriptor(sg, desc_type, desc_dict, target_path):
    """
    Cache the given descriptor into a new bundle cache.

    :param sg: Shotgun API instance
    :param desc_type: Descriptor.ENGINE | Descriptor.APP | Descriptor.FRAMEWORK
    :param desc_dict: descriptor dict or uri
    :param target_path: bundle cache root to cache into
    """
    desc = create_descriptor(sg, desc_type, desc_dict, fallback_roots=[target_path])
    desc.ensure_local()
    desc_size_kb = filesystem.compute_folder_size(desc.get_path()) / 1024
    logger.info("Caching %s into plugin bundle cache (size %d KiB)" % (desc, desc_size_kb))
    if not desc._io_descriptor.is_immutable():
        logger.warning("Descriptor %r may not work for other users using the plugin!" % desc)
    desc.clone_cache(target_path)
コード例 #4
0
def _cache_descriptor(sg, desc_type, desc_dict, target_path):
    """
    Cache the given descriptor into a new bundle cache.

    :param sg: Shotgun API instance
    :param desc_type: Descriptor.ENGINE | Descriptor.APP | Descriptor.FRAMEWORK
    :param desc_dict: descriptor dict or uri
    :param target_path: bundle cache root to cache into
    """
    desc = create_descriptor(sg, desc_type, desc_dict, fallback_roots=[target_path])
    desc.ensure_local()
    desc_size_kb = filesystem.compute_folder_size(desc.get_path()) / 1024
    logger.info("Caching %s into plugin bundle cache (size %d KiB)" % (desc, desc_size_kb))
    if not desc._io_descriptor.is_immutable():
        logger.warning("Descriptor %r may not work for other users using the plugin!" % desc)
    desc.clone_cache(target_path)
コード例 #5
0
ファイル: caching.py プロジェクト: chenqizhi1/tk-core
def cache_apps(sg_connection,
               cfg_descriptor,
               bundle_cache_root,
               should_skip_caching_callable=None):
    """
    Iterates over all environments within the given configuration descriptor
    and caches all items into the bundle cache root.

    :param sg_connection: Shotgun connection
    :param cfg_descriptor: Config descriptor
    :param bundle_cache_root: Root where to cache payload
    :param should_skip_caching_callable: Optional callable that will check the descriptor
        and return `True` or `False` to indicate whether it should be cached.
        Callable must accept a single parameter, the descriptor as a `dict`, and
        return a `bool`. If `None`, the default :func:`_should_skip_caching()` will be used
    """
    # introspect the config and cache everything
    logger.info("Introspecting environments...")
    env_path = os.path.join(cfg_descriptor.get_path(), "env")

    # set the function to call when checking whether to cache the bundle
    if should_skip_caching_callable:
        _skip_caching = should_skip_caching_callable
    else:
        _skip_caching = _should_skip_caching

    # find all environment files
    env_filenames = []
    for filename in os.listdir(env_path):
        if filename.endswith(".yml"):
            # matching the env filter (or no filter set)
            logger.info("> found %s" % filename)
            env_filenames.append(os.path.join(env_path, filename))

    processed = set()
    for desc, bundle_type in _iterate_environment(env_filenames):
        # This will avoid us from attempting to process
        # the same descriptor twice.
        if repr(desc) in processed:
            continue
        # Mark it as processed now, so we don't have to evaluate skip caching
        # multiple times
        processed.add(repr(desc))

        # We should not attempt to cache descriptors that are path-based. Not only don't they
        # need to be cached, but they might be using special tokens like CONFIG_FOLDER
        # that can't be understood outside a pipeline configuration. We also skip caching
        # app_store descriptors in sparse configs since SG Desktop will take care of downloading
        # these automatically from the app store at runtime.

        if desc["type"] in ["dev", "path"]:
            logger.warning("'%s' will not be cached inside the configuration.",
                           desc)
            continue

        if desc["type"] == "manual":
            logger.warning(
                "'%s' will not be cached because they are deprecated. "
                "We recommend you convert them to a git-based or path/dev descriptors.",
                desc,
            )
            continue

        if _skip_caching(desc):
            continue

        _cache_descriptor(sg_connection, bundle_type, desc, bundle_cache_root)

    logger.info("Total size of bundle cache: %d KiB" %
                (filesystem.compute_folder_size(bundle_cache_root) // 1024))
コード例 #6
0
def cache_apps(sg_connection,
               cfg_descriptor,
               bundle_cache_root,
               should_skip_caching_callable=None):
    """
    Iterates over all environments within the given configuration descriptor
    and caches all items into the bundle cache root.

    :param sg_connection: Shotgun connection
    :param cfg_descriptor: Config descriptor
    :param bundle_cache_root: Root where to cache payload
    :param should_skip_caching_callable: Optional callable that will check the descriptor
        and return `True` or `False` to indicate whether it should be cached.
        Callable must accept a single parameter, the descriptor as a `dict`, and
        return a `bool`. If `None`, the default :func:`_should_skip_caching()` will be used
    """
    # introspect the config and cache everything
    logger.info("Introspecting environments...")
    env_path = os.path.join(cfg_descriptor.get_path(), "env")

    # set the function to call when checking whether to cache the bundle
    if should_skip_caching_callable:
        _skip_caching = should_skip_caching_callable
    else:
        _skip_caching = _should_skip_caching

    # find all environment files
    env_filenames = []
    for filename in os.listdir(env_path):
        if filename.endswith(".yml"):
            # matching the env filter (or no filter set)
            logger.info("> found %s" % filename)
            env_filenames.append(os.path.join(env_path, filename))

    # traverse and cache
    for env_path in env_filenames:
        logger.info("Processing %s..." % env_path)
        env = environment.Environment(env_path)

        for eng in env.get_engines():
            desc = env.get_engine_descriptor_dict(eng)

            if not _skip_caching(desc):
                # resolve descriptor and clone cache into bundle cache
                _cache_descriptor(sg_connection, Descriptor.ENGINE, desc,
                                  bundle_cache_root)

            for app in env.get_apps(eng):
                desc = env.get_app_descriptor_dict(eng, app)
                if _skip_caching(desc):
                    continue
                # resolve descriptor and clone cache into bundle cache
                _cache_descriptor(sg_connection, Descriptor.APP, desc,
                                  bundle_cache_root)

        for framework in env.get_frameworks():
            desc = env.get_framework_descriptor_dict(framework)
            if _skip_caching(desc):
                continue
            _cache_descriptor(sg_connection, Descriptor.FRAMEWORK, desc,
                              bundle_cache_root)

    logger.info("Total size of bundle cache: %d KiB" %
                (filesystem.compute_folder_size(bundle_cache_root) / 1024))