def cache_apps(sg_connection, cfg_descriptor, bundle_cache_root): """ Iterates over all environments within the given configuration descriptor and caches all items into the bundle cache root. :param sg_connection: Shotgun connection :param cfg_descriptor: Config descriptor :param bundle_cache_root: Root where to cache payload """ # introspect the config and cache everything logger.info("Introspecting environments...") env_path = os.path.join(cfg_descriptor.get_path(), "env") # find all environment files env_filenames = [] for filename in os.listdir(env_path): if filename.endswith(".yml"): # matching the env filter (or no filter set) logger.info("> found %s" % filename) env_filenames.append(os.path.join(env_path, filename)) # traverse and cache for env_path in env_filenames: logger.info("Processing %s..." % env_path) env = environment.Environment(env_path) for eng in env.get_engines(): # resolve descriptor and clone cache into bundle cache _cache_descriptor( sg_connection, Descriptor.ENGINE, env.get_engine_descriptor_dict(eng), bundle_cache_root ) for app in env.get_apps(eng): # resolve descriptor and clone cache into bundle cache _cache_descriptor( sg_connection, Descriptor.APP, env.get_app_descriptor_dict(eng, app), bundle_cache_root ) for framework in env.get_frameworks(): _cache_descriptor( sg_connection, Descriptor.FRAMEWORK, env.get_framework_descriptor_dict(framework), bundle_cache_root ) logger.info("Total size of bundle cache: %d KiB" % (filesystem.compute_folder_size(bundle_cache_root) / 1024))
def _cache_apps(sg_connection, cfg_descriptor, bundle_cache_root): """ Iterates over all environments within the given configuration descriptor and caches all items into the bundle cache root. :param sg_connection: Shotgun connection :param cfg_descriptor: Config descriptor :param bundle_cache_root: Root where to cache payload """ # introspect the config and cache everything logger.info("Introspecting environments...") env_path = os.path.join(cfg_descriptor.get_path(), "env") # find all environment files env_filenames = [] for filename in os.listdir(env_path): if filename.endswith(".yml"): # matching the env filter (or no filter set) logger.info("> found %s" % filename) env_filenames.append(os.path.join(env_path, filename)) # traverse and cache for env_path in env_filenames: logger.info("Processing %s..." % env_path) env = environment.Environment(env_path) for eng in env.get_engines(): # resolve descriptor and clone cache into bundle cache _cache_descriptor( sg_connection, Descriptor.ENGINE, env.get_engine_descriptor_dict(eng), bundle_cache_root ) for app in env.get_apps(eng): # resolve descriptor and clone cache into bundle cache _cache_descriptor( sg_connection, Descriptor.APP, env.get_app_descriptor_dict(eng, app), bundle_cache_root ) for framework in env.get_frameworks(): _cache_descriptor( sg_connection, Descriptor.FRAMEWORK, env.get_framework_descriptor_dict(framework), bundle_cache_root ) logger.info("Total size of bundle cache: %d KiB" % (filesystem.compute_folder_size(bundle_cache_root) / 1024))
def _cache_descriptor(sg, desc_type, desc_dict, target_path): """ Cache the given descriptor into a new bundle cache. :param sg: Shotgun API instance :param desc_type: Descriptor.ENGINE | Descriptor.APP | Descriptor.FRAMEWORK :param desc_dict: descriptor dict or uri :param target_path: bundle cache root to cache into """ desc = create_descriptor(sg, desc_type, desc_dict, fallback_roots=[target_path]) desc.ensure_local() desc_size_kb = filesystem.compute_folder_size(desc.get_path()) / 1024 logger.info("Caching %s into plugin bundle cache (size %d KiB)" % (desc, desc_size_kb)) if not desc._io_descriptor.is_immutable(): logger.warning("Descriptor %r may not work for other users using the plugin!" % desc) desc.clone_cache(target_path)
def cache_apps(sg_connection, cfg_descriptor, bundle_cache_root, should_skip_caching_callable=None): """ Iterates over all environments within the given configuration descriptor and caches all items into the bundle cache root. :param sg_connection: Shotgun connection :param cfg_descriptor: Config descriptor :param bundle_cache_root: Root where to cache payload :param should_skip_caching_callable: Optional callable that will check the descriptor and return `True` or `False` to indicate whether it should be cached. Callable must accept a single parameter, the descriptor as a `dict`, and return a `bool`. If `None`, the default :func:`_should_skip_caching()` will be used """ # introspect the config and cache everything logger.info("Introspecting environments...") env_path = os.path.join(cfg_descriptor.get_path(), "env") # set the function to call when checking whether to cache the bundle if should_skip_caching_callable: _skip_caching = should_skip_caching_callable else: _skip_caching = _should_skip_caching # find all environment files env_filenames = [] for filename in os.listdir(env_path): if filename.endswith(".yml"): # matching the env filter (or no filter set) logger.info("> found %s" % filename) env_filenames.append(os.path.join(env_path, filename)) processed = set() for desc, bundle_type in _iterate_environment(env_filenames): # This will avoid us from attempting to process # the same descriptor twice. if repr(desc) in processed: continue # Mark it as processed now, so we don't have to evaluate skip caching # multiple times processed.add(repr(desc)) # We should not attempt to cache descriptors that are path-based. Not only don't they # need to be cached, but they might be using special tokens like CONFIG_FOLDER # that can't be understood outside a pipeline configuration. We also skip caching # app_store descriptors in sparse configs since SG Desktop will take care of downloading # these automatically from the app store at runtime. if desc["type"] in ["dev", "path"]: logger.warning("'%s' will not be cached inside the configuration.", desc) continue if desc["type"] == "manual": logger.warning( "'%s' will not be cached because they are deprecated. " "We recommend you convert them to a git-based or path/dev descriptors.", desc, ) continue if _skip_caching(desc): continue _cache_descriptor(sg_connection, bundle_type, desc, bundle_cache_root) logger.info("Total size of bundle cache: %d KiB" % (filesystem.compute_folder_size(bundle_cache_root) // 1024))
def cache_apps(sg_connection, cfg_descriptor, bundle_cache_root, should_skip_caching_callable=None): """ Iterates over all environments within the given configuration descriptor and caches all items into the bundle cache root. :param sg_connection: Shotgun connection :param cfg_descriptor: Config descriptor :param bundle_cache_root: Root where to cache payload :param should_skip_caching_callable: Optional callable that will check the descriptor and return `True` or `False` to indicate whether it should be cached. Callable must accept a single parameter, the descriptor as a `dict`, and return a `bool`. If `None`, the default :func:`_should_skip_caching()` will be used """ # introspect the config and cache everything logger.info("Introspecting environments...") env_path = os.path.join(cfg_descriptor.get_path(), "env") # set the function to call when checking whether to cache the bundle if should_skip_caching_callable: _skip_caching = should_skip_caching_callable else: _skip_caching = _should_skip_caching # find all environment files env_filenames = [] for filename in os.listdir(env_path): if filename.endswith(".yml"): # matching the env filter (or no filter set) logger.info("> found %s" % filename) env_filenames.append(os.path.join(env_path, filename)) # traverse and cache for env_path in env_filenames: logger.info("Processing %s..." % env_path) env = environment.Environment(env_path) for eng in env.get_engines(): desc = env.get_engine_descriptor_dict(eng) if not _skip_caching(desc): # resolve descriptor and clone cache into bundle cache _cache_descriptor(sg_connection, Descriptor.ENGINE, desc, bundle_cache_root) for app in env.get_apps(eng): desc = env.get_app_descriptor_dict(eng, app) if _skip_caching(desc): continue # resolve descriptor and clone cache into bundle cache _cache_descriptor(sg_connection, Descriptor.APP, desc, bundle_cache_root) for framework in env.get_frameworks(): desc = env.get_framework_descriptor_dict(framework) if _skip_caching(desc): continue _cache_descriptor(sg_connection, Descriptor.FRAMEWORK, desc, bundle_cache_root) logger.info("Total size of bundle cache: %d KiB" % (filesystem.compute_folder_size(bundle_cache_root) / 1024))