def bake_config(sg_connection, config_uri, target_path, do_zip, skip_bundles_types):
"""
Bake a Toolkit Pipeline configuration.
This will ensure a local copy of the configuration, copy it over into target
path and then establish a bundle cache containing a reflection of all items
required by the config.
:param sg_connection: Shotgun connection
:param config_uri: A TK config descriptor uri.
:param target_path: Path to build
:param do_zip: Optionally zip up config once it's baked.
:param skip_bundles: Bundle types to skip.
"""
logger.info("Your Toolkit config '%s' will be processed." % config_uri)
logger.info("Baking into '%s'" % (target_path))
should_skip_functor = functools.partial(_should_skip, skip_bundles_types)
config_descriptor = _process_configuration(sg_connection, config_uri)
# Control the output path by adding a folder based on the
# configuration descriptor and version.
target_path = os.path.join(
target_path,
"%s-%s" % (config_descriptor.system_name, config_descriptor.version,),
)
# Check that target path doesn't exist
if os.path.exists(target_path):
logger.info("The folder '%s' already exists on disk. Removing it" % target_path)
wipe_folder(target_path)
# Create target path
filesystem.ensure_folder_exists(target_path)
# Copy the config data
logger.info("Copying config data across...")
filesystem.copy_folder(config_descriptor.get_path(), target_path)
# Create bundle cache and cache all apps, engines and frameworks
logger.info("Creating bundle cache folder...")
bundle_cache_root = os.path.join(target_path, BUNDLE_CACHE_ROOT_FOLDER_NAME)
filesystem.ensure_folder_exists(bundle_cache_root)
logger.info("Downloading and caching config...")
config_descriptor.clone_cache(bundle_cache_root)
# If sparse_caching is True, we use our own descriptor filter which skips
# app_store descriptors to keep our bundle cache small and lets Toolkit
# download the bundles from the app store at runtime.
cache_apps(sg_connection, config_descriptor, bundle_cache_root, should_skip_functor)
# Now analyze what core the config needs and cache it if needed.
core_descriptor = config_descriptor.associated_core_descriptor
if core_descriptor:
logger.info("Config defines a specific core in config/core/core_api.yml.")
logger.info("This will be used when the config is executing.")
if not should_skip_functor(core_descriptor):
logger.info("Ensuring this core (%s) is cached..." % core_descriptor)
associated_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE,
core_descriptor,
bundle_cache_root_override=bundle_cache_root,
)
associated_core_desc.ensure_local()
# Remove unwanted files, e.g. git history.
cleanup_bundle_cache(bundle_cache_root)
logger.info("")
logger.info("Bake complete")
logger.info("")
logger.info(
"- Your configuration %r is ready in '%s'" % (config_descriptor, target_path)
)
logger.info("- All dependencies have been baked out into the bundle_cache folder")
logger.info("")
logger.info("")
logger.info("")
if do_zip:
logger.info("Zip archiving the baked configuration...")
archive_path = shutil.make_archive(target_path, "zip", root_dir=target_path)
logger.info("Zip archive available here: %s" % archive_path)
def build_plugin(sg_connection,
source_path,
target_path,
buildable,
bootstrap_core_uri=None):
"""
Perform a build of a plugin.
This will introspect the info.yml in the source path,
copy over everything in the source path into target path
and then establish a bundle cache containing a reflection
of all items required by the config.
:param sg_connection: Shotgun connection
:param source_path: Path to plugin.
:param target_path: Path to build
:param buildable: True if the resulting plugin build should be buildable
:param bootstrap_core_uri: Custom bootstrap core uri. If None,
the latest core from the app store will be used.
"""
logger.info("Your toolkit plugin in '%s' will be processed." % source_path)
logger.info("The build will generated into '%s'" % target_path)
# check for existence
if not os.path.exists(source_path):
raise TankError("Source path '%s' cannot be found on disk!" %
source_path)
# check that target path doesn't exist
if os.path.exists(target_path):
logger.info(
"The folder '%s' already exists on disk. Moving it to backup location"
% target_path)
filesystem.backup_folder(target_path)
shutil.rmtree(target_path)
# try to create target path
filesystem.ensure_folder_exists(target_path)
# check manifest
manifest_data = _validate_manifest(source_path)
# copy all plugin data across
# skip info.yml, this is baked into the manifest python code
logger.info("Copying plugin data across...")
skip_list = [] if buildable else [".git", "info.yml"]
filesystem.copy_folder(source_path, target_path, skip_list=skip_list)
# create bundle cache
logger.info("Creating bundle cache folder...")
bundle_cache_root = os.path.join(target_path,
BUNDLE_CACHE_ROOT_FOLDER_NAME)
filesystem.ensure_folder_exists(bundle_cache_root)
# resolve config descriptor
# the config_uri_str returned by the method contains the fully resolved
# uri to use at runtime - in the case of baked descriptors, the config_uri_str
# contains a manual descriptor uri.
(cfg_descriptor,
config_uri_str) = _process_configuration(sg_connection, source_path,
target_path, bundle_cache_root,
manifest_data, buildable)
# cache config in bundle cache
logger.info("Downloading and caching config...")
# copy the config payload across to the plugin bundle cache
cfg_descriptor.clone_cache(bundle_cache_root)
# cache all apps, engines and frameworks
_cache_apps(sg_connection, cfg_descriptor, bundle_cache_root)
# get latest core - cache it directly into the plugin root folder
if bootstrap_core_uri:
logger.info("Caching custom core for boostrap (%s)" %
bootstrap_core_uri)
bootstrap_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE,
bootstrap_core_uri,
bundle_cache_root_override=bundle_cache_root)
else:
# by default, use latest core for bootstrap
logger.info(
"Caching latest official core to use when bootstrapping plugin.")
logger.info(
"(To use a specific config instead, specify a --bootstrap-core-uri flag.)"
)
bootstrap_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE, {
"type": "app_store",
"name": "tk-core"
},
resolve_latest=True,
bundle_cache_root_override=bundle_cache_root)
# cache it
bootstrap_core_desc.ensure_local()
# make a python folder where we put our manifest
logger.info("Creating configuration manifest...")
# bake out the manifest into python files.
_bake_manifest(manifest_data, config_uri_str, bootstrap_core_desc,
target_path)
# now analyze what core the config needs
if cfg_descriptor.associated_core_descriptor:
logger.info(
"Config is specifying a custom core in config/core/core_api.yml.")
logger.info("This will be used when the config is executing.")
logger.info("Ensuring this core (%s) is cached..." %
cfg_descriptor.associated_core_descriptor)
associated_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE,
cfg_descriptor.associated_core_descriptor,
bundle_cache_root_override=bundle_cache_root)
associated_core_desc.ensure_local()
logger.info("")
logger.info("Build complete!")
logger.info("")
logger.info("- Your plugin is ready in '%s'" % target_path)
logger.info("- Plugin uses config %r" % cfg_descriptor)
logger.info("- Bootstrap core is %r" % bootstrap_core_desc)
logger.info(
"- All dependencies have been baked out into the bundle_cache folder")
if buildable:
logger.info(
"- The plugin can be used as a source for building further plugins."
)
logger.info("")
logger.info("")
logger.info("")
def build_plugin(sg_connection,
source_path,
target_path,
bootstrap_core_uri=None,
do_bake=False,
use_system_core=False):
"""
Perform a build of a plugin.
This will introspect the info.yml in the source path,
copy over everything in the source path into target path
and then establish a bundle cache containing a reflection
of all items required by the config.
:param sg_connection: Shotgun connection
:param source_path: Path to plugin.
:param target_path: Path to build
:param bootstrap_core_uri: Custom bootstrap core uri. If None,
the latest core from the app store will be used.
:param bool do_bake: If True, bake the plugin prior to building it.
:param bool use_system_core: If True, use a globally installed tk-core instead
of the one specified in the configuration.
"""
logger.info("Your toolkit plugin in '%s' will be processed." % source_path)
logger.info("The build will %s into '%s'" %
(["generated", "baked"][do_bake], target_path))
# check for existence
if not os.path.exists(source_path):
raise TankError("Source path '%s' cannot be found on disk!" %
source_path)
# check manifest
manifest_data = _validate_manifest(source_path)
if do_bake:
baked_descriptor = _bake_configuration(
sg_connection,
manifest_data,
)
# When baking we control the output path by adding a folder based on the
# configuration descriptor and version.
target_path = os.path.join(
target_path,
"%s-%s" % (baked_descriptor["name"], baked_descriptor["version"]))
# check that target path doesn't exist
if os.path.exists(target_path):
logger.info("The folder '%s' already exists on disk. Removing it" %
target_path)
wipe_folder(target_path)
# try to create target path
filesystem.ensure_folder_exists(target_path)
# copy all plugin data across
# skip info.yml, this is baked into the manifest python code
logger.info("Copying plugin data across...")
filesystem.copy_folder(source_path, target_path)
# create bundle cache
logger.info("Creating bundle cache folder...")
bundle_cache_root = os.path.join(target_path,
BUNDLE_CACHE_ROOT_FOLDER_NAME)
filesystem.ensure_folder_exists(bundle_cache_root)
# resolve config descriptor
# the config_uri_str returned by the method contains the fully resolved
# uri to use at runtime - in the case of baked descriptors, the config_uri_str
# contains a manual descriptor uri and install_path is set with the baked
# folder.
(cfg_descriptor, config_uri_str,
install_path) = _process_configuration(sg_connection, source_path,
target_path, bundle_cache_root,
manifest_data, use_system_core)
# cache config in bundle cache
logger.info("Downloading and caching config...")
# copy the config payload across to the plugin bundle cache
cfg_descriptor.clone_cache(bundle_cache_root)
# cache all apps, engines and frameworks
cache_apps(sg_connection, cfg_descriptor, bundle_cache_root)
if use_system_core:
logger.info(
"An external core will be used for this plugin, not caching it")
bootstrap_core_desc = None
else:
# get core - cache it directly into the plugin root folder
if bootstrap_core_uri:
logger.info("Caching custom core for boostrap (%s)" %
bootstrap_core_uri)
bootstrap_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE,
bootstrap_core_uri,
resolve_latest=is_descriptor_version_missing(
bootstrap_core_uri),
bundle_cache_root_override=bundle_cache_root)
# cache it
bootstrap_core_desc.ensure_local()
elif not cfg_descriptor.associated_core_descriptor:
# by default, use latest core for bootstrap
logger.info(
"Caching latest official core to use when bootstrapping plugin."
)
logger.info(
"(To use a specific config instead, specify a --bootstrap-core-uri flag.)"
)
bootstrap_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE, {
"type": "app_store",
"name": "tk-core"
},
resolve_latest=True,
bundle_cache_root_override=bundle_cache_root)
# cache it
bootstrap_core_desc.ensure_local()
else:
# The bootstrap core will be derived from the associated core desc below.
bootstrap_core_desc = None
# now analyze what core the config needs
if not use_system_core and cfg_descriptor.associated_core_descriptor:
logger.info(
"Config is specifying a custom core in config/core/core_api.yml.")
logger.info("This will be used when the config is executing.")
logger.info("Ensuring this core (%s) is cached..." %
cfg_descriptor.associated_core_descriptor)
associated_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE,
cfg_descriptor.associated_core_descriptor,
bundle_cache_root_override=bundle_cache_root)
associated_core_desc.ensure_local()
if bootstrap_core_desc is None:
# Use the same version as the one specified by the config.
if install_path:
# Install path is set only if the config was baked. We re-use the
# install path as an optimisation to avoid core swapping when the
# config is bootstrapped.
logger.info(
"Bootstrapping will use installed %s required by the config"
% associated_core_desc)
# If the core was installed we directly use it.
bootstrap_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE, {
"type": "path",
"name": "tk-core",
"path": os.path.join(install_path, "install", "core"),
"version": associated_core_desc.version,
},
resolve_latest=False,
bundle_cache_root_override=bundle_cache_root)
else:
logger.info(
"Bootstrapping will use core %s required by the config" %
associated_core_desc)
bootstrap_core_desc = associated_core_desc
# make a python folder where we put our manifest
logger.info("Creating configuration manifest...")
# bake out the manifest into python files.
_bake_manifest(manifest_data, config_uri_str, bootstrap_core_desc,
target_path)
cleanup_bundle_cache(bundle_cache_root)
logger.info("")
logger.info("Build complete!")
logger.info("")
logger.info("- Your plugin is ready in '%s'" % target_path)
logger.info("- Plugin uses config %r" % cfg_descriptor)
if bootstrap_core_desc:
logger.info("- Bootstrap core is %r" % bootstrap_core_desc)
else:
logger.info("- Plugin will need an external installed core.")
logger.info(
"- All dependencies have been baked out into the bundle_cache folder")
logger.info("")
logger.info("")
logger.info("")
def build_plugin(sg_connection, source_path, target_path, buildable, bootstrap_core_uri=None):
"""
Perform a build of a plugin.
This will introspect the info.yml in the source path,
copy over everything in the source path into target path
and then establish a bundle cache containing a reflection
of all items required by the config.
:param sg_connection: Shotgun connection
:param source_path: Path to plugin.
:param target_path: Path to build
:param buildable: True if the resulting plugin build should be buildable
:param bootstrap_core_uri: Custom bootstrap core uri. If None,
the latest core from the app store will be used.
"""
logger.info("Your toolkit plugin in '%s' will be processed." % source_path)
logger.info("The build will generated into '%s'" % target_path)
# check for existence
if not os.path.exists(source_path):
raise TankError("Source path '%s' cannot be found on disk!" % source_path)
# check that target path doesn't exist
if os.path.exists(target_path):
logger.info("The folder '%s' already exists on disk. Moving it to backup location" % target_path)
filesystem.backup_folder(target_path)
shutil.rmtree(target_path)
# try to create target path
filesystem.ensure_folder_exists(target_path)
# check manifest
manifest_data = _validate_manifest(source_path)
# copy all plugin data across
# skip info.yml, this is baked into the manifest python code
logger.info("Copying plugin data across...")
skip_list = [] if buildable else [".git", "info.yml"]
filesystem.copy_folder(source_path, target_path, skip_list=skip_list)
# create bundle cache
logger.info("Creating bundle cache folder...")
bundle_cache_root = os.path.join(target_path, BUNDLE_CACHE_ROOT_FOLDER_NAME)
filesystem.ensure_folder_exists(bundle_cache_root)
# resolve config descriptor
# the config_uri_str returned by the method contains the fully resolved
# uri to use at runtime - in the case of baked descriptors, the config_uri_str
# contains a manual descriptor uri.
(cfg_descriptor, config_uri_str) = _process_configuration(
sg_connection,
source_path,
target_path,
bundle_cache_root,
manifest_data,
buildable
)
# cache config in bundle cache
logger.info("Downloading and caching config...")
# copy the config payload across to the plugin bundle cache
cfg_descriptor.clone_cache(bundle_cache_root)
# cache all apps, engines and frameworks
_cache_apps(sg_connection, cfg_descriptor, bundle_cache_root)
# get latest core - cache it directly into the plugin root folder
if bootstrap_core_uri:
logger.info("Caching custom core for boostrap (%s)" % bootstrap_core_uri)
bootstrap_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE,
bootstrap_core_uri,
bundle_cache_root_override=bundle_cache_root
)
else:
# by default, use latest core for bootstrap
logger.info("Caching latest official core to use when bootstrapping plugin.")
logger.info("(To use a specific config instead, specify a --bootstrap-core-uri flag.)")
bootstrap_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE,
{"type": "app_store", "name": "tk-core"},
resolve_latest=True,
bundle_cache_root_override=bundle_cache_root
)
# cache it
bootstrap_core_desc.ensure_local()
# make a python folder where we put our manifest
logger.info("Creating configuration manifest...")
# bake out the manifest into python files.
_bake_manifest(
manifest_data,
config_uri_str,
bootstrap_core_desc,
target_path
)
# now analyze what core the config needs
if cfg_descriptor.associated_core_descriptor:
logger.info("Config is specifying a custom core in config/core/core_api.yml.")
logger.info("This will be used when the config is executing.")
logger.info("Ensuring this core (%s) is cached..." % cfg_descriptor.associated_core_descriptor)
associated_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE,
cfg_descriptor.associated_core_descriptor,
bundle_cache_root_override=bundle_cache_root
)
associated_core_desc.ensure_local()
logger.info("")
logger.info("Build complete!")
logger.info("")
logger.info("- Your plugin is ready in '%s'" % target_path)
logger.info("- Plugin uses config %r" % cfg_descriptor)
logger.info("- Bootstrap core is %r" % bootstrap_core_desc)
logger.info("- All dependencies have been baked out into the bundle_cache folder")
if buildable:
logger.info("- The plugin can be used as a source for building further plugins.")
logger.info("")
logger.info("")
logger.info("")
def build_plugin(sg_connection, source_path, target_path, bootstrap_core_uri=None, do_bake=False, use_system_core=False):
"""
Perform a build of a plugin.
This will introspect the info.yml in the source path,
copy over everything in the source path into target path
and then establish a bundle cache containing a reflection
of all items required by the config.
:param sg_connection: Shotgun connection
:param source_path: Path to plugin.
:param target_path: Path to build
:param bootstrap_core_uri: Custom bootstrap core uri. If None,
the latest core from the app store will be used.
:param bool do_bake: If True, bake the plugin prior to building it.
:param bool use_system_core: If True, use a globally installed tk-core instead
of the one specified in the configuration.
"""
logger.info("Your toolkit plugin in '%s' will be processed." % source_path)
logger.info("The build will %s into '%s'" % (["generated", "baked"][do_bake], target_path))
# check for existence
if not os.path.exists(source_path):
raise TankError("Source path '%s' cannot be found on disk!" % source_path)
# check manifest
manifest_data = _validate_manifest(source_path)
if do_bake:
baked_descriptor = _bake_configuration(
sg_connection,
manifest_data,
)
# When baking we control the output path by adding a folder based on the
# configuration descriptor and version.
target_path = os.path.join(target_path, "%s-%s" % (
baked_descriptor["name"],
baked_descriptor["version"]
))
# check that target path doesn't exist
if os.path.exists(target_path):
logger.info("The folder '%s' already exists on disk. Removing it" % target_path)
wipe_folder(target_path)
# try to create target path
filesystem.ensure_folder_exists(target_path)
# copy all plugin data across
# skip info.yml, this is baked into the manifest python code
logger.info("Copying plugin data across...")
filesystem.copy_folder(source_path, target_path)
# create bundle cache
logger.info("Creating bundle cache folder...")
bundle_cache_root = os.path.join(target_path, BUNDLE_CACHE_ROOT_FOLDER_NAME)
filesystem.ensure_folder_exists(bundle_cache_root)
# resolve config descriptor
# the config_uri_str returned by the method contains the fully resolved
# uri to use at runtime - in the case of baked descriptors, the config_uri_str
# contains a manual descriptor uri and install_path is set with the baked
# folder.
(cfg_descriptor, config_uri_str, install_path) = _process_configuration(
sg_connection,
source_path,
target_path,
bundle_cache_root,
manifest_data,
use_system_core
)
# cache config in bundle cache
logger.info("Downloading and caching config...")
# copy the config payload across to the plugin bundle cache
cfg_descriptor.clone_cache(bundle_cache_root)
# cache all apps, engines and frameworks
cache_apps(sg_connection, cfg_descriptor, bundle_cache_root)
if use_system_core:
logger.info("An external core will be used for this plugin, not caching it")
bootstrap_core_desc = None
else:
# get core - cache it directly into the plugin root folder
if bootstrap_core_uri:
logger.info("Caching custom core for boostrap (%s)" % bootstrap_core_uri)
bootstrap_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE,
bootstrap_core_uri,
resolve_latest=is_descriptor_version_missing(bootstrap_core_uri),
bundle_cache_root_override=bundle_cache_root
)
# cache it
bootstrap_core_desc.ensure_local()
elif not cfg_descriptor.associated_core_descriptor:
# by default, use latest core for bootstrap
logger.info("Caching latest official core to use when bootstrapping plugin.")
logger.info("(To use a specific config instead, specify a --bootstrap-core-uri flag.)")
bootstrap_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE,
{"type": "app_store", "name": "tk-core"},
resolve_latest=True,
bundle_cache_root_override=bundle_cache_root
)
# cache it
bootstrap_core_desc.ensure_local()
else:
# The bootstrap core will be derived from the associated core desc below.
bootstrap_core_desc = None
# now analyze what core the config needs
if not use_system_core and cfg_descriptor.associated_core_descriptor:
logger.info("Config is specifying a custom core in config/core/core_api.yml.")
logger.info("This will be used when the config is executing.")
logger.info("Ensuring this core (%s) is cached..." % cfg_descriptor.associated_core_descriptor)
associated_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE,
cfg_descriptor.associated_core_descriptor,
bundle_cache_root_override=bundle_cache_root
)
associated_core_desc.ensure_local()
if bootstrap_core_desc is None:
# Use the same version as the one specified by the config.
if install_path:
# Install path is set only if the config was baked. We re-use the
# install path as an optimisation to avoid core swapping when the
# config is bootstrapped.
logger.info(
"Bootstrapping will use installed %s required by the config" %
associated_core_desc
)
# If the core was installed we directly use it.
bootstrap_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE, {
"type": "path",
"name": "tk-core",
"path": os.path.join(install_path, "install", "core"),
"version": associated_core_desc.version,
},
resolve_latest=False,
bundle_cache_root_override=bundle_cache_root
)
else:
logger.info(
"Bootstrapping will use core %s required by the config" %
associated_core_desc
)
bootstrap_core_desc = associated_core_desc
# make a python folder where we put our manifest
logger.info("Creating configuration manifest...")
# bake out the manifest into python files.
_bake_manifest(
manifest_data,
config_uri_str,
bootstrap_core_desc,
target_path
)
cleanup_bundle_cache(bundle_cache_root)
logger.info("")
logger.info("Build complete!")
logger.info("")
logger.info("- Your plugin is ready in '%s'" % target_path)
logger.info("- Plugin uses config %r" % cfg_descriptor)
if bootstrap_core_desc:
logger.info("- Bootstrap core is %r" % bootstrap_core_desc)
else:
logger.info("- Plugin will need an external installed core.")
logger.info("- All dependencies have been baked out into the bundle_cache folder")
logger.info("")
logger.info("")
logger.info("")