def bake_config(sg_connection, config_uri, target_path, do_zip, skip_bundles_types):
"""
Bake a Toolkit Pipeline configuration.
This will ensure a local copy of the configuration, copy it over into target
path and then establish a bundle cache containing a reflection of all items
required by the config.
:param sg_connection: Shotgun connection
:param config_uri: A TK config descriptor uri.
:param target_path: Path to build
:param do_zip: Optionally zip up config once it's baked.
:param skip_bundles: Bundle types to skip.
"""
logger.info("Your Toolkit config '%s' will be processed." % config_uri)
logger.info("Baking into '%s'" % (target_path))
should_skip_functor = functools.partial(_should_skip, skip_bundles_types)
config_descriptor = _process_configuration(sg_connection, config_uri)
# Control the output path by adding a folder based on the
# configuration descriptor and version.
target_path = os.path.join(
target_path,
"%s-%s" % (config_descriptor.system_name, config_descriptor.version,),
)
# Check that target path doesn't exist
if os.path.exists(target_path):
logger.info("The folder '%s' already exists on disk. Removing it" % target_path)
wipe_folder(target_path)
# Create target path
filesystem.ensure_folder_exists(target_path)
# Copy the config data
logger.info("Copying config data across...")
filesystem.copy_folder(config_descriptor.get_path(), target_path)
# Create bundle cache and cache all apps, engines and frameworks
logger.info("Creating bundle cache folder...")
bundle_cache_root = os.path.join(target_path, BUNDLE_CACHE_ROOT_FOLDER_NAME)
filesystem.ensure_folder_exists(bundle_cache_root)
logger.info("Downloading and caching config...")
config_descriptor.clone_cache(bundle_cache_root)
# If sparse_caching is True, we use our own descriptor filter which skips
# app_store descriptors to keep our bundle cache small and lets Toolkit
# download the bundles from the app store at runtime.
cache_apps(sg_connection, config_descriptor, bundle_cache_root, should_skip_functor)
# Now analyze what core the config needs and cache it if needed.
core_descriptor = config_descriptor.associated_core_descriptor
if core_descriptor:
logger.info("Config defines a specific core in config/core/core_api.yml.")
logger.info("This will be used when the config is executing.")
if not should_skip_functor(core_descriptor):
logger.info("Ensuring this core (%s) is cached..." % core_descriptor)
associated_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE,
core_descriptor,
bundle_cache_root_override=bundle_cache_root,
)
associated_core_desc.ensure_local()
# Remove unwanted files, e.g. git history.
cleanup_bundle_cache(bundle_cache_root)
logger.info("")
logger.info("Bake complete")
logger.info("")
logger.info(
"- Your configuration %r is ready in '%s'" % (config_descriptor, target_path)
)
logger.info("- All dependencies have been baked out into the bundle_cache folder")
logger.info("")
logger.info("")
logger.info("")
if do_zip:
logger.info("Zip archiving the baked configuration...")
archive_path = shutil.make_archive(target_path, "zip", root_dir=target_path)
logger.info("Zip archive available here: %s" % archive_path)
def build_plugin(sg_connection,
source_path,
target_path,
bootstrap_core_uri=None,
do_bake=False,
use_system_core=False):
"""
Perform a build of a plugin.
This will introspect the info.yml in the source path,
copy over everything in the source path into target path
and then establish a bundle cache containing a reflection
of all items required by the config.
:param sg_connection: Shotgun connection
:param source_path: Path to plugin.
:param target_path: Path to build
:param bootstrap_core_uri: Custom bootstrap core uri. If None,
the latest core from the app store will be used.
:param bool do_bake: If True, bake the plugin prior to building it.
:param bool use_system_core: If True, use a globally installed tk-core instead
of the one specified in the configuration.
"""
logger.info("Your toolkit plugin in '%s' will be processed." % source_path)
logger.info("The build will %s into '%s'" %
(["generated", "baked"][do_bake], target_path))
# check for existence
if not os.path.exists(source_path):
raise TankError("Source path '%s' cannot be found on disk!" %
source_path)
# check manifest
manifest_data = _validate_manifest(source_path)
if do_bake:
baked_descriptor = _bake_configuration(
sg_connection,
manifest_data,
)
# When baking we control the output path by adding a folder based on the
# configuration descriptor and version.
target_path = os.path.join(
target_path,
"%s-%s" % (baked_descriptor["name"], baked_descriptor["version"]))
# check that target path doesn't exist
if os.path.exists(target_path):
logger.info("The folder '%s' already exists on disk. Removing it" %
target_path)
wipe_folder(target_path)
# try to create target path
filesystem.ensure_folder_exists(target_path)
# copy all plugin data across
# skip info.yml, this is baked into the manifest python code
logger.info("Copying plugin data across...")
filesystem.copy_folder(source_path, target_path)
# create bundle cache
logger.info("Creating bundle cache folder...")
bundle_cache_root = os.path.join(target_path,
BUNDLE_CACHE_ROOT_FOLDER_NAME)
filesystem.ensure_folder_exists(bundle_cache_root)
# resolve config descriptor
# the config_uri_str returned by the method contains the fully resolved
# uri to use at runtime - in the case of baked descriptors, the config_uri_str
# contains a manual descriptor uri and install_path is set with the baked
# folder.
(cfg_descriptor, config_uri_str,
install_path) = _process_configuration(sg_connection, source_path,
target_path, bundle_cache_root,
manifest_data, use_system_core)
# cache config in bundle cache
logger.info("Downloading and caching config...")
# copy the config payload across to the plugin bundle cache
cfg_descriptor.clone_cache(bundle_cache_root)
# cache all apps, engines and frameworks
cache_apps(sg_connection, cfg_descriptor, bundle_cache_root)
if use_system_core:
logger.info(
"An external core will be used for this plugin, not caching it")
bootstrap_core_desc = None
else:
# get core - cache it directly into the plugin root folder
if bootstrap_core_uri:
logger.info("Caching custom core for boostrap (%s)" %
bootstrap_core_uri)
bootstrap_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE,
bootstrap_core_uri,
resolve_latest=is_descriptor_version_missing(
bootstrap_core_uri),
bundle_cache_root_override=bundle_cache_root)
# cache it
bootstrap_core_desc.ensure_local()
elif not cfg_descriptor.associated_core_descriptor:
# by default, use latest core for bootstrap
logger.info(
"Caching latest official core to use when bootstrapping plugin."
)
logger.info(
"(To use a specific config instead, specify a --bootstrap-core-uri flag.)"
)
bootstrap_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE, {
"type": "app_store",
"name": "tk-core"
},
resolve_latest=True,
bundle_cache_root_override=bundle_cache_root)
# cache it
bootstrap_core_desc.ensure_local()
else:
# The bootstrap core will be derived from the associated core desc below.
bootstrap_core_desc = None
# now analyze what core the config needs
if not use_system_core and cfg_descriptor.associated_core_descriptor:
logger.info(
"Config is specifying a custom core in config/core/core_api.yml.")
logger.info("This will be used when the config is executing.")
logger.info("Ensuring this core (%s) is cached..." %
cfg_descriptor.associated_core_descriptor)
associated_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE,
cfg_descriptor.associated_core_descriptor,
bundle_cache_root_override=bundle_cache_root)
associated_core_desc.ensure_local()
if bootstrap_core_desc is None:
# Use the same version as the one specified by the config.
if install_path:
# Install path is set only if the config was baked. We re-use the
# install path as an optimisation to avoid core swapping when the
# config is bootstrapped.
logger.info(
"Bootstrapping will use installed %s required by the config"
% associated_core_desc)
# If the core was installed we directly use it.
bootstrap_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE, {
"type": "path",
"name": "tk-core",
"path": os.path.join(install_path, "install", "core"),
"version": associated_core_desc.version,
},
resolve_latest=False,
bundle_cache_root_override=bundle_cache_root)
else:
logger.info(
"Bootstrapping will use core %s required by the config" %
associated_core_desc)
bootstrap_core_desc = associated_core_desc
# make a python folder where we put our manifest
logger.info("Creating configuration manifest...")
# bake out the manifest into python files.
_bake_manifest(manifest_data, config_uri_str, bootstrap_core_desc,
target_path)
cleanup_bundle_cache(bundle_cache_root)
logger.info("")
logger.info("Build complete!")
logger.info("")
logger.info("- Your plugin is ready in '%s'" % target_path)
logger.info("- Plugin uses config %r" % cfg_descriptor)
if bootstrap_core_desc:
logger.info("- Bootstrap core is %r" % bootstrap_core_desc)
else:
logger.info("- Plugin will need an external installed core.")
logger.info(
"- All dependencies have been baked out into the bundle_cache folder")
logger.info("")
logger.info("")
logger.info("")
def build_plugin(sg_connection,
source_path,
target_path,
bootstrap_core_uri=None):
"""
Perform a build of a plugin.
This will introspect the info.yml in the source path,
copy over everything in the source path into target path
and then establish a bundle cache containing a reflection
of all items required by the config.
:param sg_connection: Shotgun connection
:param source_path: Path to plugin.
:param target_path: Path to build
:param bootstrap_core_uri: Custom bootstrap core uri. If None,
the latest core from the app store will be used.
"""
logger.info("Your toolkit plugin in '%s' will be processed." % source_path)
logger.info("The build will generated into '%s'" % target_path)
# check for existence
if not os.path.exists(source_path):
raise TankError("Source path '%s' cannot be found on disk!" %
source_path)
# check that target path doesn't exist
if os.path.exists(target_path):
logger.info("The folder '%s' already exists on disk. Removing it" %
target_path)
wipe_folder(target_path)
# try to create target path
filesystem.ensure_folder_exists(target_path)
# check manifest
manifest_data = _validate_manifest(source_path)
# copy all plugin data across
# skip info.yml, this is baked into the manifest python code
logger.info("Copying plugin data across...")
filesystem.copy_folder(source_path, target_path)
# create bundle cache
logger.info("Creating bundle cache folder...")
bundle_cache_root = os.path.join(target_path,
BUNDLE_CACHE_ROOT_FOLDER_NAME)
filesystem.ensure_folder_exists(bundle_cache_root)
# resolve config descriptor
# the config_uri_str returned by the method contains the fully resolved
# uri to use at runtime - in the case of baked descriptors, the config_uri_str
# contains a manual descriptor uri.
(cfg_descriptor,
config_uri_str) = _process_configuration(sg_connection, source_path,
target_path, bundle_cache_root,
manifest_data)
# cache config in bundle cache
logger.info("Downloading and caching config...")
# copy the config payload across to the plugin bundle cache
cfg_descriptor.clone_cache(bundle_cache_root)
# cache all apps, engines and frameworks
cache_apps(sg_connection, cfg_descriptor, bundle_cache_root)
# get latest core - cache it directly into the plugin root folder
if bootstrap_core_uri:
logger.info("Caching custom core for boostrap (%s)" %
bootstrap_core_uri)
bootstrap_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE,
bootstrap_core_uri,
bundle_cache_root_override=bundle_cache_root)
else:
# by default, use latest core for bootstrap
logger.info(
"Caching latest official core to use when bootstrapping plugin.")
logger.info(
"(To use a specific config instead, specify a --bootstrap-core-uri flag.)"
)
bootstrap_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE, {
"type": "app_store",
"name": "tk-core"
},
resolve_latest=True,
bundle_cache_root_override=bundle_cache_root)
# cache it
bootstrap_core_desc.ensure_local()
# make a python folder where we put our manifest
logger.info("Creating configuration manifest...")
# bake out the manifest into python files.
_bake_manifest(manifest_data, config_uri_str, bootstrap_core_desc,
target_path)
# now analyze what core the config needs
if cfg_descriptor.associated_core_descriptor:
logger.info(
"Config is specifying a custom core in config/core/core_api.yml.")
logger.info("This will be used when the config is executing.")
logger.info("Ensuring this core (%s) is cached..." %
cfg_descriptor.associated_core_descriptor)
associated_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE,
cfg_descriptor.associated_core_descriptor,
bundle_cache_root_override=bundle_cache_root)
associated_core_desc.ensure_local()
cleanup_bundle_cache(bundle_cache_root)
logger.info("")
logger.info("Build complete!")
logger.info("")
logger.info("- Your plugin is ready in '%s'" % target_path)
logger.info("- Plugin uses config %r" % cfg_descriptor)
logger.info("- Bootstrap core is %r" % bootstrap_core_desc)
logger.info(
"- All dependencies have been baked out into the bundle_cache folder")
logger.info("")
logger.info("")
logger.info("")
def _process_configuration(sg_connection, source_path, target_path,
bundle_cache_root, manifest_data, use_system_core):
"""
Given data in the plugin manifest, download resolve and
cache the configuration.
:param sg_connection: Shotgun connection
:param source_path: Root path of plugin source.
:param target_path: Build target path
:param bundle_cache_root: Bundle cache root
:param manifest_data: Manifest data as a dictionary
:param bool use_system_core: If True, use a globally installed tk-core instead
of the one specified in the configuration.
:return: (Resolved config descriptor object, config descriptor uri to use at runtime, install path)
"""
logger.info("Analyzing configuration")
install_path = None
# get config def from info yml and generate both
# dict and string uris.
base_config_def = manifest_data["base_configuration"]
if isinstance(base_config_def, str):
# convert to dict so we can introspect
base_config_uri_dict = descriptor_uri_to_dict(base_config_def)
base_config_uri_str = base_config_def
else:
base_config_uri_dict = base_config_def
base_config_uri_str = descriptor_dict_to_uri(base_config_def)
# Special case - check for the 'baked' descriptor type
# and process it. A baked descriptor is a special concept
# that only exists in the build script. The baked descriptor
# takes a single path parameter which can be a local or absolute
# path. The path is copied across by the build script into a
# manual descriptor, with a version number based on the current date.
# This ensures that the manual descriptor will be correctly
# re-cached at bootstrap time.
if base_config_uri_dict[
"type"] == bootstrap_constants.BAKED_DESCRIPTOR_TYPE:
logger.info("Baked descriptor detected.")
baked_path = os.path.expanduser(
os.path.expandvars(base_config_uri_dict["path"]))
# if it's a relative path, expand it
if not os.path.isabs(baked_path):
full_baked_path = os.path.abspath(
os.path.join(source_path, baked_path))
# if it's a relative path, we have already copied it to the build
# target location. In this case, attempt to locate it and remove it.
baked_target_path = os.path.abspath(
os.path.join(target_path, baked_path))
if baked_target_path.startswith(baked_target_path):
logger.debug("Removing '%s' from build" % baked_target_path)
shutil.rmtree(baked_target_path)
else:
# path is absolute
full_baked_path = os.path.abspath(baked_path)
logger.info("Will bake an immutable config into the plugin from '%s'" %
full_baked_path)
# A baked config descriptor does not require a name nor a version, so
# if these keys are not available, use the current date time for the
# version and an arbitrary name for the config. Please note that this
# only happens if the baked descriptor was set in the original config.
# When baking a plugin with the --bake option, which is the recommended
# workflow, these values are automatically set.
baked_name = base_config_uri_dict.get("name") or "tk-config-plugin"
baked_version = (base_config_uri_dict.get("version")
or datetime.datetime.now().strftime("%Y%m%d_%H%M%S"))
install_path = os.path.join(
bundle_cache_root,
bootstrap_constants.BAKED_DESCRIPTOR_FOLDER_NAME, baked_name,
baked_version)
cfg_descriptor = create_descriptor(sg_connection, Descriptor.CONFIG, {
"type": "path",
"path": full_baked_path
})
BakedConfiguration.bake_config_scaffold(install_path, sg_connection,
manifest_data["plugin_id"],
cfg_descriptor)
# now lastly,
base_config_uri_str = descriptor_dict_to_uri({
"type":
bootstrap_constants.BAKED_DESCRIPTOR_TYPE,
"name":
baked_name,
"version":
baked_version
})
if use_system_core:
# If asked to use a globally installed tk-core instead of the one
# specified by the config, we remove the local copy which was created
# in the scaffold step.
logger.info("Removing core reference in %s" % install_path)
wipe_folder(os.path.join(install_path, "install"))
# And make sure we don't have any reference to a tk-core in the config,
# otherwise it would be picked up when bootstrapping.
filesystem.safe_delete_file(
os.path.join(install_path, "config", "core", "core_api.yml"))
else:
# Workaround for tk-core bootstrap needing a shotgun.yml file: when swapping
# tk-core, this file is checked to see if a script user was specified and
# should be used in place of the authenticated user. So we create a dummy
# file with an "unspecified" host, as the key is required by the tk-core
# code parsing the file.
# It is not clear if this workaround is needed for non baked configs as
# their workflow is different, so for now we just keep it for bake configs
# only.
shotgun_yaml_path = os.path.join(install_path, "config", "core",
"shotgun.yml")
if not os.path.exists(shotgun_yaml_path):
logger.info("Patching %s" % shotgun_yaml_path)
with open(shotgun_yaml_path, "w") as pf:
pf.write(
"# Workaround for tk-core bootstrap\nhost: unspecified"
)
else:
# if the descriptor in the config contains a version number
# we will go into a fixed update mode.
using_latest_config = is_descriptor_version_missing(
base_config_uri_dict)
if using_latest_config:
logger.info(
"Your configuration definition does not contain a version number. "
"This means that the plugin will attempt to auto update at startup."
)
else:
logger.info(
"Your configuration definition contains a version number. "
"This means that the plugin will be frozen and no automatic updates "
"will be performed at startup.")
cfg_descriptor = create_descriptor(sg_connection,
Descriptor.CONFIG,
base_config_uri_dict,
resolve_latest=using_latest_config)
logger.info("Resolved config %r" % cfg_descriptor)
logger.info("Runtime config descriptor uri will be %s" %
base_config_uri_str)
if install_path:
logger.info("The config was baked in %s" % install_path)
return cfg_descriptor, base_config_uri_str, install_path
def _build_bundle_cache(sg_connection, target_path, config_descriptor_uri):
"""
Perform a build of the bundle cache.
This will build the bundle cache for a given config descriptor.
:param sg_connection: Shotgun connection
:param target_path: Path to build
:param config_descriptor_uri: Descriptor of the configuration to cache.
"""
logger.info("The build will generated into '%s'" % target_path)
bundle_cache_root = os.path.join(target_path, BUNDLE_CACHE_ROOT_FOLDER_NAME)
# check that target path doesn't exist
if os.path.exists(bundle_cache_root):
logger.info("The folder '%s' already exists on disk. Removing it" % bundle_cache_root)
wipe_folder(bundle_cache_root)
# try to create target path
logger.info("Creating bundle cache folder...")
filesystem.ensure_folder_exists(bundle_cache_root)
# Resolve the configuration
cfg_descriptor = create_descriptor(
sg_connection,
Descriptor.CONFIG,
config_descriptor_uri,
# If the user hasn't specified the version to retrieve, resolve the latest from Shotgun.
resolve_latest="version" not in config_descriptor_uri
)
logger.info("Resolved config %r" % cfg_descriptor)
logger.info("Runtime config descriptor uri will be %s" % config_descriptor_uri)
# cache config in bundle cache
logger.info("Downloading and caching config...")
cfg_descriptor.ensure_local()
# copy the config payload across to the plugin bundle cache
cfg_descriptor.clone_cache(bundle_cache_root)
# cache all apps, engines and frameworks
cache_apps(sg_connection, cfg_descriptor, bundle_cache_root)
if cfg_descriptor.associated_core_descriptor:
logger.info("Config is specifying a custom core in config/core/core_api.yml.")
logger.info("This will be used when the config is executing.")
logger.info("Ensuring this core (%s) is cached..." % cfg_descriptor.associated_core_descriptor)
bootstrap_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE,
cfg_descriptor.associated_core_descriptor,
bundle_cache_root_override=bundle_cache_root
)
# cache it
bootstrap_core_desc.ensure_local()
bootstrap_core_desc.clone_cache(bundle_cache_root)
cleanup_bundle_cache(bundle_cache_root)
logger.info("")
logger.info("Build complete!")
logger.info("")
logger.info("- Your bundle cache is ready in '%s'" % target_path)
logger.info("- All dependencies have been baked out into the bundle_cache folder")
logger.info("")
def build_plugin(sg_connection, source_path, target_path, bootstrap_core_uri=None):
"""
Perform a build of a plugin.
This will introspect the info.yml in the source path,
copy over everything in the source path into target path
and then establish a bundle cache containing a reflection
of all items required by the config.
:param sg_connection: Shotgun connection
:param source_path: Path to plugin.
:param target_path: Path to build
:param bootstrap_core_uri: Custom bootstrap core uri. If None,
the latest core from the app store will be used.
"""
logger.info("Your toolkit plugin in '%s' will be processed." % source_path)
logger.info("The build will generated into '%s'" % target_path)
# check for existence
if not os.path.exists(source_path):
raise TankError("Source path '%s' cannot be found on disk!" % source_path)
# check that target path doesn't exist
if os.path.exists(target_path):
logger.info("The folder '%s' already exists on disk. Removing it" % target_path)
wipe_folder(target_path)
# try to create target path
filesystem.ensure_folder_exists(target_path)
# check manifest
manifest_data = _validate_manifest(source_path)
# copy all plugin data across
# skip info.yml, this is baked into the manifest python code
logger.info("Copying plugin data across...")
filesystem.copy_folder(source_path, target_path)
# create bundle cache
logger.info("Creating bundle cache folder...")
bundle_cache_root = os.path.join(target_path, BUNDLE_CACHE_ROOT_FOLDER_NAME)
filesystem.ensure_folder_exists(bundle_cache_root)
# resolve config descriptor
# the config_uri_str returned by the method contains the fully resolved
# uri to use at runtime - in the case of baked descriptors, the config_uri_str
# contains a manual descriptor uri.
(cfg_descriptor, config_uri_str) = _process_configuration(
sg_connection,
source_path,
target_path,
bundle_cache_root,
manifest_data
)
# cache config in bundle cache
logger.info("Downloading and caching config...")
# copy the config payload across to the plugin bundle cache
cfg_descriptor.clone_cache(bundle_cache_root)
# cache all apps, engines and frameworks
cache_apps(sg_connection, cfg_descriptor, bundle_cache_root)
# get latest core - cache it directly into the plugin root folder
if bootstrap_core_uri:
logger.info("Caching custom core for boostrap (%s)" % bootstrap_core_uri)
bootstrap_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE,
bootstrap_core_uri,
resolve_latest=is_descriptor_version_missing(bootstrap_core_uri),
bundle_cache_root_override=bundle_cache_root
)
else:
# by default, use latest core for bootstrap
logger.info("Caching latest official core to use when bootstrapping plugin.")
logger.info("(To use a specific config instead, specify a --bootstrap-core-uri flag.)")
bootstrap_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE,
{"type": "app_store", "name": "tk-core"},
resolve_latest=True,
bundle_cache_root_override=bundle_cache_root
)
# cache it
bootstrap_core_desc.ensure_local()
# make a python folder where we put our manifest
logger.info("Creating configuration manifest...")
# bake out the manifest into python files.
_bake_manifest(
manifest_data,
config_uri_str,
bootstrap_core_desc,
target_path
)
# now analyze what core the config needs
if cfg_descriptor.associated_core_descriptor:
logger.info("Config is specifying a custom core in config/core/core_api.yml.")
logger.info("This will be used when the config is executing.")
logger.info("Ensuring this core (%s) is cached..." % cfg_descriptor.associated_core_descriptor)
associated_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE,
cfg_descriptor.associated_core_descriptor,
bundle_cache_root_override=bundle_cache_root
)
associated_core_desc.ensure_local()
cleanup_bundle_cache(bundle_cache_root)
logger.info("")
logger.info("Build complete!")
logger.info("")
logger.info("- Your plugin is ready in '%s'" % target_path)
logger.info("- Plugin uses config %r" % cfg_descriptor)
logger.info("- Bootstrap core is %r" % bootstrap_core_desc)
logger.info("- All dependencies have been baked out into the bundle_cache folder")
logger.info("")
logger.info("")
logger.info("")
def build_plugin(sg_connection, source_path, target_path, bootstrap_core_uri=None, do_bake=False, use_system_core=False):
"""
Perform a build of a plugin.
This will introspect the info.yml in the source path,
copy over everything in the source path into target path
and then establish a bundle cache containing a reflection
of all items required by the config.
:param sg_connection: Shotgun connection
:param source_path: Path to plugin.
:param target_path: Path to build
:param bootstrap_core_uri: Custom bootstrap core uri. If None,
the latest core from the app store will be used.
:param bool do_bake: If True, bake the plugin prior to building it.
:param bool use_system_core: If True, use a globally installed tk-core instead
of the one specified in the configuration.
"""
logger.info("Your toolkit plugin in '%s' will be processed." % source_path)
logger.info("The build will %s into '%s'" % (["generated", "baked"][do_bake], target_path))
# check for existence
if not os.path.exists(source_path):
raise TankError("Source path '%s' cannot be found on disk!" % source_path)
# check manifest
manifest_data = _validate_manifest(source_path)
if do_bake:
baked_descriptor = _bake_configuration(
sg_connection,
manifest_data,
)
# When baking we control the output path by adding a folder based on the
# configuration descriptor and version.
target_path = os.path.join(target_path, "%s-%s" % (
baked_descriptor["name"],
baked_descriptor["version"]
))
# check that target path doesn't exist
if os.path.exists(target_path):
logger.info("The folder '%s' already exists on disk. Removing it" % target_path)
wipe_folder(target_path)
# try to create target path
filesystem.ensure_folder_exists(target_path)
# copy all plugin data across
# skip info.yml, this is baked into the manifest python code
logger.info("Copying plugin data across...")
filesystem.copy_folder(source_path, target_path)
# create bundle cache
logger.info("Creating bundle cache folder...")
bundle_cache_root = os.path.join(target_path, BUNDLE_CACHE_ROOT_FOLDER_NAME)
filesystem.ensure_folder_exists(bundle_cache_root)
# resolve config descriptor
# the config_uri_str returned by the method contains the fully resolved
# uri to use at runtime - in the case of baked descriptors, the config_uri_str
# contains a manual descriptor uri and install_path is set with the baked
# folder.
(cfg_descriptor, config_uri_str, install_path) = _process_configuration(
sg_connection,
source_path,
target_path,
bundle_cache_root,
manifest_data,
use_system_core
)
# cache config in bundle cache
logger.info("Downloading and caching config...")
# copy the config payload across to the plugin bundle cache
cfg_descriptor.clone_cache(bundle_cache_root)
# cache all apps, engines and frameworks
cache_apps(sg_connection, cfg_descriptor, bundle_cache_root)
if use_system_core:
logger.info("An external core will be used for this plugin, not caching it")
bootstrap_core_desc = None
else:
# get core - cache it directly into the plugin root folder
if bootstrap_core_uri:
logger.info("Caching custom core for boostrap (%s)" % bootstrap_core_uri)
bootstrap_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE,
bootstrap_core_uri,
resolve_latest=is_descriptor_version_missing(bootstrap_core_uri),
bundle_cache_root_override=bundle_cache_root
)
# cache it
bootstrap_core_desc.ensure_local()
elif not cfg_descriptor.associated_core_descriptor:
# by default, use latest core for bootstrap
logger.info("Caching latest official core to use when bootstrapping plugin.")
logger.info("(To use a specific config instead, specify a --bootstrap-core-uri flag.)")
bootstrap_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE,
{"type": "app_store", "name": "tk-core"},
resolve_latest=True,
bundle_cache_root_override=bundle_cache_root
)
# cache it
bootstrap_core_desc.ensure_local()
else:
# The bootstrap core will be derived from the associated core desc below.
bootstrap_core_desc = None
# now analyze what core the config needs
if not use_system_core and cfg_descriptor.associated_core_descriptor:
logger.info("Config is specifying a custom core in config/core/core_api.yml.")
logger.info("This will be used when the config is executing.")
logger.info("Ensuring this core (%s) is cached..." % cfg_descriptor.associated_core_descriptor)
associated_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE,
cfg_descriptor.associated_core_descriptor,
bundle_cache_root_override=bundle_cache_root
)
associated_core_desc.ensure_local()
if bootstrap_core_desc is None:
# Use the same version as the one specified by the config.
if install_path:
# Install path is set only if the config was baked. We re-use the
# install path as an optimisation to avoid core swapping when the
# config is bootstrapped.
logger.info(
"Bootstrapping will use installed %s required by the config" %
associated_core_desc
)
# If the core was installed we directly use it.
bootstrap_core_desc = create_descriptor(
sg_connection,
Descriptor.CORE, {
"type": "path",
"name": "tk-core",
"path": os.path.join(install_path, "install", "core"),
"version": associated_core_desc.version,
},
resolve_latest=False,
bundle_cache_root_override=bundle_cache_root
)
else:
logger.info(
"Bootstrapping will use core %s required by the config" %
associated_core_desc
)
bootstrap_core_desc = associated_core_desc
# make a python folder where we put our manifest
logger.info("Creating configuration manifest...")
# bake out the manifest into python files.
_bake_manifest(
manifest_data,
config_uri_str,
bootstrap_core_desc,
target_path
)
cleanup_bundle_cache(bundle_cache_root)
logger.info("")
logger.info("Build complete!")
logger.info("")
logger.info("- Your plugin is ready in '%s'" % target_path)
logger.info("- Plugin uses config %r" % cfg_descriptor)
if bootstrap_core_desc:
logger.info("- Bootstrap core is %r" % bootstrap_core_desc)
else:
logger.info("- Plugin will need an external installed core.")
logger.info("- All dependencies have been baked out into the bundle_cache folder")
logger.info("")
logger.info("")
logger.info("")
def _process_configuration(sg_connection, source_path, target_path, bundle_cache_root, manifest_data, use_system_core):
"""
Given data in the plugin manifest, download resolve and
cache the configuration.
:param sg_connection: Shotgun connection
:param source_path: Root path of plugin source.
:param target_path: Build target path
:param bundle_cache_root: Bundle cache root
:param manifest_data: Manifest data as a dictionary
:param bool use_system_core: If True, use a globally installed tk-core instead
of the one specified in the configuration.
:return: (Resolved config descriptor object, config descriptor uri to use at runtime, install path)
"""
logger.info("Analyzing configuration")
install_path = None
# get config def from info yml and generate both
# dict and string uris.
base_config_def = manifest_data["base_configuration"]
if isinstance(base_config_def, str):
# convert to dict so we can introspect
base_config_uri_dict = descriptor_uri_to_dict(base_config_def)
base_config_uri_str = base_config_def
else:
base_config_uri_dict = base_config_def
base_config_uri_str = descriptor_dict_to_uri(base_config_def)
# Special case - check for the 'baked' descriptor type
# and process it. A baked descriptor is a special concept
# that only exists in the build script. The baked descriptor
# takes a single path parameter which can be a local or absolute
# path. The path is copied across by the build script into a
# manual descriptor, with a version number based on the current date.
# This ensures that the manual descriptor will be correctly
# re-cached at bootstrap time.
if base_config_uri_dict["type"] == bootstrap_constants.BAKED_DESCRIPTOR_TYPE:
logger.info("Baked descriptor detected.")
baked_path = os.path.expanduser(os.path.expandvars(base_config_uri_dict["path"]))
# if it's a relative path, expand it
if not os.path.isabs(baked_path):
full_baked_path = os.path.abspath(os.path.join(source_path, baked_path))
# if it's a relative path, we have already copied it to the build
# target location. In this case, attempt to locate it and remove it.
baked_target_path = os.path.abspath(os.path.join(target_path, baked_path))
if baked_target_path.startswith(baked_target_path):
logger.debug("Removing '%s' from build" % baked_target_path)
shutil.rmtree(baked_target_path)
else:
# path is absolute
full_baked_path = os.path.abspath(baked_path)
logger.info("Will bake an immutable config into the plugin from '%s'" % full_baked_path)
# A baked config descriptor does not require a name nor a version, so
# if these keys are not available, use the current date time for the
# version and an arbitrary name for the config. Please note that this
# only happens if the baked descriptor was set in the original config.
# When baking a plugin with the --bake option, which is the recommended
# workflow, these values are automatically set.
baked_name = base_config_uri_dict.get("name") or "tk-config-plugin"
baked_version = (
base_config_uri_dict.get("version") or
datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
)
install_path = os.path.join(
bundle_cache_root,
bootstrap_constants.BAKED_DESCRIPTOR_FOLDER_NAME,
baked_name,
baked_version
)
cfg_descriptor = create_descriptor(
sg_connection,
Descriptor.CONFIG,
{"type": "path", "path": full_baked_path}
)
BakedConfiguration.bake_config_scaffold(
install_path,
sg_connection,
manifest_data["plugin_id"],
cfg_descriptor
)
# now lastly,
base_config_uri_str = descriptor_dict_to_uri(
{
"type": bootstrap_constants.BAKED_DESCRIPTOR_TYPE,
"name": baked_name,
"version": baked_version
}
)
if use_system_core:
# If asked to use a globally installed tk-core instead of the one
# specified by the config, we remove the local copy which was created
# in the scaffold step.
logger.info("Removing core reference in %s" % install_path)
wipe_folder(os.path.join(install_path, "install"))
# And make sure we don't have any reference to a tk-core in the config,
# otherwise it would be picked up when bootstrapping.
filesystem.safe_delete_file(os.path.join(install_path, "config", "core", "core_api.yml"))
else:
# Workaround for tk-core bootstrap needing a shotgun.yml file: when swapping
# tk-core, this file is checked to see if a script user was specified and
# should be used in place of the authenticated user. So we create a dummy
# file with an "unspecified" host, as the key is required by the tk-core
# code parsing the file.
# It is not clear if this workaround is needed for non baked configs as
# their workflow is different, so for now we just keep it for bake configs
# only.
shotgun_yaml_path = os.path.join(install_path, "config", "core", "shotgun.yml")
if not os.path.exists(shotgun_yaml_path):
logger.info("Patching %s" % shotgun_yaml_path)
with open(shotgun_yaml_path, "w") as pf:
pf.write("# Workaround for tk-core bootstrap\nhost: unspecified")
else:
# if the descriptor in the config contains a version number
# we will go into a fixed update mode.
using_latest_config = is_descriptor_version_missing(base_config_uri_dict)
if using_latest_config:
logger.info(
"Your configuration definition does not contain a version number. "
"This means that the plugin will attempt to auto update at startup."
)
else:
logger.info(
"Your configuration definition contains a version number. "
"This means that the plugin will be frozen and no automatic updates "
"will be performed at startup."
)
cfg_descriptor = create_descriptor(
sg_connection,
Descriptor.CONFIG,
base_config_uri_dict,
resolve_latest=using_latest_config
)
logger.info("Resolved config %r" % cfg_descriptor)
logger.info("Runtime config descriptor uri will be %s" % base_config_uri_str)
if install_path:
logger.info("The config was baked in %s" % install_path)
return cfg_descriptor, base_config_uri_str, install_path