from tank.descriptor.errors import TankDescriptorError
from tank.bootstrap import constants as bootstrap_constants
import functools
from utils import (
cache_apps,
authenticate,
add_authentication_options,
OptionParserLineBreakingEpilog,
cleanup_bundle_cache,
wipe_folder,
automated_setup_documentation,
)
# Set up logging
logger = LogManager.get_logger("bake_config")
# The folder where all items will be cached
BUNDLE_CACHE_ROOT_FOLDER_NAME = "bundle_cache"
def _should_skip(types, desc):
"""
Check if a descriptor should be skipped.
:param list types: List of descriptor type names that should be skipped.
:param dict desc: Descriptor dictionary to check.
:returns: ``True`` if the contents should be skipped, ``False`` otherwise.
"""
return desc["type"] in types
def main():
"""
Main entry point for script.
Handles argument parsing and validation and then calls the script payload.
"""
usage = "%prog [options] config_descriptor target_path"
desc = "Bake a self contained Toolkit config from a descriptor"
epilog = """
Details and Examples
--------------------
In its simplest form, just provide a local path and target folder for the build.
> python bake_config.py ~/dev/tk-config-myconfig /tmp/baked_configurations
Or you can specify a version with a Toolkit config descriptor uri.
> python bake_config.py "sgtk:descriptor:dev?version=v1.0.9&path=../tk-config-myconfig" /tmp/baked_configurations
Any type of Toolkit config descriptor uri can be used, if a version is not specified, the latest for the descriptor is resolved.
> python bake_config.py "sgtk:descriptor:app_store?name=tk-config-basic" /tmp/baked_configurations
By default, all bundle types are cached. If you want to omit certain types, simply provide a comma seperated list
of bundle types to skip, e.g. --skip-bundle-types=app_store,shotgun,github_release.
{automated_setup_documentation}
For information about the various descriptors that can be used, see
http://developer.shotgridsoftware.com/tk-core/descriptor
""".format(
automated_setup_documentation=automated_setup_documentation
).format(
script_name="bake_config.py"
)
parser = OptionParserLineBreakingEpilog(
usage=usage, description=desc, epilog=epilog
)
parser.add_option(
"-d", "--debug", default=False, action="store_true", help="Enable debug logging"
)
parser.add_option(
"-z", "--zip", default=False, action="store_true", help="Zip archive the config"
)
parser.add_option(
"--skip-bundle-types",
# You can't have an empty default optional value, so we'll pick something
# and treat it accordingly.
default="none",
help="Comma separated list of bundle types to skip. Possible values are 'app_store', "
"'git', 'git_branch', 'github_release', 'shotgun'. Empty by default.",
)
add_authentication_options(parser)
# parse cmd line
(options, remaining_args) = parser.parse_args()
logger.info("Welcome to the Toolkit config baker.")
logger.info("")
if options.debug:
LogManager().global_debug = True
if len(remaining_args) != 2:
parser.print_help()
return 2
# Get config descriptor
config_descriptor = remaining_args[0]
# Try to parse it, check if it is a local path if it fails
try:
descriptor_uri_to_dict(config_descriptor)
except TankDescriptorError:
# Check if it is a local path
path = os.path.abspath(
os.path.expanduser(os.path.expandvars(config_descriptor))
)
if os.path.isdir(path):
logger.info("Using a dev descriptor for local path %s" % path)
# Forge a dev descriptor, using "latest" for the version.
# TODO: try to retrieve a valid version from the folder, e.g. with a
# git tag from the folder.
config_descriptor = "sgtk:descriptor:dev?name=%s&path=%s&version=latest" % (
os.path.basename(path),
path,
)
else:
logger.error(
"%s is not a valid descriptor nor a local path." % config_descriptor
)
raise
# Get output path
target_path = remaining_args[1]
target_path = os.path.expanduser(os.path.expandvars(target_path))
sg_user = authenticate(options)
sg_connection = sg_user.create_sg_connection()
# make sure we are properly connected
try:
sg_connection.find_one("HumanUser", [])
except Exception as e:
logger.error("Could not communicate with ShotGrid: %s" % e)
return 3
# Strip any extra whitespaces and make sure every bundle type exists.
skip_bundle_types = options.skip_bundle_types.split(",")
skip_bundle_types = [bundle_type.strip() for bundle_type in skip_bundle_types]
for bundle_type in skip_bundle_types:
if bundle_type not in [
"app_store",
"git",
"git_branch",
"github_release",
"shotgun",
"none",
]:
logger.error("Unknown bundle type: %s" % bundle_type)
return 4
# we are all set.
bake_config(
sg_connection, config_descriptor, target_path, options.zip, skip_bundle_types,
)
# all good!
return 0
def main():
"""
Main entry point for script.
Handles argument parsing and validation and then calls the script payload.
"""
usage = "%prog [options] source_path target_path"
desc = "Builds a standard toolkit plugin structure ready for testing and deploy"
epilog = """
Details and Examples
--------------------
In its simplest form, just provide a source and target folder for the build.
> python build_plugin.py ~/dev/tk-maya/plugins/basic /tmp/maya-plugin
For automated build setups, you can provide a specific shotgun API script name and
and corresponding script key:
> python build_plugin.py
--shotgun-host='https://mysite.shotgunstudio.com'
--shotgun-script-name='plugin_build'
--shotgun-script-key='<script-key-here>'
~/dev/tk-maya/plugins/basic /tmp/maya-plugin
By default, the build script will use the latest app store core for its bootstrapping.
If you want to use a specific core for the bootstrap, this can be specified via the
--bootstrap-core-uri option:
> python build_plugin.py
--bootstrap-core-uri='sgtk:descriptor:dev?path=~/dev/tk-core'
~/dev/tk-maya/plugins/basic /tmp/maya-plugin
For information about the various descriptors that can be used, see
http://developer.shotgunsoftware.com/tk-core/descriptor
"""
parser = OptionParserLineBreakingEpilog(usage=usage,
description=desc,
epilog=epilog)
parser.add_option("-d",
"--debug",
default=False,
action="store_true",
help="Enable debug logging")
parser.add_option(
"-b",
"--buildable",
default=False,
action="store_true",
help=
("Don't cull config files as part of the build process. Enabling this setting "
"means that the built plugin can be used as a source for another build."
))
parser.add_option(
"-c",
"--bootstrap-core-uri",
default=None,
action="store",
help=
("Specify which version of core to be used by the bootstrap process. "
"If not specified, defaults to the most recently released core."))
group = optparse.OptionGroup(
parser, "Shotgun Authentication",
"In order to download content from the Toolkit app store, the script will need to authenticate "
"against any shotgun site. By default, it will use the toolkit authentication APIs stored "
"credentials, and if such are not found, it will prompt for site, username and password."
)
group.add_option("-s",
"--shotgun-host",
default=None,
action="store",
help="Shotgun host to authenticate with.")
group.add_option("-n",
"--shotgun-script-name",
default=None,
action="store",
help="Script to use to authenticate with the given host.")
group.add_option(
"-k",
"--shotgun-script-key",
default=None,
action="store",
help="Script key to use to authenticate with the given host.")
parser.add_option_group(group)
# parse cmd line
(options, remaining_args) = parser.parse_args()
logger.info("Welcome to the Toolkit plugin builder.")
logger.info("")
if options.debug:
LogManager().global_debug = True
if options.bootstrap_core_uri:
bootstrap_core_uri = options.bootstrap_core_uri
else:
# default
bootstrap_core_uri = None
if len(remaining_args) != 2:
parser.print_help()
return 2
# get paths
source_path = remaining_args[0]
target_path = remaining_args[1]
# convert any env vars and tildes
source_path = os.path.expanduser(os.path.expandvars(source_path))
target_path = os.path.expanduser(os.path.expandvars(target_path))
# now authenticate to shotgun
sg_auth = ShotgunAuthenticator()
if options.shotgun_host:
script_name = options.shotgun_script_name
script_key = options.shotgun_script_key
if script_name is None or script_key is None:
logger.error(
"Need to provide, host, script name and script key! Run with -h for more info."
)
return 2
logger.info("Connecting to %s using script user %s..." %
(options.shotgun_host, script_name))
sg_user = sg_auth.create_script_user(script_name, script_key,
options.shotgun_host)
else:
# get user, prompt if necessary
sg_user = sg_auth.get_user()
sg_connection = sg_user.create_sg_connection()
# make sure we are properly connected
try:
sg_connection.find_one("HumanUser", [])
except Exception, e:
logger.error("Could not communicate with Shotgun: %s" % e)
return 3
import shutil
import contextlib
import logging
from mock import Mock
from tank_test.tank_test_base import TankTestBase, temp_env_var
from tank_test.tank_test_base import setUpModule # noqa
from tank import path_cache
from tank import folder
from tank import constants
from tank import LogManager
import tank
log = LogManager.get_logger(__name__)
def add_item_to_cache(path_cache, entity, path, primary=True):
data = [{
"entity": entity,
"path": path,
"primary": primary,
"metadata": {}
}]
# Last two parameters are only used for debug logging, they can be empty.
path_cache.add_mappings(data, None, [])
def sync_path_cache(tk, force_full_sync=False):
"""
def main():
"""
Main entry point for script.
Handles argument parsing and validation and then calls the script payload.
"""
usage = "%prog [options] source_path target_path"
desc = "Builds a standard toolkit plugin structure ready for testing and deploy"
epilog = """
Details and Examples
--------------------
In its simplest form, just provide a source and target folder for the build.
> python build_plugin.py ~/dev/tk-maya/plugins/basic /tmp/maya-plugin
For automated build setups, you can provide a specific shotgun API script name and
and corresponding script key:
> python build_plugin.py
--shotgun-host='https://mysite.shotgunstudio.com'
--shotgun-script-name='plugin_build'
--shotgun-script-key='<script-key-here>'
~/dev/tk-maya/plugins/basic /tmp/maya-plugin
By default, the build script will use the latest app store core for its bootstrapping.
If you want to use a specific core for the bootstrap, this can be specified via the
--bootstrap-core-uri option:
> python build_plugin.py
--bootstrap-core-uri='sgtk:descriptor:dev?path=~/dev/tk-core'
~/dev/tk-maya/plugins/basic /tmp/maya-plugin
For information about the various descriptors that can be used, see
http://developer.shotgunsoftware.com/tk-core/descriptor
"""
parser = OptionParserLineBreakingEpilog(usage=usage,
description=desc,
epilog=epilog)
parser.add_option("-d",
"--debug",
default=False,
action="store_true",
help="Enable debug logging")
parser.add_option(
"-c",
"--bootstrap-core-uri",
default=None,
action="store",
help=
("Specify which version of core to be used by the bootstrap process. "
"If not specified, defaults to the most recently released core."))
add_authentication_options(parser)
# parse cmd line
(options, remaining_args) = parser.parse_args()
logger.info("Welcome to the Toolkit plugin builder.")
logger.info("")
if options.debug:
LogManager().global_debug = True
if options.bootstrap_core_uri:
bootstrap_core_uri = options.bootstrap_core_uri
else:
# default
bootstrap_core_uri = None
if len(remaining_args) != 2:
parser.print_help()
return 2
# get paths
source_path = remaining_args[0]
target_path = remaining_args[1]
# convert any env vars and tildes
source_path = os.path.expanduser(os.path.expandvars(source_path))
target_path = os.path.expanduser(os.path.expandvars(target_path))
sg_user = authenticate(options)
sg_connection = sg_user.create_sg_connection()
# make sure we are properly connected
try:
sg_connection.find_one("HumanUser", [])
except Exception, e:
logger.error("Could not communicate with Shotgun: %s" % e)
return 3
python_folder = os.path.abspath(os.path.join(this_folder, "..", "python"))
sys.path.append(python_folder)
# sgtk imports
from tank import LogManager
from tank.util import filesystem
from tank.errors import TankError
from tank.platform import environment
from tank.descriptor import Descriptor, descriptor_uri_to_dict, descriptor_dict_to_uri, create_descriptor
from tank.authentication import ShotgunAuthenticator
from tank.bootstrap.baked_configuration import BakedConfiguration
from tank.bootstrap import constants as bootstrap_constants
from tank_vendor import yaml
# set up logging
logger = LogManager.get_logger("build_plugin")
# required keys in the info.yml plugin manifest file
REQUIRED_MANIFEST_PARAMETERS = ["base_configuration", "plugin_id"]
# the folder where all items will be cached
BUNDLE_CACHE_ROOT_FOLDER_NAME = "bundle_cache"
# when we are baking a config, use these settings
BAKED_BUNDLE_NAME = "tk-config-plugin"
BAKED_BUNDLE_VERSION = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
# generation of the build syntax
BUILD_GENERATION = 2
# By accessing, using, copying or modifying this work you indicate your
# agreement to the Shotgun Pipeline Toolkit Source Code License. All rights
# not expressly granted therein are reserved by Shotgun Software Inc.
import os
import glob
import shutil
import stat
from tank.util import filesystem
from tank.platform import environment
from tank.descriptor import Descriptor, create_descriptor
from tank import LogManager
logger = LogManager.get_logger("utils.caching")
def _cache_descriptor(sg, desc_type, desc_dict, target_path):
"""
Cache the given descriptor into a new bundle cache.
:param sg: Shotgun API instance
:param desc_type: Descriptor.ENGINE | Descriptor.APP | Descriptor.FRAMEWORK
:param desc_dict: descriptor dict or uri
:param target_path: bundle cache root to cache into
"""
desc = create_descriptor(sg, desc_type, desc_dict, fallback_roots=[target_path])
desc.ensure_local()
desc_size_kb = filesystem.compute_folder_size(desc.get_path()) / 1024
logger.info("Caching %s into plugin bundle cache (size %d KiB)" % (desc, desc_size_kb))
def main():
"""
Main entry point for script.
Handles argument parsing and validation and then calls the script payload.
"""
usage = "%prog [options] config_descriptor target_path"
desc = "Bake a self contained Toolkit config from a descriptor"
epilog = """
Details and Examples
--------------------
In its simplest form, just provide a local path and target folder for the build.
> python bake_config.py ~/dev/tk-config-myconfig /tmp/baked_configurations
Or you can specify a version with a Toolkit config descriptor uri.
> python bake_config.py "sgtk:descriptor:dev?version=v1.0.9&path=../tk-config-myconfig" /tmp/baked_configurations
Any type of Toolkit config descriptor uri can be used, if a version is not specified, the latest for the descriptor is resolved.
> python bake_config.py "sgtk:descriptor:app_store?name=tk-config-basic" /tmp/baked_configurations
{automated_setup_documentation}
For information about the various descriptors that can be used, see
http://developer.shotgunsoftware.com/tk-core/descriptor
""".format(automated_setup_documentation=automated_setup_documentation)
parser = OptionParserLineBreakingEpilog(usage=usage,
description=desc,
epilog=epilog)
parser.add_option("-d",
"--debug",
default=False,
action="store_true",
help="Enable debug logging")
parser.add_option("-z",
"--zip",
default=False,
action="store_true",
help="Zip archive the config")
parser.add_option("-r",
"--sparse",
default=False,
action="store_true",
help="Don't cache any app_store bundles")
add_authentication_options(parser)
# parse cmd line
(options, remaining_args) = parser.parse_args()
logger.info("Welcome to the Toolkit config baker.")
logger.info("")
if options.debug:
LogManager().global_debug = True
if len(remaining_args) != 2:
parser.print_help()
return 2
# Get config descriptor
config_descriptor = remaining_args[0]
# Try to parse it, check if it is a local path if it fails
try:
descriptor_uri_to_dict(config_descriptor)
except TankDescriptorError as e:
# Check if it is a local path
path = os.path.abspath(
os.path.expanduser(os.path.expandvars(config_descriptor)))
if os.path.isdir(path):
logger.info("Using a dev descriptor for local path %s" % path)
# Forge a dev descriptor, using "latest" for the version.
# TODO: try to retrieve a valid version from the folder, e.g. with a
# git tag from the folder.
config_descriptor = "sgtk:descriptor:dev?name=%s&path=%s&version=latest" % (
os.path.basename(path), path)
else:
logger.error("%s is not a valid descriptor nor a local path." %
config_descriptor)
raise
# Get output path
target_path = remaining_args[1]
target_path = os.path.expanduser(os.path.expandvars(target_path))
sg_user = authenticate(options)
sg_connection = sg_user.create_sg_connection()
# make sure we are properly connected
try:
sg_connection.find_one("HumanUser", [])
except Exception, e:
logger.error("Could not communicate with Shotgun: %s" % e)
return 3
python_folder = os.path.abspath(os.path.join(this_folder, "..", "python"))
sys.path.append(python_folder)
# sgtk imports
from tank import LogManager
from tank.util import filesystem
from tank.errors import TankError
from tank.platform import environment
from tank.descriptor import Descriptor, descriptor_uri_to_dict, descriptor_dict_to_uri, create_descriptor
from tank.authentication import ShotgunAuthenticator
from tank.bootstrap.baked_configuration import BakedConfiguration
from tank.bootstrap import constants as bootstrap_constants
from tank_vendor import yaml
# set up logging
logger = LogManager.get_logger("build_plugin")
# required keys in the info.yml plugin manifest file
REQUIRED_MANIFEST_PARAMETERS = ["base_configuration", "plugin_id"]
# the folder where all items will be cached
BUNDLE_CACHE_ROOT_FOLDER_NAME = "bundle_cache"
# when we are baking a config, use these settings
BAKED_BUNDLE_NAME = "tk-config-plugin"
BAKED_BUNDLE_VERSION = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
# generation of the build syntax
BUILD_GENERATION = 2
class OptionParserLineBreakingEpilog(optparse.OptionParser):
def main():
"""
Main entry point for script.
Handles argument parsing and validation and then calls the script payload.
"""
usage = "%prog [options] source_path target_path"
desc = "Builds a standard toolkit plugin structure ready for testing and deploy"
epilog = """
Details and Examples
--------------------
In its simplest form, just provide a source and target folder for the build.
> python build_plugin.py ~/dev/tk-maya/plugins/basic /tmp/maya-plugin
By default, the build script will use the latest app store core for its bootstrapping.
If you want to use a specific core for the bootstrap, this can be specified via the
--bootstrap-core-uri option:
> python build_plugin.py
--bootstrap-core-uri='sgtk:descriptor:dev?path=~/dev/tk-core'
~/dev/tk-maya/plugins/basic /tmp/maya-plugin
By using the '--bake' option, you can build a plugin with an immutable configuration
where every single Toolkit component is cached and frozen to the version retrieved at
build time. This can be useful to distribute a self contained plugin to third party
users.
> python build_plugin.py
~/dev/tk-maya/plugins/basic /tmp/maya-plugin
--bake
{automated_setup_documentation}
For information about the various descriptors that can be used, see
http://developer.shotgunsoftware.com/tk-core/descriptor
""".format(automated_setup_documentation=automated_setup_documentation)
parser = OptionParserLineBreakingEpilog(usage=usage,
description=desc,
epilog=epilog)
parser.add_option("-d",
"--debug",
default=False,
action="store_true",
help="Enable debug logging")
parser.add_option(
"-c",
"--bootstrap-core-uri",
default=None,
action="store",
help=
("Specify which version of core to be used by the bootstrap process. "
"If not specified, defaults to the most recently released core."))
parser.add_option("--bake",
default=False,
action="store_true",
help="Bake the plugin with an immutable configuration.")
parser.add_option(
"--system-core",
default=False,
action="store_true",
help=
"Use tk-core installed on the system rather than a private copy in the config."
)
add_authentication_options(parser)
# parse cmd line
(options, remaining_args) = parser.parse_args()
logger.info("Welcome to the Toolkit plugin builder.")
logger.info("")
if options.debug:
LogManager().global_debug = True
if options.system_core:
if options.bootstrap_core_uri:
parser.error(
"bootstrap-core-uri and system-core options are incompatible. "
"Please use one or the other but not both.")
if not options.bake:
parser.error(
"system-core option can only be used for baked plugins. "
"Please use the --bake option or do not use --system-core.")
if options.bootstrap_core_uri:
bootstrap_core_uri = options.bootstrap_core_uri
else:
# default
bootstrap_core_uri = None
if len(remaining_args) != 2:
parser.print_help()
return 2
# get paths
source_path = remaining_args[0]
target_path = remaining_args[1]
# convert any env vars and tildes
source_path = os.path.expanduser(os.path.expandvars(source_path))
target_path = os.path.expanduser(os.path.expandvars(target_path))
sg_user = authenticate(options)
sg_connection = sg_user.create_sg_connection()
# make sure we are properly connected
try:
sg_connection.find_one("HumanUser", [])
except Exception, e:
logger.error("Could not communicate with Shotgun: %s" % e)
return 3
# CONFIDENTIAL AND PROPRIETARY
#
# This work is provided "AS IS" and subject to the Shotgun Pipeline Toolkit
# Source Code License included in this distribution package. See LICENSE.
# By accessing, using, copying or modifying this work you indicate your
# agreement to the Shotgun Pipeline Toolkit Source Code License. All rights
# not expressly granted therein are reserved by Shotgun Software Inc.
import optparse
import os
from tank import LogManager
from tank.authentication import ShotgunAuthenticator
logger = LogManager.get_logger("utils.authentication")
automated_setup_documentation = """For automated build setups, you can provide a specific shotgun API script name and
and corresponding script key:
> python populate_bundle_cache.py
--shotgun-host='https://mysite.shotgunstudio.com'
--shotgun-script-name='plugin_build'
--shotgun-script-key='<script-key-here>'
"sgtk:descriptor:app_store?version=v0.3.6&name=tk-config-basic" /tmp
You can also use the SHOTGUN_HOST, SHOTGUN_SCRIPT_NAME and SHOTGUN_SCRIPT_KEY environment
variables to authenticate."""
def add_authentication_options(parser):
# CONFIDENTIAL AND PROPRIETARY
#
# This work is provided "AS IS" and subject to the Shotgun Pipeline Toolkit
# Source Code License included in this distribution package. See LICENSE.
# By accessing, using, copying or modifying this work you indicate your
# agreement to the Shotgun Pipeline Toolkit Source Code License. All rights
# not expressly granted therein are reserved by Shotgun Software Inc.
import optparse
import os
from tank import LogManager
from tank.authentication import ShotgunAuthenticator
logger = LogManager.get_logger("utils.authentication")
automated_setup_documentation = """For automated build setups, you can provide a specific shotgun API script name and
and corresponding script key:
> python populate_bundle_cache.py
--shotgun-host='https://mysite.shotgunstudio.com'
--shotgun-script-name='plugin_build'
--shotgun-script-key='<script-key-here>'
"sgtk:descriptor:app_store?version=v0.3.6&name=tk-config-basic" /tmp
You can also use the SHOTGUN_HOST, SHOTGUN_SCRIPT_NAME and SHOTGUN_SCRIPT_KEY environment
variables to authenticate."""
def add_authentication_options(parser):
# By accessing, using, copying or modifying this work you indicate your
# agreement to the Shotgun Pipeline Toolkit Source Code License. All rights
# not expressly granted therein are reserved by Shotgun Software Inc.
import os
import glob
import shutil
import stat
from tank.util import filesystem
from tank.platform import environment
from tank.descriptor import Descriptor, create_descriptor
from tank import LogManager
logger = LogManager.get_logger("utils.caching")
def _cache_descriptor(sg, desc_type, desc_dict, target_path):
"""
Cache the given descriptor into a new bundle cache.
:param sg: Shotgun API instance
:param desc_type: Descriptor.ENGINE | Descriptor.APP | Descriptor.FRAMEWORK
:param desc_dict: descriptor dict or uri
:param target_path: bundle cache root to cache into
"""
desc = create_descriptor(sg,
desc_type,
desc_dict,
fallback_roots=[target_path])
