def _validate_manifest(source_path): """ Validate that the manifest file is present and valid. :param source_path: Source path to plugin :return: parsed yaml content of manifest file """ # check for source manifest file manifest_path = os.path.join(source_path, "info.yml") if not os.path.exists(manifest_path): raise TankError("Cannot find plugin manifest '%s'" % manifest_path) logger.debug("Reading %s" % manifest_path) try: with open(manifest_path, "rt") as fh: manifest_data = yaml.load(fh) except Exception, e: raise TankError("Cannot parse info.yml manifest: %s" % e)
def _validate_manifest(source_path): """ Validate that the manifest file is present and valid. :param source_path: Source path to plugin :return: parsed yaml content of manifest file """ # check for source manifest file manifest_path = os.path.join(source_path, "info.yml") if not os.path.exists(manifest_path): raise TankError("Cannot find plugin manifest '%s'" % manifest_path) logger.debug("Reading %s" % manifest_path) try: with open(manifest_path, "rt") as fh: manifest_data = yaml.load(fh) except Exception as e: raise TankError("Cannot parse info.yml manifest: %s" % e) logger.debug("Validating manifest...") # legacy check - if we find entry_point, convert it across # to be plugin_id if "entry_point" in manifest_data: logger.warning( "Found legacy entry_point syntax. Please upgrade to use plugin_id instead." ) manifest_data["plugin_id"] = manifest_data["entry_point"] for parameter in REQUIRED_MANIFEST_PARAMETERS: if parameter not in manifest_data: raise TankError( "Required plugin manifest parameter '%s' missing in '%s'" % (parameter, manifest_path)) # plugin_id needs to be alpha numeric + period if re.search(r"^[a-zA-Z0-9_\.]+$", manifest_data["plugin_id"]) is None: raise TankError( "Plugin id can only contain alphanumerics, period and underscore characters." ) return manifest_data
def _bake_manifest(manifest_data, config_uri, core_descriptor, plugin_root): """ Bake the info.yml manifest into a python file. :param manifest_data: info.yml manifest data :param config_uri: Configuration descriptor uri string to use at runtime :param core_descriptor: descriptor object pointing at core to use for bootstrap :param plugin_root: Root path for plugin """ # suffix our generated python module with plugin id for uniqueness # replace all non-alphanumeric chars with underscores. module_name = "sgtk_plugin_%s" % re.sub("\W", "_", manifest_data["plugin_id"]) full_module_path = os.path.join(plugin_root, "python", module_name) filesystem.ensure_folder_exists(full_module_path) # write __init__.py try: with open(os.path.join(full_module_path, "__init__.py"), "wt") as fh: fh.write("# this file was auto generated.\n") fh.write("from . import manifest\n") fh.write("# end of file.\n") except Exception, e: raise TankError("Cannot write __init__.py file: %s" % e)
def build_plugin(sg_connection, source_path, target_path, buildable, bootstrap_core_uri=None): """ Perform a build of a plugin. This will introspect the info.yml in the source path, copy over everything in the source path into target path and then establish a bundle cache containing a reflection of all items required by the config. :param sg_connection: Shotgun connection :param source_path: Path to plugin. :param target_path: Path to build :param buildable: True if the resulting plugin build should be buildable :param bootstrap_core_uri: Custom bootstrap core uri. If None, the latest core from the app store will be used. """ logger.info("Your toolkit plugin in '%s' will be processed." % source_path) logger.info("The build will generated into '%s'" % target_path) # check for existence if not os.path.exists(source_path): raise TankError("Source path '%s' cannot be found on disk!" % source_path) # check that target path doesn't exist if os.path.exists(target_path): logger.info( "The folder '%s' already exists on disk. Moving it to backup location" % target_path) filesystem.backup_folder(target_path) shutil.rmtree(target_path) # try to create target path filesystem.ensure_folder_exists(target_path) # check manifest manifest_data = _validate_manifest(source_path) # copy all plugin data across # skip info.yml, this is baked into the manifest python code logger.info("Copying plugin data across...") skip_list = [] if buildable else [".git", "info.yml"] filesystem.copy_folder(source_path, target_path, skip_list=skip_list) # create bundle cache logger.info("Creating bundle cache folder...") bundle_cache_root = os.path.join(target_path, BUNDLE_CACHE_ROOT_FOLDER_NAME) filesystem.ensure_folder_exists(bundle_cache_root) # resolve config descriptor # the config_uri_str returned by the method contains the fully resolved # uri to use at runtime - in the case of baked descriptors, the config_uri_str # contains a manual descriptor uri. (cfg_descriptor, config_uri_str) = _process_configuration(sg_connection, source_path, target_path, bundle_cache_root, manifest_data, buildable) # cache config in bundle cache logger.info("Downloading and caching config...") # copy the config payload across to the plugin bundle cache cfg_descriptor.clone_cache(bundle_cache_root) # cache all apps, engines and frameworks _cache_apps(sg_connection, cfg_descriptor, bundle_cache_root) # get latest core - cache it directly into the plugin root folder if bootstrap_core_uri: logger.info("Caching custom core for boostrap (%s)" % bootstrap_core_uri) bootstrap_core_desc = create_descriptor( sg_connection, Descriptor.CORE, bootstrap_core_uri, bundle_cache_root_override=bundle_cache_root) else: # by default, use latest core for bootstrap logger.info( "Caching latest official core to use when bootstrapping plugin.") logger.info( "(To use a specific config instead, specify a --bootstrap-core-uri flag.)" ) bootstrap_core_desc = create_descriptor( sg_connection, Descriptor.CORE, { "type": "app_store", "name": "tk-core" }, resolve_latest=True, bundle_cache_root_override=bundle_cache_root) # cache it bootstrap_core_desc.ensure_local() # make a python folder where we put our manifest logger.info("Creating configuration manifest...") # bake out the manifest into python files. _bake_manifest(manifest_data, config_uri_str, bootstrap_core_desc, target_path) # now analyze what core the config needs if cfg_descriptor.associated_core_descriptor: logger.info( "Config is specifying a custom core in config/core/core_api.yml.") logger.info("This will be used when the config is executing.") logger.info("Ensuring this core (%s) is cached..." % cfg_descriptor.associated_core_descriptor) associated_core_desc = create_descriptor( sg_connection, Descriptor.CORE, cfg_descriptor.associated_core_descriptor, bundle_cache_root_override=bundle_cache_root) associated_core_desc.ensure_local() logger.info("") logger.info("Build complete!") logger.info("") logger.info("- Your plugin is ready in '%s'" % target_path) logger.info("- Plugin uses config %r" % cfg_descriptor) logger.info("- Bootstrap core is %r" % bootstrap_core_desc) logger.info( "- All dependencies have been baked out into the bundle_cache folder") if buildable: logger.info( "- The plugin can be used as a source for building further plugins." ) logger.info("") logger.info("") logger.info("")
raise TankError("Cannot parse info.yml manifest: %s" % e) logger.debug("Validating manifest...") # legacy check - if we find entry_point, convert it across # to be plugin_id if "entry_point" in manifest_data: logger.warning( "Found legacy entry_point syntax. Please upgrade to use plugin_id instead." ) manifest_data["plugin_id"] = manifest_data["entry_point"] for parameter in REQUIRED_MANIFEST_PARAMETERS: if parameter not in manifest_data: raise TankError( "Required plugin manifest parameter '%s' missing in '%s'" % (parameter, manifest_path)) # plugin_id needs to be alpha numeric + period if re.search("^[a-zA-Z0-9_\.]+$", manifest_data["plugin_id"]) is None: raise TankError( "Plugin id can only contain alphanumerics, period and underscore characters." ) return manifest_data def _bake_manifest(manifest_data, config_uri, core_descriptor, plugin_root): """ Bake the info.yml manifest into a python file.
def build_plugin(sg_connection, source_path, target_path, bootstrap_core_uri=None, do_bake=False, use_system_core=False): """ Perform a build of a plugin. This will introspect the info.yml in the source path, copy over everything in the source path into target path and then establish a bundle cache containing a reflection of all items required by the config. :param sg_connection: Shotgun connection :param source_path: Path to plugin. :param target_path: Path to build :param bootstrap_core_uri: Custom bootstrap core uri. If None, the latest core from the app store will be used. :param bool do_bake: If True, bake the plugin prior to building it. :param bool use_system_core: If True, use a globally installed tk-core instead of the one specified in the configuration. """ logger.info("Your toolkit plugin in '%s' will be processed." % source_path) logger.info("The build will %s into '%s'" % (["generated", "baked"][do_bake], target_path)) # check for existence if not os.path.exists(source_path): raise TankError("Source path '%s' cannot be found on disk!" % source_path) # check manifest manifest_data = _validate_manifest(source_path) if do_bake: baked_descriptor = _bake_configuration( sg_connection, manifest_data, ) # When baking we control the output path by adding a folder based on the # configuration descriptor and version. target_path = os.path.join( target_path, "%s-%s" % (baked_descriptor["name"], baked_descriptor["version"])) # check that target path doesn't exist if os.path.exists(target_path): logger.info("The folder '%s' already exists on disk. Removing it" % target_path) wipe_folder(target_path) # try to create target path filesystem.ensure_folder_exists(target_path) # copy all plugin data across # skip info.yml, this is baked into the manifest python code logger.info("Copying plugin data across...") filesystem.copy_folder(source_path, target_path) # create bundle cache logger.info("Creating bundle cache folder...") bundle_cache_root = os.path.join(target_path, BUNDLE_CACHE_ROOT_FOLDER_NAME) filesystem.ensure_folder_exists(bundle_cache_root) # resolve config descriptor # the config_uri_str returned by the method contains the fully resolved # uri to use at runtime - in the case of baked descriptors, the config_uri_str # contains a manual descriptor uri and install_path is set with the baked # folder. (cfg_descriptor, config_uri_str, install_path) = _process_configuration(sg_connection, source_path, target_path, bundle_cache_root, manifest_data, use_system_core) # cache config in bundle cache logger.info("Downloading and caching config...") # copy the config payload across to the plugin bundle cache cfg_descriptor.clone_cache(bundle_cache_root) # cache all apps, engines and frameworks cache_apps(sg_connection, cfg_descriptor, bundle_cache_root) if use_system_core: logger.info( "An external core will be used for this plugin, not caching it") bootstrap_core_desc = None else: # get core - cache it directly into the plugin root folder if bootstrap_core_uri: logger.info("Caching custom core for boostrap (%s)" % bootstrap_core_uri) bootstrap_core_desc = create_descriptor( sg_connection, Descriptor.CORE, bootstrap_core_uri, resolve_latest=is_descriptor_version_missing( bootstrap_core_uri), bundle_cache_root_override=bundle_cache_root) # cache it bootstrap_core_desc.ensure_local() elif not cfg_descriptor.associated_core_descriptor: # by default, use latest core for bootstrap logger.info( "Caching latest official core to use when bootstrapping plugin." ) logger.info( "(To use a specific config instead, specify a --bootstrap-core-uri flag.)" ) bootstrap_core_desc = create_descriptor( sg_connection, Descriptor.CORE, { "type": "app_store", "name": "tk-core" }, resolve_latest=True, bundle_cache_root_override=bundle_cache_root) # cache it bootstrap_core_desc.ensure_local() else: # The bootstrap core will be derived from the associated core desc below. bootstrap_core_desc = None # now analyze what core the config needs if not use_system_core and cfg_descriptor.associated_core_descriptor: logger.info( "Config is specifying a custom core in config/core/core_api.yml.") logger.info("This will be used when the config is executing.") logger.info("Ensuring this core (%s) is cached..." % cfg_descriptor.associated_core_descriptor) associated_core_desc = create_descriptor( sg_connection, Descriptor.CORE, cfg_descriptor.associated_core_descriptor, bundle_cache_root_override=bundle_cache_root) associated_core_desc.ensure_local() if bootstrap_core_desc is None: # Use the same version as the one specified by the config. if install_path: # Install path is set only if the config was baked. We re-use the # install path as an optimisation to avoid core swapping when the # config is bootstrapped. logger.info( "Bootstrapping will use installed %s required by the config" % associated_core_desc) # If the core was installed we directly use it. bootstrap_core_desc = create_descriptor( sg_connection, Descriptor.CORE, { "type": "path", "name": "tk-core", "path": os.path.join(install_path, "install", "core"), "version": associated_core_desc.version, }, resolve_latest=False, bundle_cache_root_override=bundle_cache_root) else: logger.info( "Bootstrapping will use core %s required by the config" % associated_core_desc) bootstrap_core_desc = associated_core_desc # make a python folder where we put our manifest logger.info("Creating configuration manifest...") # bake out the manifest into python files. _bake_manifest(manifest_data, config_uri_str, bootstrap_core_desc, target_path) cleanup_bundle_cache(bundle_cache_root) logger.info("") logger.info("Build complete!") logger.info("") logger.info("- Your plugin is ready in '%s'" % target_path) logger.info("- Plugin uses config %r" % cfg_descriptor) if bootstrap_core_desc: logger.info("- Bootstrap core is %r" % bootstrap_core_desc) else: logger.info("- Plugin will need an external installed core.") logger.info( "- All dependencies have been baked out into the bundle_cache folder") logger.info("") logger.info("") logger.info("")
def _bake_manifest(manifest_data, config_uri, core_descriptor, plugin_root): """ Bake the info.yml manifest into a python file. :param manifest_data: info.yml manifest data :param config_uri: Configuration descriptor uri string to use at runtime :param core_descriptor: descriptor object pointing at core to use for bootstrap :param plugin_root: Root path for plugin """ # suffix our generated python module with plugin id for uniqueness # replace all non-alphanumeric chars with underscores. module_name = "sgtk_plugin_%s" % re.sub(r"\W", "_", manifest_data["plugin_id"]) full_module_path = os.path.join(plugin_root, "python", module_name) filesystem.ensure_folder_exists(full_module_path) # write __init__.py try: with open(os.path.join(full_module_path, "__init__.py"), "wt") as fh: fh.write("# this file was auto generated.\n") fh.write("from . import manifest\n") fh.write("# end of file.\n") except Exception as e: raise TankError("Cannot write __init__.py file: %s" % e) # now bake out the manifest into code params_path = os.path.join(full_module_path, "manifest.py") try: with open(params_path, "wt") as fh: fh.write("# this file was auto generated.\n\n\n") fh.write('base_configuration="%s"\n' % config_uri) for (parameter, value) in manifest_data.items(): if parameter == "base_configuration": continue if isinstance(value, str): fh.write('%s="%s"\n' % (parameter, value.replace('"', "'"))) elif isinstance(value, int): fh.write("%s=%d\n" % (parameter, value)) elif isinstance(value, bool): fh.write("%s=%s\n" % (parameter, value)) else: raise ValueError( "Invalid manifest value %s: %s - data type not supported!" % (parameter, value)) fh.write("\n\n# system generated parameters\n") fh.write('BUILD_DATE="%s"\n' % datetime.datetime.now().strftime("%Y%m%d_%H%M%S")) fh.write("BUILD_GENERATION=%d\n" % BUILD_GENERATION) # Write out helper function 'get_sgtk_pythonpath()'. # this makes it easy for a plugin to import sgtk if not core_descriptor: # If we don't have core_descriptor, the plugin will use the # system installed tk-core. Arguably in that case we don't need # this method, but let's keep things consistent. fh.write("\n\n") fh.write("def get_sgtk_pythonpath(plugin_root):\n") fh.write(' """ \n') fh.write( " Auto generated helper method which returns the \n") fh.write(" path to the core bundled with the plugin.\n") fh.write(" \n") fh.write(" For more information, see the documentation.\n") fh.write(' """ \n') fh.write(" import os\n") fh.write(" import sgtk\n") fh.write( " return os.path.dirname(os.path.dirname(sgtk.__file__))\n" ) fh.write("\n\n") elif core_descriptor.get_path().startswith(plugin_root): # The core descriptor is cached inside our plugin, build a relative # path from the plugin root. core_path_parts = os.path.normpath( core_descriptor.get_path()).split(os.path.sep) core_path_relative_parts = core_path_parts[ core_path_parts.index(BUNDLE_CACHE_ROOT_FOLDER_NAME):] core_path_relative_parts.append("python") fh.write("\n\n") fh.write("def get_sgtk_pythonpath(plugin_root):\n") fh.write(' """ \n') fh.write( " Auto generated helper method which returns the \n") fh.write(" path to the core bundled with the plugin.\n") fh.write(" \n") fh.write(" For more information, see the documentation.\n") fh.write(' """ \n') fh.write(" import os\n") fh.write(" return os.path.join(plugin_root, %s)\n" % ", ".join('"%s"' % dir for dir in core_path_relative_parts)) fh.write("\n\n") else: # the core descriptor is outside of bundle cache! logger.warning( "Your core %r has its payload outside the plugin bundle cache. " "This plugin cannot be distributed to others." % core_descriptor) core_path_parts = os.path.normpath( core_descriptor.get_path()).split(os.path.sep) core_path_parts.append("python") # because we are using an external core, the plugin_root parameter # is simply ignored in any calls from the plugin code to # get_sgtk_pythonpath() fh.write("\n\n") fh.write("def get_sgtk_pythonpath(plugin_root):\n") fh.write( " # NOTE - this was built with a core that is not part of the plugin. \n" ) fh.write( " # The plugin_root parameter is therefore ignored.\n") fh.write( " # This is normally only done during development and \n" ) fh.write( " # typically means that the plugin cannot run on other machines \n" ) fh.write(" # than the one where it was built. \n") fh.write(" # \n") fh.write( " # For more information, see the documentation.\n") fh.write(" # \n") fh.write(" return r'%s'\n" % os.path.sep.join(core_path_parts)) fh.write("\n\n") # Write out helper function 'initialize_manager()'. # This method is a convenience method to make it easier to # set up the bootstrap manager given a plugin config fh.write("\n\n") fh.write("def initialize_manager(manager, plugin_root):\n") fh.write(' """ \n') fh.write(" Auto generated helper method which initializes\n") fh.write(" a toolkit manager with common plugin parameters.\n") fh.write(" \n") fh.write(" For more information, see the documentation.\n") fh.write(' """ \n') fh.write(" import os\n") # set base configuration fh.write(" manager.base_configuration = '%s'\n" % config_uri) # set entry point fh.write(" manager.plugin_id = '%s'\n" % manifest_data["plugin_id"]) # set shotgun config lookup flag if defined if "do_shotgun_config_lookup" in manifest_data: fh.write(" manager.do_shotgun_config_lookup = %s\n" % manifest_data["do_shotgun_config_lookup"]) # set bundle cache fallback path fh.write( " bundle_cache_path = os.path.join(plugin_root, 'bundle_cache')\n" ) fh.write( " manager.bundle_cache_fallback_paths = [bundle_cache_path]\n" ) fh.write(" return manager\n") fh.write("\n\n") fh.write("# end of file.\n") except Exception as e: logger.exception(e) raise TankError("Cannot write manifest file: %s" % e)