def import_manifest(self, manifest):
if manifest.config:
_manifest.read_string(manifest.config)
from jolt.loader import JoltLoader
JoltLoader.get().load_plugins()
for param in manifest.parameters:
if param.key.startswith("config."):
set("params", param.key.split(".", 1)[1], param.value)
def include(joltfile):
""" Include another Python file """
try:
from os import path
from sys import _getframe
from jolt.loader import JoltLoader
filepath = _getframe().f_back.f_code.co_filename
filepath = path.dirname(filepath)
filepath = path.join(filepath, joltfile)
JoltLoader.get()._load_file(filepath)
except Exception as e:
from jolt.error import raise_error
raise_error("failed to load '{0}': {1}", joltfile, str(e))
def __init__(self, *args, **kwargs):
super(GitSrc, self).__init__(*args, **kwargs)
self.joltdir = JoltLoader.get().joltdir
self.relpath = str(self.path) or self._get_name()
self.abspath = fs.path.join(self.joltdir, self.relpath)
self.refspecs = kwargs.get("refspecs", [])
self.git = new_git(self.url, self.abspath, self.relpath, self.refspecs)
def _autocomplete_tasks(ctx, args, incomplete):
manifest = JoltManifest()
utils.call_and_catch(manifest.parse)
manifest.process_import()
tasks = JoltLoader.get().load()
tasks = [
task.name for task in tasks if task.name.startswith(incomplete or '')
]
return sorted(tasks)
def get_command(self, ctx, cmd_name):
if cmd_name == "info":
cmd_name = "inspect"
if cmd_name in ["export", "inspect"]:
log.set_level(log.SILENCE)
elif ctx.params.get("verbose", False):
log.set_level(log.VERBOSE)
elif ctx.params.get("extra_verbose", False):
log.set_level(log.DEBUG)
config_files = ctx.params.get("config_file") or []
for config_file in config_files:
log.verbose("Config: {0}", config_file)
config.load_or_set(config_file)
# Load configured plugins
JoltLoader.get().load_plugins()
return click.Group.get_command(self, ctx, cmd_name)
def freeze(ctx, task, default, output, remove):
"""
Freeze the identity of a task.
<WIP>
"""
manifest = ctx.obj["manifest"]
options = JoltOptions(default=default)
acache = cache.ArtifactCache.get(options)
scheduler.ExecutorRegistry.get(options)
registry = TaskRegistry.get()
for params in default:
registry.set_default_parameters(params)
gb = graph.GraphBuilder(registry, manifest)
dag = gb.build(task)
available_in_cache = [
(t.is_available_locally(acache)
or (t.is_available_remotely(acache) and acache.download_enabled()), t)
for t in dag.tasks if t.is_cacheable()
]
for available, task in available_in_cache:
raise_task_error_if(
not remove and not available, task,
"task artifact is not available in any cache, build it first")
for task in dag.tasks:
if task.is_resource() or not task.is_cacheable():
continue
manifest_task = manifest.find_task(task)
if remove and manifest_task:
manifest.remove_task(manifest_task)
continue
if not remove:
if not manifest_task:
manifest_task = manifest.create_task()
manifest_task.name = task.qualified_name
manifest_task.identity = task.identity
manifest.write(fs.path.join(JoltLoader.get().joltdir, output))
def inspect(ctx, task, influence=False, artifact=False, salt=None):
"""
View information about a task.
This command displays information about a task, such as its class
documentation, parameters and their accepted values, requirements,
task class origin (file/line), influence attributes, artifact identity,
cache status, and more. Default parameter values, if any, are highlighted.
"""
task_name = task
task_cls_name, task_params = utils.parse_task_name(task_name)
task_registry = TaskRegistry.get()
task = task_registry.get_task_class(task_cls_name)
raise_task_error_if(not task, task_name, "no such task")
from jolt import inspection
print()
print(" {0}".format(task.name))
print()
if task.__doc__:
print(" {0}".format(task.__doc__.strip()))
print()
print(" Parameters")
has_param = False
params = {
key: getattr(task, key)
for key in dir(task)
if isinstance(utils.getattr_safe(task, key), Parameter)
}
for item, param in params.items():
has_param = True
print(" {0:<15} {1}".format(item, param.help or ""))
if not has_param:
print(" None")
print()
print(" Definition")
print(" {0:<15} {1} ({2})".format(
"File",
fs.path.relpath(inspection.getfile(task),
JoltLoader.get().joltdir), inspection.getlineno(task)))
print()
print(" Requirements")
manifest = ctx.obj["manifest"]
try:
task = task_registry.get_task(task_name, manifest=manifest)
for req in sorted(
utils.as_list(utils.call_or_return(task, task.requires))):
print(" {0}".format(task.tools.expand(req)))
if not task.requires:
print(" None")
print()
except Exception as e:
log.exception()
if "has not been set" in str(e):
print(" Unavailable (parameters must be set)")
print()
return
print(" Unavailable (exception during evaluation)")
print()
return
if salt:
task.taint = salt
if artifact:
acache = cache.ArtifactCache.get()
builder = graph.GraphBuilder(task_registry, manifest)
dag = builder.build([task.qualified_name])
tasks = dag.select(lambda graph, node: node.task is task)
assert len(tasks) == 1, "graph produced multiple tasks, one expected"
proxy = tasks[0]
task = proxy.task
print(" Cache")
print(" Identity {0}".format(proxy.identity))
if acache.is_available_locally(proxy):
with acache.get_artifact(proxy) as artifact:
print(" Location {0}".format(artifact.path))
print(" Local True ({0})".format(
utils.as_human_size(acache.get_artifact(proxy).get_size())))
else:
print(" Local False")
print(" Remote {0}".format(
acache.is_available_remotely(proxy)))
print()
if influence:
print(" Influence")
for string in HashInfluenceRegistry.get().get_strings(task):
string = string.split(":", 1)
print(" {:<18}{}".format(string[0][10:], string[1].strip()))
def cli(ctx, verbose, extra_verbose, config_file, debugger, profile, force,
salt, debug, network, local, keep_going, jobs):
"""
A task execution tool.
When invoked without any commands and arguments, Jolt by default tries
to execute and build the artifact of a task called `default`. To build
artifacts of other tasks use the build subcommand.
The Jolt command line interface is hierarchical. One set of options
can be passed to the top-level command and a different set of options
to the subcommands, simultaneously. For example, verbose output is
a top-level option while forced rebuild is a build command option.
They may combined like this:
$ jolt --verbose build --force taskname
Most build command options are available also at the top-level when
build is invoked implicitly for the default task.
"""
global debug_enabled
debug_enabled = debugger
log.verbose("Jolt command: {}",
" ".join([fs.path.basename(sys.argv[0])] + sys.argv[1:]))
log.verbose("Jolt host: {}", environ.get("HOSTNAME", "localhost"))
log.verbose("Jolt install path: {}", fs.path.dirname(__file__))
if ctx.invoked_subcommand in ["config"]:
# Don't attempt to load any task recipes as they might require
# plugins that are not yet configured.
return
if ctx.invoked_subcommand is None:
build = ctx.command.get_command(ctx, "build")
manifest = JoltManifest()
utils.call_and_catch(manifest.parse)
manifest.process_import()
ctx.obj["manifest"] = manifest
if manifest.version:
from jolt.version_utils import requirement, version
req = requirement(manifest.version)
ver = version(__version__)
raise_error_if(not req.satisfied(ver),
"this project requires Jolt version {} (running {})",
req, __version__)
loader = JoltLoader.get()
tasks = loader.load()
for cls in tasks:
TaskRegistry.get().add_task_class(cls)
if ctx.invoked_subcommand in ["build", "clean"] and loader.joltdir:
ctx.obj["workspace_lock"] = utils.LockFile(
fs.path.join(loader.joltdir, "build"), log.info,
"Workspace is locked by another process, please wait...")
atexit.register(ctx.obj["workspace_lock"].close)
# If no command is given, we default to building the default task.
# If the default task doesn't exist, help is printed inside build().
if ctx.invoked_subcommand is None:
task = config.get("jolt", "default", "default")
taskname, _ = utils.parse_task_name(task)
if TaskRegistry.get().get_task_class(taskname) is not None:
ctx.invoke(build,
task=[task],
force=force,
salt=salt,
debug=debug,
network=network,
local=local,
keep_going=keep_going,
jobs=jobs)
else:
print(cli.get_help(ctx))
sys.exit(1)
def loaderdir(self):
return JoltLoader.get().joltdir
