def create_executor(self, task):
if task.is_resource():
return self.executors.create_local(task)
if task.is_alias():
return self.executors.create_skipper(task)
raise_task_error_if(
not self.cache.upload_enabled(), task,
"artifact upload must be enabled for workers, fix configuration")
if not task.is_cacheable():
return self.executors.create_local(task)
if task.is_available_locally(self.cache):
if task.is_goal() and not task.is_available_remotely(self.cache):
# Unpacked artifacts may become unpacked before we manage to upload.
# To keep the implementation simple we take the easy road and rebuild
# all artifacts that have not been unpacked, even if they are uploadable.
if task.is_unpacked(self.cache) and task.is_uploadable(
self.cache):
return self.executors.create_uploader(task)
else:
return self.executors.create_local(task, force=True)
return self.executors.create_skipper(task)
if not self.cache.download_enabled():
return self.executors.create_local(task)
if task.is_available_remotely(self.cache):
return self.executors.create_downloader(task)
return self.executors.create_local(task)
def import_manifest(self, manifest):
loader = JoltLoader.get()
loader.set_joltdir(manifest.joltdir)
for recipe in manifest.recipes:
recipe = Recipe(recipe.path, source=recipe.source)
recipe.save()
for project in manifest.projects:
for recipe in project.recipes:
loader._add_project_recipe(project.name, recipe.joltdir,
recipe.src)
for resource in project.resources:
loader._add_project_resource(project.name, resource.name,
resource.text)
# Acquire resource immediately
task = TaskRegistry.get().get_task(resource.text,
manifest=manifest)
raise_task_error_if(
not isinstance(task, WorkspaceResource), task,
"only workspace resources are allowed in manifest")
task.acquire_ws()
for module in project.modules:
loader._add_project_module(project.name, module.src)
sys.path.append(fs.path.join(manifest.joltdir, module.src))
def acquire(self, artifact, deps, tools):
raise_task_error_if(not self._user(tools), self, "Username has not been configured")
raise_task_error_if(not self._password(tools), self, "Password has not been configured")
with tools.cwd(tools.builddir()):
tools.write_file("docker-credential", self._password(tools))
tools.run("cat docker-credential | docker login -u {user} --password-stdin {server}", user=self._user(tools))
def run(self, deps, tools):
raise_task_error_if(not self.cmakelists, self,
"cmakelists attribute has not been defined")
cmake = tools.cmake()
cmake.configure(
tools.expand(self.cmakelists),
*["-D" + tools.expand(option) for option in self.options])
cmake.build()
cmake.install()
def _upload(self, env, task):
if self.is_aborted():
return
try:
task.started("Upload")
raise_task_error_if(not env.cache.upload(task), task,
"failed to upload task artifact")
except Exception as e:
task.failed("Upload")
raise e
else:
task.finished("Upload")
def finished(self, what="Execution"):
raise_task_error_if(self.is_completed() and not self.is_extension(),
self, "task has already been completed")
self._completed = True
try:
self.graph.remove_node(self)
except KeyError:
self.warning("Pruned task was executed")
self.task.info(
colors.green(what + " finished after {0} {1}" + self.log_name),
self.duration_running,
self.duration_queued.diff(self.duration_running))
hooks.task_finished(self)
def _download(self, env, task):
if self.is_aborted():
return
if not task.is_downloadable():
return
try:
task.started("Download")
raise_task_error_if(not env.cache.download(task), task,
"failed to download task artifact")
except Exception as e:
task.failed("Download")
raise e
else:
task.finished("Download")
def submit(self, cache, task):
if self._aborted:
return None
env = JoltEnvironment(cache=cache)
executor = self.strategy.create_executor(task)
raise_task_error_if(
not executor, task, "no executor can execute the task; "
"requesting a distributed network build without proper configuration?"
)
task.set_in_progress()
future = executor.submit(env)
self.futures[future] = task
return future
def _export(ctx, task):
acache = cache.ArtifactCache.get()
task = [utils.stable_task_name(t) for t in task]
registry = TaskRegistry.get()
executors = scheduler.ExecutorRegistry.get()
strategy = scheduler.LocalStrategy(executors, acache)
dag = graph.GraphBuilder(registry, ctx.obj["manifest"])
dag = dag.build(task)
gp = graph.GraphPruner(strategy)
dag = gp.prune(dag)
class Export(object):
def __init__(self):
self.environ = {}
self.prepend_environ = {}
def setenv(self, name, value):
self.environ[name] = value
class Context(object):
def __init__(self, tasks):
self.tasks = tasks
self.environ = set()
self.exports = {}
def add_export(self, task, visitor):
self.exports[task] = visitor
self.environ.update(set(visitor.environ.keys()))
tasks = list(
filter(lambda t: t.is_cacheable(), reversed(dag.topological_nodes)))
context = Context(tasks)
for task in context.tasks:
artifact = acache.get_artifact(task)
raise_task_error_if(
artifact.is_temporary(), task,
"Task artifact not found in local cache, build first")
visitor = Export()
cache.visit_artifact(task, artifact, visitor)
context.add_export(task, visitor)
script = utils.render("export.sh.template", ctx=context)
print(script)
def _upload(self, env, task):
if self.is_aborted():
return
try:
task.started("Upload")
hooks.task_started_upload(task)
raise_task_error_if(not env.cache.upload(task), task,
"Failed to upload task artifact")
except Exception as e:
with task.task.report() as report:
report.add_exception(e)
task.failed("Upload")
raise e
else:
hooks.task_finished_upload(task)
task.finished("Upload")
def run(self, deps, tools):
raise_task_error_if(not tools.which("conan"), self,
"Conan: Conan is not installed in the PATH")
raise_task_error_if(
self.conanfile
and (self._generators() or self._packages() or self._options()),
self,
"Conan: 'conanfile' attribute cannot be used with other attributes"
)
conanfile = tools.expand_path(
self.conanfile) if self.conanfile else None
with tools.cwd(tools.builddir()):
if conanfile is None or not path.exists(conanfile):
conanfile = "conanfile.txt"
self.info("Creating conanfile.txt")
self.tools.write_file(conanfile, "[requires]\n")
for pkg in self._packages():
self.tools.append_file(conanfile, pkg + "\n")
with tools.environ(CONAN_USER_HOME=tools.builddir(
"conan", incremental=self.incremental)):
for remote, url in self._remotes().items():
self.info("Registering remote '{}'", remote)
tools.run("conan remote add -f {} {}",
remote,
url,
output_on_error=True)
self.info("Installing packages into the Conan cache")
generators = " ".join(
["-g " + gen for gen in self._generators()])
options = " ".join(["-o " + opt for opt in self._options()])
settings = " ".join(["-s " + opt for opt in self._settings()])
tools.run("conan install --build=missing -u -if . {} {} {} {}",
generators, options, settings, conanfile)
self.info("Parsing manifest")
self._manifest = json.loads(tools.read_file("conanbuildinfo.json"))
for dep in self._manifest["dependencies"]:
self.info("Collecting '{}' files from: {}", dep["name"],
dep["rootpath"])
tools.copy(dep["rootpath"], dep["name"])
def _acquire_ws(self):
if not self.git.is_cloned():
self.git.clone()
if not self._revision.is_imported:
self.git.diff_unchecked()
rev = self._get_revision()
if rev is not None:
raise_task_error_if(
not self._revision.is_imported and not self.sha.is_unset()
and self.git.diff(), self,
"explicit sha requested but git repo '{0}' has local changes",
self.git.relpath)
# Should be safe to do this now
rev = self.git.rev_parse(rev)
if not self.git.is_head(rev) or self._revision.is_imported:
self.git.checkout(rev)
self.git.clean()
self.git.patch(self._diff.value)
def freeze(ctx, task, default, output, remove):
"""
Freeze the identity of a task.
<WIP>
"""
manifest = ctx.obj["manifest"]
options = JoltOptions(default=default)
acache = cache.ArtifactCache.get(options)
scheduler.ExecutorRegistry.get(options)
registry = TaskRegistry.get()
for params in default:
registry.set_default_parameters(params)
gb = graph.GraphBuilder(registry, manifest)
dag = gb.build(task)
available_in_cache = [
(t.is_available_locally(acache)
or (t.is_available_remotely(acache) and acache.download_enabled()), t)
for t in dag.tasks if t.is_cacheable()
]
for available, task in available_in_cache:
raise_task_error_if(
not remove and not available, task,
"task artifact is not available in any cache, build it first")
for task in dag.tasks:
if task.is_resource() or not task.is_cacheable():
continue
manifest_task = manifest.find_task(task)
if remove and manifest_task:
manifest.remove_task(manifest_task)
continue
if not remove:
if not manifest_task:
manifest_task = manifest.create_task()
manifest_task.name = task.qualified_name
manifest_task.identity = task.identity
manifest.write(fs.path.join(JoltLoader.get().joltdir, output))
def inspect(ctx, task, influence=False, artifact=False, salt=None):
"""
View information about a task.
This command displays information about a task, such as its class
documentation, parameters and their accepted values, requirements,
task class origin (file/line), influence attributes, artifact identity,
cache status, and more. Default parameter values, if any, are highlighted.
"""
task_name = task
task_cls_name, task_params = utils.parse_task_name(task_name)
task_registry = TaskRegistry.get()
task = task_registry.get_task_class(task_cls_name)
raise_task_error_if(not task, task_name, "no such task")
from jolt import inspection
print()
print(" {0}".format(task.name))
print()
if task.__doc__:
print(" {0}".format(task.__doc__.strip()))
print()
print(" Parameters")
has_param = False
params = {
key: getattr(task, key)
for key in dir(task)
if isinstance(utils.getattr_safe(task, key), Parameter)
}
for item, param in params.items():
has_param = True
print(" {0:<15} {1}".format(item, param.help or ""))
if not has_param:
print(" None")
print()
print(" Definition")
print(" {0:<15} {1} ({2})".format(
"File",
fs.path.relpath(inspection.getfile(task),
JoltLoader.get().joltdir), inspection.getlineno(task)))
print()
print(" Requirements")
manifest = ctx.obj["manifest"]
try:
task = task_registry.get_task(task_name, manifest=manifest)
for req in sorted(
utils.as_list(utils.call_or_return(task, task.requires))):
print(" {0}".format(task.tools.expand(req)))
if not task.requires:
print(" None")
print()
except Exception as e:
log.exception()
if "has not been set" in str(e):
print(" Unavailable (parameters must be set)")
print()
return
print(" Unavailable (exception during evaluation)")
print()
return
if salt:
task.taint = salt
if artifact:
acache = cache.ArtifactCache.get()
builder = graph.GraphBuilder(task_registry, manifest)
dag = builder.build([task.qualified_name])
tasks = dag.select(lambda graph, node: node.task is task)
assert len(tasks) == 1, "graph produced multiple tasks, one expected"
proxy = tasks[0]
task = proxy.task
print(" Cache")
print(" Identity {0}".format(proxy.identity))
if acache.is_available_locally(proxy):
with acache.get_artifact(proxy) as artifact:
print(" Location {0}".format(artifact.path))
print(" Local True ({0})".format(
utils.as_human_size(acache.get_artifact(proxy).get_size())))
else:
print(" Local False")
print(" Remote {0}".format(
acache.is_available_remotely(proxy)))
print()
if influence:
print(" Influence")
for string in HashInfluenceRegistry.get().get_strings(task):
string = string.split(":", 1)
print(" {:<18}{}".format(string[0][10:], string[1].strip()))
def _run(self, env):
timeout = int(config.getint("amqp", "timeout", 300))
manifest, routing_key = self._create_manifest()
self.connect()
self.publish_request(manifest, routing_key)
log.debug("[AMQP] Queued {0}", self.task.short_qualified_name)
self.task.running()
for extension in self.task.extensions:
extension.running()
while self.response is None:
try:
self.connection.process_data_events(time_limit=timeout)
if self.response is None:
self.task.info(
"Remote execution still in progress after {}",
self.task.duration_queued)
except (ConnectionError, AMQPConnectionError):
log.warning("[AMQP] Lost server connection")
self.connect()
log.debug("[AMQP] Finished {0}", self.task.short_qualified_name)
manifest = JoltManifest()
with raise_task_error_on_exception(
self.task, "failed to parse build result manifest"):
manifest.parsestring(self.response)
self.task.running(utils.duration() - float(manifest.duration))
if manifest.result != "SUCCESS":
output = []
if manifest.stdout:
output.extend(manifest.stdout.split("\n"))
if manifest.stderr:
output.extend(manifest.stderr.split("\n"))
for line in output:
log.transfer(line, self.task.identity[:8])
for task in [self.task] + self.task.extensions:
with task.task.report() as report:
remote_report = manifest.find_task(task.qualified_name)
if remote_report:
for error in remote_report.errors:
report.manifest.append(error)
raise_error("[AMQP] remote build failed with status: {0}".format(
manifest.result))
raise_task_error_if(
not env.cache.is_available_remotely(self.task), self.task,
"no task artifact available in any cache, check configuration")
raise_task_error_if(
not env.cache.download(self.task) and env.cache.download_enabled(),
self.task, "failed to download task artifact")
for extension in self.task.extensions:
raise_task_error_if(
not env.cache.download(extension)
and env.cache.download_enabled(), self.task,
"failed to download task artifact")
return self.task
def run(self, cache, force_upload=False, force_build=False):
with self.tools:
tasks = [self] + self.extensions
available_locally = available_remotely = False
for child in self.children:
if not child.has_artifact():
continue
if not cache.is_available_locally(child):
raise_task_error_if(not cache.download(child), child,
"failed to download task artifact")
if not force_build:
available_locally = all(map(cache.is_available_locally, tasks))
if available_locally and not force_upload:
return
available_remotely = cache.download_enabled() and \
all(map(cache.is_available_remotely, tasks))
if not available_locally and available_remotely:
available_locally = cache.download(self)
if force_build or not available_locally:
with log.threadsink() as buildlog:
if self.task.is_runnable():
log.verbose("Host: {0}",
getenv("HOSTNAME", "localhost"))
with cache.get_locked_artifact(
self, discard=force_build) as artifact:
if not cache.is_available_locally(
self) or self.has_extensions():
with cache.get_context(self) as context:
self.running()
with self.tools.cwd(self.task.joltdir):
hooks.task_prerun(self, context,
self.tools)
if self.is_goal() and self.options.debug:
log.info("Entering debug shell")
self.task.shell(context, self.tools)
self.task.run(context, self.tools)
hooks.task_postrun(self, context,
self.tools)
if not cache.is_available_locally(self):
with self.tools.cwd(self.task.joltdir):
hooks.task_prepublish(
self, artifact, self.tools)
self.task.publish(artifact, self.tools)
self.task._verify_influence(
context, artifact, self.tools)
hooks.task_postpublish(
self, artifact, self.tools)
with open(
fs.path.join(
artifact.path, ".build.log"),
"w") as f:
f.write(buildlog.getvalue())
cache.commit(artifact)
else:
self.info(
"Publication skipped, already in local cache"
)
else:
self.info(
"Execution skipped, already in local cache")
# Must upload the artifact while still holding its lock, otherwise the
# artifact may become unpack():ed before we have a chance to.
if force_upload or force_build or not available_remotely:
raise_task_error_if(
not cache.upload(
self, force=force_upload, locked=False)
and cache.upload_enabled(), self,
"failed to upload task artifact")
elif force_upload or not available_remotely:
raise_task_error_if(
not cache.upload(self, force=force_upload)
and cache.upload_enabled(), self,
"failed to upload task artifact")
for extension in self.extensions:
try:
extension.started()
with hooks.task_run(extension):
extension.run(cache, force_upload, force_build)
except Exception as e:
extension.failed()
raise e
else:
extension.finished()
