def create_executor(self, task): if task.is_resource(): return self.executors.create_local(task) if task.is_alias(): return self.executors.create_skipper(task) raise_task_error_if( not self.cache.upload_enabled(), task, "artifact upload must be enabled for workers, fix configuration") if not task.is_cacheable(): return self.executors.create_local(task) if task.is_available_locally(self.cache): if task.is_goal() and not task.is_available_remotely(self.cache): # Unpacked artifacts may become unpacked before we manage to upload. # To keep the implementation simple we take the easy road and rebuild # all artifacts that have not been unpacked, even if they are uploadable. if task.is_unpacked(self.cache) and task.is_uploadable( self.cache): return self.executors.create_uploader(task) else: return self.executors.create_local(task, force=True) return self.executors.create_skipper(task) if not self.cache.download_enabled(): return self.executors.create_local(task) if task.is_available_remotely(self.cache): return self.executors.create_downloader(task) return self.executors.create_local(task)
def import_manifest(self, manifest): loader = JoltLoader.get() loader.set_joltdir(manifest.joltdir) for recipe in manifest.recipes: recipe = Recipe(recipe.path, source=recipe.source) recipe.save() for project in manifest.projects: for recipe in project.recipes: loader._add_project_recipe(project.name, recipe.joltdir, recipe.src) for resource in project.resources: loader._add_project_resource(project.name, resource.name, resource.text) # Acquire resource immediately task = TaskRegistry.get().get_task(resource.text, manifest=manifest) raise_task_error_if( not isinstance(task, WorkspaceResource), task, "only workspace resources are allowed in manifest") task.acquire_ws() for module in project.modules: loader._add_project_module(project.name, module.src) sys.path.append(fs.path.join(manifest.joltdir, module.src))
def acquire(self, artifact, deps, tools): raise_task_error_if(not self._user(tools), self, "Username has not been configured") raise_task_error_if(not self._password(tools), self, "Password has not been configured") with tools.cwd(tools.builddir()): tools.write_file("docker-credential", self._password(tools)) tools.run("cat docker-credential | docker login -u {user} --password-stdin {server}", user=self._user(tools))
def run(self, deps, tools): raise_task_error_if(not self.cmakelists, self, "cmakelists attribute has not been defined") cmake = tools.cmake() cmake.configure( tools.expand(self.cmakelists), *["-D" + tools.expand(option) for option in self.options]) cmake.build() cmake.install()
def _upload(self, env, task): if self.is_aborted(): return try: task.started("Upload") raise_task_error_if(not env.cache.upload(task), task, "failed to upload task artifact") except Exception as e: task.failed("Upload") raise e else: task.finished("Upload")
def finished(self, what="Execution"): raise_task_error_if(self.is_completed() and not self.is_extension(), self, "task has already been completed") self._completed = True try: self.graph.remove_node(self) except KeyError: self.warning("Pruned task was executed") self.task.info( colors.green(what + " finished after {0} {1}" + self.log_name), self.duration_running, self.duration_queued.diff(self.duration_running)) hooks.task_finished(self)
def _download(self, env, task): if self.is_aborted(): return if not task.is_downloadable(): return try: task.started("Download") raise_task_error_if(not env.cache.download(task), task, "failed to download task artifact") except Exception as e: task.failed("Download") raise e else: task.finished("Download")
def submit(self, cache, task): if self._aborted: return None env = JoltEnvironment(cache=cache) executor = self.strategy.create_executor(task) raise_task_error_if( not executor, task, "no executor can execute the task; " "requesting a distributed network build without proper configuration?" ) task.set_in_progress() future = executor.submit(env) self.futures[future] = task return future
def _export(ctx, task): acache = cache.ArtifactCache.get() task = [utils.stable_task_name(t) for t in task] registry = TaskRegistry.get() executors = scheduler.ExecutorRegistry.get() strategy = scheduler.LocalStrategy(executors, acache) dag = graph.GraphBuilder(registry, ctx.obj["manifest"]) dag = dag.build(task) gp = graph.GraphPruner(strategy) dag = gp.prune(dag) class Export(object): def __init__(self): self.environ = {} self.prepend_environ = {} def setenv(self, name, value): self.environ[name] = value class Context(object): def __init__(self, tasks): self.tasks = tasks self.environ = set() self.exports = {} def add_export(self, task, visitor): self.exports[task] = visitor self.environ.update(set(visitor.environ.keys())) tasks = list( filter(lambda t: t.is_cacheable(), reversed(dag.topological_nodes))) context = Context(tasks) for task in context.tasks: artifact = acache.get_artifact(task) raise_task_error_if( artifact.is_temporary(), task, "Task artifact not found in local cache, build first") visitor = Export() cache.visit_artifact(task, artifact, visitor) context.add_export(task, visitor) script = utils.render("export.sh.template", ctx=context) print(script)
def _upload(self, env, task): if self.is_aborted(): return try: task.started("Upload") hooks.task_started_upload(task) raise_task_error_if(not env.cache.upload(task), task, "Failed to upload task artifact") except Exception as e: with task.task.report() as report: report.add_exception(e) task.failed("Upload") raise e else: hooks.task_finished_upload(task) task.finished("Upload")
def run(self, deps, tools): raise_task_error_if(not tools.which("conan"), self, "Conan: Conan is not installed in the PATH") raise_task_error_if( self.conanfile and (self._generators() or self._packages() or self._options()), self, "Conan: 'conanfile' attribute cannot be used with other attributes" ) conanfile = tools.expand_path( self.conanfile) if self.conanfile else None with tools.cwd(tools.builddir()): if conanfile is None or not path.exists(conanfile): conanfile = "conanfile.txt" self.info("Creating conanfile.txt") self.tools.write_file(conanfile, "[requires]\n") for pkg in self._packages(): self.tools.append_file(conanfile, pkg + "\n") with tools.environ(CONAN_USER_HOME=tools.builddir( "conan", incremental=self.incremental)): for remote, url in self._remotes().items(): self.info("Registering remote '{}'", remote) tools.run("conan remote add -f {} {}", remote, url, output_on_error=True) self.info("Installing packages into the Conan cache") generators = " ".join( ["-g " + gen for gen in self._generators()]) options = " ".join(["-o " + opt for opt in self._options()]) settings = " ".join(["-s " + opt for opt in self._settings()]) tools.run("conan install --build=missing -u -if . {} {} {} {}", generators, options, settings, conanfile) self.info("Parsing manifest") self._manifest = json.loads(tools.read_file("conanbuildinfo.json")) for dep in self._manifest["dependencies"]: self.info("Collecting '{}' files from: {}", dep["name"], dep["rootpath"]) tools.copy(dep["rootpath"], dep["name"])
def _acquire_ws(self): if not self.git.is_cloned(): self.git.clone() if not self._revision.is_imported: self.git.diff_unchecked() rev = self._get_revision() if rev is not None: raise_task_error_if( not self._revision.is_imported and not self.sha.is_unset() and self.git.diff(), self, "explicit sha requested but git repo '{0}' has local changes", self.git.relpath) # Should be safe to do this now rev = self.git.rev_parse(rev) if not self.git.is_head(rev) or self._revision.is_imported: self.git.checkout(rev) self.git.clean() self.git.patch(self._diff.value)
def freeze(ctx, task, default, output, remove): """ Freeze the identity of a task. <WIP> """ manifest = ctx.obj["manifest"] options = JoltOptions(default=default) acache = cache.ArtifactCache.get(options) scheduler.ExecutorRegistry.get(options) registry = TaskRegistry.get() for params in default: registry.set_default_parameters(params) gb = graph.GraphBuilder(registry, manifest) dag = gb.build(task) available_in_cache = [ (t.is_available_locally(acache) or (t.is_available_remotely(acache) and acache.download_enabled()), t) for t in dag.tasks if t.is_cacheable() ] for available, task in available_in_cache: raise_task_error_if( not remove and not available, task, "task artifact is not available in any cache, build it first") for task in dag.tasks: if task.is_resource() or not task.is_cacheable(): continue manifest_task = manifest.find_task(task) if remove and manifest_task: manifest.remove_task(manifest_task) continue if not remove: if not manifest_task: manifest_task = manifest.create_task() manifest_task.name = task.qualified_name manifest_task.identity = task.identity manifest.write(fs.path.join(JoltLoader.get().joltdir, output))
def inspect(ctx, task, influence=False, artifact=False, salt=None): """ View information about a task. This command displays information about a task, such as its class documentation, parameters and their accepted values, requirements, task class origin (file/line), influence attributes, artifact identity, cache status, and more. Default parameter values, if any, are highlighted. """ task_name = task task_cls_name, task_params = utils.parse_task_name(task_name) task_registry = TaskRegistry.get() task = task_registry.get_task_class(task_cls_name) raise_task_error_if(not task, task_name, "no such task") from jolt import inspection print() print(" {0}".format(task.name)) print() if task.__doc__: print(" {0}".format(task.__doc__.strip())) print() print(" Parameters") has_param = False params = { key: getattr(task, key) for key in dir(task) if isinstance(utils.getattr_safe(task, key), Parameter) } for item, param in params.items(): has_param = True print(" {0:<15} {1}".format(item, param.help or "")) if not has_param: print(" None") print() print(" Definition") print(" {0:<15} {1} ({2})".format( "File", fs.path.relpath(inspection.getfile(task), JoltLoader.get().joltdir), inspection.getlineno(task))) print() print(" Requirements") manifest = ctx.obj["manifest"] try: task = task_registry.get_task(task_name, manifest=manifest) for req in sorted( utils.as_list(utils.call_or_return(task, task.requires))): print(" {0}".format(task.tools.expand(req))) if not task.requires: print(" None") print() except Exception as e: log.exception() if "has not been set" in str(e): print(" Unavailable (parameters must be set)") print() return print(" Unavailable (exception during evaluation)") print() return if salt: task.taint = salt if artifact: acache = cache.ArtifactCache.get() builder = graph.GraphBuilder(task_registry, manifest) dag = builder.build([task.qualified_name]) tasks = dag.select(lambda graph, node: node.task is task) assert len(tasks) == 1, "graph produced multiple tasks, one expected" proxy = tasks[0] task = proxy.task print(" Cache") print(" Identity {0}".format(proxy.identity)) if acache.is_available_locally(proxy): with acache.get_artifact(proxy) as artifact: print(" Location {0}".format(artifact.path)) print(" Local True ({0})".format( utils.as_human_size(acache.get_artifact(proxy).get_size()))) else: print(" Local False") print(" Remote {0}".format( acache.is_available_remotely(proxy))) print() if influence: print(" Influence") for string in HashInfluenceRegistry.get().get_strings(task): string = string.split(":", 1) print(" {:<18}{}".format(string[0][10:], string[1].strip()))
def _run(self, env): timeout = int(config.getint("amqp", "timeout", 300)) manifest, routing_key = self._create_manifest() self.connect() self.publish_request(manifest, routing_key) log.debug("[AMQP] Queued {0}", self.task.short_qualified_name) self.task.running() for extension in self.task.extensions: extension.running() while self.response is None: try: self.connection.process_data_events(time_limit=timeout) if self.response is None: self.task.info( "Remote execution still in progress after {}", self.task.duration_queued) except (ConnectionError, AMQPConnectionError): log.warning("[AMQP] Lost server connection") self.connect() log.debug("[AMQP] Finished {0}", self.task.short_qualified_name) manifest = JoltManifest() with raise_task_error_on_exception( self.task, "failed to parse build result manifest"): manifest.parsestring(self.response) self.task.running(utils.duration() - float(manifest.duration)) if manifest.result != "SUCCESS": output = [] if manifest.stdout: output.extend(manifest.stdout.split("\n")) if manifest.stderr: output.extend(manifest.stderr.split("\n")) for line in output: log.transfer(line, self.task.identity[:8]) for task in [self.task] + self.task.extensions: with task.task.report() as report: remote_report = manifest.find_task(task.qualified_name) if remote_report: for error in remote_report.errors: report.manifest.append(error) raise_error("[AMQP] remote build failed with status: {0}".format( manifest.result)) raise_task_error_if( not env.cache.is_available_remotely(self.task), self.task, "no task artifact available in any cache, check configuration") raise_task_error_if( not env.cache.download(self.task) and env.cache.download_enabled(), self.task, "failed to download task artifact") for extension in self.task.extensions: raise_task_error_if( not env.cache.download(extension) and env.cache.download_enabled(), self.task, "failed to download task artifact") return self.task
def run(self, cache, force_upload=False, force_build=False): with self.tools: tasks = [self] + self.extensions available_locally = available_remotely = False for child in self.children: if not child.has_artifact(): continue if not cache.is_available_locally(child): raise_task_error_if(not cache.download(child), child, "failed to download task artifact") if not force_build: available_locally = all(map(cache.is_available_locally, tasks)) if available_locally and not force_upload: return available_remotely = cache.download_enabled() and \ all(map(cache.is_available_remotely, tasks)) if not available_locally and available_remotely: available_locally = cache.download(self) if force_build or not available_locally: with log.threadsink() as buildlog: if self.task.is_runnable(): log.verbose("Host: {0}", getenv("HOSTNAME", "localhost")) with cache.get_locked_artifact( self, discard=force_build) as artifact: if not cache.is_available_locally( self) or self.has_extensions(): with cache.get_context(self) as context: self.running() with self.tools.cwd(self.task.joltdir): hooks.task_prerun(self, context, self.tools) if self.is_goal() and self.options.debug: log.info("Entering debug shell") self.task.shell(context, self.tools) self.task.run(context, self.tools) hooks.task_postrun(self, context, self.tools) if not cache.is_available_locally(self): with self.tools.cwd(self.task.joltdir): hooks.task_prepublish( self, artifact, self.tools) self.task.publish(artifact, self.tools) self.task._verify_influence( context, artifact, self.tools) hooks.task_postpublish( self, artifact, self.tools) with open( fs.path.join( artifact.path, ".build.log"), "w") as f: f.write(buildlog.getvalue()) cache.commit(artifact) else: self.info( "Publication skipped, already in local cache" ) else: self.info( "Execution skipped, already in local cache") # Must upload the artifact while still holding its lock, otherwise the # artifact may become unpack():ed before we have a chance to. if force_upload or force_build or not available_remotely: raise_task_error_if( not cache.upload( self, force=force_upload, locked=False) and cache.upload_enabled(), self, "failed to upload task artifact") elif force_upload or not available_remotely: raise_task_error_if( not cache.upload(self, force=force_upload) and cache.upload_enabled(), self, "failed to upload task artifact") for extension in self.extensions: try: extension.started() with hooks.task_run(extension): extension.run(cache, force_upload, force_build) except Exception as e: extension.failed() raise e else: extension.finished()