def load(self): super(NativeRecipe, self).load() name = utils.canonical(self.path) loader = SourceFileLoader("joltfile_{0}".format(name), self.path) module = ModuleType(loader.name) module.__file__ = self.path loader.exec_module(module) sys.modules[loader.name] = module classes = inspection.getmoduleclasses(module, [Task, TaskGenerator], NativeRecipe._is_abstract) generators = [] for cls in classes[TaskGenerator]: cls.joltdir = self.joltdir or os.path.dirname(self.path) generators.append(cls()) for generator in generators: generated_tasks = utils.as_list(generator.generate()) classes[Task] += filter(NativeRecipe._is_task, generated_tasks) for task in classes[Task]: task.name = task.name or task.__name__.lower() task.joltdir = self.joltdir or os.path.dirname(self.path) task.joltproject = self.project self.tasks.append(task) log.verbose("Loaded: {0}", self.path)
def upload(self, node, force=False): if not self._upload and not force: return True with self._cache.get_artifact(node) as artifact: path = self._get_path(node, artifact) temp = self._get_temp(node, artifact) try: log.verbose("[VOLUME] Copying {}", path) fs.copy(artifact.get_archive_path(), temp) # To avoid race-condition, make sure that the artifact still is missing before moving it into place. if not fs.exists(path): fs.rename(temp, path) else: fs.unlink(temp) return True except OSError as e: if e.errno != errno.EEXIST: log.verbose("[VOLUME] Failed to copy artifact, errno={}", os.strerror(e.errno)) return e.errno == errno.EEXIST except Exception: log.exception() finally: fs.unlink(temp, ignore_errors=True) return False
def _preload(self): with open(self.path) as f: root = yaml.safe_load_all(f) root = [o for o in root] for doc in root: for key, obj in doc.items(): builder = _builders.get(key) if not builder: log.verbose("Unknown yaml task type: {}", key) continue task = builder.build(obj) task.joltdir = os.path.dirname(self.path) self.tasks.append(task)
def parse(self, filename="default.joltxmanifest"): path = os.getcwd() filepath = fs.path.join(path, filename) while not fs.path.exists(filepath): opath = path path = fs.path.dirname(path) if path == opath: break filepath = fs.path.join(path, filename) if path == fs.sep: raise Exception("couldn't find manifest file") with open(filepath) as f: self.parsestring(f.read()) log.verbose("Loaded: {0}", filepath) self.path = filepath return self raise Exception("failed to parse xml file")
def get_command(self, ctx, cmd_name): if cmd_name == "info": cmd_name = "inspect" if cmd_name in ["export", "inspect"]: log.set_level(log.SILENCE) elif ctx.params.get("verbose", False): log.set_level(log.VERBOSE) elif ctx.params.get("extra_verbose", False): log.set_level(log.DEBUG) config_files = ctx.params.get("config_file") or [] for config_file in config_files: log.verbose("Config: {0}", config_file) config.load_or_set(config_file) # Load configured plugins JoltLoader.get().load_plugins() return click.Group.get_command(self, ctx, cmd_name)
def load(self): super(NativeRecipe, self).load() name = utils.canonical(self.path) module = imp.load_source("joltfile_{0}".format(name), self.path) classes = inspect.getmoduleclasses(module, [Task, TaskGenerator], NativeRecipe._is_abstract) for cls in classes[TaskGenerator]: cls.joltdir = self.joltdir or os.path.dirname(self.path) generated_tasks = utils.as_list(cls().generate()) classes[Task] += filter(NativeRecipe._is_task, generated_tasks) for task in classes[Task]: task.name = task.name or task.__name__.lower() task.joltdir = self.joltdir or os.path.dirname(self.path) task.joltproject = self.project self.tasks.append(task) log.verbose("Loaded: {0}", self.path)
def download(self, node, force=False): if not self._download and not force: return False with self._cache.get_artifact(node) as artifact: path = self._get_path(node, artifact) try: log.verbose("[VOLUME] Copying {}", path) fs.copy(path, artifact.get_archive_path()) return True except OSError as e: if e.errno == errno.ESTALE: log.verbose("[VOLUME] got stale file handle, retrying...") raise StaleFileHandleError(e) else: log.exception() except Exception: log.exception() return False
def build(self, task_list, influence=True): with self._progress("Building graph", len(self.graph.tasks), "tasks") as progress: goals = [self._get_node(progress, task) for task in task_list] self.graph._nodes_by_name = self.nodes if influence: topological_nodes = self.graph.topological_nodes with self._progress("Collecting task influence", len(self.graph.tasks), "tasks") as p: for node in reversed(topological_nodes): node.finalize(self.graph, self.manifest) p.update(1) max_time = 0 min_time = 0 for node in topological_nodes: max_time += node.task.weight node.task.weight += max([a.weight for a in node.ancestors] + [0]) min_time = max(node.task.weight, min_time) log.verbose("Time estimate: {}- {}", utils.duration_diff(min_time), utils.duration_diff(max_time)) self.graph.requested_goals = goals self.graph.goals = [] for goal in goals: goal.set_goal() self.graph.goals.append(goal) if goal.is_alias(): for goal_alias in goal.neighbors: goal_alias.set_goal() self.graph.goals.append(goal_alias) return self.graph
import functools import sys from jolt import cache from jolt import config from jolt import filesystem as fs from jolt import log from jolt import utils from jolt.hooks import TaskHook, TaskHookFactory from jolt.influence import StringInfluence from jolt.plugins import ninja from jolt.plugins import ninjacli log.verbose("[NinjaCache] Loaded") class CacheHooks(TaskHook): def task_created(self, task): if not isinstance(task.task, ninja.CXXLibrary) or task.task.shared: return task.task.influence.append(StringInfluence("NinjaCache")) task.task._write_ninja_cache = functools.partial( self.task_post_ninja_file, task) def task_post_ninja_file(self, task, deps, tools): if not isinstance(task.task, ninja.CXXLibrary) or task.task.shared: return cli = fs.path.join(fs.path.dirname(__file__), "ninjacli.py") disabled = config.getboolean("ninja-cache", "disable", False)
def run(self, cache, force_upload=False, force_build=False): with self.tools: tasks = [self] + self.extensions available_locally = available_remotely = False for child in self.children: if not child.has_artifact(): continue if not cache.is_available_locally(child): raise_task_error_if(not cache.download(child), child, "failed to download task artifact") if not force_build: available_locally = all(map(cache.is_available_locally, tasks)) if available_locally and not force_upload: return available_remotely = cache.download_enabled() and \ all(map(cache.is_available_remotely, tasks)) if not available_locally and available_remotely: available_locally = cache.download(self) if force_build or not available_locally: with log.threadsink() as buildlog: if self.task.is_runnable(): log.verbose("Host: {0}", getenv("HOSTNAME", "localhost")) with cache.get_locked_artifact( self, discard=force_build) as artifact: if not cache.is_available_locally( self) or self.has_extensions(): with cache.get_context(self) as context: self.running() with self.tools.cwd(self.task.joltdir): hooks.task_prerun(self, context, self.tools) if self.is_goal() and self.options.debug: log.info("Entering debug shell") self.task.shell(context, self.tools) self.task.run(context, self.tools) hooks.task_postrun(self, context, self.tools) if not cache.is_available_locally(self): with self.tools.cwd(self.task.joltdir): hooks.task_prepublish( self, artifact, self.tools) self.task.publish(artifact, self.tools) self.task._verify_influence( context, artifact, self.tools) hooks.task_postpublish( self, artifact, self.tools) with open( fs.path.join( artifact.path, ".build.log"), "w") as f: f.write(buildlog.getvalue()) cache.commit(artifact) else: self.info( "Publication skipped, already in local cache" ) else: self.info( "Execution skipped, already in local cache") # Must upload the artifact while still holding its lock, otherwise the # artifact may become unpack():ed before we have a chance to. if force_upload or force_build or not available_remotely: raise_task_error_if( not cache.upload( self, force=force_upload, locked=False) and cache.upload_enabled(), self, "failed to upload task artifact") elif force_upload or not available_remotely: raise_task_error_if( not cache.upload(self, force=force_upload) and cache.upload_enabled(), self, "failed to upload task artifact") for extension in self.extensions: try: extension.started() with hooks.task_run(extension): extension.run(cache, force_upload, force_build) except Exception as e: extension.failed() raise e else: extension.finished()
def verbose(self, fmt, *args, **kwargs): log.verbose(fmt + " " + self.log_name, *args, **kwargs)
self.task.failed(TYPE) raise_error("Lost connection to AMQP server") except Exception as e: log.exception() for extension in self.task.extensions: extension.failed(TYPE) self.task.failed(TYPE) raise e finally: if self.connection is not None: utils.call_and_catch(self.connection.close) return self.task @scheduler.ExecutorFactory.Register class AmqpExecutorFactory(scheduler.NetworkExecutorFactory): def __init__(self, options): workers = config.getint(NAME, "workers", 16) super(AmqpExecutorFactory, self).__init__(max_workers=workers) self._options = options @property def options(self): return self._options def create(self, task): return AmqpExecutor(self, task) log.verbose("[AMQP] Loaded")
def info(self, fmt, *args, **kwargs): log.verbose(fmt, *args, **kwargs)
import glob import os import yaml from jolt.tasks import Task from jolt import log from jolt import loader from jolt.error import raise_error_if from jolt import influence log.verbose("[YamlTask] Loaded") @influence.attribute("yaml") class YamlTask(Task): name = None requires = [] cacheable = True commands = [] collects = [] extends = "" fast = False influence = [] joltdir = "." joltproject = None metadata = {} selfsustained = False taint = False weight = False yaml = None
from jolt import Parameter, Export from jolt import log from jolt.tasks import TaskRegistry from jolt.plugins import git log.verbose("[Gerrit] Loaded") class GerritSrc(git.GitSrc): name = "gerrit-src" url = Parameter(help="URL to the Gerrit git repo to be cloned. Required.") sha = Parameter(required=False, help="Specific commit or tag to be checked out. Optional.") path = Parameter(required=False, help="Local path where the repository should be cloned.") _revision = Export( value=lambda self: self._get_revision() or self.git.head()) def __init__(self, *args, **kwargs): refspec1 = '+refs/changes/*:refs/remotes/origin/changes/*' super(GerritSrc, self).__init__(*args, refspecs=[refspec1], **kwargs) class Gerrit(git.Git): name = "gerrit" url = Parameter(help="URL to the Gerrit git repo to be cloned. Required.") sha = Parameter(required=False, help="Specific commit or tag to be checked out. Optional.") path = Parameter(required=False, help="Local path where the repository should be cloned.") _revision = Export(
from jolt import config from jolt import error from jolt import log from jolt.hooks import TaskHookFactory from jolt.plugins import telemetry log.verbose("[Dashboard] Loaded") class DashboardHooks(telemetry.TelemetryHooks): def __init__(self, uri=None): uri = config.get("dashboard", "uri", "http://dashboard") error.raise_error_if(not uri, "dashboard.uri not configured") super().__init__(plugin="dashboard", uri=uri + "/api/v1/tasks", local=False) @TaskHookFactory.register class DashboardFactory(TaskHookFactory): def create(self, env): return DashboardHooks()
import re from jolt.tasks import BooleanParameter, Export, Parameter, TaskRegistry, WorkspaceResource from jolt.influence import FileInfluence, HashInfluenceRegistry from jolt.tools import Tools from jolt.loader import JoltLoader from jolt import config from jolt import filesystem as fs from jolt import log from jolt import utils from jolt.error import JoltCommandError from jolt.error import raise_error from jolt.error import raise_error_if from jolt.error import raise_task_error_if log.verbose("[Git] Loaded") class GitRepository(object): def __init__(self, url, path, relpath, refspecs=None): self.path = path self.relpath = relpath self.tools = Tools() self.url = url self.default_refspecs = [ '+refs/heads/*:refs/remotes/origin/*', '+refs/tags/*:refs/remotes/origin/*', ] self.refspecs = refspecs or [] self._tree_hash = {} self._original_head = True
from socket import gethostname from requests.exceptions import RequestException from jolt import config from jolt import log from jolt import utils from jolt.error import raise_error_if from jolt.hooks import TaskHook, TaskHookFactory log.verbose("[Telemetry] Loaded") class TelemetryHooks(TaskHook): def __init__(self, plugin="telemetry", uri=None, local=True, network=True, queued=True, started=True, failed=True, finished=True): self._uri = uri or config.get(plugin, "uri", uri) self._network = config.getboolean(plugin, "network", network) self._local = config.getboolean(plugin, "local", local) self._queued = config.getboolean(plugin, "queued", queued) self._started = config.getboolean(plugin, "started", started) self._failed = config.getboolean(plugin, "failed", failed) self._finished = config.getboolean(plugin, "finished", finished) raise_error_if(not self._uri, "telemetry.uri not configured")
from contextlib import contextmanager from datetime import datetime from jolt import config from jolt import filesystem as fs from jolt import log from jolt.error import raise_error_if from jolt.hooks import TaskHook, TaskHookFactory log.verbose("[LogStash] Loaded") class LogStashHooks(TaskHook): def __init__(self): self._uri = config.get("logstash", "http.uri") self._failed_enabled = config.getboolean("logstash", "failed", False) self._finished_enabled = config.getboolean("logstash", "finished", False) raise_error_if(not self._uri, "logstash.http.uri not configured") def _get_uri(self, task): return "{}/{}-{}.txt".format( self._uri, datetime.now().strftime("%Y-%m-%d_%H%M%S.%f"), task.canonical_name) def _stash_log(self, task, logbuffer): with task.tools.tmpdir("logstash") as t: filepath = fs.path.join(t.path, "log") with open(filepath, "w") as f: f.write(logbuffer) task.logstash = self._get_uri(task)
def build(ctx, task, network, keep_going, default, local, no_download, no_upload, download, upload, worker, force, salt, copy, debug, result, jobs): """ Build task artifact. TASK is the name of the task to execute. It is optionally followed by a colon and parameter value assignments. Assignments are separated by commas. Example: taskname:param1=value1,param2=value2 Default parameter values can be overridden for any task in the dependency tree with --default. DEFAULT is a qualified task name, just like TASK, but parameter assignments change default values. By default, a task is executed locally and the resulting artifact is stored in the local artifact cache. If an artifact is already available in the cache, no execution takes place. Artifacts are identified with a hash digest, constructed from hashing task attributes. When remote cache providers are configured, artifacts may be downloaded from and/or uploaded to the remote cache as execution progresses. Several options exist to control the behavior, such as --local which disables all remote caches. Distributed task execution is enabled by passing the --network option. Tasks are then distributed to and executed by a pool of workers, if one has been configured. Rebuilds can be forced with either --force or --salt. --force rebuilds the requested task, but not its dependencies. --salt affects the entire dependency tree. Both add an extra attribute to the task hash calculation in order to taint the identity and induce a cache miss. In both cases, existing intermediate files in build directories are removed before execution starts. """ raise_error_if(network and local, "The -n and -l flags are mutually exclusive") raise_error_if(network and debug, "The -g and -n flags are mutually exclusive") raise_error_if( no_download and download, "The --download and --no-download flags are mutually exclusive") raise_error_if( no_upload and upload, "The --upload and --no-upload flags are mutually exclusive") duration = utils.duration() task = list(task) task = [utils.stable_task_name(t) for t in task] if network: _download = config.getboolean("network", "download", True) _upload = config.getboolean("network", "upload", True) else: _download = config.getboolean("jolt", "download", True) _upload = config.getboolean("jolt", "upload", True) if local: _download = False _upload = False else: if no_download: _download = False if no_upload: _upload = False if download: _download = True if upload: _upload = True options = JoltOptions(network=network, local=local, download=_download, upload=_upload, keep_going=keep_going, default=default, worker=worker, debug=debug, salt=salt, jobs=jobs) acache = cache.ArtifactCache.get(options) executors = scheduler.ExecutorRegistry.get(options) if worker: log.set_worker() log.verbose("Local build as a worker") strategy = scheduler.WorkerStrategy(executors, acache) elif network: log.verbose("Distributed build as a user") strategy = scheduler.DistributedStrategy(executors, acache) else: log.verbose("Local build as a user") strategy = scheduler.LocalStrategy(executors, acache) hooks.TaskHookRegistry.get(options) registry = TaskRegistry.get(options) for params in default: registry.set_default_parameters(params) manifest = ctx.obj["manifest"] for mb in manifest.builds: for mt in mb.tasks: task.append(mt.name) for mt in mb.defaults: registry.set_default_parameters(mt.name) if force: for goal in task: registry.get_task(goal, manifest=manifest).taint = uuid.uuid4() gb = graph.GraphBuilder(registry, manifest, options, progress=True) dag = gb.build(task) gp = graph.GraphPruner(strategy) dag = gp.prune(dag) goal_tasks = dag.goals goal_task_duration = 0 queue = scheduler.TaskQueue(strategy) try: if not dag.has_tasks(): return progress = log.progress( "Progress", dag.number_of_tasks(filterfn=lambda t: not t.is_resource()), " tasks", estimates=False, debug=debug) with progress: while dag.has_tasks(): # Find all tasks ready to be executed leafs = dag.select(lambda graph, task: task.is_ready()) # Order the tasks by their weights to improve build times leafs.sort(key=lambda x: x.weight) while leafs: task = leafs.pop() queue.submit(acache, task) task, error = queue.wait() if not task: dag.debug() break elif task.is_goal() and task.duration_running: goal_task_duration += task.duration_running.seconds if not task.is_resource(): progress.update(1) if not keep_going and error is not None: queue.abort() raise error if dag.failed: log.error("List of failed tasks") for failed in dag.failed: log.error("- {}", failed.log_name.strip("()")) raise_error("no more tasks could be executed") for goal in goal_tasks: if acache.is_available_locally(goal): with acache.get_artifact(goal) as artifact: log.info("Location: {0}", artifact.path) if copy: artifact.copy("*", utils.as_dirpath( fs.path.join( workdir, click.format_filename(copy))), symlinks=True) except KeyboardInterrupt: print() log.warning("Interrupted by user") try: queue.abort() sys.exit(1) except KeyboardInterrupt: print() log.warning("Interrupted again, exiting") _exit(1) finally: log.info("Total execution time: {0} {1}", str(duration), str(queue.duration_acc) if network else '') if result: with report.update() as manifest: manifest.duration = str(goal_task_duration) manifest.write(result)
from jolt.graph import GraphBuilder from jolt.error import raise_error_if from jolt.manifest import JoltManifest from jolt.scheduler import JoltEnvironment from jolt.scheduler import LocalExecutor from jolt.scheduler import LocalExecutorFactory from jolt.scheduler import NetworkExecutorExtension from jolt.scheduler import NetworkExecutorExtensionFactory from jolt.loader import JoltLoader from jolt import config from jolt import filesystem as fs from jolt import influence from jolt import log from jolt import utils log.verbose("[SelfDeploy] Loaded") _path = fs.path.dirname(__file__) _path = fs.path.dirname(_path) _path = fs.path.dirname(_path) @influence.files(fs.path.join(_path, "**", "*.py")) @influence.files(fs.path.join(_path, "**", "*.sh")) @influence.files(fs.path.join(_path, "**", "*.xslt")) @influence.files(fs.path.join(_path, "**", "*.template")) class Jolt(Task): name = "jolt" def __init__(self, *args, **kwargs): super(Jolt, self).__init__(*args, **kwargs)
def create(cache): log.verbose("[Http] Loaded") return Http(cache)
from jolt import cache from jolt import config from jolt import filesystem as fs from jolt import log from jolt import utils from jolt.error import raise_error_if from jolt.hooks import TaskHook, TaskHookFactory log.verbose("[Symlinks] Loaded") class SymlinkHooks(TaskHook): def __init__(self): self._path = config.get("symlinks", "path", "artifacts") raise_error_if(not self._path, "symlinks.path not configured") def task_finished(self, task): if not task.has_artifact(): return srcpath = cache.ArtifactCache.get().get_path(task) destpath = fs.path.join( task.task.joltdir, self._path, utils.canonical(task.short_qualified_name)) if fs.path.exists(srcpath): fs.unlink(destpath, ignore_errors=True) fs.makedirs(fs.path.dirname(destpath))
from jolt import config from jolt import log from jolt import tasks log.verbose("[Alias] Loaded") _registry = tasks.TaskRegistry.get() for key, value in config.options("alias"): class ConfigAlias(tasks.Alias): name = key requires = value.split() _registry.add_task_class(ConfigAlias)
def cli(ctx, verbose, extra_verbose, config_file, debugger, profile, force, salt, debug, network, local, keep_going, jobs): """ A task execution tool. When invoked without any commands and arguments, Jolt by default tries to execute and build the artifact of a task called `default`. To build artifacts of other tasks use the build subcommand. The Jolt command line interface is hierarchical. One set of options can be passed to the top-level command and a different set of options to the subcommands, simultaneously. For example, verbose output is a top-level option while forced rebuild is a build command option. They may combined like this: $ jolt --verbose build --force taskname Most build command options are available also at the top-level when build is invoked implicitly for the default task. """ global debug_enabled debug_enabled = debugger log.verbose("Jolt command: {}", " ".join([fs.path.basename(sys.argv[0])] + sys.argv[1:])) log.verbose("Jolt host: {}", environ.get("HOSTNAME", "localhost")) log.verbose("Jolt install path: {}", fs.path.dirname(__file__)) if ctx.invoked_subcommand in ["config"]: # Don't attempt to load any task recipes as they might require # plugins that are not yet configured. return if ctx.invoked_subcommand is None: build = ctx.command.get_command(ctx, "build") manifest = JoltManifest() utils.call_and_catch(manifest.parse) manifest.process_import() ctx.obj["manifest"] = manifest if manifest.version: from jolt.version_utils import requirement, version req = requirement(manifest.version) ver = version(__version__) raise_error_if(not req.satisfied(ver), "this project requires Jolt version {} (running {})", req, __version__) loader = JoltLoader.get() tasks = loader.load() for cls in tasks: TaskRegistry.get().add_task_class(cls) if ctx.invoked_subcommand in ["build", "clean"] and loader.joltdir: ctx.obj["workspace_lock"] = utils.LockFile( fs.path.join(loader.joltdir, "build"), log.info, "Workspace is locked by another process, please wait...") atexit.register(ctx.obj["workspace_lock"].close) # If no command is given, we default to building the default task. # If the default task doesn't exist, help is printed inside build(). if ctx.invoked_subcommand is None: task = config.get("jolt", "default", "default") taskname, _ = utils.parse_task_name(task) if TaskRegistry.get().get_task_class(taskname) is not None: ctx.invoke(build, task=[task], force=force, salt=salt, debug=debug, network=network, local=local, keep_going=keep_going, jobs=jobs) else: print(cli.get_help(ctx)) sys.exit(1)
def create(cache): log.verbose("[Volume] Loaded") return DiskVolume(cache)
def create(cache): log.verbose("[Ftp] Loaded") return FtpStorage(cache)
from jolt import cache from jolt import cli from jolt import config from jolt import filesystem as fs from jolt import graph from jolt import log from jolt import scheduler from jolt import utils from jolt import loader from jolt.hooks import TaskHook, TaskHookFactory, TaskHookRegistry from jolt.influence import StringInfluence from jolt.options import JoltOptions from jolt.plugins import ninja from jolt.tasks import TaskRegistry log.verbose("[NinjaCompDB] Loaded") def joltdir(): return loader.JoltLoader.get().joltdir def patch(command, attrib, search, replace): command[attrib] = command[attrib].replace(search, replace) class CompDB(object): def __init__(self, path="compile_commands.json", artifact=None): self.commands = [] self.attribs = {} if artifact: