Beispiel #1
0
    def load(self):
        super(NativeRecipe, self).load()

        name = utils.canonical(self.path)
        loader = SourceFileLoader("joltfile_{0}".format(name), self.path)
        module = ModuleType(loader.name)
        module.__file__ = self.path
        loader.exec_module(module)
        sys.modules[loader.name] = module

        classes = inspection.getmoduleclasses(module, [Task, TaskGenerator],
                                              NativeRecipe._is_abstract)
        generators = []

        for cls in classes[TaskGenerator]:
            cls.joltdir = self.joltdir or os.path.dirname(self.path)
            generators.append(cls())

        for generator in generators:
            generated_tasks = utils.as_list(generator.generate())
            classes[Task] += filter(NativeRecipe._is_task, generated_tasks)

        for task in classes[Task]:
            task.name = task.name or task.__name__.lower()
            task.joltdir = self.joltdir or os.path.dirname(self.path)
            task.joltproject = self.project
            self.tasks.append(task)

        log.verbose("Loaded: {0}", self.path)
Beispiel #2
0
 def upload(self, node, force=False):
     if not self._upload and not force:
         return True
     with self._cache.get_artifact(node) as artifact:
         path = self._get_path(node, artifact)
         temp = self._get_temp(node, artifact)
         try:
             log.verbose("[VOLUME] Copying {}", path)
             fs.copy(artifact.get_archive_path(), temp)
             # To avoid race-condition, make sure that the artifact still is missing before moving it into place.
             if not fs.exists(path):
                 fs.rename(temp, path)
             else:
                 fs.unlink(temp)
             return True
         except OSError as e:
             if e.errno != errno.EEXIST:
                 log.verbose("[VOLUME] Failed to copy artifact, errno={}",
                             os.strerror(e.errno))
             return e.errno == errno.EEXIST
         except Exception:
             log.exception()
         finally:
             fs.unlink(temp, ignore_errors=True)
     return False
Beispiel #3
0
    def _preload(self):
        with open(self.path) as f:
            root = yaml.safe_load_all(f)
            root = [o for o in root]

        for doc in root:
            for key, obj in doc.items():
                builder = _builders.get(key)
                if not builder:
                    log.verbose("Unknown yaml task type: {}", key)
                    continue
                task = builder.build(obj)
                task.joltdir = os.path.dirname(self.path)
                self.tasks.append(task)
Beispiel #4
0
 def parse(self, filename="default.joltxmanifest"):
     path = os.getcwd()
     filepath = fs.path.join(path, filename)
     while not fs.path.exists(filepath):
         opath = path
         path = fs.path.dirname(path)
         if path == opath:
             break
         filepath = fs.path.join(path, filename)
     if path == fs.sep:
         raise Exception("couldn't find manifest file")
     with open(filepath) as f:
         self.parsestring(f.read())
         log.verbose("Loaded: {0}", filepath)
         self.path = filepath
         return self
     raise Exception("failed to parse xml file")
Beispiel #5
0
Datei: cli.py Projekt: srand/jolt
    def get_command(self, ctx, cmd_name):
        if cmd_name == "info":
            cmd_name = "inspect"

        if cmd_name in ["export", "inspect"]:
            log.set_level(log.SILENCE)
        elif ctx.params.get("verbose", False):
            log.set_level(log.VERBOSE)
        elif ctx.params.get("extra_verbose", False):
            log.set_level(log.DEBUG)

        config_files = ctx.params.get("config_file") or []
        for config_file in config_files:
            log.verbose("Config: {0}", config_file)
            config.load_or_set(config_file)

        # Load configured plugins
        JoltLoader.get().load_plugins()

        return click.Group.get_command(self, ctx, cmd_name)
Beispiel #6
0
    def load(self):
        super(NativeRecipe, self).load()

        name = utils.canonical(self.path)
        module = imp.load_source("joltfile_{0}".format(name), self.path)
        classes = inspect.getmoduleclasses(module, [Task, TaskGenerator],
                                           NativeRecipe._is_abstract)

        for cls in classes[TaskGenerator]:
            cls.joltdir = self.joltdir or os.path.dirname(self.path)
            generated_tasks = utils.as_list(cls().generate())
            classes[Task] += filter(NativeRecipe._is_task, generated_tasks)

        for task in classes[Task]:
            task.name = task.name or task.__name__.lower()
            task.joltdir = self.joltdir or os.path.dirname(self.path)
            task.joltproject = self.project
            self.tasks.append(task)

        log.verbose("Loaded: {0}", self.path)
Beispiel #7
0
    def download(self, node, force=False):
        if not self._download and not force:
            return False

        with self._cache.get_artifact(node) as artifact:
            path = self._get_path(node, artifact)
            try:
                log.verbose("[VOLUME] Copying {}", path)
                fs.copy(path, artifact.get_archive_path())
                return True
            except OSError as e:
                if e.errno == errno.ESTALE:
                    log.verbose("[VOLUME] got stale file handle, retrying...")
                    raise StaleFileHandleError(e)
                else:
                    log.exception()
            except Exception:
                log.exception()

        return False
Beispiel #8
0
    def build(self, task_list, influence=True):
        with self._progress("Building graph", len(self.graph.tasks),
                            "tasks") as progress:
            goals = [self._get_node(progress, task) for task in task_list]
            self.graph._nodes_by_name = self.nodes

        if influence:
            topological_nodes = self.graph.topological_nodes
            with self._progress("Collecting task influence",
                                len(self.graph.tasks), "tasks") as p:
                for node in reversed(topological_nodes):
                    node.finalize(self.graph, self.manifest)
                    p.update(1)

            max_time = 0
            min_time = 0
            for node in topological_nodes:
                max_time += node.task.weight
                node.task.weight += max([a.weight
                                         for a in node.ancestors] + [0])
                min_time = max(node.task.weight, min_time)

            log.verbose("Time estimate: {}- {}", utils.duration_diff(min_time),
                        utils.duration_diff(max_time))

        self.graph.requested_goals = goals
        self.graph.goals = []
        for goal in goals:
            goal.set_goal()
            self.graph.goals.append(goal)
            if goal.is_alias():
                for goal_alias in goal.neighbors:
                    goal_alias.set_goal()
                    self.graph.goals.append(goal_alias)

        return self.graph
Beispiel #9
0
import functools
import sys

from jolt import cache
from jolt import config
from jolt import filesystem as fs
from jolt import log
from jolt import utils
from jolt.hooks import TaskHook, TaskHookFactory
from jolt.influence import StringInfluence
from jolt.plugins import ninja
from jolt.plugins import ninjacli

log.verbose("[NinjaCache] Loaded")


class CacheHooks(TaskHook):
    def task_created(self, task):
        if not isinstance(task.task, ninja.CXXLibrary) or task.task.shared:
            return
        task.task.influence.append(StringInfluence("NinjaCache"))
        task.task._write_ninja_cache = functools.partial(
            self.task_post_ninja_file, task)

    def task_post_ninja_file(self, task, deps, tools):
        if not isinstance(task.task, ninja.CXXLibrary) or task.task.shared:
            return

        cli = fs.path.join(fs.path.dirname(__file__), "ninjacli.py")
        disabled = config.getboolean("ninja-cache", "disable", False)
Beispiel #10
0
    def run(self, cache, force_upload=False, force_build=False):
        with self.tools:
            tasks = [self] + self.extensions
            available_locally = available_remotely = False

            for child in self.children:
                if not child.has_artifact():
                    continue
                if not cache.is_available_locally(child):
                    raise_task_error_if(not cache.download(child), child,
                                        "failed to download task artifact")

            if not force_build:
                available_locally = all(map(cache.is_available_locally, tasks))
                if available_locally and not force_upload:
                    return
                available_remotely = cache.download_enabled() and \
                    all(map(cache.is_available_remotely, tasks))
                if not available_locally and available_remotely:
                    available_locally = cache.download(self)

            if force_build or not available_locally:
                with log.threadsink() as buildlog:
                    if self.task.is_runnable():
                        log.verbose("Host: {0}",
                                    getenv("HOSTNAME", "localhost"))

                    with cache.get_locked_artifact(
                            self, discard=force_build) as artifact:
                        if not cache.is_available_locally(
                                self) or self.has_extensions():
                            with cache.get_context(self) as context:
                                self.running()
                                with self.tools.cwd(self.task.joltdir):
                                    hooks.task_prerun(self, context,
                                                      self.tools)
                                    if self.is_goal() and self.options.debug:
                                        log.info("Entering debug shell")
                                        self.task.shell(context, self.tools)
                                    self.task.run(context, self.tools)
                                    hooks.task_postrun(self, context,
                                                       self.tools)

                                if not cache.is_available_locally(self):
                                    with self.tools.cwd(self.task.joltdir):
                                        hooks.task_prepublish(
                                            self, artifact, self.tools)
                                        self.task.publish(artifact, self.tools)
                                        self.task._verify_influence(
                                            context, artifact, self.tools)
                                        hooks.task_postpublish(
                                            self, artifact, self.tools)
                                    with open(
                                            fs.path.join(
                                                artifact.path, ".build.log"),
                                            "w") as f:
                                        f.write(buildlog.getvalue())
                                    cache.commit(artifact)
                                else:
                                    self.info(
                                        "Publication skipped, already in local cache"
                                    )
                        else:
                            self.info(
                                "Execution skipped, already in local cache")

                        # Must upload the artifact while still holding its lock, otherwise the
                        # artifact may become unpack():ed before we have a chance to.
                        if force_upload or force_build or not available_remotely:
                            raise_task_error_if(
                                not cache.upload(
                                    self, force=force_upload, locked=False)
                                and cache.upload_enabled(), self,
                                "failed to upload task artifact")
            elif force_upload or not available_remotely:
                raise_task_error_if(
                    not cache.upload(self, force=force_upload)
                    and cache.upload_enabled(), self,
                    "failed to upload task artifact")

            for extension in self.extensions:
                try:
                    extension.started()
                    with hooks.task_run(extension):
                        extension.run(cache, force_upload, force_build)
                except Exception as e:
                    extension.failed()
                    raise e
                else:
                    extension.finished()
Beispiel #11
0
 def verbose(self, fmt, *args, **kwargs):
     log.verbose(fmt + " " + self.log_name, *args, **kwargs)
Beispiel #12
0
            self.task.failed(TYPE)
            raise_error("Lost connection to AMQP server")
        except Exception as e:
            log.exception()
            for extension in self.task.extensions:
                extension.failed(TYPE)
            self.task.failed(TYPE)
            raise e
        finally:
            if self.connection is not None:
                utils.call_and_catch(self.connection.close)
        return self.task


@scheduler.ExecutorFactory.Register
class AmqpExecutorFactory(scheduler.NetworkExecutorFactory):
    def __init__(self, options):
        workers = config.getint(NAME, "workers", 16)
        super(AmqpExecutorFactory, self).__init__(max_workers=workers)
        self._options = options

    @property
    def options(self):
        return self._options

    def create(self, task):
        return AmqpExecutor(self, task)


log.verbose("[AMQP] Loaded")
Beispiel #13
0
 def info(self, fmt, *args, **kwargs):
     log.verbose(fmt, *args, **kwargs)
Beispiel #14
0
import glob
import os
import yaml

from jolt.tasks import Task
from jolt import log
from jolt import loader
from jolt.error import raise_error_if
from jolt import influence


log.verbose("[YamlTask] Loaded")


@influence.attribute("yaml")
class YamlTask(Task):
    name = None
    requires = []
    cacheable = True
    commands = []
    collects = []
    extends = ""
    fast = False
    influence = []
    joltdir = "."
    joltproject = None
    metadata = {}
    selfsustained = False
    taint = False
    weight = False
    yaml = None
Beispiel #15
0
from jolt import Parameter, Export
from jolt import log
from jolt.tasks import TaskRegistry
from jolt.plugins import git

log.verbose("[Gerrit] Loaded")


class GerritSrc(git.GitSrc):
    name = "gerrit-src"
    url = Parameter(help="URL to the Gerrit git repo to be cloned. Required.")
    sha = Parameter(required=False,
                    help="Specific commit or tag to be checked out. Optional.")
    path = Parameter(required=False,
                     help="Local path where the repository should be cloned.")
    _revision = Export(
        value=lambda self: self._get_revision() or self.git.head())

    def __init__(self, *args, **kwargs):
        refspec1 = '+refs/changes/*:refs/remotes/origin/changes/*'
        super(GerritSrc, self).__init__(*args, refspecs=[refspec1], **kwargs)


class Gerrit(git.Git):
    name = "gerrit"
    url = Parameter(help="URL to the Gerrit git repo to be cloned. Required.")
    sha = Parameter(required=False,
                    help="Specific commit or tag to be checked out. Optional.")
    path = Parameter(required=False,
                     help="Local path where the repository should be cloned.")
    _revision = Export(
Beispiel #16
0
from jolt import config
from jolt import error
from jolt import log
from jolt.hooks import TaskHookFactory
from jolt.plugins import telemetry


log.verbose("[Dashboard] Loaded")


class DashboardHooks(telemetry.TelemetryHooks):
    def __init__(self, uri=None):
        uri = config.get("dashboard", "uri", "http://dashboard")
        error.raise_error_if(not uri, "dashboard.uri not configured")
        super().__init__(plugin="dashboard", uri=uri + "/api/v1/tasks", local=False)


@TaskHookFactory.register
class DashboardFactory(TaskHookFactory):
    def create(self, env):
        return DashboardHooks()
Beispiel #17
0
import re

from jolt.tasks import BooleanParameter, Export, Parameter, TaskRegistry, WorkspaceResource
from jolt.influence import FileInfluence, HashInfluenceRegistry
from jolt.tools import Tools
from jolt.loader import JoltLoader
from jolt import config
from jolt import filesystem as fs
from jolt import log
from jolt import utils
from jolt.error import JoltCommandError
from jolt.error import raise_error
from jolt.error import raise_error_if
from jolt.error import raise_task_error_if

log.verbose("[Git] Loaded")


class GitRepository(object):
    def __init__(self, url, path, relpath, refspecs=None):
        self.path = path
        self.relpath = relpath
        self.tools = Tools()
        self.url = url
        self.default_refspecs = [
            '+refs/heads/*:refs/remotes/origin/*',
            '+refs/tags/*:refs/remotes/origin/*',
        ]
        self.refspecs = refspecs or []
        self._tree_hash = {}
        self._original_head = True
Beispiel #18
0
from socket import gethostname
from requests.exceptions import RequestException

from jolt import config
from jolt import log
from jolt import utils
from jolt.error import raise_error_if
from jolt.hooks import TaskHook, TaskHookFactory

log.verbose("[Telemetry] Loaded")


class TelemetryHooks(TaskHook):
    def __init__(self,
                 plugin="telemetry",
                 uri=None,
                 local=True,
                 network=True,
                 queued=True,
                 started=True,
                 failed=True,
                 finished=True):
        self._uri = uri or config.get(plugin, "uri", uri)
        self._network = config.getboolean(plugin, "network", network)
        self._local = config.getboolean(plugin, "local", local)
        self._queued = config.getboolean(plugin, "queued", queued)
        self._started = config.getboolean(plugin, "started", started)
        self._failed = config.getboolean(plugin, "failed", failed)
        self._finished = config.getboolean(plugin, "finished", finished)
        raise_error_if(not self._uri, "telemetry.uri not configured")
Beispiel #19
0
from contextlib import contextmanager
from datetime import datetime

from jolt import config
from jolt import filesystem as fs
from jolt import log
from jolt.error import raise_error_if
from jolt.hooks import TaskHook, TaskHookFactory

log.verbose("[LogStash] Loaded")


class LogStashHooks(TaskHook):
    def __init__(self):
        self._uri = config.get("logstash", "http.uri")
        self._failed_enabled = config.getboolean("logstash", "failed", False)
        self._finished_enabled = config.getboolean("logstash", "finished",
                                                   False)
        raise_error_if(not self._uri, "logstash.http.uri not configured")

    def _get_uri(self, task):
        return "{}/{}-{}.txt".format(
            self._uri,
            datetime.now().strftime("%Y-%m-%d_%H%M%S.%f"), task.canonical_name)

    def _stash_log(self, task, logbuffer):
        with task.tools.tmpdir("logstash") as t:
            filepath = fs.path.join(t.path, "log")
            with open(filepath, "w") as f:
                f.write(logbuffer)
            task.logstash = self._get_uri(task)
Beispiel #20
0
Datei: cli.py Projekt: srand/jolt
def build(ctx, task, network, keep_going, default, local, no_download,
          no_upload, download, upload, worker, force, salt, copy, debug,
          result, jobs):
    """
    Build task artifact.

    TASK is the name of the task to execute. It is optionally followed by a colon and
    parameter value assignments. Assignments are separated by commas. Example:

       taskname:param1=value1,param2=value2

    Default parameter values can be overridden for any task in the dependency tree
    with --default. DEFAULT is a qualified task name, just like TASK, but parameter
    assignments change default values.

    By default, a task is executed locally and the resulting artifact is stored
    in the local artifact cache. If an artifact is already available in the cache,
    no execution takes place. Artifacts are identified with a hash digest,
    constructed from hashing task attributes.

    When remote cache providers are configured, artifacts may be downloaded from and/or
    uploaded to the remote cache as execution progresses. Several options exist to control
    the behavior, such as --local which disables all remote caches.

    Distributed task execution is enabled by passing the --network option. Tasks are then
    distributed to and executed by a pool of workers, if one has been configured.

    Rebuilds can be forced with either --force or --salt. --force rebuilds the requested
    task, but not its dependencies. --salt affects the entire dependency tree. Both add
    an extra attribute to the task hash calculation in order to taint the identity and
    induce a cache miss. In both cases, existing intermediate files in build directories
    are removed before execution starts.

    """
    raise_error_if(network and local,
                   "The -n and -l flags are mutually exclusive")

    raise_error_if(network and debug,
                   "The -g and -n flags are mutually exclusive")

    raise_error_if(
        no_download and download,
        "The --download and --no-download flags are mutually exclusive")

    raise_error_if(
        no_upload and upload,
        "The --upload and --no-upload flags are mutually exclusive")

    duration = utils.duration()

    task = list(task)
    task = [utils.stable_task_name(t) for t in task]

    if network:
        _download = config.getboolean("network", "download", True)
        _upload = config.getboolean("network", "upload", True)
    else:
        _download = config.getboolean("jolt", "download", True)
        _upload = config.getboolean("jolt", "upload", True)

    if local:
        _download = False
        _upload = False
    else:
        if no_download:
            _download = False
        if no_upload:
            _upload = False
        if download:
            _download = True
        if upload:
            _upload = True

    options = JoltOptions(network=network,
                          local=local,
                          download=_download,
                          upload=_upload,
                          keep_going=keep_going,
                          default=default,
                          worker=worker,
                          debug=debug,
                          salt=salt,
                          jobs=jobs)

    acache = cache.ArtifactCache.get(options)

    executors = scheduler.ExecutorRegistry.get(options)
    if worker:
        log.set_worker()
        log.verbose("Local build as a worker")
        strategy = scheduler.WorkerStrategy(executors, acache)
    elif network:
        log.verbose("Distributed build as a user")
        strategy = scheduler.DistributedStrategy(executors, acache)
    else:
        log.verbose("Local build as a user")
        strategy = scheduler.LocalStrategy(executors, acache)

    hooks.TaskHookRegistry.get(options)
    registry = TaskRegistry.get(options)

    for params in default:
        registry.set_default_parameters(params)

    manifest = ctx.obj["manifest"]

    for mb in manifest.builds:
        for mt in mb.tasks:
            task.append(mt.name)
        for mt in mb.defaults:
            registry.set_default_parameters(mt.name)

    if force:
        for goal in task:
            registry.get_task(goal, manifest=manifest).taint = uuid.uuid4()

    gb = graph.GraphBuilder(registry, manifest, options, progress=True)
    dag = gb.build(task)

    gp = graph.GraphPruner(strategy)
    dag = gp.prune(dag)

    goal_tasks = dag.goals
    goal_task_duration = 0

    queue = scheduler.TaskQueue(strategy)

    try:
        if not dag.has_tasks():
            return

        progress = log.progress(
            "Progress",
            dag.number_of_tasks(filterfn=lambda t: not t.is_resource()),
            " tasks",
            estimates=False,
            debug=debug)

        with progress:
            while dag.has_tasks():
                # Find all tasks ready to be executed
                leafs = dag.select(lambda graph, task: task.is_ready())

                # Order the tasks by their weights to improve build times
                leafs.sort(key=lambda x: x.weight)

                while leafs:
                    task = leafs.pop()
                    queue.submit(acache, task)

                task, error = queue.wait()

                if not task:
                    dag.debug()
                    break
                elif task.is_goal() and task.duration_running:
                    goal_task_duration += task.duration_running.seconds

                if not task.is_resource():
                    progress.update(1)

                if not keep_going and error is not None:
                    queue.abort()
                    raise error

        if dag.failed:
            log.error("List of failed tasks")
            for failed in dag.failed:
                log.error("- {}", failed.log_name.strip("()"))
            raise_error("no more tasks could be executed")

        for goal in goal_tasks:
            if acache.is_available_locally(goal):
                with acache.get_artifact(goal) as artifact:
                    log.info("Location: {0}", artifact.path)
                    if copy:
                        artifact.copy("*",
                                      utils.as_dirpath(
                                          fs.path.join(
                                              workdir,
                                              click.format_filename(copy))),
                                      symlinks=True)
    except KeyboardInterrupt:
        print()
        log.warning("Interrupted by user")
        try:
            queue.abort()
            sys.exit(1)
        except KeyboardInterrupt:
            print()
            log.warning("Interrupted again, exiting")
            _exit(1)
    finally:
        log.info("Total execution time: {0} {1}", str(duration),
                 str(queue.duration_acc) if network else '')
        if result:
            with report.update() as manifest:
                manifest.duration = str(goal_task_duration)
                manifest.write(result)
Beispiel #21
0
from jolt.graph import GraphBuilder
from jolt.error import raise_error_if
from jolt.manifest import JoltManifest
from jolt.scheduler import JoltEnvironment
from jolt.scheduler import LocalExecutor
from jolt.scheduler import LocalExecutorFactory
from jolt.scheduler import NetworkExecutorExtension
from jolt.scheduler import NetworkExecutorExtensionFactory
from jolt.loader import JoltLoader
from jolt import config
from jolt import filesystem as fs
from jolt import influence
from jolt import log
from jolt import utils

log.verbose("[SelfDeploy] Loaded")

_path = fs.path.dirname(__file__)
_path = fs.path.dirname(_path)
_path = fs.path.dirname(_path)


@influence.files(fs.path.join(_path, "**", "*.py"))
@influence.files(fs.path.join(_path, "**", "*.sh"))
@influence.files(fs.path.join(_path, "**", "*.xslt"))
@influence.files(fs.path.join(_path, "**", "*.template"))
class Jolt(Task):
    name = "jolt"

    def __init__(self, *args, **kwargs):
        super(Jolt, self).__init__(*args, **kwargs)
Beispiel #22
0
 def create(cache):
     log.verbose("[Http] Loaded")
     return Http(cache)
Beispiel #23
0
from jolt import cache
from jolt import config
from jolt import filesystem as fs
from jolt import log
from jolt import utils
from jolt.error import raise_error_if
from jolt.hooks import TaskHook, TaskHookFactory


log.verbose("[Symlinks] Loaded")


class SymlinkHooks(TaskHook):
    def __init__(self):
        self._path = config.get("symlinks", "path", "artifacts")
        raise_error_if(not self._path, "symlinks.path not configured")

    def task_finished(self, task):
        if not task.has_artifact():
            return

        srcpath = cache.ArtifactCache.get().get_path(task)
        destpath = fs.path.join(
            task.task.joltdir,
            self._path,
            utils.canonical(task.short_qualified_name))

        if fs.path.exists(srcpath):
            fs.unlink(destpath, ignore_errors=True)
            fs.makedirs(fs.path.dirname(destpath))
Beispiel #24
0
from jolt import config
from jolt import log
from jolt import tasks


log.verbose("[Alias] Loaded")


_registry = tasks.TaskRegistry.get()

for key, value in config.options("alias"):
    class ConfigAlias(tasks.Alias):
        name = key
        requires = value.split()
    _registry.add_task_class(ConfigAlias)
Beispiel #25
0
Datei: cli.py Projekt: srand/jolt
def cli(ctx, verbose, extra_verbose, config_file, debugger, profile, force,
        salt, debug, network, local, keep_going, jobs):
    """
    A task execution tool.

    When invoked without any commands and arguments, Jolt by default tries
    to execute and build the artifact of a task called `default`. To build
    artifacts of other tasks use the build subcommand.

    The Jolt command line interface is hierarchical. One set of options
    can be passed to the top-level command and a different set of options
    to the subcommands, simultaneously. For example, verbose output is
    a top-level option while forced rebuild is a build command option.
    They may combined like this:

      $ jolt --verbose build --force taskname

    Most build command options are available also at the top-level when
    build is invoked implicitly for the default task.

    """

    global debug_enabled
    debug_enabled = debugger

    log.verbose("Jolt command: {}",
                " ".join([fs.path.basename(sys.argv[0])] + sys.argv[1:]))
    log.verbose("Jolt host: {}", environ.get("HOSTNAME", "localhost"))
    log.verbose("Jolt install path: {}", fs.path.dirname(__file__))

    if ctx.invoked_subcommand in ["config"]:
        # Don't attempt to load any task recipes as they might require
        # plugins that are not yet configured.
        return

    if ctx.invoked_subcommand is None:
        build = ctx.command.get_command(ctx, "build")

    manifest = JoltManifest()
    utils.call_and_catch(manifest.parse)
    manifest.process_import()
    ctx.obj["manifest"] = manifest

    if manifest.version:
        from jolt.version_utils import requirement, version
        req = requirement(manifest.version)
        ver = version(__version__)
        raise_error_if(not req.satisfied(ver),
                       "this project requires Jolt version {} (running {})",
                       req, __version__)

    loader = JoltLoader.get()
    tasks = loader.load()
    for cls in tasks:
        TaskRegistry.get().add_task_class(cls)

    if ctx.invoked_subcommand in ["build", "clean"] and loader.joltdir:
        ctx.obj["workspace_lock"] = utils.LockFile(
            fs.path.join(loader.joltdir, "build"), log.info,
            "Workspace is locked by another process, please wait...")
        atexit.register(ctx.obj["workspace_lock"].close)

    # If no command is given, we default to building the default task.
    # If the default task doesn't exist, help is printed inside build().
    if ctx.invoked_subcommand is None:
        task = config.get("jolt", "default", "default")
        taskname, _ = utils.parse_task_name(task)
        if TaskRegistry.get().get_task_class(taskname) is not None:
            ctx.invoke(build,
                       task=[task],
                       force=force,
                       salt=salt,
                       debug=debug,
                       network=network,
                       local=local,
                       keep_going=keep_going,
                       jobs=jobs)
        else:
            print(cli.get_help(ctx))
            sys.exit(1)
Beispiel #26
0
 def create(cache):
     log.verbose("[Volume] Loaded")
     return DiskVolume(cache)
Beispiel #27
0
 def create(cache):
     log.verbose("[Ftp] Loaded")
     return FtpStorage(cache)
Beispiel #28
0
from jolt import cache
from jolt import cli
from jolt import config
from jolt import filesystem as fs
from jolt import graph
from jolt import log
from jolt import scheduler
from jolt import utils
from jolt import loader
from jolt.hooks import TaskHook, TaskHookFactory, TaskHookRegistry
from jolt.influence import StringInfluence
from jolt.options import JoltOptions
from jolt.plugins import ninja
from jolt.tasks import TaskRegistry

log.verbose("[NinjaCompDB] Loaded")


def joltdir():
    return loader.JoltLoader.get().joltdir


def patch(command, attrib, search, replace):
    command[attrib] = command[attrib].replace(search, replace)


class CompDB(object):
    def __init__(self, path="compile_commands.json", artifact=None):
        self.commands = []
        self.attribs = {}
        if artifact: