def __init__(self, cache): super(FtpStorage, self).__init__() self._cache = cache self._uri = config.get(NAME, "host") raise_error_if(not self._uri, "ftp URI not configured") self._path = config.get(NAME, "path", "") self._upload = config.getboolean(NAME, "upload", True) self._download = config.getboolean(NAME, "download", True) self._tls = config.getboolean(NAME, "tls", False) self._disabled = False
def _get_auth(): service = config.get(NAME, "keyring.service", NAME) username = config.get(NAME, "keyring.username") if not username: return "guest", "guest" password = config.get(NAME, "keyring.password") or keyring.get_password(NAME, username) if not password: password = getpass.getpass(NAME + " password: "******"no password in keyring for " + NAME keyring.set_password(service, username, password) return username, password
def __init__(self, *args, **kwargs): super(Jolt, self).__init__(*args, **kwargs) self.influence.append( influence.StringInfluence(config.get("selfdeploy", "requires", ""))) for e in self.extras: self.influence.append(influence.DirectoryInfluence(e))
def __init__(self, amqp_url): """Create a new instance of the consumer class, passing in the AMQP URL used to connect to RabbitMQ. :param str amqp_url: The AMQP url to connect with """ self.should_reconnect = False self.was_consuming = False self._connection = None self._channel = None self._closing = False self._consumer_tag = None self._url = amqp_url self._consuming = False # In production, experiment with higher prefetch values # for higher consumer throughput self._prefetch_count = 1 self._job = None self._routing_key = self.ROUTING_KEY_PREFIX + config.get( "amqp", "routing_key", os.getenv("RABBITMQ_ROUTING_KEY", self.ROUTING_KEY_REQUEST)) self._queue = self.QUEUE + "_" + self._routing_key self._max_priority = config.getint("amqp", "max-priority", 1)
def __init__(self, cache): super(DiskVolume, self).__init__() self._cache = cache self._path = config.get(NAME, "path") raise_error_if(not self._path, "volume path not configured") fs.makedirs(self._path) self._upload = config.getboolean(NAME, "upload", True) self._download = config.getboolean(NAME, "download", True)
def _get_auth(self): service = config.get(NAME, "keyring.service") if not service: return None, None username = config.get(NAME, "keyring.username") if not username: username = input(NAME + " username: "******"no username configured for " + NAME) config.set(NAME, "keyring.username", username) config.save() password = config.get( NAME, "keyring.password") or keyring.get_password(NAME, username) if not password: password = getpass.getpass(NAME + " password: "******"no password in keyring for " + NAME) keyring.set_password(service, username, password) return username, password
def __init__(self, cache): super(Http, self).__init__() self._cache = cache self._uri = config.get(NAME, "uri") raise_error_if(not self._uri, "HTTP URI not configured") if self._uri[-1] != "/": self._uri += "/" self._upload = config.getboolean(NAME, "upload", True) self._download = config.getboolean(NAME, "download", True) self._disabled = False
def load_plugins(self): searchpath = config.get("jolt", "pluginpath") searchpath = searchpath.split(":") if searchpath else [] searchpath.append(fs.path.join(fs.path.dirname(__file__), "plugins")) for plugin in config.plugins(): for path in searchpath: if "jolt.plugins." + plugin not in sys.modules: module = fs.path.join(fs.path.dirname(__file__), path, plugin + ".py") if fs.path.exists(module): self.load_plugin(module) continue
def __init__(self, plugin="telemetry", uri=None, local=True, network=True, queued=True, started=True, failed=True, finished=True): self._uri = uri or config.get(plugin, "uri", uri) self._network = config.getboolean(plugin, "network", network) self._local = config.getboolean(plugin, "local", local) self._queued = config.getboolean(plugin, "queued", queued) self._started = config.getboolean(plugin, "started", started) self._failed = config.getboolean(plugin, "failed", failed) self._finished = config.getboolean(plugin, "finished", finished) raise_error_if(not self._uri, "telemetry.uri not configured")
def _log(follow, delete): """ Display the Jolt log file. """ if follow: subprocess.call("tail -f {0}".format(logfile), shell=True) elif delete: fs.unlink(logfile) else: t = tools.Tools() configured_pager = config.get("jolt", "pager", environ.get("PAGER", None)) for pager in [configured_pager, "less", "more", "cat"]: if pager and t.which(pager): return subprocess.call("{1} {0}".format(logfile, pager), shell=True) print(t.read_file(logfile))
def __init__(self): self._server = config.get("email", "server") self._to = config.get("email", "to") self._from = config.get("email", "from", "jolt@localhost") self._subject = config.get("email", "subject", "Jolt Build Report") self._stylesheet = config.get( "email", "stylesheet", fs.path.join(fs.path.dirname(__file__), "email.xslt")) self._artifact = config.get("email", "artifact") self._failure = config.getboolean("email", "on_failure", True) self._success = config.getboolean("email", "on_success", True) raise_error_if(not self._server, "email.server not configured") raise_error_if(not self._server, "email.to not configured")
def get_parameters(self, task): registry = TaskRegistry() registry.add_task_class(Jolt) acache = ArtifactCache.get() env = JoltEnvironment(cache=acache) gb = GraphBuilder(registry, JoltManifest()) dag = gb.build(["jolt"]) task = dag.select(lambda graph, task: True) assert len(task) == 1, "too many selfdeploy tasks found" task = task[0] if not acache.is_available_remotely(task): factory = LocalExecutorFactory() executor = LocalExecutor(factory, task, force_upload=True) executor.run(env) jolt_url = acache.location(task) raise_error_if(not jolt_url, "failed to deploy jolt to a remote cache") return { "jolt_url": jolt_url, "jolt_identity": task.identity[:8], "jolt_requires": config.get("selfdeploy", "requires", "") }
def extra_dependencies(self): req = config.get("selfdeploy", "requires", "") return req.split() if req else []
def docs(): """ Opens the Jolt documentation in the default webbrowser. """ webbrowser.open(config.get("jolt", "docs", "http://jolt.readthedocs.io/"))
def _print_key(section, opt): value = config.get(section, opt, alias=alias) raise_error_if(value is None, "no such key: {}".format(key)) print("{} = {}".format(key, value))
def cli(ctx, verbose, extra_verbose, config_file, debugger, profile, force, salt, debug, network, local, keep_going, jobs): """ A task execution tool. When invoked without any commands and arguments, Jolt by default tries to execute and build the artifact of a task called `default`. To build artifacts of other tasks use the build subcommand. The Jolt command line interface is hierarchical. One set of options can be passed to the top-level command and a different set of options to the subcommands, simultaneously. For example, verbose output is a top-level option while forced rebuild is a build command option. They may combined like this: $ jolt --verbose build --force taskname Most build command options are available also at the top-level when build is invoked implicitly for the default task. """ global debug_enabled debug_enabled = debugger log.verbose("Jolt command: {}", " ".join([fs.path.basename(sys.argv[0])] + sys.argv[1:])) log.verbose("Jolt host: {}", environ.get("HOSTNAME", "localhost")) log.verbose("Jolt install path: {}", fs.path.dirname(__file__)) if ctx.invoked_subcommand in ["config"]: # Don't attempt to load any task recipes as they might require # plugins that are not yet configured. return if ctx.invoked_subcommand is None: build = ctx.command.get_command(ctx, "build") manifest = JoltManifest() utils.call_and_catch(manifest.parse) manifest.process_import() ctx.obj["manifest"] = manifest if manifest.version: from jolt.version_utils import requirement, version req = requirement(manifest.version) ver = version(__version__) raise_error_if(not req.satisfied(ver), "this project requires Jolt version {} (running {})", req, __version__) loader = JoltLoader.get() tasks = loader.load() for cls in tasks: TaskRegistry.get().add_task_class(cls) if ctx.invoked_subcommand in ["build", "clean"] and loader.joltdir: ctx.obj["workspace_lock"] = utils.LockFile( fs.path.join(loader.joltdir, "build"), log.info, "Workspace is locked by another process, please wait...") atexit.register(ctx.obj["workspace_lock"].close) # If no command is given, we default to building the default task. # If the default task doesn't exist, help is printed inside build(). if ctx.invoked_subcommand is None: task = config.get("jolt", "default", "default") taskname, _ = utils.parse_task_name(task) if TaskRegistry.get().get_task_class(taskname) is not None: ctx.invoke(build, task=[task], force=force, salt=salt, debug=debug, network=network, local=local, keep_going=keep_going, jobs=jobs) else: print(cli.get_help(ctx)) sys.exit(1)
def __init__(self, uri=None): uri = config.get("dashboard", "uri", "http://dashboard") error.raise_error_if(not uri, "dashboard.uri not configured") super().__init__(plugin="dashboard", uri=uri + "/api/v1/tasks", local=False)
def _get_url(): host = config.get("amqp", "host", "amqp-service") port = int(config.get("amqp", "port", 5672)) username, password = _get_auth() return "amqp://{}:{}@{}:{}/%2F".format(username, password, host, port)
def run(self): with open("default.joltxmanifest", "wb") as f: f.write(self.body) log.info("Manifest written") tools = Tools() for recipe in tools.glob("*.jolt"): tools.unlink(recipe) try: jolt = self.selfdeploy() config_file = config.get("amqp", "config", "") if config_file: config_file = "-c " + config_file log.info("Running jolt") tools.run( "{} -vv {} build --worker --result result.joltxmanifest", jolt, config_file, output_stdio=True) except JoltCommandError as e: self.response = "" try: manifest = JoltManifest() try: manifest.parse("result.joltxmanifest") except Exception: manifest.duration = "0" manifest.result = "FAILED" manifest.stdout = "\n".join(e.stdout) manifest.stderr = "\n".join(e.stderr) self.response = manifest.format() except Exception: log.exception() log.error("Task failed") except Exception: log.exception() self.response = "" try: manifest = JoltManifest() try: manifest.parse("result.joltxmanifest") except Exception: manifest.duration = "0" manifest.result = "FAILED" self.response = manifest.format() except Exception: log.exception() log.error("Task failed") else: self.response = "" try: manifest = JoltManifest() try: manifest.parse("result.joltxmanifest") except Exception: manifest.duration = "0" manifest.result = "SUCCESS" self.response = manifest.format() except Exception: log.exception() log.info("Task succeeded") utils.call_and_catch(tools.unlink, "result.joltxmanifest") self.consumer.add_on_job_completed_callback(self)
def __init__(self): self._path = config.get("symlinks", "path", "artifacts") raise_error_if(not self._path, "symlinks.path not configured")
def extras(self): ext = config.get("selfdeploy", "extra", "") return [fs.path.join(self.loaderdir, e) for e in ext.split(",")] if ext else []
import threading import logging import logging.handlers from contextlib import contextmanager try: from StringIO import StringIO except Exception: from io import StringIO from jolt import config from jolt.error import JoltError from jolt import filesystem as fs from jolt import colors default_path = fs.path.join(config.get_logpath(), "jolt.log") logfile = config.get("jolt", "logfile", default_path) logsize = config.getsize("jolt", "logsize", os.environ.get("JOLT_LOGSIZE", 10 * 1024**2)) # 10MiB logcount = config.getint("jolt", "logcount", os.environ.get("JOLT_LOGCOUNT", 1)) dirpath = fs.path.dirname(logfile) if not fs.path.exists(dirpath): fs.makedirs(dirpath) with open(logfile, "a") as f: f.write( "--------------------------------------------------------------------------------\n" ) ################################################################################
def __init__(self): self._uri = config.get("logstash", "http.uri") self._failed_enabled = config.getboolean("logstash", "failed", False) self._finished_enabled = config.getboolean("logstash", "finished", False) raise_error_if(not self._uri, "logstash.http.uri not configured")