def __init__(self, cache):
super(FtpStorage, self).__init__()
self._cache = cache
self._uri = config.get(NAME, "host")
raise_error_if(not self._uri, "ftp URI not configured")
self._path = config.get(NAME, "path", "")
self._upload = config.getboolean(NAME, "upload", True)
self._download = config.getboolean(NAME, "download", True)
self._tls = config.getboolean(NAME, "tls", False)
self._disabled = False
def _get_auth():
service = config.get(NAME, "keyring.service", NAME)
username = config.get(NAME, "keyring.username")
if not username:
return "guest", "guest"
password = config.get(NAME, "keyring.password") or keyring.get_password(NAME, username)
if not password:
password = getpass.getpass(NAME + " password: "******"no password in keyring for " + NAME
keyring.set_password(service, username, password)
return username, password
def __init__(self, *args, **kwargs):
super(Jolt, self).__init__(*args, **kwargs)
self.influence.append(
influence.StringInfluence(config.get("selfdeploy", "requires",
"")))
for e in self.extras:
self.influence.append(influence.DirectoryInfluence(e))
def __init__(self, amqp_url):
"""Create a new instance of the consumer class, passing in the AMQP
URL used to connect to RabbitMQ.
:param str amqp_url: The AMQP url to connect with
"""
self.should_reconnect = False
self.was_consuming = False
self._connection = None
self._channel = None
self._closing = False
self._consumer_tag = None
self._url = amqp_url
self._consuming = False
# In production, experiment with higher prefetch values
# for higher consumer throughput
self._prefetch_count = 1
self._job = None
self._routing_key = self.ROUTING_KEY_PREFIX + config.get(
"amqp", "routing_key",
os.getenv("RABBITMQ_ROUTING_KEY", self.ROUTING_KEY_REQUEST))
self._queue = self.QUEUE + "_" + self._routing_key
self._max_priority = config.getint("amqp", "max-priority", 1)
def __init__(self, cache):
super(DiskVolume, self).__init__()
self._cache = cache
self._path = config.get(NAME, "path")
raise_error_if(not self._path, "volume path not configured")
fs.makedirs(self._path)
self._upload = config.getboolean(NAME, "upload", True)
self._download = config.getboolean(NAME, "download", True)
def _get_auth(self):
service = config.get(NAME, "keyring.service")
if not service:
return None, None
username = config.get(NAME, "keyring.username")
if not username:
username = input(NAME + " username: "******"no username configured for " + NAME)
config.set(NAME, "keyring.username", username)
config.save()
password = config.get(
NAME, "keyring.password") or keyring.get_password(NAME, username)
if not password:
password = getpass.getpass(NAME + " password: "******"no password in keyring for " + NAME)
keyring.set_password(service, username, password)
return username, password
def __init__(self, cache):
super(Http, self).__init__()
self._cache = cache
self._uri = config.get(NAME, "uri")
raise_error_if(not self._uri, "HTTP URI not configured")
if self._uri[-1] != "/":
self._uri += "/"
self._upload = config.getboolean(NAME, "upload", True)
self._download = config.getboolean(NAME, "download", True)
self._disabled = False
def load_plugins(self):
searchpath = config.get("jolt", "pluginpath")
searchpath = searchpath.split(":") if searchpath else []
searchpath.append(fs.path.join(fs.path.dirname(__file__), "plugins"))
for plugin in config.plugins():
for path in searchpath:
if "jolt.plugins." + plugin not in sys.modules:
module = fs.path.join(fs.path.dirname(__file__), path,
plugin + ".py")
if fs.path.exists(module):
self.load_plugin(module)
continue
def __init__(self,
plugin="telemetry",
uri=None,
local=True,
network=True,
queued=True,
started=True,
failed=True,
finished=True):
self._uri = uri or config.get(plugin, "uri", uri)
self._network = config.getboolean(plugin, "network", network)
self._local = config.getboolean(plugin, "local", local)
self._queued = config.getboolean(plugin, "queued", queued)
self._started = config.getboolean(plugin, "started", started)
self._failed = config.getboolean(plugin, "failed", failed)
self._finished = config.getboolean(plugin, "finished", finished)
raise_error_if(not self._uri, "telemetry.uri not configured")
def _log(follow, delete):
"""
Display the Jolt log file.
"""
if follow:
subprocess.call("tail -f {0}".format(logfile), shell=True)
elif delete:
fs.unlink(logfile)
else:
t = tools.Tools()
configured_pager = config.get("jolt", "pager",
environ.get("PAGER", None))
for pager in [configured_pager, "less", "more", "cat"]:
if pager and t.which(pager):
return subprocess.call("{1} {0}".format(logfile, pager),
shell=True)
print(t.read_file(logfile))
def __init__(self):
self._server = config.get("email", "server")
self._to = config.get("email", "to")
self._from = config.get("email", "from", "jolt@localhost")
self._subject = config.get("email", "subject", "Jolt Build Report")
self._stylesheet = config.get(
"email", "stylesheet",
fs.path.join(fs.path.dirname(__file__), "email.xslt"))
self._artifact = config.get("email", "artifact")
self._failure = config.getboolean("email", "on_failure", True)
self._success = config.getboolean("email", "on_success", True)
raise_error_if(not self._server, "email.server not configured")
raise_error_if(not self._server, "email.to not configured")
def get_parameters(self, task):
registry = TaskRegistry()
registry.add_task_class(Jolt)
acache = ArtifactCache.get()
env = JoltEnvironment(cache=acache)
gb = GraphBuilder(registry, JoltManifest())
dag = gb.build(["jolt"])
task = dag.select(lambda graph, task: True)
assert len(task) == 1, "too many selfdeploy tasks found"
task = task[0]
if not acache.is_available_remotely(task):
factory = LocalExecutorFactory()
executor = LocalExecutor(factory, task, force_upload=True)
executor.run(env)
jolt_url = acache.location(task)
raise_error_if(not jolt_url, "failed to deploy jolt to a remote cache")
return {
"jolt_url": jolt_url,
"jolt_identity": task.identity[:8],
"jolt_requires": config.get("selfdeploy", "requires", "")
}
def extra_dependencies(self):
req = config.get("selfdeploy", "requires", "")
return req.split() if req else []
def docs():
"""
Opens the Jolt documentation in the default webbrowser.
"""
webbrowser.open(config.get("jolt", "docs", "http://jolt.readthedocs.io/"))
def _print_key(section, opt):
value = config.get(section, opt, alias=alias)
raise_error_if(value is None, "no such key: {}".format(key))
print("{} = {}".format(key, value))
def cli(ctx, verbose, extra_verbose, config_file, debugger, profile, force,
salt, debug, network, local, keep_going, jobs):
"""
A task execution tool.
When invoked without any commands and arguments, Jolt by default tries
to execute and build the artifact of a task called `default`. To build
artifacts of other tasks use the build subcommand.
The Jolt command line interface is hierarchical. One set of options
can be passed to the top-level command and a different set of options
to the subcommands, simultaneously. For example, verbose output is
a top-level option while forced rebuild is a build command option.
They may combined like this:
$ jolt --verbose build --force taskname
Most build command options are available also at the top-level when
build is invoked implicitly for the default task.
"""
global debug_enabled
debug_enabled = debugger
log.verbose("Jolt command: {}",
" ".join([fs.path.basename(sys.argv[0])] + sys.argv[1:]))
log.verbose("Jolt host: {}", environ.get("HOSTNAME", "localhost"))
log.verbose("Jolt install path: {}", fs.path.dirname(__file__))
if ctx.invoked_subcommand in ["config"]:
# Don't attempt to load any task recipes as they might require
# plugins that are not yet configured.
return
if ctx.invoked_subcommand is None:
build = ctx.command.get_command(ctx, "build")
manifest = JoltManifest()
utils.call_and_catch(manifest.parse)
manifest.process_import()
ctx.obj["manifest"] = manifest
if manifest.version:
from jolt.version_utils import requirement, version
req = requirement(manifest.version)
ver = version(__version__)
raise_error_if(not req.satisfied(ver),
"this project requires Jolt version {} (running {})",
req, __version__)
loader = JoltLoader.get()
tasks = loader.load()
for cls in tasks:
TaskRegistry.get().add_task_class(cls)
if ctx.invoked_subcommand in ["build", "clean"] and loader.joltdir:
ctx.obj["workspace_lock"] = utils.LockFile(
fs.path.join(loader.joltdir, "build"), log.info,
"Workspace is locked by another process, please wait...")
atexit.register(ctx.obj["workspace_lock"].close)
# If no command is given, we default to building the default task.
# If the default task doesn't exist, help is printed inside build().
if ctx.invoked_subcommand is None:
task = config.get("jolt", "default", "default")
taskname, _ = utils.parse_task_name(task)
if TaskRegistry.get().get_task_class(taskname) is not None:
ctx.invoke(build,
task=[task],
force=force,
salt=salt,
debug=debug,
network=network,
local=local,
keep_going=keep_going,
jobs=jobs)
else:
print(cli.get_help(ctx))
sys.exit(1)
def __init__(self, uri=None):
uri = config.get("dashboard", "uri", "http://dashboard")
error.raise_error_if(not uri, "dashboard.uri not configured")
super().__init__(plugin="dashboard", uri=uri + "/api/v1/tasks", local=False)
def _get_url():
host = config.get("amqp", "host", "amqp-service")
port = int(config.get("amqp", "port", 5672))
username, password = _get_auth()
return "amqp://{}:{}@{}:{}/%2F".format(username, password, host, port)
def run(self):
with open("default.joltxmanifest", "wb") as f:
f.write(self.body)
log.info("Manifest written")
tools = Tools()
for recipe in tools.glob("*.jolt"):
tools.unlink(recipe)
try:
jolt = self.selfdeploy()
config_file = config.get("amqp", "config", "")
if config_file:
config_file = "-c " + config_file
log.info("Running jolt")
tools.run(
"{} -vv {} build --worker --result result.joltxmanifest",
jolt,
config_file,
output_stdio=True)
except JoltCommandError as e:
self.response = ""
try:
manifest = JoltManifest()
try:
manifest.parse("result.joltxmanifest")
except Exception:
manifest.duration = "0"
manifest.result = "FAILED"
manifest.stdout = "\n".join(e.stdout)
manifest.stderr = "\n".join(e.stderr)
self.response = manifest.format()
except Exception:
log.exception()
log.error("Task failed")
except Exception:
log.exception()
self.response = ""
try:
manifest = JoltManifest()
try:
manifest.parse("result.joltxmanifest")
except Exception:
manifest.duration = "0"
manifest.result = "FAILED"
self.response = manifest.format()
except Exception:
log.exception()
log.error("Task failed")
else:
self.response = ""
try:
manifest = JoltManifest()
try:
manifest.parse("result.joltxmanifest")
except Exception:
manifest.duration = "0"
manifest.result = "SUCCESS"
self.response = manifest.format()
except Exception:
log.exception()
log.info("Task succeeded")
utils.call_and_catch(tools.unlink, "result.joltxmanifest")
self.consumer.add_on_job_completed_callback(self)
def __init__(self):
self._path = config.get("symlinks", "path", "artifacts")
raise_error_if(not self._path, "symlinks.path not configured")
def extras(self):
ext = config.get("selfdeploy", "extra", "")
return [fs.path.join(self.loaderdir, e)
for e in ext.split(",")] if ext else []
import threading
import logging
import logging.handlers
from contextlib import contextmanager
try:
from StringIO import StringIO
except Exception:
from io import StringIO
from jolt import config
from jolt.error import JoltError
from jolt import filesystem as fs
from jolt import colors
default_path = fs.path.join(config.get_logpath(), "jolt.log")
logfile = config.get("jolt", "logfile", default_path)
logsize = config.getsize("jolt", "logsize",
os.environ.get("JOLT_LOGSIZE", 10 * 1024**2)) # 10MiB
logcount = config.getint("jolt", "logcount",
os.environ.get("JOLT_LOGCOUNT", 1))
dirpath = fs.path.dirname(logfile)
if not fs.path.exists(dirpath):
fs.makedirs(dirpath)
with open(logfile, "a") as f:
f.write(
"--------------------------------------------------------------------------------\n"
)
################################################################################
def __init__(self):
self._uri = config.get("logstash", "http.uri")
self._failed_enabled = config.getboolean("logstash", "failed", False)
self._finished_enabled = config.getboolean("logstash", "finished",
False)
raise_error_if(not self._uri, "logstash.http.uri not configured")
