def run(self, env):
try:
self.task.started(TYPE)
for extension in self.task.extensions:
extension.started(TYPE)
with hooks.task_run([self.task] + self.task.extensions):
self._run(env)
for extension in self.task.extensions:
extension.finished(TYPE)
self.task.finished(TYPE)
except (ConnectionError, AMQPConnectionError):
log.exception()
for extension in self.task.extensions:
extension.failed(TYPE)
self.task.failed(TYPE)
raise_error("Lost connection to AMQP server")
except Exception as e:
log.exception()
for extension in self.task.extensions:
extension.failed(TYPE)
self.task.failed(TYPE)
raise e
finally:
if self.connection is not None:
utils.call_and_catch(self.connection.close)
return self.task
def _get_ftp(self):
try:
username, password = self._get_auth()
if self._tls:
ftp = FTP_TLS(self._uri, timeout=CONNECT_TIMEOUT)
else:
ftp = FTP(self._uri, timeout=CONNECT_TIMEOUT)
ftp.login(username, password)
if self._tls:
ftp.prot_d()
if not catch(ftp.cwd, self._path):
if self._path.startswith("/"):
ftp.cwd("/")
components = self._path.split("/")
for component in components:
if not catch(ftp.cwd, component):
ftp.mkd(component)
ftp.cwd(component)
return ftp
except Exception:
log.exception()
log.warning(
"[FTP] failed to establish server connection, disabled")
self._disabled = True
return None
def catch(func, *args, **kwargs):
try:
val = func(*args, **kwargs)
return val if val is not None else True
except Exception:
log.exception()
return False
def upload(self, node, force=False):
if not self._upload and not force:
return True
with self._cache.get_artifact(node) as artifact:
path = self._get_path(node, artifact)
temp = self._get_temp(node, artifact)
try:
log.verbose("[VOLUME] Copying {}", path)
fs.copy(artifact.get_archive_path(), temp)
# To avoid race-condition, make sure that the artifact still is missing before moving it into place.
if not fs.exists(path):
fs.rename(temp, path)
else:
fs.unlink(temp)
return True
except OSError as e:
if e.errno != errno.EEXIST:
log.verbose("[VOLUME] Failed to copy artifact, errno={}",
os.strerror(e.errno))
return e.errno == errno.EEXIST
except Exception:
log.exception()
finally:
fs.unlink(temp, ignore_errors=True)
return False
def get_influence(self, task):
self.tools = Tools(task, task.joltdir)
try:
manifest_path = fs.path.join(task.joltdir, task.expand(self.path))
manifest = RepoManifest(task, manifest_path)
manifest.parse(fs.path.join(manifest_path, ".repo",
"manifest.xml"))
result = []
for project in sorted(manifest.projects, key=lambda p: p.name):
if self.include is not None and project.path_or_name not in self.include:
continue
if self.exclude is not None and project.path_or_name in self.exclude:
continue
with _git_repo_lock:
gip = _git_repos.get(project.path_or_name)
if gip is None:
gip = git.GitInfluenceProvider(project.path_or_name)
_git_repos[project.path_or_name] = gip
result.append(gip.get_influence(task))
return "\n".join(result)
except KeyError:
log.exception()
assert False, "failed to calculate hash influence for repo manifest at {0}".format(
self.path)
def publish(self, artifact, tools):
with tools.cwd(tools.builddir()):
try:
pinned_reqs = self.dependencies
if pinned_reqs:
tools.write_file("requirements.txt",
"\n".join(pinned_reqs))
artifact.collect("requirements.txt")
except Exception:
log.exception()
with tools.cwd(_path):
artifact.collect('README.rst')
artifact.collect('setup.py')
artifact.collect('jolt/*.py')
artifact.collect('jolt/*.sh')
artifact.collect('jolt/*/*.py')
artifact.collect('jolt/*/*/*.py')
artifact.collect('jolt/*/*.xslt')
artifact.collect('jolt/*/*.template')
artifact.collect('jolt/plugins/selfdeploy/README.rst',
flatten=True)
artifact.collect('jolt/plugins/selfdeploy/setup.py', flatten=True)
for e in self.extra_files:
with tools.cwd(fs.path.dirname(e)):
artifact.collect(fs.path.basename(e))
def call_and_catch_and_log(f, *args, **kwargs):
try:
return f(*args, **kwargs)
except KeyboardInterrupt as e:
raise e
except Exception:
from jolt import log
log.exception()
return None
def wait(self):
for future in as_completed(self.futures):
task = self.futures[future]
try:
future.result()
except Exception as error:
log.exception()
return task, error
finally:
self.duration_acc += task.duration_running or 0
del self.futures[future]
return task, None
return None, None
def run(self, env):
if self.is_aborted():
return
try:
self.task.started()
with hooks.task_run(self.task):
self.task.run(env.cache,
force_build=self.force_build,
force_upload=self.force_upload)
except Exception as e:
log.exception()
self.task.failed()
raise e
else:
self.task.finished()
return self.task
def download(self, node, force=False):
if not self._download and not force:
return False
with self._cache.get_artifact(node) as artifact:
path = self._get_path(node, artifact)
try:
log.verbose("[VOLUME] Copying {}", path)
fs.copy(path, artifact.get_archive_path())
return True
except OSError as e:
if e.errno == errno.ESTALE:
log.verbose("[VOLUME] got stale file handle, retrying...")
raise StaleFileHandleError(e)
else:
log.exception()
except Exception:
log.exception()
return False
def selfdeploy(self):
""" Installs the correct version of Jolt as specified in execution request. """
tools = Tools()
manifest = JoltManifest()
try:
manifest.parse()
ident = manifest.get_parameter("jolt_identity")
url = manifest.get_parameter("jolt_url")
if not ident or not url:
return "jolt"
requires = manifest.get_parameter("jolt_requires")
log.info("Jolt version: {}", ident)
src = "build/selfdeploy/{}/src".format(ident)
env = "build/selfdeploy/{}/env".format(ident)
if not fs.path.exists(env):
try:
fs.makedirs(src)
tools.run("curl {} | tar zx -C {}", url, src)
tools.run("virtualenv {}", env)
tools.run(". {}/bin/activate && pip install -e {}",
env, src)
if requires:
tools.run(
". {}/bin/activate && pip install {}", env,
requires)
except Exception as e:
tools.rmtree("build/selfdeploy/{}",
ident,
ignore_errors=True)
raise e
return ". {}/bin/activate && jolt".format(env)
except Exception as e:
log.exception()
raise e
def inspect(ctx, task, influence=False, artifact=False, salt=None):
"""
View information about a task.
This command displays information about a task, such as its class
documentation, parameters and their accepted values, requirements,
task class origin (file/line), influence attributes, artifact identity,
cache status, and more. Default parameter values, if any, are highlighted.
"""
task_name = task
task_cls_name, task_params = utils.parse_task_name(task_name)
task_registry = TaskRegistry.get()
task = task_registry.get_task_class(task_cls_name)
raise_task_error_if(not task, task_name, "no such task")
from jolt import inspection
print()
print(" {0}".format(task.name))
print()
if task.__doc__:
print(" {0}".format(task.__doc__.strip()))
print()
print(" Parameters")
has_param = False
params = {
key: getattr(task, key)
for key in dir(task)
if isinstance(utils.getattr_safe(task, key), Parameter)
}
for item, param in params.items():
has_param = True
print(" {0:<15} {1}".format(item, param.help or ""))
if not has_param:
print(" None")
print()
print(" Definition")
print(" {0:<15} {1} ({2})".format(
"File",
fs.path.relpath(inspection.getfile(task),
JoltLoader.get().joltdir), inspection.getlineno(task)))
print()
print(" Requirements")
manifest = ctx.obj["manifest"]
try:
task = task_registry.get_task(task_name, manifest=manifest)
for req in sorted(
utils.as_list(utils.call_or_return(task, task.requires))):
print(" {0}".format(task.tools.expand(req)))
if not task.requires:
print(" None")
print()
except Exception as e:
log.exception()
if "has not been set" in str(e):
print(" Unavailable (parameters must be set)")
print()
return
print(" Unavailable (exception during evaluation)")
print()
return
if salt:
task.taint = salt
if artifact:
acache = cache.ArtifactCache.get()
builder = graph.GraphBuilder(task_registry, manifest)
dag = builder.build([task.qualified_name])
tasks = dag.select(lambda graph, node: node.task is task)
assert len(tasks) == 1, "graph produced multiple tasks, one expected"
proxy = tasks[0]
task = proxy.task
print(" Cache")
print(" Identity {0}".format(proxy.identity))
if acache.is_available_locally(proxy):
with acache.get_artifact(proxy) as artifact:
print(" Location {0}".format(artifact.path))
print(" Local True ({0})".format(
utils.as_human_size(acache.get_artifact(proxy).get_size())))
else:
print(" Local False")
print(" Remote {0}".format(
acache.is_available_remotely(proxy)))
print()
if influence:
print(" Influence")
for string in HashInfluenceRegistry.get().get_strings(task):
string = string.split(":", 1)
print(" {:<18}{}".format(string[0][10:], string[1].strip()))
def _config(ctx, list, delete, global_, user, key, value):
"""
Configure Jolt.
You can query/set/replace/unset configuration keys with this command.
Key strings are constructed from the configuration section and the
option separated by a dot.
There are tree different configuration sources:
- A global configuration file
- A user configuration file
- Temporary configuration passed on the command line.
When reading, the values are read from all configuration sources.
The options --global and --user can be used to tell the command to read
from only one of the sources. If a configuration key is available from
multiple sources, temporary CLI configuration has priority followed by
the user configuration file and lastly the global configuration file.
When writing, the new values are written to the user configuration by default.
The options --global and --user can be used to tell the command to write
to only one of the sources.
When removing keys, the values are removed from all sources.
The options --global and --user can be used to restrict removal to one of
the sources.
To assign a value to a key:
$ jolt config jolt.default all # Change name of the default task
To list existing keys:
$ jolt config -l # List all existing keys
$ jolt config -l -g # List keys in the global config file
$ jolt config jolt.colors # Display the value of a key.
To delete an existing key:
$ jolt config -d jolt.colors
To pass temporary configuration:
$ jolt -c jolt.colors=true config -l
"""
if delete and not key:
raise click.UsageError("--delete requires KEY")
if not key and not list and not key:
print(ctx.get_help())
sys.exit(1)
if global_ and user:
raise click.UsageError("--global and --user are mutually exclusive")
alias = None
if global_:
alias = "global"
if user:
alias = "user"
def _print_key(section, opt):
value = config.get(section, opt, alias=alias)
raise_error_if(value is None, "no such key: {}".format(key))
print("{} = {}".format(key, value))
def _print_section(section):
print("{}".format(section))
if list:
for section, option, value in config.items(alias):
if option:
print("{}.{} = {}".format(section, option, value))
else:
print(section)
elif delete:
raise_error_if(config.delete(key, alias) <= 0, "no such key: {}", key)
config.save()
elif key:
section, opt = config.split(key)
if value:
raise_error_if(opt is None,
"invalid configuration key: {}".format(key))
config.set(section, opt, value, alias)
try:
config.save()
except Exception as e:
log.exception()
raise_error("failed to write configuration file: {}".format(e))
else:
if opt:
_print_key(section, opt)
else:
_print_section(section)
def run(self):
with open("default.joltxmanifest", "wb") as f:
f.write(self.body)
log.info("Manifest written")
tools = Tools()
for recipe in tools.glob("*.jolt"):
tools.unlink(recipe)
try:
jolt = self.selfdeploy()
config_file = config.get("amqp", "config", "")
if config_file:
config_file = "-c " + config_file
log.info("Running jolt")
tools.run(
"{} -vv {} build --worker --result result.joltxmanifest",
jolt,
config_file,
output_stdio=True)
except JoltCommandError as e:
self.response = ""
try:
manifest = JoltManifest()
try:
manifest.parse("result.joltxmanifest")
except Exception:
manifest.duration = "0"
manifest.result = "FAILED"
manifest.stdout = "\n".join(e.stdout)
manifest.stderr = "\n".join(e.stderr)
self.response = manifest.format()
except Exception:
log.exception()
log.error("Task failed")
except Exception:
log.exception()
self.response = ""
try:
manifest = JoltManifest()
try:
manifest.parse("result.joltxmanifest")
except Exception:
manifest.duration = "0"
manifest.result = "FAILED"
self.response = manifest.format()
except Exception:
log.exception()
log.error("Task failed")
else:
self.response = ""
try:
manifest = JoltManifest()
try:
manifest.parse("result.joltxmanifest")
except Exception:
manifest.duration = "0"
manifest.result = "SUCCESS"
self.response = manifest.format()
except Exception:
log.exception()
log.info("Task succeeded")
utils.call_and_catch(tools.unlink, "result.joltxmanifest")
self.consumer.add_on_job_completed_callback(self)