def _get_ftp(self):
try:
username, password = self._get_auth()
if self._tls:
ftp = FTP_TLS(self._uri, timeout=CONNECT_TIMEOUT)
else:
ftp = FTP(self._uri, timeout=CONNECT_TIMEOUT)
ftp.login(username, password)
if self._tls:
ftp.prot_d()
if not catch(ftp.cwd, self._path):
if self._path.startswith("/"):
ftp.cwd("/")
components = self._path.split("/")
for component in components:
if not catch(ftp.cwd, component):
ftp.mkd(component)
ftp.cwd(component)
return ftp
except Exception:
log.exception()
log.warning(
"[FTP] failed to establish server connection, disabled")
self._disabled = True
return None
def onerror_warning(func, path, exc_info):
from jolt import log
if type(exc_info[1]) == OSError:
msg = exc_info[1].strerror
else:
msg = "Reason unknown"
if os.path.exists(path):
log.warning("Could not remove file or directory: {} ({})", path, msg)
def on_channel_closed(self, channel, reason):
"""Invoked by pika when RabbitMQ unexpectedly closes the channel.
Channels are usually closed if you attempt to do something that
violates the protocol, such as re-declare an exchange or queue with
different parameters. In this case, we'll close the connection
to shutdown the object.
:param pika.channel.Channel: The closed channel
:param Exception reason: why the channel was closed
"""
log.warning('Channel {} was closed: {}', channel, reason)
self.close_connection()
def on_connection_closed(self, _unused_connection, reason):
"""This method is invoked by pika when the connection to RabbitMQ is
closed unexpectedly. Since it is unexpected, we will reconnect to
RabbitMQ if it disconnects.
:param pika.connection.Connection connection: The closed connection obj
:param Exception reason: exception representing reason for loss of
connection.
"""
self._channel = None
if self._closing:
self._connection.ioloop.stop()
else:
log.warning('Connection closed, reconnect necessary: {}', reason)
self.reconnect()
def location(self, node):
if self._disabled:
return False
with self._cache.get_artifact(node) as artifact:
from requests.api import head
url = self._get_url(node, artifact)
try:
response = head(url, stream=True, timeout=TIMEOUT_HEAD)
except ConnectTimeout:
self._disabled = True
log.warning(
"[HTTP] failed to establish server connection, disabled")
return False
log.debug("[HTTP] Head: {0}", url)
log.debug("[HTTP] Response: {0}", response.status_code)
return url if response.status_code == 200 else ''
return False
def build(ctx, task, network, keep_going, default, local, no_download,
no_upload, download, upload, worker, force, salt, copy, debug,
result, jobs):
"""
Build task artifact.
TASK is the name of the task to execute. It is optionally followed by a colon and
parameter value assignments. Assignments are separated by commas. Example:
taskname:param1=value1,param2=value2
Default parameter values can be overridden for any task in the dependency tree
with --default. DEFAULT is a qualified task name, just like TASK, but parameter
assignments change default values.
By default, a task is executed locally and the resulting artifact is stored
in the local artifact cache. If an artifact is already available in the cache,
no execution takes place. Artifacts are identified with a hash digest,
constructed from hashing task attributes.
When remote cache providers are configured, artifacts may be downloaded from and/or
uploaded to the remote cache as execution progresses. Several options exist to control
the behavior, such as --local which disables all remote caches.
Distributed task execution is enabled by passing the --network option. Tasks are then
distributed to and executed by a pool of workers, if one has been configured.
Rebuilds can be forced with either --force or --salt. --force rebuilds the requested
task, but not its dependencies. --salt affects the entire dependency tree. Both add
an extra attribute to the task hash calculation in order to taint the identity and
induce a cache miss. In both cases, existing intermediate files in build directories
are removed before execution starts.
"""
raise_error_if(network and local,
"The -n and -l flags are mutually exclusive")
raise_error_if(network and debug,
"The -g and -n flags are mutually exclusive")
raise_error_if(
no_download and download,
"The --download and --no-download flags are mutually exclusive")
raise_error_if(
no_upload and upload,
"The --upload and --no-upload flags are mutually exclusive")
duration = utils.duration()
task = list(task)
task = [utils.stable_task_name(t) for t in task]
if network:
_download = config.getboolean("network", "download", True)
_upload = config.getboolean("network", "upload", True)
else:
_download = config.getboolean("jolt", "download", True)
_upload = config.getboolean("jolt", "upload", True)
if local:
_download = False
_upload = False
else:
if no_download:
_download = False
if no_upload:
_upload = False
if download:
_download = True
if upload:
_upload = True
options = JoltOptions(network=network,
local=local,
download=_download,
upload=_upload,
keep_going=keep_going,
default=default,
worker=worker,
debug=debug,
salt=salt,
jobs=jobs)
acache = cache.ArtifactCache.get(options)
executors = scheduler.ExecutorRegistry.get(options)
if worker:
log.set_worker()
log.verbose("Local build as a worker")
strategy = scheduler.WorkerStrategy(executors, acache)
elif network:
log.verbose("Distributed build as a user")
strategy = scheduler.DistributedStrategy(executors, acache)
else:
log.verbose("Local build as a user")
strategy = scheduler.LocalStrategy(executors, acache)
hooks.TaskHookRegistry.get(options)
registry = TaskRegistry.get(options)
for params in default:
registry.set_default_parameters(params)
manifest = ctx.obj["manifest"]
for mb in manifest.builds:
for mt in mb.tasks:
task.append(mt.name)
for mt in mb.defaults:
registry.set_default_parameters(mt.name)
if force:
for goal in task:
registry.get_task(goal, manifest=manifest).taint = uuid.uuid4()
gb = graph.GraphBuilder(registry, manifest, options, progress=True)
dag = gb.build(task)
gp = graph.GraphPruner(strategy)
dag = gp.prune(dag)
goal_tasks = dag.goals
goal_task_duration = 0
queue = scheduler.TaskQueue(strategy)
try:
if not dag.has_tasks():
return
progress = log.progress(
"Progress",
dag.number_of_tasks(filterfn=lambda t: not t.is_resource()),
" tasks",
estimates=False,
debug=debug)
with progress:
while dag.has_tasks():
# Find all tasks ready to be executed
leafs = dag.select(lambda graph, task: task.is_ready())
# Order the tasks by their weights to improve build times
leafs.sort(key=lambda x: x.weight)
while leafs:
task = leafs.pop()
queue.submit(acache, task)
task, error = queue.wait()
if not task:
dag.debug()
break
elif task.is_goal() and task.duration_running:
goal_task_duration += task.duration_running.seconds
if not task.is_resource():
progress.update(1)
if not keep_going and error is not None:
queue.abort()
raise error
if dag.failed:
log.error("List of failed tasks")
for failed in dag.failed:
log.error("- {}", failed.log_name.strip("()"))
raise_error("no more tasks could be executed")
for goal in goal_tasks:
if acache.is_available_locally(goal):
with acache.get_artifact(goal) as artifact:
log.info("Location: {0}", artifact.path)
if copy:
artifact.copy("*",
utils.as_dirpath(
fs.path.join(
workdir,
click.format_filename(copy))),
symlinks=True)
except KeyboardInterrupt:
print()
log.warning("Interrupted by user")
try:
queue.abort()
sys.exit(1)
except KeyboardInterrupt:
print()
log.warning("Interrupted again, exiting")
_exit(1)
finally:
log.info("Total execution time: {0} {1}", str(duration),
str(queue.duration_acc) if network else '')
if result:
with report.update() as manifest:
manifest.duration = str(goal_task_duration)
manifest.write(result)
def _run(self, env):
timeout = int(config.getint("amqp", "timeout", 300))
manifest, routing_key = self._create_manifest()
self.connect()
self.publish_request(manifest, routing_key)
log.debug("[AMQP] Queued {0}", self.task.short_qualified_name)
self.task.running()
for extension in self.task.extensions:
extension.running()
while self.response is None:
try:
self.connection.process_data_events(time_limit=timeout)
if self.response is None:
self.task.info(
"Remote execution still in progress after {}",
self.task.duration_queued)
except (ConnectionError, AMQPConnectionError):
log.warning("[AMQP] Lost server connection")
self.connect()
log.debug("[AMQP] Finished {0}", self.task.short_qualified_name)
manifest = JoltManifest()
with raise_task_error_on_exception(
self.task, "failed to parse build result manifest"):
manifest.parsestring(self.response)
self.task.running(utils.duration() - float(manifest.duration))
if manifest.result != "SUCCESS":
output = []
if manifest.stdout:
output.extend(manifest.stdout.split("\n"))
if manifest.stderr:
output.extend(manifest.stderr.split("\n"))
for line in output:
log.transfer(line, self.task.identity[:8])
for task in [self.task] + self.task.extensions:
with task.task.report() as report:
remote_report = manifest.find_task(task.qualified_name)
if remote_report:
for error in remote_report.errors:
report.manifest.append(error)
raise_error("[AMQP] remote build failed with status: {0}".format(
manifest.result))
raise_task_error_if(
not env.cache.is_available_remotely(self.task), self.task,
"no task artifact available in any cache, check configuration")
raise_task_error_if(
not env.cache.download(self.task) and env.cache.download_enabled(),
self.task, "failed to download task artifact")
for extension in self.task.extensions:
raise_task_error_if(
not env.cache.download(extension)
and env.cache.download_enabled(), self.task,
"failed to download task artifact")
return self.task
def deprecation_warning(*args, **kwargs):
from jolt import log
log.warning("Called method is deprecated: {}", func.__qualname__)
return func(*args, **kwargs)
def compdb(ctx, task, default):
"""
Generate a compilation database for a task.
Aggregates compilation databases found in artifacts of the specified task and
its dependencies. The commands are then post-processed and localized to the
current workspace.
All task artifacts are sandboxed and their directory trees are recreated
using symlinks pointing to the origin of collected files. When opening a
file, an IDE can then follow the symlinks into the workspace instead of
opening files in the artifact cache.
The database must be regenerated if dependencies or the directory tree
of an artifact change.
"""
manifest = ctx.obj["manifest"]
options = JoltOptions(default=default)
acache = cache.ArtifactCache.get(options)
TaskHookRegistry.get(options)
executors = scheduler.ExecutorRegistry.get(options)
registry = TaskRegistry.get()
strategy = scheduler.DownloadStrategy(executors, acache)
queue = scheduler.TaskQueue(strategy)
for params in default:
registry.set_default_parameters(params)
gb = graph.GraphBuilder(registry, manifest, options, progress=True)
dag = gb.build(task)
try:
with log.progress("Progress",
dag.number_of_tasks(),
" tasks",
estimates=False,
debug=False) as p:
while dag.has_tasks():
leafs = dag.select(lambda graph, task: task.is_ready())
# Order the tasks by their weights to improve build times
leafs.sort(key=lambda x: x.weight)
while leafs:
task = leafs.pop()
queue.submit(acache, task)
task, error = queue.wait()
p.update(1)
except KeyboardInterrupt:
print()
log.warning("Interrupted by user")
try:
queue.abort()
sys.exit(1)
except KeyboardInterrupt:
print()
log.warning("Interrupted again, exiting")
os._exit(1)
for goal in dag.goals:
artifact, deps = get_task_artifacts(goal)
db = CompDB("all_compile_commands.json", artifact)
db.read()
db.relocate(goal, sandboxes=True)
outdir = goal.tools.builddir("compdb", incremental=True)
dbpath = fs.path.join(outdir, "all_compile_commands.json")
db.write(dbpath, force=True)
stage_artifacts(deps + [artifact], goal.tools)
log.info("Compilation DB: {}", dbpath)
