def __init__(self, amqp_url):
"""Create a new instance of the consumer class, passing in the AMQP
URL used to connect to RabbitMQ.
:param str amqp_url: The AMQP url to connect with
"""
self.should_reconnect = False
self.was_consuming = False
self._connection = None
self._channel = None
self._closing = False
self._consumer_tag = None
self._url = amqp_url
self._consuming = False
# In production, experiment with higher prefetch values
# for higher consumer throughput
self._prefetch_count = 1
self._job = None
self._routing_key = self.ROUTING_KEY_PREFIX + config.get(
"amqp", "routing_key",
os.getenv("RABBITMQ_ROUTING_KEY", self.ROUTING_KEY_REQUEST))
self._queue = self.QUEUE + "_" + self._routing_key
self._max_priority = config.getint("amqp", "max-priority", 1)
def __init__(self, options=None):
max_workers = config.getint(
"jolt", "parallel_tasks",
os.getenv("JOLT_PARALLEL_TASKS",
1 if options is None else options.jobs))
super(LocalExecutorFactory, self).__init__(options=options,
max_workers=max_workers)
def __init__(self, factory, task):
super(AmqpExecutor, self).__init__(factory)
self.factory = factory
self.callback_queue = None
self.connection = None
self.priority = config.getint("amqp", "priority", 0)
self.task = task
def task_pre_ninja_file(self, task, deps, tools):
if not isinstance(task.task, ninja.CXXLibrary) or task.task.shared:
return
cli = fs.path.join(fs.path.dirname(__file__), "ninjacli.py")
disabled = config.getboolean("ninja-cache", "disable", False)
tools.setenv("CCWRAP", "{} {} -- ".format(sys.executable, cli))
tools.setenv("CXXWRAP", "{} {} -- ".format(sys.executable, cli))
tools.setenv("JOLT_CACHEDIR", cache.ArtifactCache.get().root)
tools.setenv("JOLT_CANONTASK", utils.canonical(task.task.name))
tools.setenv("NINJACACHE_DISABLE", "1" if disabled else "0")
tools.setenv("NINJACACHE_MAXARTIFACTS",
config.getint("ninja-cache", "maxartifacts", 0))
if log.is_verbose():
tools.setenv("NINJACACHE_VERBOSE", "1")
def __init__(self, options):
workers = config.getint(NAME, "workers", 16)
super(AmqpExecutorFactory, self).__init__(max_workers=workers)
self._options = options
def _run(self, env):
timeout = int(config.getint("amqp", "timeout", 300))
manifest, routing_key = self._create_manifest()
self.connect()
self.publish_request(manifest, routing_key)
log.debug("[AMQP] Queued {0}", self.task.short_qualified_name)
self.task.running()
for extension in self.task.extensions:
extension.running()
while self.response is None:
try:
self.connection.process_data_events(time_limit=timeout)
if self.response is None:
self.task.info(
"Remote execution still in progress after {}",
self.task.duration_queued)
except (ConnectionError, AMQPConnectionError):
log.warning("[AMQP] Lost server connection")
self.connect()
log.debug("[AMQP] Finished {0}", self.task.short_qualified_name)
manifest = JoltManifest()
with raise_task_error_on_exception(
self.task, "failed to parse build result manifest"):
manifest.parsestring(self.response)
self.task.running(utils.duration() - float(manifest.duration))
if manifest.result != "SUCCESS":
output = []
if manifest.stdout:
output.extend(manifest.stdout.split("\n"))
if manifest.stderr:
output.extend(manifest.stderr.split("\n"))
for line in output:
log.transfer(line, self.task.identity[:8])
for task in [self.task] + self.task.extensions:
with task.task.report() as report:
remote_report = manifest.find_task(task.qualified_name)
if remote_report:
for error in remote_report.errors:
report.manifest.append(error)
raise_error("[AMQP] remote build failed with status: {0}".format(
manifest.result))
raise_task_error_if(
not env.cache.is_available_remotely(self.task), self.task,
"no task artifact available in any cache, check configuration")
raise_task_error_if(
not env.cache.download(self.task) and env.cache.download_enabled(),
self.task, "failed to download task artifact")
for extension in self.task.extensions:
raise_task_error_if(
not env.cache.download(extension)
and env.cache.download_enabled(), self.task,
"failed to download task artifact")
return self.task
from contextlib import contextmanager
try:
from StringIO import StringIO
except Exception:
from io import StringIO
from jolt import config
from jolt.error import JoltError
from jolt import filesystem as fs
from jolt import colors
default_path = fs.path.join(config.get_logpath(), "jolt.log")
logfile = config.get("jolt", "logfile", default_path)
logsize = config.getsize("jolt", "logsize",
os.environ.get("JOLT_LOGSIZE", 10 * 1024**2)) # 10MiB
logcount = config.getint("jolt", "logcount",
os.environ.get("JOLT_LOGCOUNT", 1))
dirpath = fs.path.dirname(logfile)
if not fs.path.exists(dirpath):
fs.makedirs(dirpath)
with open(logfile, "a") as f:
f.write(
"--------------------------------------------------------------------------------\n"
)
################################################################################
ERROR = logging.ERROR
WARNING = logging.WARNING
INFO = logging.INFO
VERBOSE = 15