def set_broker_in_app(self):
from firexapp.engine.celery import app
broker_url = self.broker.get_url()
BrokerFactory.set_broker_env(broker_url)
app.conf.result_backend = broker_url
app.conf.broker_url = broker_url
app.conf.mc = BrokerFactory.get_hostname_port_from_url(broker_url)[0]
def get_broker(cmd_output):
logs_dir = get_log_dir_from_output(cmd_output)
file_exists = wait_until(
lambda: os.path.exists(RedisManager.get_metadata_file(logs_dir)),
timeout=10,
sleep_for=0.5)
assert file_exists, "No broker metadata file."
broker = BrokerFactory.broker_manager_from_logs_dir(logs_dir)
return broker
def shutdown_run(logs_dir,
celery_shutdown_timeout,
reason='No reason provided'):
logger.info(f"Shutting down due to reason: {reason}")
logger.info(f"Shutting down with logs: {logs_dir}.")
broker = BrokerFactory.broker_manager_from_logs_dir(logs_dir)
logger.info(f"Shutting down with broker: {broker.broker_url_safe_print}.")
celery_manager = CeleryManager(logs_dir=logs_dir, broker=broker)
celery_app = Celery(broker=broker.broker_url,
accept_content=['pickle', 'json'])
try:
if is_celery_responsive(broker, celery_app):
revoke_active_tasks(broker, celery_app)
logger.info("Found active Celery; sending Celery shutdown.")
celery_app.control.shutdown()
celery_shutdown_success = celery_manager.wait_for_shutdown(
celery_shutdown_timeout)
if not celery_shutdown_success:
logger.warning(
f"Celery not shutdown after {celery_shutdown_timeout} secs, force killing instead."
)
celery_manager.shutdown()
else:
logger.debug("Confirmed Celery shutdown successfully.")
elif celery_manager.find_all_procs():
logger.info(
"Celery not active, but found celery processes to force shutdown."
)
celery_manager.shutdown()
else:
logger.info("No active Celery processes.")
finally:
if broker.is_alive():
logger.info("Broker is alive; sending redis shutdown.")
broker.shutdown()
wait_for_broker_shutdown(broker)
else:
logger.info("No active Broker.")
logger.info("Shutdown of %s complete." % logs_dir)
def start(self, args, uid=None, **kwargs)->{}:
# assemble startup cmd
if args.recording:
dest = args.recording
else:
dest = FileRegistry().get_file(DEFAULT_RECORDER_DEST_REGISTRY_KEY, uid.logs_dir)
if not os.path.isdir(os.path.dirname(dest)):
silent_mkdir(os.path.dirname(dest))
cmd = "firex_recorder"
cmd += ' --destination ' + dest
cmd += ' --broker ' + BrokerFactory.get_broker_url()
if self.default_timeout:
cmd += ' --timeout ' + str(self.default_timeout)
cmd += " &"
# start the recording service
logger.debug("Starting Recorder...")
recorder_stdout = FileRegistry().get_file(RECORDER_LOG_REGISTRY_KEY, uid.logs_dir)
with open(recorder_stdout, 'wb') as out:
subprocess.check_call(cmd, shell=True, stdout=out, stderr=subprocess.STDOUT)
def __init__(self, plugins=None, logs_dir=None, worker_log_level='debug', cap_concurrency=None,
app='firexapp.engine', env=None, broker=None):
if not broker:
self.broker = BrokerFactory.get_broker_url(assert_if_not_set=True)
self.hostname = gethostname()
self.plugins = plugins
self.logs_dir = logs_dir
self.worker_log_level = worker_log_level
self.cap_concurrency = cap_concurrency
self.app = app
self.env = os.environ.copy()
self.env['CELERY_RDBSIG'] = '1'
self.update_env(self.get_plugins_env(plugins))
if env:
self.update_env(env)
self.pid_files = dict()
self._celery_logs_dir = None
self._celery_pids_dir = None
self._workers_logs_dir = None
def start_broker(self, args):
from firexapp.broker_manager.broker_factory import BrokerFactory
self.broker = BrokerFactory.create_new_broker_manager(logs_dir=self.uid.logs_dir)
self.broker.start()
add_custom_log_levels()
add_hostname_to_log_records()
# logging formats
timestamp_format = "<small>[%(asctime)s]"
process_format = "[%(levelname)s/%(processName)-13s]"
task_format = "[%(task_id).8s-%(task_name)s]"
message_format = ":</small> %(message)s"
worker_log_format = timestamp_format + process_format + message_format
worker_task_log_format = timestamp_format + process_format + task_format + message_format
logger = get_task_logger(__name__)
broker_url = BrokerFactory.get_broker_url()
result_backend = broker_url
# find default tasks
logger.debug("Beginning bundle discovery")
bundles = find_firex_task_bundles()
logger.debug("Bundle discovery completed.")
if bundles:
logger.debug('Bundles discovered:\n' + '\n'.join([f'\t - {b}' for b in bundles]))
# Plugins are imported via firexapp.plugins._worker_init_signal()
imports = tuple(bundles) + tuple(["firexapp.tasks.example",
"firexapp.tasks.core_tasks",
"firexapp.tasks.root_tasks",
"firexapp.submit.report_trigger",
"firexapp.reporters.json_reporter"