def test_uid_object(self):
try:
uid = Uid()
key = 'key3'
relative_path = 'some_relative_path/something3.txt'
FileRegistry().register_file(key, relative_path)
with self.subTest('Basic functionality'):
self.assertEqual(FileRegistry().get_file(key, uid), os.path.join(uid.logs_dir, relative_path))
finally:
shutil.rmtree(uid.logs_dir)
def test_registered_keys(self):
key = 'key1'
filename = 'something.txt'
some_path = '/nobackup/user'
FileRegistry().register_file(key, filename)
with self.subTest('Basic functionality'):
self.assertEqual(FileRegistry().get_file(key, some_path), os.path.join(some_path, filename))
with self.subTest('Testing duplicate key registrations'):
with self.assertRaises(KeyAlreadyRegistered):
FileRegistry().register_file(key, 'something_else')
self.assertEqual(FileRegistry().get_file(key, some_path), os.path.join(some_path, filename))
def submit(self, args_from_first_pass: argparse.Namespace, other_args_from_first_pass: list):
uid = Uid()
self.uid = uid
logger.info("FireX ID: %s", uid)
logger.info('Logs: %s', uid.logs_dir)
self.install_configs = load_new_install_configs(uid.identifier,
uid.logs_dir,
args_from_first_pass.install_configs)
args, others = self.resolve_install_configs_args(args_from_first_pass, other_args_from_first_pass)
chain_args = self.process_other_chain_args(args, others)
chain_args['uid'] = uid
if args.logs_link:
self.create_logs_link(args.logs_link)
if self.install_configs.has_viewer():
uid.add_viewers(logs_url=self.install_configs.get_logs_root_url())
logger.info(f'Logs URL: {uid.logs_url}')
# Create an env file for debugging
with open(FileRegistry().get_file(ENVIRON_FILE_REGISTRY_KEY, self.uid.logs_dir), 'w') as f:
json.dump(dict(os.environ), fp=f, skipkeys=True, sort_keys=True, indent=4)
chain_args = self.convert_chain_args(chain_args)
chain_args = self.start_engine(args=args, chain_args=chain_args, uid=uid)
# Execute chain
try:
root_task_name = app.conf.get("root_task")
if root_task_name is None:
raise NotRegistered("No root task configured")
root_task = get_app_task(root_task_name)
except NotRegistered as e:
logger.error(e)
self.main_error_exit_handler(reason=str(e))
sys.exit(-1)
self.wait_tracking_services_task_ready()
safe_create_initial_run_json(**chain_args)
# AsyncResult objects cannot be in memory after the broker (i.e. backend) shutdowns, otherwise errors are
# produced when they are garbage collected. We therefore monkey patch AsyncResults to track all instances
# (e.g. from unpickle, instantiated directly, etc) so that disable_all_async_results can disable their
# references to the backend.
monkey_patch_async_result_to_track_instances()
root_task_result_promise = root_task.s(submit_app=self, **chain_args).delay()
self.copy_submission_log()
if args.sync:
logger.info("Waiting for chain to complete...")
chain_results = self.process_sync(root_task_result_promise, chain_args)
results_str = self.format_results_str(chain_results)
self.log_results(results_str)
self.self_destruct(chain_details=(root_task_result_promise, chain_args),
reason="Sync run: completed successfully")
def store_debug_info():
setup, pydev = get_pydev_debug_setup()
data = {"pydev": pydev, "setup": setup, "debug_host": gethostname()}
from firexapp.engine.celery import app
logs_dir = app.backend.get('logs_dir').decode()
json_path = FileRegistry().get_file(PYDEV_REGISTRY_KEY, logs_dir)
print("Storing debugging information", file=sys.stdout)
with open(json_path, 'w') as outfile:
json.dump(data, outfile)
def start(self, args, uid=None, **kwargs)->{}:
# assemble startup cmd
if args.recording:
dest = args.recording
else:
dest = FileRegistry().get_file(DEFAULT_RECORDER_DEST_REGISTRY_KEY, uid.logs_dir)
if not os.path.isdir(os.path.dirname(dest)):
silent_mkdir(os.path.dirname(dest))
cmd = "firex_recorder"
cmd += ' --destination ' + dest
cmd += ' --broker ' + BrokerFactory.get_broker_url()
if self.default_timeout:
cmd += ' --timeout ' + str(self.default_timeout)
cmd += " &"
# start the recording service
logger.debug("Starting Recorder...")
recorder_stdout = FileRegistry().get_file(RECORDER_LOG_REGISTRY_KEY, uid.logs_dir)
with open(recorder_stdout, 'wb') as out:
subprocess.check_call(cmd, shell=True, stdout=out, stderr=subprocess.STDOUT)
def assert_expected_firex_output(self, cmd_output, cmd_err):
rec_file = os.path.join(self.results_folder, "recording.rec")
assert os.path.isfile(rec_file), 'recording file not created'
logs_dir = get_log_dir_from_output(cmd_output)
recorder_stdout = FileRegistry().get_file(RECORDER_LOG_REGISTRY_KEY, logs_dir)
assert os.path.isfile(recorder_stdout), 'recorder output not found'
with open(recorder_stdout) as f:
for line in f:
if line.strip().endswith("Shutting down on broker and root task completion"):
break
else:
assert False, "Recorder did not log it's own shut down"
def assert_expected_firex_output(self, cmd_output, cmd_err):
logs_dir = get_log_dir_from_output(cmd_output)
recorder_stdout = FileRegistry().get_file(RECORDER_LOG_REGISTRY_KEY, logs_dir)
assert os.path.isfile(recorder_stdout), 'recorder output not found'
for _ in range(0, 6):
# there could be a delay in shutting down. Give it a change to finish
with open(recorder_stdout) as f:
for line in f:
if line.strip().endswith("Exiting on timeout"):
return
else:
import time
time.sleep(0.5)
else:
assert False, "Recorder did not log it's own shut down"
def test_dump_and_read_from_file(self):
FileRegistry().destroy()
registry = {'key1': 'value1',
'key2': 'value2'}
for k, v in registry.items():
FileRegistry().register_file(k, v)
file_registry = tempfile.NamedTemporaryFile().name
FileRegistry().dump_to_file(file_registry)
FileRegistry().destroy()
FileRegistry(from_file=file_registry)
self.assertDictEqual(FileRegistry().file_registry, registry)
def restart_celery_in_debug():
from firexapp.engine.celery import app
firex_logs_dir = app.backend.get('logs_dir').decode()
json_path = FileRegistry().get_file(PYDEV_REGISTRY_KEY, firex_logs_dir)
if not os.path.isfile(json_path):
print("No debugging information found", file=sys.stdout)
return
with open(json_path, "r") as infile:
data = json.load(infile)
setup = data['setup']
pydev = data['pydev']
debug_host = data['debug_host']
celery_argsv = sys.argv
debug_celery = get_pydev_command(celery_argsv,
setup=setup,
pydev=pydev,
debug_host=debug_host)
print("restarting celery in debug mode", file=sys.stdout)
os.execl(debug_celery[0], *debug_celery)
def get_submission_file(logs_dir: str):
submission_file = FileRegistry().get_file(SUBMISSION_FILE_REGISTRY_KEY,
logs_dir)
assert os.path.isfile(submission_file), "submission file missing not there"
return submission_file
def copy_submission_log(self):
if self.submission_tmp_file and os.path.isfile(self.submission_tmp_file) and self.uid:
copyfile(self.submission_tmp_file, FileRegistry().get_file(SUBMISSION_FILE_REGISTRY_KEY, self.uid.logs_dir))
from firexapp.submit.console import setup_console_logging
from firexapp.application import import_microservices, get_app_tasks, get_app_task
from firexapp.engine.celery import app
from firexapp.broker_manager.broker_factory import BrokerFactory
from firexapp.submit.shutdown import launch_background_shutdown, DEFAULT_CELERY_SHUTDOWN_TIMEOUT
from firexapp.submit.install_configs import load_new_install_configs, FireXInstallConfigs, INSTALL_CONFIGS_ENV_NAME
from firexapp.submit.arguments import whitelist_arguments
from firexapp.common import dict2str, silent_mkdir, create_link
from firexapp.reporters.json_reporter import FireXJsonReportGenerator
add_hostname_to_log_records()
logger = setup_console_logging(__name__)
SUBMISSION_FILE_REGISTRY_KEY = 'firex_submission'
FileRegistry().register_file(SUBMISSION_FILE_REGISTRY_KEY, os.path.join(Uid.debug_dirname, 'submission.txt'))
ENVIRON_FILE_REGISTRY_KEY = 'env'
FileRegistry().register_file(ENVIRON_FILE_REGISTRY_KEY, os.path.join(Uid.debug_dirname, 'environ.json'))
class JsonFileAction(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
if not os.path.isabs(values):
values = os.path.join(os.getcwd(), values)
assert not os.path.isdir(values), f'{values} is a directory, you must provide a filename or path'
if os.path.islink(values) or os.path.isfile(values):
logger.print(f'--json_file {values} exists; removing it')
import subprocess
from functools import partial
from firexapp.fileregistry import FileRegistry
from firexapp.submit.uid import Uid
from socket import gethostname
from urllib.parse import urlsplit
from logging import ERROR, INFO
from psutil import Process
from pathlib import Path
from firexapp.broker_manager import BrokerManager
from firexapp.common import get_available_port, wait_until, silent_mkdir
REDIS_DIR_REGISTRY_KEY = 'REDIS_DIR_REGISTRY_KEY'
FileRegistry().register_file(REDIS_DIR_REGISTRY_KEY,
os.path.join(Uid.debug_dirname, 'redis'))
REDIS_LOG_REGISTRY_KEY = 'REDIS_LOG_REGISTRY_KEY'
FileRegistry().register_file(
REDIS_LOG_REGISTRY_KEY,
os.path.join(FileRegistry().get_relative_path(REDIS_DIR_REGISTRY_KEY),
'redis.stdout.txt'))
REDIS_PID_REGISTRY_KEY = 'REDIS_PID_REGISTRY_KEY'
FileRegistry().register_file(
REDIS_PID_REGISTRY_KEY,
os.path.join(FileRegistry().get_relative_path(REDIS_DIR_REGISTRY_KEY),
'redis.pid'))
REDIS_METADATA_REGISTRY_KEY = 'REDIS_METADATA_REGISTRY_KEY'
FileRegistry().register_file(
def get_worker_logs_dir(logs_dir):
return FileRegistry().get_file(MICROSERVICE_LOGS_REGISTRY_KEY, logs_dir)
import inspect
import json
import os
import sys
from _socket import gethostname
from firexapp.fileregistry import FileRegistry
from firexapp.submit.uid import Uid
PYDEV_REGISTRY_KEY = 'pydev_debug'
FileRegistry().register_file(
PYDEV_REGISTRY_KEY, os.path.join(Uid.debug_dirname, 'pydev_debug.json'))
def get_pydev_debug_setup():
frame = inspect.currentframe()
all_frames = inspect.getouterframes(frame)
for i in range(1, len(all_frames)):
setup_frame = all_frames[-i]
setup = setup_frame.frame.f_locals.get('setup')
if setup:
return setup, setup_frame.filename
def is_debugging():
frame = inspect.currentframe()
top_frame = inspect.getouterframes(frame)[-1]
return "pydevd.py" in top_frame.filename
def is_firex_submit():
import os
import re
import subprocess
import psutil
from firexapp.broker_manager.broker_factory import BrokerFactory
from socket import gethostname
from firexapp.common import poll_until_file_not_empty, poll_until_dir_empty, find_procs
from firexapp.plugins import PLUGGING_ENV_NAME, cdl2list
from firexapp.fileregistry import FileRegistry
from collections.abc import Iterable
from firexapp.common import qualify_firex_bin
logger = setup_console_logging(__name__)
CELERY_LOGS_REGISTRY_KEY = 'celery_logs'
FileRegistry().register_file(CELERY_LOGS_REGISTRY_KEY, os.path.join(Uid.debug_dirname, 'celery'))
CELERY_PIDS_REGISTRY_KEY = 'celery_pids'
FileRegistry().register_file(CELERY_PIDS_REGISTRY_KEY,
os.path.join(FileRegistry().get_relative_path(CELERY_LOGS_REGISTRY_KEY), 'pids'))
MICROSERVICE_LOGS_REGISTRY_KEY = 'microservice_logs'
FileRegistry().register_file(MICROSERVICE_LOGS_REGISTRY_KEY, 'microservice_logs')
class CeleryWorkerStartFailed(Exception):
pass
class CeleryManager(object):
celery_bin_name = 'celery'
def get_celery_pids_dir(logs_dir):
return FileRegistry().get_file(CELERY_PIDS_REGISTRY_KEY, logs_dir)
def get_pid_file(logs_dir):
return FileRegistry().get_file(REDIS_PID_REGISTRY_KEY, logs_dir)
def get_log_file(logs_dir):
return FileRegistry().get_file(REDIS_LOG_REGISTRY_KEY, logs_dir)
import os
import subprocess
from celery.utils.log import get_task_logger
from firexapp.broker_manager.broker_factory import BrokerFactory
from firexapp.common import silent_mkdir
from firexapp.fileregistry import FileRegistry
from firexapp.submit.uid import Uid
from firexapp.submit.tracking_service import TrackingService
logger = get_task_logger(__name__)
RECORDER_LOG_REGISTRY_KEY = 'RECORDER_OUTPUT_REGISTRY_KEY'
FileRegistry().register_file(RECORDER_LOG_REGISTRY_KEY, os.path.join(Uid.debug_dirname, 'recorder.stdout'))
DEFAULT_RECORDER_DEST_REGISTRY_KEY = 'DEFAULT_RECORDER_DEST_REGISTRY_KEY'
FileRegistry().register_file(DEFAULT_RECORDER_DEST_REGISTRY_KEY, os.path.join(Uid.debug_dirname, 'firex.rec'))
class RecorderLauncher(TrackingService):
default_timeout = None
def extra_cli_arguments(self, arg_parser):
arg_parser.add_argument('--recording', help='A file to record celery events', default=None)
def start(self, args, uid=None, **kwargs)->{}:
# assemble startup cmd
if args.recording:
dest = args.recording
def get_metadata_file(logs_dir):
return FileRegistry().get_file(REDIS_METADATA_REGISTRY_KEY, logs_dir)
def test_unregistered_key(self):
some_path = '/nobackup/user'
with self.assertRaises(KeyNotRegistered):
FileRegistry().get_file('unregistered_key', some_path)
def get_password_file(logs_dir):
return FileRegistry().get_file(REDIS_CREDS_REGISTRY_KEY, logs_dir)
def tearDown(self):
FileRegistry().destroy()
self.assertDictEqual(FileRegistry().file_registry, {})