def run():
"""
Start the Horizon agent and logger.
"""
if not isdir(settings.PID_PATH):
print('pid directory does not exist at %s' % settings.PID_PATH)
sys.exit(1)
if not isdir(settings.LOG_PATH):
print('log directory does not exist at %s' % settings.LOG_PATH)
sys.exit(1)
# @added 20201103 - Feature #3820: HORIZON_SHARDS
try:
HORIZON_SHARDS = settings.HORIZON_SHARDS.copy()
except:
HORIZON_SHARDS = {}
horizon_shards_validated = True
if HORIZON_SHARDS:
for shard_host in HORIZON_SHARDS:
try:
int(HORIZON_SHARDS[shard_host])
except:
horizon_shards_validated = False
if not horizon_shards_validated:
print(
'the HORIZON_SHARDS dict in settings.py does not have valid shard numbers'
% str(settings.HORIZON_SHARDS))
sys.exit(1)
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s :: %(process)s :: %(message)s",
datefmt="%Y-%m-%d %H:%M:%S")
handler = logging.handlers.TimedRotatingFileHandler(logfile,
when="midnight",
interval=1,
backupCount=5)
memory_handler = logging.handlers.MemoryHandler(100,
flushLevel=logging.DEBUG,
target=handler)
handler.setFormatter(formatter)
logger.addHandler(memory_handler)
# Validate settings variables
valid_settings = validate_settings_variables(skyline_app)
if not valid_settings:
print('error :: invalid variables in settings.py - cannot start')
sys.exit(1)
horizon = Horizon()
if len(sys.argv) > 1 and sys.argv[1] == 'run':
horizon.run()
else:
daemon_runner = runner.DaemonRunner(horizon)
daemon_runner.daemon_context.files_preserve = [handler.stream]
daemon_runner.do_action()
def run():
"""
Start the Mirage agent.
"""
if not isdir(settings.PID_PATH):
print ('pid directory does not exist at %s' % settings.PID_PATH)
sys.exit(1)
if not isdir(settings.LOG_PATH):
print ('log directory does not exist at %s' % settings.LOG_PATH)
sys.exit(1)
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s :: %(process)s :: %(message)s", datefmt="%Y-%m-%d %H:%M:%S")
handler = logging.handlers.TimedRotatingFileHandler(
logfile,
when="midnight",
interval=1,
backupCount=5)
handler.setFormatter(formatter)
logger.addHandler(handler)
# Validate settings variables
valid_settings = validate_settings_variables(skyline_app)
if not valid_settings:
print ('error :: invalid variables in settings.py - cannot start')
sys.exit(1)
# Make sure we can run all the algorithms
try:
timeseries = map(list, zip(map(float, range(int(time()) - 86400, int(time()) + 1)), [1] * 86401))
second_order_resolution_seconds = 86400
ensemble = [globals()[algorithm](timeseries, second_order_resolution_seconds) for algorithm in settings.MIRAGE_ALGORITHMS]
except KeyError as e:
print ('Algorithm %s deprecated or not defined; check settings.MIRAGE_ALGORITHMS' % e)
sys.exit(1)
except Exception as e:
print ('Algorithm test run failed.')
traceback.print_exc()
sys.exit(1)
logger.info('Tested algorithms')
del timeseries
del ensemble
mirage = MirageAgent()
if len(sys.argv) > 1 and sys.argv[1] == 'run':
mirage.run()
else:
daemon_runner = runner.DaemonRunner(mirage)
daemon_runner.daemon_context.files_preserve = [handler.stream]
daemon_runner.do_action()
def run():
"""
Start the Vista.
Start the logger.
"""
if not isdir(settings.PID_PATH):
print('pid directory does not exist at %s' % settings.PID_PATH)
sys.exit(1)
if not isdir(settings.LOG_PATH):
print('log directory does not exist at %s' % settings.LOG_PATH)
sys.exit(1)
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s :: %(process)s :: %(message)s",
datefmt="%Y-%m-%d %H:%M:%S")
handler = logging.handlers.TimedRotatingFileHandler(logfile,
when="midnight",
interval=1,
backupCount=5)
memory_handler = logging.handlers.MemoryHandler(256,
flushLevel=logging.DEBUG,
target=handler)
handler.setFormatter(formatter)
logger.addHandler(memory_handler)
# Validate settings variables
valid_settings = validate_settings_variables(skyline_app)
if not valid_settings:
print(
'error :: agent :: invalid variables in settings.py - cannot start'
)
sys.exit(1)
vista = Vista()
logger.info('agent :: starting vista')
memory_handler.flush
if len(sys.argv) > 1 and sys.argv[1] == 'run':
vista.run()
else:
daemon_runner = runner.DaemonRunner(vista)
daemon_runner.daemon_context.files_preserve = [handler.stream]
daemon_runner.do_action()
logger.info('stopping vista')
memory_handler.flush
def run():
"""
Check that all the `ALGORITHMS` can be run.
Start the AnalyzerAgent.
Start the logger.
"""
if not isdir(settings.PID_PATH):
print('pid directory does not exist at %s' % settings.PID_PATH)
sys.exit(1)
if not isdir(settings.LOG_PATH):
print('log directory does not exist at %s' % settings.LOG_PATH)
sys.exit(1)
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s :: %(process)s :: %(message)s",
datefmt="%Y-%m-%d %H:%M:%S")
handler = logging.handlers.TimedRotatingFileHandler(logfile,
when="midnight",
interval=1,
backupCount=5)
memory_handler = logging.handlers.MemoryHandler(256,
flushLevel=logging.DEBUG,
target=handler)
handler.setFormatter(formatter)
logger.addHandler(memory_handler)
# Validate settings variables
valid_settings = validate_settings_variables(skyline_app)
if not valid_settings:
print('error :: invalid variables in settings.py - cannot start')
sys.exit(1)
# Make sure we can run all the algorithms
try:
# from analyzer import algorithms
import algorithms
logger.info('Testing algorithms')
# @modified 20200723 - Task #3608: Update Skyline to Python 3.8.3 and deps
# Use a shorter timeseries for quicker start up
# timeseries = map(list, zip(map(float, range(int(time()) - 86400, int(time()) + 1)), [1] * 86401))
timeseries = map(
list,
zip(map(float, range(int(time()) - 1440,
int(time()) + 1)), [1] * 1440))
# @added 20191024 - Branch #3262: py3
# Convert map to list
if python_version == 3:
if isinstance(timeseries, map):
timeseries = list(timeseries)
# ensemble = [globals()[algorithm](timeseries) for algorithm in settings.ALGORITHMS]
ensemble = [
getattr(algorithms, algorithm)(timeseries)
for algorithm in settings.ALGORITHMS
]
except KeyError as e:
print(
'Algorithm %s deprecated or not defined; check settings.ALGORITHMS'
% e)
sys.exit(1)
except Exception as e:
print('Algorithm test run failed.')
traceback.print_exc()
sys.exit(1)
logger.info('Tested algorithms')
del timeseries
del ensemble
analyzer = AnalyzerAgent()
logger.info('starting analyzer.run')
memory_handler.flush
if len(sys.argv) > 1 and sys.argv[1] == 'run':
analyzer.run()
else:
daemon_runner = runner.DaemonRunner(analyzer)
daemon_runner.daemon_context.files_preserve = [handler.stream]
daemon_runner.do_action()
logger.info('stopping analyzer')
memory_handler.flush
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s :: %(process)s :: %(message)s",
datefmt="%Y-%m-%d %H:%M:%S")
handler = logging.handlers.TimedRotatingFileHandler(logfile,
when="midnight",
interval=1,
backupCount=5)
memory_handler = logging.handlers.MemoryHandler(256,
flushLevel=logging.DEBUG,
target=handler)
handler.setFormatter(formatter)
logger.addHandler(memory_handler)
# Validate settings variables
valid_settings = validate_settings_variables(skyline_app)
if not valid_settings:
print('error :: invalid variables in settings.py - cannot start')
sys.exit(1)
# @added 20191031 - Feature #3310: gracefully handle db failure
# Branch 3262: py3
try:
start_if_no_db = settings.START_IF_NO_DB
except:
start_if_no_db = False
# Make sure mysql is available
mysql_up = False
try:
def run():
"""
Start the Crucible agent and ensure all the required directories exist,
creating the crucible directories if they do not exist
"""
if not isdir(settings.PID_PATH):
print('pid directory does not exist at %s' % settings.PID_PATH)
sys.exit(1)
if not isdir(settings.LOG_PATH):
print('log directory does not exist at %s' % settings.LOG_PATH)
sys.exit(1)
# Make sure the required directories exists
if not os.path.exists(settings.CRUCIBLE_CHECK_PATH):
try:
os.makedirs(settings.CRUCIBLE_CHECK_PATH, mode=0o755)
except:
print('failed to create directory - %s' %
settings.CRUCIBLE_CHECK_PATH)
sys.exit(1)
if not os.path.exists(settings.CRUCIBLE_DATA_FOLDER):
try:
os.makedirs(settings.CRUCIBLE_DATA_FOLDER, mode=0o755)
except:
print('failed to create directory - %s' %
settings.CRUCIBLE_DATA_FOLDER)
sys.exit(1)
failed_checks_dir = settings.CRUCIBLE_DATA_FOLDER + '/failed_checks'
if not os.path.exists(failed_checks_dir):
try:
os.makedirs(failed_checks_dir, mode=0o755)
except:
print('failed to create directory - %s' % failed_checks_dir)
sys.exit(1)
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s :: %(process)s :: %(message)s",
datefmt="%Y-%m-%d %H:%M:%S")
handler = logging.handlers.TimedRotatingFileHandler(logfile,
when="midnight",
interval=1,
backupCount=5)
memory_handler = logging.handlers.MemoryHandler(256,
flushLevel=logging.DEBUG,
target=handler)
handler.setFormatter(formatter)
logger.addHandler(memory_handler)
# Validate settings variables
valid_settings = validate_settings_variables(skyline_app)
if not valid_settings:
print('error :: invalid variables in settings.py - cannot start')
sys.exit(1)
crucible = CrucibleAgent()
if len(sys.argv) > 1 and sys.argv[1] == 'run':
logger.info('starting skyline crucible via run')
crucible.run()
else:
logger.info('starting skyline crucible via daemon')
daemon_runner = runner.DaemonRunner(crucible)
daemon_runner.daemon_context.files_preserve = [handler.stream]
daemon_runner.do_action()
def run():
"""
Start the Boundary agent.
"""
if not isdir(settings.PID_PATH):
print ('pid directory does not exist at %s' % settings.PID_PATH)
sys.exit(1)
if not isdir(settings.LOG_PATH):
print ('log directory does not exist at %s' % settings.LOG_PATH)
sys.exit(1)
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s :: %(process)s :: %(message)s", datefmt="%Y-%m-%d %H:%M:%S")
handler = logging.handlers.TimedRotatingFileHandler(
logfile,
when="midnight",
interval=1,
backupCount=5)
memory_handler = logging.handlers.MemoryHandler(256,
flushLevel=logging.DEBUG,
target=handler)
handler.setFormatter(formatter)
logger.addHandler(memory_handler)
# Validate settings variables
valid_settings = validate_settings_variables(skyline_app)
if not valid_settings:
print ('error :: invalid variables in settings.py - cannot start')
sys.exit(1)
# Make sure all the BOUNDARY_ALGORITHMS are valid
try:
if settings.BOUNDARY_ALGORITHMS:
configuration_error = False
for algorithm in settings.BOUNDARY_ALGORITHMS:
valid = True
if not isinstance(algorithm, str):
valid = False
if not valid:
configuration_error = True
print ('configuration error in tuple, expected: str')
print('configuration error in BOUNDARY_ALGORITHMS tuple: %s' % str(algorithm))
except:
try:
if configuration_error:
print ('There are configuration issues in BOUNDARY_ALGORITHMS in settings.py')
except:
print ('There are no BOUNDARY_ALGORITHMS in settings.py. try adding some, nothing to do')
sys.exit(1)
# Make sure we can run all the algorithms
try:
timeseries = map(list, zip(map(float, range(int(time()) - 86400, int(time()) + 1)), [1] * 86401))
ensemble = [globals()[algorithm](
timeseries, 'test', 3600, 100, 300, 1
) for algorithm in settings.BOUNDARY_ALGORITHMS]
except KeyError as e:
print ('Algorithm %s deprecated or not defined; check settings.BOUNDARY_ALGORITHMS' % e)
sys.exit(1)
except Exception as e:
print ('Algorithm test run failed.')
traceback.print_exc()
sys.exit(1)
# Make sure all the BOUNDARY_METRICS are valid
try:
if settings.BOUNDARY_METRICS:
configuration_error = False
for metric in settings.BOUNDARY_METRICS:
valid = True
strings = []
strings.append(metric[0])
strings.append(metric[1])
strings.append(metric[7])
for string in strings:
if not isinstance(string, str):
valid = False
values = []
values.append(metric[2])
values.append(metric[3])
values.append(metric[4])
values.append(metric[5])
values.append(metric[6])
for value in values:
if not isinstance(value, int):
valid = False
alert_via = metric[7]
for alerter in alert_via.split("|"):
if not isinstance(alerter, str):
valid = False
if not valid:
configuration_error = True
print ('configuration error in tuple, expected: str, str, int, int, int, int, str')
print('configuration error in BOUNDARY_METRICS tuple: %s' % str(metric))
except:
if configuration_error:
print ('There are configuration issues in BOUNDARY_METRICS in settings.py')
else:
print ('There are no BOUNDARY_METRICS in settings.py. try adding some, nothing to do')
sys.exit(1)
logger.info('Tested algorithms')
del timeseries
del ensemble
del strings
del values
boundary = BoundaryAgent()
if len(sys.argv) > 1 and sys.argv[1] == 'run':
boundary.run()
else:
daemon_runner = runner.DaemonRunner(boundary)
daemon_runner.daemon_context.files_preserve = [handler.stream]
daemon_runner.do_action()
def run():
"""
Check that all the `ALGORITHMS` can be run.
Start the AnalyzerAgent.
Start the logger.
"""
if not isdir(settings.PID_PATH):
print('pid directory does not exist at %s' % settings.PID_PATH)
sys.exit(1)
if not isdir(settings.LOG_PATH):
print('log directory does not exist at %s' % settings.LOG_PATH)
sys.exit(1)
if len(sys.argv) > 1 and sys.argv[1] == 'stop':
do_not_overwrite_log = True
# This should hopefully take care of a TODO from the bin files,
# TODO: write a real kill script
# as above @earthgecko 20160520
pidfile_path = settings.PID_PATH + '/' + skyline_app + '.pid'
pid = int(open(pidfile_path).read())
try:
kill(pid, signal.SIGTERM)
print('%s pid %s stopped' % (skyline_app, str(pid)))
sys.exit(0)
except OSError as e:
print('Failed to kill pid %s - OSError - %s' % (str(pid), str(e)))
sys.exit(1)
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s :: %(process)s :: %(message)s",
datefmt="%Y-%m-%d %H:%M:%S")
if len(sys.argv) > 1 and sys.argv[1] == 'stop':
handler = logging.FileHandler(settings.LOG_PATH + '/' + skyline_app +
'.stop.log',
mode='a',
delay=False)
else:
handler = logging.handlers.TimedRotatingFileHandler(logfile,
when="midnight",
interval=1,
backupCount=5)
memory_handler = logging.handlers.MemoryHandler(256,
flushLevel=logging.DEBUG,
target=handler)
handler.setFormatter(formatter)
logger.addHandler(memory_handler)
# Validate settings variables
valid_settings = validate_settings_variables(skyline_app)
if not valid_settings:
print('error :: invalid variables in settings.py - cannot start')
sys.exit(1)
if len(sys.argv) > 1 and sys.argv[1] == 'stop':
do_not_overwrite_log = True
else:
# Make sure we can run all the algorithms
try:
# from analyzer import algorithms
import algorithms_dev
logger.info('Testing algorithms')
timeseries = map(
list,
zip(map(float, range(int(time()) - 86400,
int(time()) + 1)), [1] * 86401))
# ensemble = [globals()[algorithm](timeseries) for algorithm in settings.ALGORITHMS]
ensemble = [
getattr(algorithms_dev, algorithm)(timeseries)
for algorithm in settings.ALGORITHMS
]
logger.info('Tested algorithms OK')
logger.info('ensemble: %s' % str(ensemble))
except KeyError as e:
print(
'Algorithm %s deprecated or not defined; check settings.ALGORITHMS'
% e)
sys.exit(1)
except Exception as e:
print('Algorithm test run failed.')
traceback.print_exc()
sys.exit(1)
logger.info('Tested algorithms')
del timeseries
del ensemble
analyzer = AnalyzerDevAgent()
if len(sys.argv) > 1 and sys.argv[1] == 'stop':
do_not_overwrite_log = True
else:
logger.info('starting analyzer_dev.run')
memory_handler.flush
if len(sys.argv) > 1 and sys.argv[1] == 'run':
analyzer.run()
else:
daemon_runner = runner.DaemonRunner(analyzer)
daemon_runner.daemon_context.files_preserve = [handler.stream]
daemon_runner.do_action()
if len(sys.argv) > 1 and sys.argv[1] == 'stop':
do_not_overwrite_log = True
else:
logger.info('stopped analyzer_dev')
def run():
"""
Check that all the `ALGORITHMS` can be run.
Start the AnalyzerAgent.
Start the logger.
"""
if not isdir(settings.PID_PATH):
print ('pid directory does not exist at %s' % settings.PID_PATH)
sys.exit(1)
if not isdir(settings.LOG_PATH):
print ('log directory does not exist at %s' % settings.LOG_PATH)
sys.exit(1)
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s :: %(process)s :: %(message)s", datefmt="%Y-%m-%d %H:%M:%S")
handler = logging.handlers.TimedRotatingFileHandler(
logfile,
when="midnight",
interval=1,
backupCount=5)
memory_handler = logging.handlers.MemoryHandler(256,
flushLevel=logging.DEBUG,
target=handler)
handler.setFormatter(formatter)
logger.addHandler(memory_handler)
# Validate settings variables
valid_settings = validate_settings_variables(skyline_app)
if not valid_settings:
print ('error :: invalid variables in settings.py - cannot start')
sys.exit(1)
# Make sure we can run all the algorithms
try:
# from analyzer import algorithms
import algorithms
logger.info('Testing algorithms')
timeseries = map(list, zip(map(float, range(int(time()) - 86400, int(time()) + 1)), [1] * 86401))
# ensemble = [globals()[algorithm](timeseries) for algorithm in settings.ALGORITHMS]
ensemble = [getattr(algorithms, algorithm)(timeseries) for algorithm in settings.ALGORITHMS]
except KeyError as e:
print ('Algorithm %s deprecated or not defined; check settings.ALGORITHMS' % e)
sys.exit(1)
except Exception as e:
print ('Algorithm test run failed.')
traceback.print_exc()
sys.exit(1)
logger.info('Tested algorithms')
del timeseries
del ensemble
analyzer = AnalyzerAgent()
logger.info('starting analyzer.run')
memory_handler.flush
if len(sys.argv) > 1 and sys.argv[1] == 'run':
analyzer.run()
else:
daemon_runner = runner.DaemonRunner(analyzer)
daemon_runner.daemon_context.files_preserve = [handler.stream]
daemon_runner.do_action()
logger.info('stopping analyzer')
memory_handler.flush
def run():
"""
Start the Webapp server
"""
if not isdir(settings.PID_PATH):
print('pid directory does not exist at %s' % settings.PID_PATH)
sys.exit(1)
if not isdir(settings.LOG_PATH):
print('log directory does not exist at %s' % settings.LOG_PATH)
sys.exit(1)
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s :: %(process)s :: %(message)s",
datefmt="%Y-%m-%d %H:%M:%S")
handler = logging.handlers.TimedRotatingFileHandler(logfile,
when="midnight",
interval=1,
backupCount=5)
memory_handler = logging.handlers.MemoryHandler(100,
flushLevel=logging.DEBUG,
target=handler)
handler.setFormatter(formatter)
logger.addHandler(memory_handler)
# Validate settings variables
valid_settings = validate_settings_variables(skyline_app)
if not valid_settings:
print('error :: invalid variables in settings.py - cannot start')
sys.exit(1)
try:
settings.WEBAPP_SERVER
except:
logger.error('error :: failed to determine %s from settings.py' %
str('WEBAPP_SERVER'))
print('Failed to determine %s from settings.py' % str('WEBAPP_SERVER'))
sys.exit(1)
try:
settings.WEBAPP_IP
except:
logger.error('error :: failed to determine %s from settings.py' %
str('WEBAPP_IP'))
print('Failed to determine %s from settings.py' % str('WEBAPP_IP'))
sys.exit(1)
try:
settings.WEBAPP_PORT
except:
logger.error('error :: failed to determine %s from settings.py' %
str('WEBAPP_PORT'))
print('Failed to determine %s from settings.py' % str('WEBAPP_PORT'))
sys.exit(1)
try:
settings.WEBAPP_AUTH_ENABLED
except:
logger.error('error :: failed to determine %s from settings.py' %
str('WEBAPP_AUTH_ENABLED'))
print('Failed to determine %s from settings.py' %
str('WEBAPP_AUTH_ENABLED'))
sys.exit(1)
try:
settings.WEBAPP_IP_RESTRICTED
except:
logger.error('error :: failed to determine %s from settings.py' %
str('WEBAPP_IP_RESTRICTED'))
print('Failed to determine %s from settings.py' %
str('WEBAPP_IP_RESTRICTED'))
sys.exit(1)
try:
settings.WEBAPP_AUTH_USER
except:
logger.error('error :: failed to determine %s from settings.py' %
str('WEBAPP_AUTH_USER'))
print('Failed to determine %s from settings.py' %
str('WEBAPP_AUTH_USER'))
sys.exit(1)
try:
settings.WEBAPP_AUTH_USER_PASSWORD
except:
logger.error('error :: failed to determine %s from settings.py' %
str('WEBAPP_AUTH_USER_PASSWORD'))
print('Failed to determine %s from settings.py' %
str('WEBAPP_AUTH_USER_PASSWORD'))
sys.exit(1)
try:
settings.WEBAPP_ALLOWED_IPS
except:
logger.error('error :: failed to determine %s from settings.py' %
str('WEBAPP_ALLOWED_IPS'))
print('Failed to determine %s from settings.py' %
str('WEBAPP_ALLOWED_IPS'))
sys.exit(1)
webapp = App()
if len(sys.argv) > 1 and sys.argv[1] == 'run':
webapp.run()
else:
daemon_runner = runner.DaemonRunner(webapp)
daemon_runner.daemon_context.files_preserve = [handler.stream]
daemon_runner.do_action()
