def run(): """ Start the Horizon agent and logger. """ if not isdir(settings.PID_PATH): print('pid directory does not exist at %s' % settings.PID_PATH) sys.exit(1) if not isdir(settings.LOG_PATH): print('log directory does not exist at %s' % settings.LOG_PATH) sys.exit(1) # @added 20201103 - Feature #3820: HORIZON_SHARDS try: HORIZON_SHARDS = settings.HORIZON_SHARDS.copy() except: HORIZON_SHARDS = {} horizon_shards_validated = True if HORIZON_SHARDS: for shard_host in HORIZON_SHARDS: try: int(HORIZON_SHARDS[shard_host]) except: horizon_shards_validated = False if not horizon_shards_validated: print( 'the HORIZON_SHARDS dict in settings.py does not have valid shard numbers' % str(settings.HORIZON_SHARDS)) sys.exit(1) logger.setLevel(logging.DEBUG) formatter = logging.Formatter("%(asctime)s :: %(process)s :: %(message)s", datefmt="%Y-%m-%d %H:%M:%S") handler = logging.handlers.TimedRotatingFileHandler(logfile, when="midnight", interval=1, backupCount=5) memory_handler = logging.handlers.MemoryHandler(100, flushLevel=logging.DEBUG, target=handler) handler.setFormatter(formatter) logger.addHandler(memory_handler) # Validate settings variables valid_settings = validate_settings_variables(skyline_app) if not valid_settings: print('error :: invalid variables in settings.py - cannot start') sys.exit(1) horizon = Horizon() if len(sys.argv) > 1 and sys.argv[1] == 'run': horizon.run() else: daemon_runner = runner.DaemonRunner(horizon) daemon_runner.daemon_context.files_preserve = [handler.stream] daemon_runner.do_action()
def run(): """ Start the Mirage agent. """ if not isdir(settings.PID_PATH): print ('pid directory does not exist at %s' % settings.PID_PATH) sys.exit(1) if not isdir(settings.LOG_PATH): print ('log directory does not exist at %s' % settings.LOG_PATH) sys.exit(1) logger.setLevel(logging.DEBUG) formatter = logging.Formatter("%(asctime)s :: %(process)s :: %(message)s", datefmt="%Y-%m-%d %H:%M:%S") handler = logging.handlers.TimedRotatingFileHandler( logfile, when="midnight", interval=1, backupCount=5) handler.setFormatter(formatter) logger.addHandler(handler) # Validate settings variables valid_settings = validate_settings_variables(skyline_app) if not valid_settings: print ('error :: invalid variables in settings.py - cannot start') sys.exit(1) # Make sure we can run all the algorithms try: timeseries = map(list, zip(map(float, range(int(time()) - 86400, int(time()) + 1)), [1] * 86401)) second_order_resolution_seconds = 86400 ensemble = [globals()[algorithm](timeseries, second_order_resolution_seconds) for algorithm in settings.MIRAGE_ALGORITHMS] except KeyError as e: print ('Algorithm %s deprecated or not defined; check settings.MIRAGE_ALGORITHMS' % e) sys.exit(1) except Exception as e: print ('Algorithm test run failed.') traceback.print_exc() sys.exit(1) logger.info('Tested algorithms') del timeseries del ensemble mirage = MirageAgent() if len(sys.argv) > 1 and sys.argv[1] == 'run': mirage.run() else: daemon_runner = runner.DaemonRunner(mirage) daemon_runner.daemon_context.files_preserve = [handler.stream] daemon_runner.do_action()
def run(): """ Start the Vista. Start the logger. """ if not isdir(settings.PID_PATH): print('pid directory does not exist at %s' % settings.PID_PATH) sys.exit(1) if not isdir(settings.LOG_PATH): print('log directory does not exist at %s' % settings.LOG_PATH) sys.exit(1) logger.setLevel(logging.DEBUG) formatter = logging.Formatter("%(asctime)s :: %(process)s :: %(message)s", datefmt="%Y-%m-%d %H:%M:%S") handler = logging.handlers.TimedRotatingFileHandler(logfile, when="midnight", interval=1, backupCount=5) memory_handler = logging.handlers.MemoryHandler(256, flushLevel=logging.DEBUG, target=handler) handler.setFormatter(formatter) logger.addHandler(memory_handler) # Validate settings variables valid_settings = validate_settings_variables(skyline_app) if not valid_settings: print( 'error :: agent :: invalid variables in settings.py - cannot start' ) sys.exit(1) vista = Vista() logger.info('agent :: starting vista') memory_handler.flush if len(sys.argv) > 1 and sys.argv[1] == 'run': vista.run() else: daemon_runner = runner.DaemonRunner(vista) daemon_runner.daemon_context.files_preserve = [handler.stream] daemon_runner.do_action() logger.info('stopping vista') memory_handler.flush
def run(): """ Check that all the `ALGORITHMS` can be run. Start the AnalyzerAgent. Start the logger. """ if not isdir(settings.PID_PATH): print('pid directory does not exist at %s' % settings.PID_PATH) sys.exit(1) if not isdir(settings.LOG_PATH): print('log directory does not exist at %s' % settings.LOG_PATH) sys.exit(1) logger.setLevel(logging.DEBUG) formatter = logging.Formatter("%(asctime)s :: %(process)s :: %(message)s", datefmt="%Y-%m-%d %H:%M:%S") handler = logging.handlers.TimedRotatingFileHandler(logfile, when="midnight", interval=1, backupCount=5) memory_handler = logging.handlers.MemoryHandler(256, flushLevel=logging.DEBUG, target=handler) handler.setFormatter(formatter) logger.addHandler(memory_handler) # Validate settings variables valid_settings = validate_settings_variables(skyline_app) if not valid_settings: print('error :: invalid variables in settings.py - cannot start') sys.exit(1) # Make sure we can run all the algorithms try: # from analyzer import algorithms import algorithms logger.info('Testing algorithms') # @modified 20200723 - Task #3608: Update Skyline to Python 3.8.3 and deps # Use a shorter timeseries for quicker start up # timeseries = map(list, zip(map(float, range(int(time()) - 86400, int(time()) + 1)), [1] * 86401)) timeseries = map( list, zip(map(float, range(int(time()) - 1440, int(time()) + 1)), [1] * 1440)) # @added 20191024 - Branch #3262: py3 # Convert map to list if python_version == 3: if isinstance(timeseries, map): timeseries = list(timeseries) # ensemble = [globals()[algorithm](timeseries) for algorithm in settings.ALGORITHMS] ensemble = [ getattr(algorithms, algorithm)(timeseries) for algorithm in settings.ALGORITHMS ] except KeyError as e: print( 'Algorithm %s deprecated or not defined; check settings.ALGORITHMS' % e) sys.exit(1) except Exception as e: print('Algorithm test run failed.') traceback.print_exc() sys.exit(1) logger.info('Tested algorithms') del timeseries del ensemble analyzer = AnalyzerAgent() logger.info('starting analyzer.run') memory_handler.flush if len(sys.argv) > 1 and sys.argv[1] == 'run': analyzer.run() else: daemon_runner = runner.DaemonRunner(analyzer) daemon_runner.daemon_context.files_preserve = [handler.stream] daemon_runner.do_action() logger.info('stopping analyzer') memory_handler.flush
logger.setLevel(logging.DEBUG) formatter = logging.Formatter("%(asctime)s :: %(process)s :: %(message)s", datefmt="%Y-%m-%d %H:%M:%S") handler = logging.handlers.TimedRotatingFileHandler(logfile, when="midnight", interval=1, backupCount=5) memory_handler = logging.handlers.MemoryHandler(256, flushLevel=logging.DEBUG, target=handler) handler.setFormatter(formatter) logger.addHandler(memory_handler) # Validate settings variables valid_settings = validate_settings_variables(skyline_app) if not valid_settings: print('error :: invalid variables in settings.py - cannot start') sys.exit(1) # @added 20191031 - Feature #3310: gracefully handle db failure # Branch 3262: py3 try: start_if_no_db = settings.START_IF_NO_DB except: start_if_no_db = False # Make sure mysql is available mysql_up = False try:
def run(): """ Start the Crucible agent and ensure all the required directories exist, creating the crucible directories if they do not exist """ if not isdir(settings.PID_PATH): print('pid directory does not exist at %s' % settings.PID_PATH) sys.exit(1) if not isdir(settings.LOG_PATH): print('log directory does not exist at %s' % settings.LOG_PATH) sys.exit(1) # Make sure the required directories exists if not os.path.exists(settings.CRUCIBLE_CHECK_PATH): try: os.makedirs(settings.CRUCIBLE_CHECK_PATH, mode=0o755) except: print('failed to create directory - %s' % settings.CRUCIBLE_CHECK_PATH) sys.exit(1) if not os.path.exists(settings.CRUCIBLE_DATA_FOLDER): try: os.makedirs(settings.CRUCIBLE_DATA_FOLDER, mode=0o755) except: print('failed to create directory - %s' % settings.CRUCIBLE_DATA_FOLDER) sys.exit(1) failed_checks_dir = settings.CRUCIBLE_DATA_FOLDER + '/failed_checks' if not os.path.exists(failed_checks_dir): try: os.makedirs(failed_checks_dir, mode=0o755) except: print('failed to create directory - %s' % failed_checks_dir) sys.exit(1) logger.setLevel(logging.DEBUG) formatter = logging.Formatter("%(asctime)s :: %(process)s :: %(message)s", datefmt="%Y-%m-%d %H:%M:%S") handler = logging.handlers.TimedRotatingFileHandler(logfile, when="midnight", interval=1, backupCount=5) memory_handler = logging.handlers.MemoryHandler(256, flushLevel=logging.DEBUG, target=handler) handler.setFormatter(formatter) logger.addHandler(memory_handler) # Validate settings variables valid_settings = validate_settings_variables(skyline_app) if not valid_settings: print('error :: invalid variables in settings.py - cannot start') sys.exit(1) crucible = CrucibleAgent() if len(sys.argv) > 1 and sys.argv[1] == 'run': logger.info('starting skyline crucible via run') crucible.run() else: logger.info('starting skyline crucible via daemon') daemon_runner = runner.DaemonRunner(crucible) daemon_runner.daemon_context.files_preserve = [handler.stream] daemon_runner.do_action()
def run(): """ Start the Boundary agent. """ if not isdir(settings.PID_PATH): print ('pid directory does not exist at %s' % settings.PID_PATH) sys.exit(1) if not isdir(settings.LOG_PATH): print ('log directory does not exist at %s' % settings.LOG_PATH) sys.exit(1) logger.setLevel(logging.DEBUG) formatter = logging.Formatter("%(asctime)s :: %(process)s :: %(message)s", datefmt="%Y-%m-%d %H:%M:%S") handler = logging.handlers.TimedRotatingFileHandler( logfile, when="midnight", interval=1, backupCount=5) memory_handler = logging.handlers.MemoryHandler(256, flushLevel=logging.DEBUG, target=handler) handler.setFormatter(formatter) logger.addHandler(memory_handler) # Validate settings variables valid_settings = validate_settings_variables(skyline_app) if not valid_settings: print ('error :: invalid variables in settings.py - cannot start') sys.exit(1) # Make sure all the BOUNDARY_ALGORITHMS are valid try: if settings.BOUNDARY_ALGORITHMS: configuration_error = False for algorithm in settings.BOUNDARY_ALGORITHMS: valid = True if not isinstance(algorithm, str): valid = False if not valid: configuration_error = True print ('configuration error in tuple, expected: str') print('configuration error in BOUNDARY_ALGORITHMS tuple: %s' % str(algorithm)) except: try: if configuration_error: print ('There are configuration issues in BOUNDARY_ALGORITHMS in settings.py') except: print ('There are no BOUNDARY_ALGORITHMS in settings.py. try adding some, nothing to do') sys.exit(1) # Make sure we can run all the algorithms try: timeseries = map(list, zip(map(float, range(int(time()) - 86400, int(time()) + 1)), [1] * 86401)) ensemble = [globals()[algorithm]( timeseries, 'test', 3600, 100, 300, 1 ) for algorithm in settings.BOUNDARY_ALGORITHMS] except KeyError as e: print ('Algorithm %s deprecated or not defined; check settings.BOUNDARY_ALGORITHMS' % e) sys.exit(1) except Exception as e: print ('Algorithm test run failed.') traceback.print_exc() sys.exit(1) # Make sure all the BOUNDARY_METRICS are valid try: if settings.BOUNDARY_METRICS: configuration_error = False for metric in settings.BOUNDARY_METRICS: valid = True strings = [] strings.append(metric[0]) strings.append(metric[1]) strings.append(metric[7]) for string in strings: if not isinstance(string, str): valid = False values = [] values.append(metric[2]) values.append(metric[3]) values.append(metric[4]) values.append(metric[5]) values.append(metric[6]) for value in values: if not isinstance(value, int): valid = False alert_via = metric[7] for alerter in alert_via.split("|"): if not isinstance(alerter, str): valid = False if not valid: configuration_error = True print ('configuration error in tuple, expected: str, str, int, int, int, int, str') print('configuration error in BOUNDARY_METRICS tuple: %s' % str(metric)) except: if configuration_error: print ('There are configuration issues in BOUNDARY_METRICS in settings.py') else: print ('There are no BOUNDARY_METRICS in settings.py. try adding some, nothing to do') sys.exit(1) logger.info('Tested algorithms') del timeseries del ensemble del strings del values boundary = BoundaryAgent() if len(sys.argv) > 1 and sys.argv[1] == 'run': boundary.run() else: daemon_runner = runner.DaemonRunner(boundary) daemon_runner.daemon_context.files_preserve = [handler.stream] daemon_runner.do_action()
def run(): """ Check that all the `ALGORITHMS` can be run. Start the AnalyzerAgent. Start the logger. """ if not isdir(settings.PID_PATH): print('pid directory does not exist at %s' % settings.PID_PATH) sys.exit(1) if not isdir(settings.LOG_PATH): print('log directory does not exist at %s' % settings.LOG_PATH) sys.exit(1) if len(sys.argv) > 1 and sys.argv[1] == 'stop': do_not_overwrite_log = True # This should hopefully take care of a TODO from the bin files, # TODO: write a real kill script # as above @earthgecko 20160520 pidfile_path = settings.PID_PATH + '/' + skyline_app + '.pid' pid = int(open(pidfile_path).read()) try: kill(pid, signal.SIGTERM) print('%s pid %s stopped' % (skyline_app, str(pid))) sys.exit(0) except OSError as e: print('Failed to kill pid %s - OSError - %s' % (str(pid), str(e))) sys.exit(1) logger.setLevel(logging.DEBUG) formatter = logging.Formatter("%(asctime)s :: %(process)s :: %(message)s", datefmt="%Y-%m-%d %H:%M:%S") if len(sys.argv) > 1 and sys.argv[1] == 'stop': handler = logging.FileHandler(settings.LOG_PATH + '/' + skyline_app + '.stop.log', mode='a', delay=False) else: handler = logging.handlers.TimedRotatingFileHandler(logfile, when="midnight", interval=1, backupCount=5) memory_handler = logging.handlers.MemoryHandler(256, flushLevel=logging.DEBUG, target=handler) handler.setFormatter(formatter) logger.addHandler(memory_handler) # Validate settings variables valid_settings = validate_settings_variables(skyline_app) if not valid_settings: print('error :: invalid variables in settings.py - cannot start') sys.exit(1) if len(sys.argv) > 1 and sys.argv[1] == 'stop': do_not_overwrite_log = True else: # Make sure we can run all the algorithms try: # from analyzer import algorithms import algorithms_dev logger.info('Testing algorithms') timeseries = map( list, zip(map(float, range(int(time()) - 86400, int(time()) + 1)), [1] * 86401)) # ensemble = [globals()[algorithm](timeseries) for algorithm in settings.ALGORITHMS] ensemble = [ getattr(algorithms_dev, algorithm)(timeseries) for algorithm in settings.ALGORITHMS ] logger.info('Tested algorithms OK') logger.info('ensemble: %s' % str(ensemble)) except KeyError as e: print( 'Algorithm %s deprecated or not defined; check settings.ALGORITHMS' % e) sys.exit(1) except Exception as e: print('Algorithm test run failed.') traceback.print_exc() sys.exit(1) logger.info('Tested algorithms') del timeseries del ensemble analyzer = AnalyzerDevAgent() if len(sys.argv) > 1 and sys.argv[1] == 'stop': do_not_overwrite_log = True else: logger.info('starting analyzer_dev.run') memory_handler.flush if len(sys.argv) > 1 and sys.argv[1] == 'run': analyzer.run() else: daemon_runner = runner.DaemonRunner(analyzer) daemon_runner.daemon_context.files_preserve = [handler.stream] daemon_runner.do_action() if len(sys.argv) > 1 and sys.argv[1] == 'stop': do_not_overwrite_log = True else: logger.info('stopped analyzer_dev')
def run(): """ Check that all the `ALGORITHMS` can be run. Start the AnalyzerAgent. Start the logger. """ if not isdir(settings.PID_PATH): print ('pid directory does not exist at %s' % settings.PID_PATH) sys.exit(1) if not isdir(settings.LOG_PATH): print ('log directory does not exist at %s' % settings.LOG_PATH) sys.exit(1) logger.setLevel(logging.DEBUG) formatter = logging.Formatter("%(asctime)s :: %(process)s :: %(message)s", datefmt="%Y-%m-%d %H:%M:%S") handler = logging.handlers.TimedRotatingFileHandler( logfile, when="midnight", interval=1, backupCount=5) memory_handler = logging.handlers.MemoryHandler(256, flushLevel=logging.DEBUG, target=handler) handler.setFormatter(formatter) logger.addHandler(memory_handler) # Validate settings variables valid_settings = validate_settings_variables(skyline_app) if not valid_settings: print ('error :: invalid variables in settings.py - cannot start') sys.exit(1) # Make sure we can run all the algorithms try: # from analyzer import algorithms import algorithms logger.info('Testing algorithms') timeseries = map(list, zip(map(float, range(int(time()) - 86400, int(time()) + 1)), [1] * 86401)) # ensemble = [globals()[algorithm](timeseries) for algorithm in settings.ALGORITHMS] ensemble = [getattr(algorithms, algorithm)(timeseries) for algorithm in settings.ALGORITHMS] except KeyError as e: print ('Algorithm %s deprecated or not defined; check settings.ALGORITHMS' % e) sys.exit(1) except Exception as e: print ('Algorithm test run failed.') traceback.print_exc() sys.exit(1) logger.info('Tested algorithms') del timeseries del ensemble analyzer = AnalyzerAgent() logger.info('starting analyzer.run') memory_handler.flush if len(sys.argv) > 1 and sys.argv[1] == 'run': analyzer.run() else: daemon_runner = runner.DaemonRunner(analyzer) daemon_runner.daemon_context.files_preserve = [handler.stream] daemon_runner.do_action() logger.info('stopping analyzer') memory_handler.flush
def run(): """ Start the Webapp server """ if not isdir(settings.PID_PATH): print('pid directory does not exist at %s' % settings.PID_PATH) sys.exit(1) if not isdir(settings.LOG_PATH): print('log directory does not exist at %s' % settings.LOG_PATH) sys.exit(1) logger.setLevel(logging.DEBUG) formatter = logging.Formatter("%(asctime)s :: %(process)s :: %(message)s", datefmt="%Y-%m-%d %H:%M:%S") handler = logging.handlers.TimedRotatingFileHandler(logfile, when="midnight", interval=1, backupCount=5) memory_handler = logging.handlers.MemoryHandler(100, flushLevel=logging.DEBUG, target=handler) handler.setFormatter(formatter) logger.addHandler(memory_handler) # Validate settings variables valid_settings = validate_settings_variables(skyline_app) if not valid_settings: print('error :: invalid variables in settings.py - cannot start') sys.exit(1) try: settings.WEBAPP_SERVER except: logger.error('error :: failed to determine %s from settings.py' % str('WEBAPP_SERVER')) print('Failed to determine %s from settings.py' % str('WEBAPP_SERVER')) sys.exit(1) try: settings.WEBAPP_IP except: logger.error('error :: failed to determine %s from settings.py' % str('WEBAPP_IP')) print('Failed to determine %s from settings.py' % str('WEBAPP_IP')) sys.exit(1) try: settings.WEBAPP_PORT except: logger.error('error :: failed to determine %s from settings.py' % str('WEBAPP_PORT')) print('Failed to determine %s from settings.py' % str('WEBAPP_PORT')) sys.exit(1) try: settings.WEBAPP_AUTH_ENABLED except: logger.error('error :: failed to determine %s from settings.py' % str('WEBAPP_AUTH_ENABLED')) print('Failed to determine %s from settings.py' % str('WEBAPP_AUTH_ENABLED')) sys.exit(1) try: settings.WEBAPP_IP_RESTRICTED except: logger.error('error :: failed to determine %s from settings.py' % str('WEBAPP_IP_RESTRICTED')) print('Failed to determine %s from settings.py' % str('WEBAPP_IP_RESTRICTED')) sys.exit(1) try: settings.WEBAPP_AUTH_USER except: logger.error('error :: failed to determine %s from settings.py' % str('WEBAPP_AUTH_USER')) print('Failed to determine %s from settings.py' % str('WEBAPP_AUTH_USER')) sys.exit(1) try: settings.WEBAPP_AUTH_USER_PASSWORD except: logger.error('error :: failed to determine %s from settings.py' % str('WEBAPP_AUTH_USER_PASSWORD')) print('Failed to determine %s from settings.py' % str('WEBAPP_AUTH_USER_PASSWORD')) sys.exit(1) try: settings.WEBAPP_ALLOWED_IPS except: logger.error('error :: failed to determine %s from settings.py' % str('WEBAPP_ALLOWED_IPS')) print('Failed to determine %s from settings.py' % str('WEBAPP_ALLOWED_IPS')) sys.exit(1) webapp = App() if len(sys.argv) > 1 and sys.argv[1] == 'run': webapp.run() else: daemon_runner = runner.DaemonRunner(webapp) daemon_runner.daemon_context.files_preserve = [handler.stream] daemon_runner.do_action()