def print_intro(): """ Print the banner information including version, loaded configuration files and any parsing errors that might have occurred when loading them. """ logging.logger.info( "outliers.py - version %s - contact: [email protected]", EE_OUTLIERS_VERSIONS) logging.logger.info("run mode: %s", settings.args.run_mode) logging.print_generic_intro("initializing") logging.logger.info("loaded %d configuration files", len(settings.loaded_config_paths)) if settings.failed_config_paths: logging.logger.error( "failed to load %d configuration files that will be " "ignored", len(settings.failed_config_paths)) for failed_config_path in settings.failed_config_paths: logging.logger.error("\t+ failed to load configuration file %s", failed_config_path) if settings.failing_regular_expressions: logging.logger.error( "failed to parse %d regular expressions in whitelist that " "will be ignored", len(settings.failing_regular_expressions)) for failed_regular_expression in settings.failing_regular_expressions: logging.logger.error("\t+ failed to parse regular expression %s", failed_regular_expression)
def evaluate_model(self): self.extract_extra_model_settings() # Train the model if self.model_settings["train_model"]: self.train_model() return w2v_model = word2vec.Word2Vec(name=self.model_name) search_query = es.filter_by_query_string(self.model_settings["es_query_filter"]) if not w2v_model.is_trained(): logging.logger.warning("model was not trained! Skipping analysis.") else: # Check if we need to run the test data instead of real data if w2v_model.use_test_data: logging.print_generic_intro("using test data instead of live data to evaluate model " + self.model_name) self.evaluate_test_sentences(w2v_model=w2v_model) return self.total_events = es.count_documents(search_query=search_query) logging.print_analysis_intro(event_type="evaluating " + self.model_name, total_events=self.total_events) logging.init_ticker(total_steps=self.total_events, desc=self.model_name + " - evaluating word2vec model") raw_docs = list() eval_sentences = list() for doc in es.scan(search_query=search_query): logging.tick() fields = es.extract_fields_from_document(doc) try: new_sentences = helpers.utils.flatten_fields_into_sentences(fields=fields, sentence_format=self.model_settings["sentence_format"]) eval_sentences.extend(new_sentences) except KeyError: logging.logger.debug("skipping event which does not contain the target and aggregator fields we are processing. - [" + self.model_name + "]") continue for _ in new_sentences: raw_docs.append(doc) # Evaluate batch of events against the model if logging.current_step == self.total_events or len(eval_sentences) >= settings.config.getint("machine_learning", "word2vec_batch_eval_size"): logging.logger.info("evaluating batch of " + str(len(eval_sentences)) + " sentences") outliers = self.evaluate_batch_for_outliers(w2v_model=w2v_model, eval_sentences=eval_sentences, raw_docs=raw_docs) if len(outliers) > 0: unique_summaries = len(set(o.outlier_dict["summary"] for o in outliers)) logging.logger.info("total outliers in batch processed: " + str(len(outliers)) + " [" + str(unique_summaries) + " unique summaries]") # Reset data structures for next batch raw_docs = list() eval_sentences = list()
def print_intro(): logging.logger.info("outliers.py started - contact: [email protected]") logging.logger.info("run mode: " + settings.args.run_mode) logging.print_generic_intro("initializing") logging.logger.info("loaded " + str(len(settings.loaded_config_paths)) + " configuration files") if settings.failed_config_paths: logging.logger.warning("failed to load " + str(len(settings.failed_config_paths)) + " configuration files") for failed_config_path in settings.failed_config_paths: logging.logger.warning("failed to load " + str(failed_config_path))
def run_daemon_mode(): """ Run outliers in daemon mode. In this mode, outliers will continue running based on the cron scheduled defined in the configuration file. """ # In daemon mode, we also want to monitor the configuration file for changes. # In case of a change, we need to make sure that we are using this new configuration file for config_file in settings.args.config: logging.logger.info("monitoring configuration file %s for changes", config_file) # Monitor configuration files for potential changes file_mod_watcher = FileModificationWatcher() file_mod_watcher.add_files(settings.args.config) # Initialize Elasticsearch connection while not es.init_connection(): time.sleep(60) # Create housekeeping job, don't start it yet housekeeping_job = HousekeepingJob() first_run = True run_succeeded_without_errors = None # The daemon should run forever, until the user kills it while True: next_run = None should_schedule_next_run = False # This loop will run for as long we don't need to perform an analysis while (next_run is None or datetime.now() < next_run) and first_run is False and \ run_succeeded_without_errors is True: # Check if we already know when to perform the analysis next; if not, we need to schedule it if next_run is None: should_schedule_next_run = True # Check for configuration file changes and load them in case it's needed if file_mod_watcher.files_changed(): logging.logger.info("configuration file changed, reloading") settings.process_configuration_files() should_schedule_next_run = True # Schedule a next rune based on the cron schedule defined in the configuration file if should_schedule_next_run: next_run = croniter(settings.config.get("daemon", "schedule"), datetime.now()).get_next(datetime) logging.logger.info( "next run scheduled on {0:%Y-%m-%d %H:%M:%S}".format( next_run)) should_schedule_next_run = False # Wait 5 seconds before checking the cron schedule again time.sleep(5) # Refresh settings in case the cron has changed for example settings.process_configuration_files() # On the first run, we might have to wipe all the existing outliers if this is set in the configuration file if first_run: first_run = False logging.logger.info( "first run, so we will start immediately - after this, we will respect the cron " "schedule defined in the configuration file") # Wipe all existing outliers if needed if settings.config.getboolean("general", "es_wipe_all_existing_outliers"): logging.logger.info( "wiping all existing outliers on first run") es.remove_all_outliers() # Make sure we are still connected to Elasticsearch before analyzing, in case something went wrong with # the connection in between runs while not es.init_connection(): time.sleep(60) # Make sure housekeeping is up and running if not housekeeping_job.is_alive(): housekeeping_job.start() # Perform analysis and print the analysis summary at the end logging.print_generic_intro("starting outlier detection") analyzed_models = perform_analysis(housekeeping_job) print_analysis_summary(analyzed_models) errored_models = [ analyzer for analyzer in analyzed_models if analyzer.unknown_error_analysis ] # Check the result of the analysis. In case an error occured, we want to re-run right away (after a minute) if errored_models: run_succeeded_without_errors = False logging.logger.warning( "ran into errors while analyzing use cases - not going to wait for the cron " "schedule, we just start analyzing again after sleeping for a minute first" ) time.sleep(60) else: run_succeeded_without_errors = True logging.print_generic_intro("finished performing outlier detection")
def run_daemon_mode(): # In daemon mode, we also want to monitor the configuration file for changes. # In case of a change, we need to make sure that we are using this new configuration file for config_file in settings.args.config: logging.logger.info("monitoring configuration file " + config_file + " for changes") file_mod_watcher = FileModificationWatcher() file_mod_watcher.add_files(settings.args.config) # Initialize Elasticsearch connection es.init_connection() # Create housekeeping job, don't start it yet housekeeping_job = HousekeepingJob() num_runs = 0 first_run = True run_succeeded_without_errors = None while True: num_runs += 1 next_run = None should_schedule_next_run = False while (next_run is None or datetime.now() < next_run) and first_run is False and \ run_succeeded_without_errors is True: if next_run is None: should_schedule_next_run = True # Check for configuration file changes and load them in case it's needed if file_mod_watcher.files_changed(): logging.logger.info("configuration file changed, reloading") settings.process_configuration_files() should_schedule_next_run = True if should_schedule_next_run: next_run = croniter(settings.config.get("daemon", "schedule"), datetime.now()).get_next(datetime) logging.logger.info( "next run scheduled on {0:%Y-%m-%d %H:%M:%S}".format( next_run)) should_schedule_next_run = False time.sleep(5) settings.process_configuration_files() # Refresh settings if first_run: first_run = False logging.logger.info( "first run, so we will start immediately - after this, we will respect the cron " + "schedule defined in the configuration file") # Wipe all existing outliers if needed if settings.config.getboolean("general", "es_wipe_all_existing_outliers"): logging.logger.info( "wiping all existing outliers on first run") es.remove_all_outliers() else: # Make sure we are still connected to Elasticsearch before analyzing, in case something went wrong with # the connection in between runs es.init_connection() # Make sure housekeeping is up and running if not housekeeping_job.is_alive(): housekeeping_job.start() # Perform analysis logging.print_generic_intro("starting outlier detection") analyzed_models = perform_analysis() print_analysis_summary(analyzed_models) errored_models = [ analyzer for analyzer in analyzed_models if analyzer.unknown_error_analysis ] # Check the result of the analysis if errored_models: run_succeeded_without_errors = False logging.logger.warning( "ran into errors while analyzing use cases - not going to wait for the cron " + "schedule, we just start analyzing again after sleeping for a minute first" ) time.sleep(60) else: run_succeeded_without_errors = True logging.print_generic_intro("finished performing outlier detection")
"machine_learning", "tensorflow_log_level") # Log Handlers LOG_FILE = settings.config.get("general", "log_file") if os.path.exists(os.path.dirname(LOG_FILE)): logging.add_file_handler(LOG_FILE) else: logging.logger.warning( "log directory for log file %s does not exist, check your settings! Only logging to stdout.", LOG_FILE) logging.logger.info("outliers.py started - contact: [email protected]") logging.logger.info("run mode: " + settings.args.run_mode) logging.print_generic_intro("initializing") logging.logger.info("loaded " + str(len(settings.loaded_config_paths)) + " configuration files") if settings.failed_config_paths: logging.logger.warning("failed to load " + str(len(settings.failed_config_paths)) + " configuration files") for failed_config_path in settings.failed_config_paths: logging.logger.warning("failed to load " + str(failed_config_path)) def perform_analysis(): """ The entrypoint for analysis """ analyzers = list()
logging.verbosity = settings.config.getint("general", "log_verbosity") logging.logger.setLevel(settings.config.get("general", "log_level")) os.environ['TF_CPP_MIN_LOG_LEVEL'] = settings.config.get("machine_learning", "tensorflow_log_level") # Log Handlers LOG_FILE = settings.config.get("general", "log_file") if os.path.exists(os.path.dirname(LOG_FILE)): logging.add_file_handler(LOG_FILE) else: logging.logger.warning("log directory for log file %s does not exist, check your settings! Only logging to stdout.", LOG_FILE) logging.logger.info("outliers.py started - contact: [email protected]") logging.logger.info("run mode: " + settings.args.run_mode) logging.print_generic_intro("initializing") logging.logger.info("loaded " + str(len(settings.loaded_config_paths)) + " configuration files") if settings.failed_config_paths: logging.logger.warning("failed to load " + str(len(settings.failed_config_paths)) + " configuration files") for failed_config_path in settings.failed_config_paths: logging.logger.warning("failed to load " + str(failed_config_path)) def perform_analysis(): """ The entrypoint for analysis """ analyzers = list() for config_section_name in settings.config.sections(): try: