def echo_get_metric_from_metrics(base_name, engine): """ Called by :func:`~echo` and returns the metric id and metric db object :param timestamp: timestamp at which learn was called :type timestamp: int :return: tuple :rtype: (int, object) """ logger = logging.getLogger(skyline_app_logger) metrics_id = 0 metric_db_object = None # Get the metrics_table metadata metrics_table = None try: metrics_table, log_msg, trace = metrics_table_meta(skyline_app, engine) logger.info('echo :: metrics_table OK for %s' % base_name) except: logger.error(traceback.format_exc()) logger.error( 'error :: echo :: failed to get metrics_table meta for %s' % base_name) return False try: connection = engine.connect() stmt = select([metrics_table ]).where(metrics_table.c.metric == base_name) result = connection.execute(stmt) row = result.fetchone() metric_db_object = row metrics_id = row['id'] connection.close() except: logger.error(traceback.format_exc()) logger.error( 'error :: echo :: could not determine id from metrics table for - %s' % base_name) return False return metrics_id, metric_db_object
result = connection.execute(stmt) for row in result: if row['enabled'] != 1: continue if row['deleted'] == 1: continue fp_id = row['id'] metric_id = row['metric_id'] anomaly_timestamp = row['anomaly_timestamp'] fps_data.append([int(fp_id), int(metric_id), int(anomaly_timestamp)]) connection.close() except: print('Failed to get ionosphere data from the database') try: metrics_table, log_msg, trace = metrics_table_meta('autobuild', engine) except: print(traceback.format_exc()) # get all fp ids and metric ids from ionosphere table ionosphere_metrics_data = [] try: connection = engine.connect() stmt = select([metrics_table]).where(metrics_table.c.ionosphere_enabled > 0) result = connection.execute(stmt) for row in result: metric_id = row['id'] metric = row['metric'] ionosphere_metrics_data.append([int(metric_id), str(metric)]) connection.close() except:
def update_snab_result(snab_id, anomaly_id, snab_result): """ Update the relevant field in the snab table. :param snab_id: the snab table id :param anomaly_id: the anomaly id :param snab_result: a selected result :type snab_id: int :type anomaly_id: int :type result: str :return: snab_result_updated, base_name, anomaly_timestamp :rtype: tuple """ snab_result_updated = False base_name = None anomaly_timestamp = None connection = None logger.info( 'update_snab_result :: for snab id %s with anomaly id %s and result %s' % ( str(snab_id), str(anomaly_id), str(snab_result))) logger.info('getting MySQL engine') try: engine, fail_msg, trace = get_snab_engine() logger.info(fail_msg) except: trace = traceback.format_exc() logger.error(trace) logger.error('%s' % fail_msg) logger.error('error :: update_snab_result :: could not get a MySQL engine to get update snab table') raise # to webapp to return in the UI if not engine: trace = 'none' fail_msg = 'error :: update_snab_result :: engine not obtained' logger.error(fail_msg) raise try: snab_table, log_msg, trace = snab_table_meta(skyline_app, engine) logger.info(log_msg) logger.info('snab_table OK') except: logger.error(traceback.format_exc()) logger.error('error :: update_snab_result :: failed to get snab_table meta') if engine: snab_engine_disposal(engine) raise # to webapp to return in the UI # @modified 20201004 - Task #3748: POC SNAB # Branch #3068: SNAB # Allow results to be changed determined_result = False existing_result = None try: if not connection: connection = engine.connect() stmt = select([snab_table]).\ where(snab_table.c.id == snab_id) result = connection.execute(stmt) for row in result: tP_set = row['tP'] if tP_set: existing_result = 'tP' fP_set = row['fP'] if fP_set: existing_result = 'fP' tN_set = row['tN'] if tN_set: existing_result = 'tN' fN_set = row['fN'] if fN_set: existing_result = 'fN' unsure_set = row['unsure'] if unsure_set: existing_result = 'unsure' determined_result = True break logger.info('update_snab_result :: current result values to snab id %s - tP: %s, fP: %s, tN: %s, fN: %s, unsure: %s' % ( str(snab_id), str(tP_set), str(fP_set), str(tN_set), str(fN_set), str(unsure_set))) except: trace = traceback.format_exc() logger.error(trace) logger.error('error :: update_snab_result :: could not determine current result values for snab id %s' % ( str(snab_id))) fail_msg = 'error :: update_snab_result :: could not determine current result values for snab id %s' % ( str(snab_id)) if engine: try: connection.close() except: pass snab_engine_disposal(engine) raise if not determined_result: logger.error('error :: update_snab_result :: did not determine current result values for snab id %s' % ( str(snab_id))) else: logger.info('update_snab_result :: current existing result for snab id %s - %s' % ( str(snab_id), str(existing_result))) try: if not connection: connection = engine.connect() if snab_result == 'tP': stmt = snab_table.update().\ values(tP=1, fP=None, tN=None, fN=None, unsure=None).\ where(snab_table.c.id == int(snab_id)).\ where(snab_table.c.anomaly_id == int(anomaly_id)) if snab_result == 'fP': stmt = snab_table.update().\ values(tP=None, fP=1, tN=None, fN=None, unsure=None).\ where(snab_table.c.id == int(snab_id)).\ where(snab_table.c.anomaly_id == int(anomaly_id)) if snab_result == 'tN': stmt = snab_table.update().\ values(tP=None, fP=None, tN=1, fN=None, unsure=None).\ where(snab_table.c.id == int(snab_id)).\ where(snab_table.c.anomaly_id == int(anomaly_id)) if snab_result == 'fN': stmt = snab_table.update().\ values(tP=None, fP=None, tN=None, fN=1, unsure=None).\ where(snab_table.c.id == int(snab_id)).\ where(snab_table.c.anomaly_id == int(anomaly_id)) if snab_result == 'unsure': stmt = snab_table.update().\ values(tP=None, fP=None, tN=None, fN=None, unsure=1).\ where(snab_table.c.id == int(snab_id)).\ where(snab_table.c.anomaly_id == int(anomaly_id)) if snab_result == 'NULL': stmt = snab_table.update().\ values(tP=None, fP=None, tN=None, fN=None, unsure=None).\ where(snab_table.c.id == int(snab_id)).\ where(snab_table.c.anomaly_id == int(anomaly_id)) connection.execute(stmt) snab_result_updated = True logger.info('update_snab_result :: update result for snab id %s with anomaly id %s and result %s' % ( str(snab_id), str(anomaly_id), str(snab_result))) except: trace = traceback.format_exc() logger.error(trace) logger.error('error :: update_snab_result :: could not update result for snab id %s with anomaly id %s and result %s' % ( str(snab_id), str(anomaly_id), str(result))) fail_msg = 'error :: update_snab_result :: could not update result for snab id %s with anomaly id %s and result %s' % ( str(snab_id), str(anomaly_id), str(result)) if engine: try: connection.close() except: pass snab_engine_disposal(engine) raise try: metrics_table, log_msg, trace = metrics_table_meta(skyline_app, engine) logger.info(log_msg) logger.info('update_snab_result :: metrics_table OK') except: logger.error(traceback.format_exc()) logger.error('error :: update_snab_result :: failed to get metrics_table meta for %s' % base_name) if engine: try: connection.close() except: pass snab_engine_disposal(engine) raise # to webapp to return in the UI try: anomalies_table, log_msg, trace = anomalies_table_meta(skyline_app, engine) logger.info(log_msg) logger.info('update_snab_result :: anomalies_table OK') except: logger.error(traceback.format_exc()) logger.error('error :: update_snab_result :: failed to get anomalies_table meta') if engine: try: connection.close() except: pass snab_engine_disposal(engine) raise # to webapp to return in the UI metric_id = None try: if not connection: connection = engine.connect() stmt = select([anomalies_table]).\ where(anomalies_table.c.id == anomaly_id) result = connection.execute(stmt) for row in result: metric_id = row['metric_id'] anomaly_timestamp = row['anomaly_timestamp'] break logger.info('update_snab_result :: determined anomaly_timestamp %s for metric id %s for anomaly id %s' % ( str(anomaly_timestamp), str(metric_id), str(anomaly_id))) except: trace = traceback.format_exc() logger.error(trace) fail_msg = 'error :: update_snab_result :: could not determine anomaly timestamp or metric id from DB for anomaly id %s' % ( str(anomaly_id)) logger.error('%s' % fail_msg) if engine: try: connection.close() except: pass snab_engine_disposal(engine) raise # to webapp to return in the UI if metric_id: try: if not connection: connection = engine.connect() stmt = select([metrics_table]).where(metrics_table.c.id == int(metric_id)) result = connection.execute(stmt) for row in result: base_name = row['metric'] except: trace = traceback.format_exc() logger.error(trace) fail_msg = 'error :: could not determine metric id from metrics table' if engine: try: connection.close() except: pass snab_engine_disposal(engine) raise if connection: try: connection.close() except: pass if engine: snab_engine_disposal(engine) return snab_result_updated, base_name, anomaly_timestamp, existing_result
def metric_id_from_base_name(current_skyline_app, base_name): """ Given a base name, return the metric_id """ metric_id = 0 function_str = 'database_queries.metric_id_from_base_name' current_skyline_app_logger = current_skyline_app + 'Log' current_logger = logging.getLogger(current_skyline_app_logger) try: engine, fail_msg, trace = get_engine(current_skyline_app) except Exception as e: trace = traceback.format_exc() current_logger.error(trace) fail_msg = 'error :: %s :: could not get a MySQL engine - %s' % ( function_str, e) current_logger.error('%s' % fail_msg) return False, fail_msg, trace try: metrics_table, fail_msg, trace = metrics_table_meta( current_skyline_app, engine) current_logger.info(fail_msg) except Exception as e: trace = traceback.format_exc() current_logger.error('%s' % trace) fail_msg = 'error :: %s :: failed to get metrics_table meta for %s- %s' % ( function_str, base_name, e) current_logger.error('%s' % fail_msg) if engine: engine_disposal(current_skyline_app, engine) if current_skyline_app == 'webapp': # Raise to webapp raise return False, fail_msg, trace try: connection = engine.connect() stmt = select([metrics_table ]).where(metrics_table.c.metric == base_name) result = connection.execute(stmt) for row in result: metric_id = int(row['id']) break connection.close() current_logger.info('%s :: determined db metric id: %s' % (function_str, str(metric_id))) except Exception as e: trace = traceback.format_exc() current_logger.error(trace) fail_msg = 'error :: %s :: could not determine id of metric from DB for %s - %s' % ( function_str, base_name, e) current_logger.error('%s' % fail_msg) if engine: engine_disposal(current_skyline_app, engine) if current_skyline_app == 'webapp': # Raise to webapp raise return False, fail_msg, trace if engine: engine_disposal(current_skyline_app, engine) if not metric_id: current_logger.error('error :: %s :: no id for metric in the DB - %s' % (function_str, base_name)) return metric_id
def get_metrics_db_object(base_name): """ Returns the data of a metric from metrics table as an object and populates memcached with a dict of the object :param base_name: the metric base_name :type base_name: str :return: metrics_db_object :rtype: object """ def get_an_engine(): try: engine, log_msg, trace = get_engine(skyline_app) return engine, log_msg, trace except: logger.error(traceback.format_exc()) log_msg = 'error :: failed to get MySQL engine in spin_process' logger.error('error :: failed to get MySQL engine in spin_process') return None, log_msg, trace def engine_disposal(engine): if engine: try: engine.dispose() except: logger.error(traceback.format_exc()) logger.error('error :: calling engine.dispose()') return metrics_db_object = None memcache_metrics_db_object = None metrics_db_object_key = 'metrics_db_object.%s' % str(base_name) memcache_metric_dict = None if settings.MEMCACHE_ENABLED: memcache_metric_dict = get_memcache_metric_object( skyline_app, base_name) query_metric_table = True if memcache_metric_dict: query_metric_table = False metrics_db_object = memcache_metric_dict logger.info('using %s key data from memcache' % metrics_db_object_key) # @modified 20170825 - Task #2132: Optimise Ionosphere DB usage # If no memcache data then MySQL query_metric_table if query_metric_table: try: engine, log_msg, trace = get_an_engine() logger.info(log_msg) except: logger.error(traceback.format_exc()) logger.error( 'error :: could not get a MySQL engine to determine ionosphere_enabled' ) if not engine: logger.error( 'error :: engine not obtained to determine ionosphere_enabled') # Get the metrics_table metadata metrics_table = None try: metrics_table, log_msg, trace = metrics_table_meta( skyline_app, engine) logger.info('metrics_table OK for %s' % base_name) except: logger.error(traceback.format_exc()) logger.error('error :: failed to get metrics_table meta for %s' % base_name) try: connection = engine.connect() stmt = select([metrics_table ]).where(metrics_table.c.metric == base_name) result = connection.execute(stmt) try: result except: logger.error(traceback.format_exc()) logger.error( 'error :: got no result from MySQL from metrics table for - %s' % base_name) row = result.fetchone() # @added 20170825 - Task #2132: Optimise Ionosphere DB usage # @modified - 20180524 - Task #2132: Optimise Ionosphere DB usage # Feature #2378: Add redis auth to Skyline and rebrow # Wrapped memcache_metrics_db_object, metrics_id, # metric_ionosphere_enabled and metrics_db_object in if row # as if row is None it can fail with: # TypeError: 'NoneType' object is not iterable # memcache_metrics_db_object = dict(row) if row: memcache_metrics_db_object = dict(row) # @added 20170115 - Feature #1854: Ionosphere learn - generations # Create the metrics_db_object so it is available throughout # Here we go! Learn! metrics_db_object = row else: logger.info('could not determine metric id for %s' % base_name) connection.close() except: logger.error(traceback.format_exc()) logger.error( 'error :: could not determine ionosphere_enabled from metrics table for - %s' % base_name) # @added 20170825 - Task #2132: Optimise Ionosphere DB usage # Add the metric db object data to memcache # @modified 20191031 - Branch #3262: py3 # Task #3304: py3 - handle pymemcache bytes not str # if settings.MEMCACHE_ENABLED and query_metric_table: if settings.MEMCACHE_ENABLED and query_metric_table and memcache_metrics_db_object: try: memcache_metric_dict = {} # @modified 20191030 - Branch #3262: py3 # Task #3304: py3 - handle pymemcache bytes not str # for k, v in memcache_metrics_db_object.iteritems(): if python_version == 2: for k, v in memcache_metrics_db_object.iteritems(): key_name = str(k) key_value = str(v) memcache_metric_dict[key_name] = key_value else: for k, v in memcache_metrics_db_object.items(): key_name = str(k) key_value = str(v) memcache_metric_dict[key_name] = key_value memcache_client.set(metrics_db_object_key, memcache_metric_dict, expire=3600) logger.info('set the memcache key - %s' % metrics_db_object_key) except: logger.error(traceback.format_exc()) logger.error('error :: failed to set %s in memcache' % metrics_db_object_key) try: memcache_client.close() except: # @modified 20170913 - Task #2160: Test skyline with bandit # pass logger.error('error :: failed to close memcache_client') return metrics_db_object
def create_features_profile(current_skyline_app, requested_timestamp, data_for_metric, context, ionosphere_job, fp_parent_id, fp_generation, fp_learn): """ Add a features_profile to the Skyline ionosphere database table. :param current_skyline_app: Skyline app name :param requested_timestamp: The timestamp of the dir that the features profile data is in :param data_for_metric: The base_name of the metric :param context: The context of the caller :param ionosphere_job: The ionosphere_job name related to creation request valid jobs are ``learn_fp_human``, ``learn_fp_generation``, ``learn_fp_learnt`` and ``learn_fp_automatic``. :param fp_parent_id: The id of the parent features profile that this was learnt from, 0 being an original human generated features profile :param fp_generation: The number of generations away for the original human generated features profile, 0 being an original human generated features profile. :param fp_learn: Whether Ionosphere should learn at use_full_duration_days :type current_skyline_app: str :type requested_timestamp: int :type data_for_metric: str :type context: str :type ionosphere_job: str :type fp_parent_id: int :type fp_generation: int :type fp_learn: boolean :return: fp_id, fp_in_successful, fp_exists, fail_msg, traceback_format_exc :rtype: str, boolean, boolean, str, str """ current_skyline_app_logger = current_skyline_app + 'Log' current_logger = logging.getLogger(current_skyline_app_logger) base_name = data_for_metric.replace(settings.FULL_NAMESPACE, '', 1) if context == 'training_data': log_context = 'training data' ionosphere_learn_job = 'learn_fp_human' if context == 'features_profiles': log_context = 'features profile data' # @added 20170113 - Feature #1854: Ionosphere learn if context == 'ionosphere_learn': log_context = 'learn' current_logger.info('create_features_profile :: %s :: requested for %s at %s' % ( context, str(base_name), str(requested_timestamp))) metric_timeseries_dir = base_name.replace('.', '/') if context == 'training_data': metric_training_data_dir = '%s/%s/%s' % ( settings.IONOSPHERE_DATA_FOLDER, str(requested_timestamp), metric_timeseries_dir) if context == 'features_profiles': metric_training_data_dir = '%s/%s/%s' % ( settings.IONOSPHERE_PROFILES_FOLDER, metric_timeseries_dir, str(requested_timestamp)) # @added 20170113 - Feature #1854: Ionosphere learn if context == 'ionosphere_learn': # @modified 20170116 - Feature #1854: Ionosphere learn # Allowing ionosphere_learn to create a features profile for a training # data set that it has learnt is not anomalous if ionosphere_job != 'learn_fp_automatic': metric_training_data_dir = '%s/%s/%s' % ( settings.IONOSPHERE_LEARN_FOLDER, str(requested_timestamp), metric_timeseries_dir) else: metric_training_data_dir = '%s/%s/%s' % ( settings.IONOSPHERE_DATA_FOLDER, str(requested_timestamp), metric_timeseries_dir) features_file = '%s/%s.tsfresh.input.csv.features.transposed.csv' % ( metric_training_data_dir, base_name) features_profile_dir = '%s/%s' % ( settings.IONOSPHERE_PROFILES_FOLDER, metric_timeseries_dir) ts_features_profile_dir = '%s/%s/%s' % ( settings.IONOSPHERE_PROFILES_FOLDER, metric_timeseries_dir, str(requested_timestamp)) features_profile_created_file = '%s/%s.%s.fp.created.txt' % ( metric_training_data_dir, str(requested_timestamp), base_name) features_profile_details_file = '%s/%s.%s.fp.details.txt' % ( metric_training_data_dir, str(requested_timestamp), base_name) anomaly_check_file = '%s/%s.txt' % (metric_training_data_dir, base_name) trace = 'none' fail_msg = 'none' new_fp_id = False calculated_with_tsfresh = False calculated_time = False fcount = None fsum = None # @added 20170104 - Feature #1842: Ionosphere - Graphite now graphs # Added the ts_full_duration parameter so that the appropriate graphs can be # embedded for the user in the training data page ts_full_duration = '0' if context == 'ionosphere_learn': if not path.isfile(features_profile_details_file): current_logger.error('error :: create_features_profile :: no features_profile_details_file - %s' % features_profile_details_file) return 'none', False, False, fail_msg, trace if path.isfile(features_profile_details_file): current_logger.info('create_features_profile :: getting features profile details from from - %s' % features_profile_details_file) # Read the details file with open(features_profile_details_file, 'r') as f: fp_details_str = f.read() fp_details = literal_eval(fp_details_str) calculated_with_tsfresh = fp_details[1] calculated_time = str(fp_details[2]) fcount = str(fp_details[3]) fsum = str(fp_details[4]) try: ts_full_duration = str(fp_details[5]) except: current_logger.error('error :: create_features_profile :: could not determine the full duration from - %s' % features_profile_details_file) ts_full_duration = '0' if context != 'ionosphere_learn': if ts_full_duration == '0': if path.isfile(anomaly_check_file): current_logger.info('create_features_profile :: determining the full duration from anomaly_check_file - %s' % anomaly_check_file) # Read the details file with open(anomaly_check_file, 'r') as f: anomaly_details = f.readlines() for i, line in enumerate(anomaly_details): if 'full_duration' in line: _ts_full_duration = '%s' % str(line).split("'", 2) full_duration_array = literal_eval(_ts_full_duration) ts_full_duration = str(int(full_duration_array[1])) current_logger.info('create_features_profile :: determined the full duration as - %s' % str(ts_full_duration)) if path.isfile(features_profile_created_file): # Read the created file with open(features_profile_created_file, 'r') as f: fp_created_str = f.read() fp_created = literal_eval(fp_created_str) new_fp_id = fp_created[0] return str(new_fp_id), True, True, fail_msg, trace # Have data if path.isfile(features_file): current_logger.info('create_features_profile :: features_file exists: %s' % features_file) else: trace = traceback.format_exc() current_logger.error(trace) fail_msg = 'error :: create_features_profile :: features_file does not exist: %s' % features_file current_logger.error('%s' % fail_msg) if context == 'training' or context == 'features_profile': # Raise to webbapp I believe to provide traceback to user in UI raise else: return False, False, False, fail_msg, trace features_data = [] with open(features_file, 'rb') as fr: reader = csv.reader(fr, delimiter=',') for i, line in enumerate(reader): feature_name_item = False fname_id = False f_value = False feature_name = str(line[0]) feature_name_item = filter( lambda x: x[1] == feature_name, TSFRESH_FEATURES) if feature_name_item: feature_name_id = feature_name_item[0] if feature_name_item: feature_name_list = feature_name_item[0] fname_id = int(feature_name_list[0]) f_value = str(line[1]) if fname_id and f_value: features_data.append([fname_id, f_value]) # @added 20170113 - Feature #1854: Ionosphere learn - generations # Set the learn generations variables with the IONOSPHERE_LEARN_DEFAULT_ and any # settings.IONOSPHERE_LEARN_NAMESPACE_CONFIG values. These will later be # overridden by any database values determined for the specific metric if # they exist. # Set defaults use_full_duration_days = int(settings.IONOSPHERE_LEARN_DEFAULT_FULL_DURATION_DAYS) valid_learning_duration = int(settings.IONOSPHERE_LEARN_DEFAULT_VALID_TIMESERIES_OLDER_THAN_SECONDS) max_generations = int(settings.IONOSPHERE_LEARN_DEFAULT_MAX_GENERATIONS) max_percent_diff_from_origin = float(settings.IONOSPHERE_LEARN_DEFAULT_MAX_PERCENT_DIFF_FROM_ORIGIN) try: use_full_duration, valid_learning_duration, use_full_duration_days, max_generations, max_percent_diff_from_origin = get_ionosphere_learn_details(current_skyline_app, base_name) learn_full_duration_days = use_full_duration_days except: current_logger.error(traceback.format_exc()) current_logger.error('error :: create_features_profile :: failed to get_ionosphere_learn_details') current_logger.info('create_features_profile :: learn_full_duration_days :: %s days' % (str(learn_full_duration_days))) current_logger.info('create_features_profile :: valid_learning_duration :: %s seconds' % (str(valid_learning_duration))) current_logger.info('create_features_profile :: max_generations :: %s' % (str(max_generations))) current_logger.info('create_features_profile :: max_percent_diff_from_origin :: %s' % (str(max_percent_diff_from_origin))) current_logger.info('create_features_profile :: getting MySQL engine') try: engine, fail_msg, trace = fp_create_get_an_engine(current_skyline_app) current_logger.info(fail_msg) except: trace = traceback.format_exc() current_logger.error(trace) fail_msg = 'error :: create_features_profile :: could not get a MySQL engine' current_logger.error('%s' % fail_msg) if context == 'training' or context == 'features_profile': # Raise to webbapp I believe to provide traceback to user in UI raise else: return False, False, False, fail_msg, trace if not engine: trace = 'none' fail_msg = 'error :: create_features_profile :: engine not obtained' current_logger.error(fail_msg) if context == 'training' or context == 'features_profile': # Raise to webbapp I believe to provide traceback to user in UI raise else: return False, False, False, fail_msg, trace # Get metric details from the database metrics_id = False # Use the learn details as per config metric_learn_full_duration_days = int(use_full_duration_days) metric_learn_valid_ts_older_than = int(valid_learning_duration) metric_max_generations = int(max_generations) metric_max_percent_diff_from_origin = int(max_percent_diff_from_origin) metrics_table = None try: metrics_table, fail_msg, trace = metrics_table_meta(current_skyline_app, engine) current_logger.info(fail_msg) except: trace = traceback.format_exc() current_logger.error('%s' % trace) fail_msg = 'error :: create_features_profile :: failed to get metrics_table meta for %s' % base_name current_logger.error('%s' % fail_msg) if context == 'training' or context == 'features_profile': # @added 20170806 - Bug #2130: MySQL - Aborted_clients # Added missing disposal if engine: fp_create_engine_disposal(current_skyline_app, engine) # Raise to webbapp I believe to provide traceback to user in UI raise else: current_logger.info('create_features_profile :: disposing of any engine') fp_create_engine_disposal(current_skyline_app, engine) return False, False, False, fail_msg, trace current_logger.info('create_features_profile :: metrics_table OK') metric_db_object = None try: connection = engine.connect() # @modified 20161209 - - Branch #922: ionosphere # Task #1658: Patterning Skyline Ionosphere # result = connection.execute('select id from metrics where metric=\'%s\'' % base_name) # for row in result: # while not metrics_id: # metrics_id = row['id'] stmt = select([metrics_table]).where(metrics_table.c.metric == base_name) result = connection.execute(stmt) for row in result: metrics_id = row['id'] # @added 20170113 - Feature #1854: Ionosphere learn - generations # Added Ionosphere LEARN generation related variables try: metric_learn_full_duration_days = int(row['learn_full_duration_days']) metric_learn_valid_ts_older_than = int(row['learn_valid_ts_older_than']) metric_max_generations = int(row['max_generations']) metric_max_percent_diff_from_origin = float(row['max_percent_diff_from_origin']) except: current_logger.error('error :: create_features_profile :: failed to determine learn related values from DB for %s' % base_name) row = result.fetchone() # metric_db_object = row connection.close() current_logger.info('create_features_profile :: determined db metric id: %s' % str(metrics_id)) current_logger.info('create_features_profile :: determined db metric learn_full_duration_days: %s' % str(metric_learn_full_duration_days)) current_logger.info('create_features_profile :: determined db metric learn_valid_ts_older_than: %s' % str(metric_learn_valid_ts_older_than)) current_logger.info('create_features_profile :: determined db metric max_generations: %s' % str(metric_max_generations)) current_logger.info('create_features_profile :: determined db metric max_percent_diff_from_origin: %s' % str(metric_max_percent_diff_from_origin)) except: trace = traceback.format_exc() current_logger.error(trace) fail_msg = 'error :: create_features_profile :: could not determine id of metric from DB: %s' % base_name current_logger.error('%s' % fail_msg) if metric_learn_full_duration_days: learn_full_duration_days = metric_learn_full_duration_days # learn_full_duration = int(learn_full_duration_days) * 86400 if metric_learn_valid_ts_older_than: learn_valid_ts_older_than = metric_learn_valid_ts_older_than if metric_max_generations: max_generations = metric_max_generations if metric_max_percent_diff_from_origin: max_percent_diff_from_origin = metric_max_percent_diff_from_origin current_logger.info('create_features_profile :: generation info - learn_full_duration_days :: %s' % (str(learn_full_duration_days))) current_logger.info('create_features_profile :: generation info - learn_valid_ts_older_than :: %s' % (str(learn_valid_ts_older_than))) current_logger.info('create_features_profile :: generation info - max_generations :: %s' % (str(max_generations))) current_logger.info('create_features_profile :: generation info - max_percent_diff_from_origin :: %s' % (str(max_percent_diff_from_origin))) # @added 20170120 - Feature #1854: Ionosphere learn # Always use the timestamp from the anomaly file use_anomaly_timestamp = int(requested_timestamp) if context == 'ionosphere_learn': if path.isfile(anomaly_check_file): current_logger.info('create_features_profile :: determining the full duration from anomaly_check_file - %s' % anomaly_check_file) # Read the details file with open(anomaly_check_file, 'r') as f: anomaly_details = f.readlines() for i, line in enumerate(anomaly_details): if 'metric_timestamp' in line: _metric_timestamp = '%s' % str(line).split("'", 2) metric_timestamp_array = literal_eval(_metric_timestamp) use_anomaly_timestamp = (int(metric_timestamp_array[1])) current_logger.info('create_features_profile :: determined the anomaly metric_timestamp as - %s' % str(use_anomaly_timestamp)) ionosphere_table = None try: ionosphere_table, fail_msg, trace = ionosphere_table_meta(current_skyline_app, engine) current_logger.info(fail_msg) except: trace = traceback.format_exc() current_logger.error('%s' % trace) fail_msg = 'error :: create_features_profile :: failed to get ionosphere_table meta for %s' % base_name current_logger.error('%s' % fail_msg) if context == 'training' or context == 'features_profile': # Raise to webbapp I believe to provide traceback to user in UI # @added 20170806 - Bug #2130: MySQL - Aborted_clients # Added missing disposal if engine: fp_create_engine_disposal(current_skyline_app, engine) raise else: current_logger.info('create_features_profile :: disposing of any engine') fp_create_engine_disposal(current_skyline_app, engine) return False, False, False, fail_msg, trace current_logger.info('create_features_profile :: ionosphere_table OK') # @added 20170403 - Feature #2000: Ionosphere - validated # Set all learn_fp_human features profiles to validated. fp_validated = 0 if ionosphere_job == 'learn_fp_human': fp_validated = 1 # @added 20170424 - Feature #2000: Ionosphere - validated # Set all generation 0 and 1 as validated if int(fp_generation) <= 1: fp_validated = 1 new_fp_id = False try: connection = engine.connect() # @added 20170113 - Feature #1854: Ionosphere learn # Added learn values parent_id, generation # @modified 20170120 - Feature #1854: Ionosphere learn # Added anomaly_timestamp # @modified 20170403 - Feature #2000: Ionosphere - validated ins = ionosphere_table.insert().values( metric_id=int(metrics_id), full_duration=int(ts_full_duration), anomaly_timestamp=int(use_anomaly_timestamp), enabled=1, tsfresh_version=str(tsfresh_version), calc_time=calculated_time, features_count=fcount, features_sum=fsum, parent_id=fp_parent_id, generation=fp_generation, validated=fp_validated) result = connection.execute(ins) connection.close() new_fp_id = result.inserted_primary_key[0] current_logger.info('create_features_profile :: new ionosphere fp_id: %s' % str(new_fp_id)) except: trace = traceback.format_exc() current_logger.error('%s' % trace) fail_msg = 'error :: create_features_profile :: failed to insert a new record into the ionosphere table for %s' % base_name current_logger.error('%s' % fail_msg) if context == 'training' or context == 'features_profile': # @added 20170806 - Bug #2130: MySQL - Aborted_clients # Added missing disposal if engine: fp_create_engine_disposal(current_skyline_app, engine) # Raise to webbapp I believe to provide traceback to user in UI raise else: current_logger.info('create_features_profile :: disposing of any engine') fp_create_engine_disposal(current_skyline_app, engine) return False, False, False, fail_msg, trace if not RepresentsInt(new_fp_id): trace = traceback.format_exc() current_logger.error('%s' % trace) fail_msg = 'error :: create_features_profile :: unknown new ionosphere new_fp_id for %s' % base_name current_logger.error('%s' % fail_msg) if context == 'training' or context == 'features_profile': # @added 20170806 - Bug #2130: MySQL - Aborted_clients # Added missing disposal if engine: fp_create_engine_disposal(current_skyline_app, engine) # Raise to webbapp I believe to provide traceback to user in UI raise else: current_logger.info('create_features_profile :: disposing of any engine') fp_create_engine_disposal(current_skyline_app, engine) return False, False, False, fail_msg, trace # Create z_fp_<metric_id> table fp_table_created = False fp_table_name = 'z_fp_%s' % str(metrics_id) try: fp_meta = MetaData() # @modified 20161222 - Task #1812: z_fp table type # Changed to InnoDB from MyISAM as no files open issues and MyISAM clean # up, there can be LOTS of file_per_table z_fp_ tables/files without # the MyISAM issues. z_fp_ tables are mostly read and will be shuffled # in the table cache as required. fp_metric_table = Table( fp_table_name, fp_meta, Column('id', Integer, primary_key=True), Column('fp_id', Integer, nullable=False, key='fp_id'), Column('feature_id', Integer, nullable=False), Column('value', DOUBLE(), nullable=True), mysql_charset='utf8', mysql_key_block_size='255', mysql_engine='InnoDB') fp_metric_table.create(engine, checkfirst=True) fp_table_created = True except: trace = traceback.format_exc() current_logger.error('%s' % trace) fail_msg = 'error :: create_features_profile :: failed to create table - %s' % fp_table_name current_logger.error('%s' % fail_msg) if context == 'training' or context == 'features_profile': # @added 20170806 - Bug #2130: MySQL - Aborted_clients # Added missing disposal if engine: fp_create_engine_disposal(current_skyline_app, engine) # Raise to webbapp I believe to provide traceback to user in UI raise else: current_logger.info('create_features_profile :: %s - automated so the table should exists continuing' % context) if not fp_table_created: trace = traceback.format_exc() current_logger.error('%s' % trace) fail_msg = 'error :: create_features_profile :: failed to determine True for create table - %s' % fp_table_name current_logger.error('%s' % fail_msg) if context == 'training' or context == 'features_profile': # @added 20170806 - Bug #2130: MySQL - Aborted_clients # Added missing disposal if engine: fp_create_engine_disposal(current_skyline_app, engine) # Raise to webbapp I believe to provide traceback to user in UI raise else: current_logger.info('create_features_profile :: %s - automated so the table should exists continuing' % context) # Insert features and values insert_statement = [] for fname_id, f_value in features_data: insert_statement.append({'fp_id': new_fp_id, 'feature_id': fname_id, 'value': f_value},) if insert_statement == []: trace = traceback.format_exc() current_logger.error('%s' % trace) fail_msg = 'error :: create_features_profile :: empty insert_statement for %s inserts' % fp_table_name current_logger.error('%s' % fail_msg) # raise # else: # feature_count = sum(1 for x in a if isinstance(x, insert_statement)) # current_logger.info( # 'fp_id - %s - %s feature values in insert_statement for %s ' % # (str(feature_count), str(new_fp_id), fp_table_name)) # feature_count = sum(1 for x in a if isinstance(x, insert_statement)) # current_logger.info( # 'fp_id - %s - feature values in insert_statement for %s ' % # (str(new_fp_id), fp_table_name)) try: connection = engine.connect() connection.execute(fp_metric_table.insert(), insert_statement) connection.close() current_logger.info('create_features_profile :: fp_id - %s - feature values inserted into %s' % (str(new_fp_id), fp_table_name)) except: trace = traceback.format_exc() current_logger.error('%s' % trace) fail_msg = 'error :: create_features_profile :: failed to insert a feature values into %s' % fp_table_name current_logger.error('%s' % fail_msg) if context == 'training' or context == 'features_profile': # @added 20170806 - Bug #2130: MySQL - Aborted_clients # Added missing disposal if engine: fp_create_engine_disposal(current_skyline_app, engine) # Raise to webbapp I believe to provide traceback to user in UI raise else: current_logger.info('create_features_profile :: %s - automated so the table should exists continuing' % context) # Create metric ts table if not exists ts_<metric_id> # Create z_ts_<metric_id> table # @modified 20170121 - Feature #1854: Ionosphere learn - generations # TODO Adding the option to not save timeseries to DB, as default? # ts_table_created = False ts_table_name = 'z_ts_%s' % str(metrics_id) try: ts_meta = MetaData() # @modified 20161222 - Task #1812: z_fp table type # Changed to InnoDB from MyISAM as no files open issues and MyISAM clean # up, there can be LOTS of file_per_table z_fp_ tables/files without # the MyISAM issues. z_fp_ tables are mostly read and will be shuffled # in the table cache as required. ts_metric_table = Table( ts_table_name, ts_meta, Column('id', Integer, primary_key=True), Column('fp_id', Integer, nullable=False, key='fp_id'), Column('timestamp', Integer, nullable=False), Column('value', DOUBLE(), nullable=True), mysql_charset='utf8', mysql_key_block_size='255', mysql_engine='InnoDB') ts_metric_table.create(engine, checkfirst=True) # ts_table_created = True current_logger.info('create_features_profile :: metric ts table created OK - %s' % (ts_table_name)) except: trace = traceback.format_exc() current_logger.error('%s' % trace) fail_msg = 'error :: create_features_profile :: failed to create table - %s' % ts_table_name current_logger.error('%s' % fail_msg) if context == 'training' or context == 'features_profile': # @added 20170806 - Bug #2130: MySQL - Aborted_clients # Added missing disposal if engine: fp_create_engine_disposal(current_skyline_app, engine) # Raise to webbapp I believe to provide traceback to user in UI raise else: current_logger.info('create_features_profile :: %s - automated so the table should exists continuing' % context) # Insert timeseries that the features profile was created from raw_timeseries = [] anomaly_json = '%s/%s.json' % (metric_training_data_dir, base_name) if path.isfile(anomaly_json): current_logger.info('create_features_profile :: metric anomaly json found OK - %s' % (anomaly_json)) try: # Read the timeseries json file with open(anomaly_json, 'r') as f: raw_timeseries = f.read() except: trace = traceback.format_exc() current_logger.error(trace) fail_msg = 'error :: create_features_profile :: failed to read timeseries data from %s' % anomaly_json current_logger.error('%s' % (fail_msg)) fail_msg = 'error: failed to read timeseries data from %s' % anomaly_json # end = timer() if context == 'training' or context == 'features_profile': # @added 20170806 - Bug #2130: MySQL - Aborted_clients # Added missing disposal if engine: fp_create_engine_disposal(current_skyline_app, engine) # Raise to webbapp I believe to provide traceback to user in UI raise else: trace = 'none' fail_msg = 'error: file not found - %s' % (anomaly_json) current_logger.error(fail_msg) # raise # Convert the timeseries to csv timeseries_array_str = str(raw_timeseries).replace('(', '[').replace(')', ']') timeseries = literal_eval(timeseries_array_str) datapoints = timeseries validated_timeseries = [] for datapoint in datapoints: try: new_datapoint = [str(int(datapoint[0])), float(datapoint[1])] validated_timeseries.append(new_datapoint) # @modified 20170913 - Task #2160: Test skyline with bandit # Added nosec to exclude from bandit tests except: # nosec continue insert_statement = [] for ts, value in validated_timeseries: insert_statement.append({'fp_id': new_fp_id, 'timestamp': ts, 'value': value},) try: connection = engine.connect() connection.execute(ts_metric_table.insert(), insert_statement) connection.close() current_logger.info('create_features_profile :: fp_id - %s - timeseries inserted into %s' % (str(new_fp_id), ts_table_name)) except: trace = traceback.format_exc() current_logger.error('%s' % trace) fail_msg = 'error :: create_features_profile :: failed to insert the timeseries into %s' % ts_table_name current_logger.error('%s' % fail_msg) if context == 'training' or context == 'features_profile': # @added 20170806 - Bug #2130: MySQL - Aborted_clients # Added missing disposal if engine: fp_create_engine_disposal(current_skyline_app, engine) raise else: current_logger.info('create_features_profile :: %s - automated so the table should exists continuing' % context) # Create a created features profile file try: # data = '[%s, %s, ]' % (new_fp_id, str(int(time.time()))) # write_data_to_file(skyline_app, features_profile_created_file, 'w', data) # @modified 20170115 - Feature #1854: Ionosphere learn - generations # Added parent_id and generation data = '[%s, %s, \'%s\', %s, %s, %s, %s, %s, %s]' % ( new_fp_id, str(int(time.time())), str(tsfresh_version), str(calculated_time), str(fcount), str(fsum), str(ts_full_duration), str(fp_parent_id), str(fp_generation)) write_data_to_file(current_skyline_app, features_profile_created_file, 'w', data) except: trace = traceback.format_exc() current_logger.error('%s' % trace) fail_msg = 'error :: create_features_profile :: failed to write fp.created file' current_logger.error('%s' % fail_msg) # Set ionosphere_enabled for the metric try: # update_statement = 'UPDATE metrics SET ionosphere_enabled=1 WHERE id=%s' % str(metrics_id) connection = engine.connect() # result = connection.execute('UPDATE metrics SET ionosphere_enabled=1 WHERE id=%s' % str(metrics_id)) # connection.execute(ts_metric_table.insert(), insert_statement) connection.execute( metrics_table.update( metrics_table.c.id == metrics_id).values(ionosphere_enabled=1)) connection.close() current_logger.info('create_features_profile :: ionosphere_enabled set on metric id: %s' % str(metrics_id)) except: trace = traceback.format_exc() current_logger.error(trace) fail_msg = 'error :: create_features_profile :: could not update metrics table and set ionosphere_enabled on id %s' % str(metrics_id) current_logger.error('%s' % fail_msg) # raise # Copy data from training data dir to features_profiles dir if not path.isdir(ts_features_profile_dir): mkdir_p(ts_features_profile_dir) if path.isdir(ts_features_profile_dir): current_logger.info('create_features_profile :: fp_id - %s - features profile dir created - %s' % (str(new_fp_id), ts_features_profile_dir)) # src_files = os.listdir(src) # for file_name in src_files: # full_file_name = path.join(src, file_name) # if (path.isfile(full_file_name)): # shutil.copy(full_file_name, dest) data_files = [] try: glob_path = '%s/*.*' % metric_training_data_dir data_files = glob.glob(glob_path) except: trace = traceback.format_exc() current_logger.error('%s' % trace) current_logger.error('error :: create_features_profile :: glob - fp_id - %s - training data not copied to %s' % (str(new_fp_id), ts_features_profile_dir)) for i_file in data_files: try: shutil.copy(i_file, ts_features_profile_dir) current_logger.info('create_features_profile :: fp_id - %s - training data copied - %s' % (str(new_fp_id), i_file)) except shutil.Error as e: trace = traceback.format_exc() current_logger.error('%s' % trace) current_logger.error('error :: create_features_profile :: shutil error - fp_id - %s - training data not copied to %s' % (str(new_fp_id), ts_features_profile_dir)) current_logger.error('error :: create_features_profile :: %s' % (e)) # Any error saying that the directory doesn't exist except OSError as e: trace = traceback.format_exc() current_logger.error('%s' % trace) current_logger.error('error :: create_features_profile :: OSError error - fp_id - %s - training data not copied to %s' % (str(new_fp_id), ts_features_profile_dir)) current_logger.error('error :: create_features_profile :: %s' % (e)) current_logger.info('create_features_profile :: fp_id - %s - training data copied to %s' % (str(new_fp_id), ts_features_profile_dir)) else: current_logger.error('error :: create_features_profile :: fp_id - %s - training data not copied to %s' % (str(new_fp_id), ts_features_profile_dir)) current_logger.info('create_features_profile :: disposing of any engine') try: if engine: fp_create_engine_disposal(current_skyline_app, engine) else: current_logger.info('create_features_profile :: no engine to dispose of' % (str(new_fp_id), ts_features_profile_dir)) except: trace = traceback.format_exc() current_logger.error('%s' % trace) current_logger.error('error :: create_features_profile :: OSError error - fp_id - %s - training data not copied to %s' % (str(new_fp_id), ts_features_profile_dir)) # @added 20170113 - Feature #1854: Ionosphere learn - Redis ionosphere.learn.work namespace # Ionosphere learn needs Redis works sets # When a features profile is created there needs to be work added to a Redis # set. When a human makes a features profile, we want Ionosphere to make a # use_full_duration_days features profile valid_learning_duration (e.g. # 3361) later. if settings.IONOSPHERE_LEARN and new_fp_id: create_redis_work_item = False if context == 'training_data' and ionosphere_job == 'learn_fp_human': create_redis_work_item = True # @modified 20170120 - Feature #1854: Ionosphere learn - generations # Added fp_learn parameter to allow the user to not learn the # use_full_duration_days if not fp_learn: create_redis_work_item = False current_logger.info('fp_learn is False not adding an item to Redis ionosphere.learn.work set') if ionosphere_job == 'learn_fp_automatic': create_redis_work_item = True # @added 20170131 - Feature #1886 Ionosphere learn - child like parent with evolutionary maturity # TODO: here a check may be required to evaluate whether the origin_fp_id # had a use_full_duration features profile created, however # due to the fact that it is in learn, suggests that it did # have, not 100% sure. origin_fp_id_was_allowed_to_learn = False child_use_full_duration_count_of_origin_fp_id = 1 # TODO: Determine the state # child_use_full_duration_count_of_origin_fp_id = SELECT COUNT(id) FROM ionosphere WHERE parent_id=origin_fp_id AND full_duration=use_full_duration if child_use_full_duration_count_of_origin_fp_id == 0: current_logger.info('the origin parent was not allowed to learn not adding to Redis ionosphere.learn.work set') create_redis_work_item = False if create_redis_work_item: try: current_logger.info( 'adding work to Redis ionosphere.learn.work set - [\'Soft\', \'%s\', %s, \'%s\', %s, %s] to make a learn features profile later' % ( str(ionosphere_job), str(requested_timestamp), base_name, str(new_fp_id), str(fp_generation))) redis_conn = StrictRedis(unix_socket_path=settings.REDIS_SOCKET_PATH) redis_conn.sadd('ionosphere.learn.work', ['Soft', str(ionosphere_job), int(requested_timestamp), base_name, int(new_fp_id), int(fp_generation)]) except: current_logger.error(traceback.format_exc()) current_logger.error( 'error :: failed adding work to Redis ionosphere.learn.work set - [\'Soft\', \'%s\', %s, \'%s\', %s, %s] to make a learn features profile later' % ( str(ionosphere_job), str(requested_timestamp), base_name, str(new_fp_id), str(fp_generation))) # @added 20170806 - Bug #2130: MySQL - Aborted_clients # Added missing disposal if engine: fp_create_engine_disposal(current_skyline_app, engine) return str(new_fp_id), True, False, fail_msg, trace
def submit_crucible_job(from_timestamp, until_timestamp, metrics_list, namespaces_list, source, alert_interval, user_id, user, add_to_panorama, pad_timeseries, training_data_json, run_algorithms): """ Get a list of all the metrics passed and generate Crucible check files for each :param from_timestamp: the timestamp at which to start the time series :param until_timestamp: the timestamp at which to end the time series :param metrics_list: a list of metric names to analyse :param namespaces_list: a list of metric namespaces to analyse :param source: the source webapp making the request :param alert_interval: the alert_interval at which Crucible should trigger anomalies :param user_id: the user id of the user making the request :param user: the username making the request :param add_to_panorama: whether Crucible should add Skyline CONSENSUS anomalies to Panorama :param pad_timeseries: the amount of data to pad the time series with :param training_data_json: the full path to the training_data json file if source is training_data :param run_algorithms: list of algorithms to run :type from_timestamp: int :type until_timestamp: int :type metrics_list: list :type namespaces_list: list :type source: str :type alert_interval: int :type user_id: int :type user: str :type add_to_panorama: boolean :type pad_timeseries: str :type training_data_json: str :type run_algorithms: list :return: tuple of lists :rtype: (list, list, list, list) Returns (crucible_job_id, metrics_submitted_to_process, fail_msg, trace) """ fail_msg = None trace = None crucible_job_id = None metrics_submitted_to_process = 0 # Generate a job id based on the YMDHMS.user_id and a job directory try: jobid_timestamp = int(time()) jobid_datetimestamp = dt.datetime.fromtimestamp( jobid_timestamp).strftime('%Y%m%d%H%M%S') crucible_job_id = '%s.%s' % (str(jobid_datetimestamp), str(user_id)) except: logger.error(traceback.format_exc()) logger.error('error :: failed to determine a crucible_job_id') raise # to webapp to return in the UI # Generate a job id based on the YMDHMS.user_id and a job directory try: crucible_path = os.path.dirname(settings.CRUCIBLE_DATA_FOLDER) crucible_job_dir = '%s/jobs/%s' % (crucible_path, crucible_job_id) if not path.exists(crucible_job_dir): logger.info('creating crucible job directory - %s' % (str(crucible_job_dir))) mkdir_p(crucible_job_dir) except: trace = traceback.format_exc() fail_msg = 'error :: failed to create the crucible job directory' logger.error(trace) logger.error(fail_msg) raise # to webapp to return in the UI # TODO added checks of metric names metric_names = [] if metrics_list: logger.info('submit_crucible_job :: %s metrics passed' % str(len(metrics_list))) for metric in metrics_list: metric_names.append(metric) # TODO added checks of metric namespaces, harder to do, but so that the UI # errors to the usr rather than sending a bad or non-existent metric to # Crucible if namespaces_list: logger.info('submit_crucible_job :: %s namespaces passed' % str(len(namespaces_list))) logger.info( 'submit_crucible_job :: determine metrics for submit_crucible_job between %s and %s' % (str(from_timestamp), str(until_timestamp))) logger.info('getting MySQL engine') try: engine, fail_msg, trace = get_an_engine() logger.info(fail_msg) except: trace = traceback.format_exc() logger.error(trace) logger.error('%s' % fail_msg) logger.error( 'error :: could not get a MySQL engine to get metric names') raise # to webapp to return in the UI if not engine: trace = 'none' fail_msg = 'error :: engine not obtained' logger.error(fail_msg) raise try: metrics_table, log_msg, trace = metrics_table_meta( skyline_app, engine) logger.info(log_msg) logger.info('metrics_table OK') except: logger.error(traceback.format_exc()) logger.error('error :: failed to get metrics_table meta') if engine: engine_disposal(engine) raise # to webapp to return in the UI metrics_like_query = text( """SELECT metric FROM metrics WHERE metric LIKE :like_string""") for namespace in namespaces_list: try: connection = engine.connect() results = connection.execute(metrics_like_query, like_string=str(namespace)) connection.close() for row in results: metric_name = str(row[0]) metric_names.append(metric_name) except: trace = traceback.format_exc() logger.error(trace) logger.error( 'error :: could not determine metrics from metrics table') if engine: engine_disposal(engine) raise logger.info( 'submit_crucible_job :: %s metrics determined from passed namespaces' % str(len(metric_names))) logger.info('submit_crucible_job :: %s metrics to process' % str(len(metric_names))) metrics_submitted_to_process = [] datapoint = 0 triggered_algorithms = [ 'histogram_bins', 'first_hour_average', 'stddev_from_average', 'grubbs', 'ks_test', 'mean_subtraction_cumulation', 'median_absolute_deviation', 'stddev_from_moving_average', 'least_squares' ] added_at = int(time()) for base_name in metric_names: sane_metricname = filesafe_metricname(str(base_name)) derivative_metric = is_derivative_metric(skyline_app, base_name) if derivative_metric: target = 'nonNegativeDerivative(%s)' % base_name else: target = base_name # Generate a metric job directory crucible_anomaly_dir = '%s/%s' % (crucible_job_dir, sane_metricname) try: if not path.exists(crucible_anomaly_dir): logger.info('creating crucible metric job directory - %s' % (str(crucible_anomaly_dir))) mkdir_p(crucible_anomaly_dir) except: trace = traceback.format_exc() fail_msg = 'error :: failed to create the crucible metric job directory' logger.error(trace) logger.error(fail_msg) raise # to webapp to return in the UI if source == 'graphite': graphite_metric = True else: graphite_metric = False # @added 20200422 - Feature #3500: webapp - crucible_process_metrics # Feature #1448: Crucible web UI # In order for metrics to be analysed in Crucible like the Analyzer or # Mirage analysis, the time series data needs to be padded # Added pad_timeseries graphite_override_uri_parameters = 'from=%s&until=%s&target=%s' % ( str(from_timestamp), str(until_timestamp), target) timeseries_full_duration = int(until_timestamp) - int(from_timestamp) pad_timeseries_with = 0 if pad_timeseries == 'auto': if timeseries_full_duration > 3600: pad_timeseries_with = 3600 if timeseries_full_duration > 86400: pad_timeseries_with = 86400 if pad_timeseries == '86400': pad_timeseries_with = 86400 if pad_timeseries == '604800': pad_timeseries_with = 604800 if pad_timeseries == '0': pad_timeseries_with = 0 if pad_timeseries_with: try: padded_from_timestamp = int( from_timestamp) - pad_timeseries_with graphite_override_uri_parameters = 'from=%s&until=%s&target=%s' % ( str(padded_from_timestamp), str(until_timestamp), target) logger.info('padding time series with %s seconds - %s' % (str(pad_timeseries_with), str(graphite_override_uri_parameters))) except: logger.error(traceback.format_exc()) logger.error( 'error :: failed to construct graphite_override_uri_parameters with pad_timeseries_with %s' % str(pad_timeseries_with)) # @added 20200817 - Feature #3682: SNAB - webapp - crucible_process - run_algorithms # Allow the user to pass algorithms to run algorithms = settings.ALGORITHMS if run_algorithms: algorithms = run_algorithms # @modified 20200421 - Feature #3500: webapp - crucible_process_metrics # Feature #1448: Crucible web UI # Added add_to_panorama # @added 20200607 - Feature #3630: webapp - crucible_process_training_data # Added training_data_json crucible_anomaly_data = 'metric = \'%s\'\n' \ 'value = \'%s\'\n' \ 'from_timestamp = \'%s\'\n' \ 'metric_timestamp = \'%s\'\n' \ 'algorithms = %s\n' \ 'triggered_algorithms = %s\n' \ 'anomaly_dir = \'%s\'\n' \ 'graphite_metric = %s\n' \ 'run_crucible_tests = True\n' \ 'added_by = \'%s\'\n' \ 'added_at = \'%s\'\n' \ 'graphite_override_uri_parameters = \'%s\'\n' \ 'alert_interval = \'%s\'\n' \ 'add_to_panorama = %s\n' \ 'training_data_json = %s\n' \ % (base_name, str(datapoint), str(from_timestamp), # @modified 20200817 - Feature #3682: SNAB - webapp - crucible_process - run_algorithms # str(until_timestamp), str(settings.ALGORITHMS), str(until_timestamp), str(algorithms), triggered_algorithms, crucible_anomaly_dir, str(graphite_metric), skyline_app, str(added_at), str(graphite_override_uri_parameters), str(alert_interval), str(add_to_panorama), str(training_data_json)) # Create an anomaly file with details about the anomaly crucible_anomaly_file = '%s/%s.txt' % (crucible_anomaly_dir, sane_metricname) try: write_data_to_file(skyline_app, crucible_anomaly_file, 'w', crucible_anomaly_data) logger.info('added crucible anomaly file :: %s' % (crucible_anomaly_file)) except: logger.error(traceback.format_exc()) logger.error('error :: failed to add crucible anomaly file :: %s' % (crucible_anomaly_file)) # Create a crucible check file crucible_check_file = '%s/%s.%s.txt' % (settings.CRUCIBLE_CHECK_PATH, str(added_at), sane_metricname) try: write_data_to_file(skyline_app, crucible_check_file, 'w', crucible_anomaly_data) logger.info('added crucible check :: %s,%s' % (base_name, str(added_at))) metrics_submitted_to_process.append(base_name) except: logger.error(traceback.format_exc()) logger.error('error :: failed to add crucible check file :: %s' % (crucible_check_file)) return (crucible_job_id, metrics_submitted_to_process, fail_msg, trace)
def get_all_db_metric_names(current_skyline_app, with_ids=False): """ Given return all metric names from the database as a list """ metric_names = [] metric_names_with_ids = {} function_str = 'database_queries.get_all_db_metric_names' current_skyline_app_logger = current_skyline_app + 'Log' current_logger = logging.getLogger(current_skyline_app_logger) try: engine, fail_msg, trace = get_engine(current_skyline_app) except Exception as e: trace = traceback.format_exc() current_logger.error(trace) fail_msg = 'error :: %s :: could not get a MySQL engine - %s' % (function_str, e) current_logger.error('%s' % fail_msg) return False, fail_msg, trace try: metrics_table, fail_msg, trace = metrics_table_meta(current_skyline_app, engine) current_logger.info(fail_msg) except Exception as e: trace = traceback.format_exc() current_logger.error('%s' % trace) fail_msg = 'error :: %s :: failed to get metrics_table meta - %s' % ( function_str, e) current_logger.error('%s' % fail_msg) if engine: engine_disposal(current_skyline_app, engine) if current_skyline_app == 'webapp': # Raise to webapp raise return False, fail_msg, trace try: connection = engine.connect() if with_ids: stmt = select([metrics_table.c.id, metrics_table.c.metric]) else: stmt = select([metrics_table.c.metric]) result = connection.execute(stmt) for row in result: base_name = row['metric'] metric_names.append(base_name) if with_ids: metric_names_with_ids[base_name] = row['id'] connection.close() current_logger.info('%s :: determined metric names from the db: %s' % ( function_str, str(len(metric_names)))) except Exception as e: trace = traceback.format_exc() current_logger.error(trace) fail_msg = 'error :: %s :: could not determine metric names from DB for - %s' % ( function_str, e) current_logger.error('%s' % fail_msg) if engine: engine_disposal(current_skyline_app, engine) if current_skyline_app == 'webapp': # Raise to webapp raise return False, fail_msg, trace if engine: engine_disposal(current_skyline_app, engine) if not metric_names: current_logger.error('error :: %s :: no metric names returned from the DB' % ( function_str)) if with_ids: return metric_names, metric_names_with_ids return metric_names
def metric_ids_from_metric_like(current_skyline_app, metrics_like_str): """ Given a SQL metric name wildcard, return a list of metric_ids """ metric_ids = [] function_str = 'database_queries.metric_ids_from_metric_like' current_skyline_app_logger = current_skyline_app + 'Log' current_logger = logging.getLogger(current_skyline_app_logger) try: engine, fail_msg, trace = get_engine(current_skyline_app) except Exception as e: trace = traceback.format_exc() current_logger.error(trace) fail_msg = 'error :: %s :: could not get a MySQL engine - %s' % ( function_str, e) current_logger.error('%s' % fail_msg) if current_skyline_app == 'webapp': # Raise to webapp raise return False try: metrics_table, fail_msg, trace = metrics_table_meta( current_skyline_app, engine) current_logger.info(fail_msg) except Exception as e: trace = traceback.format_exc() current_logger.error('%s' % trace) fail_msg = 'error :: %s :: failed to get metrics_table meta - %s' % ( function_str, e) current_logger.error('%s' % fail_msg) if engine: engine_disposal(current_skyline_app, engine) if current_skyline_app == 'webapp': # Raise to webapp raise return False, fail_msg, trace try: connection = engine.connect() stmt = select([metrics_table.c.id ]).where(metrics_table.c.metric.ilike(metrics_like_str)) result = connection.execute(stmt) for row in result: metric_ids.append(int(row['id'])) connection.close() current_logger.info( '%s :: determined %s metric_ids for metric_like_str: %s' % (function_str, str(len(metric_ids)), metrics_like_str)) except Exception as e: trace = traceback.format_exc() current_logger.error(trace) fail_msg = 'error :: %s :: could not determine ids of metrics_like_str: %s -%s' % ( function_str, str(metrics_like_str), e) current_logger.error('%s' % fail_msg) if engine: engine_disposal(current_skyline_app, engine) if current_skyline_app == 'webapp': # Raise to webapp raise return False, fail_msg, trace if engine: engine_disposal(current_skyline_app, engine) if not metric_ids: current_logger.error('error :: %s :: no ids for metrics_like_str: %s' % str(metrics_like_str)) return metric_ids
def base_name_from_metric_id(current_skyline_app, metric_id, log=False): """ Given a metric id, return the base_name """ base_name = None function_str = 'database_queries.base_name_from_metric_id' def get_log(current_skyline_app): current_skyline_app_logger = current_skyline_app + 'Log' current_logger = logging.getLogger(current_skyline_app_logger) return current_logger if log: current_logger = get_log(current_skyline_app) try: engine, fail_msg, trace = get_engine(current_skyline_app) except Exception as e: if not log: current_logger = get_log(current_skyline_app) trace = traceback.format_exc() current_logger.error(trace) fail_msg = 'error :: %s :: could not get a MySQL engine - %s' % ( function_str, e) current_logger.error('%s' % fail_msg) return False, fail_msg, trace try: metrics_table, fail_msg, trace = metrics_table_meta( current_skyline_app, engine) if log: current_logger.info(fail_msg) except Exception as e: if not log: current_logger = get_log(current_skyline_app) trace = traceback.format_exc() current_logger.error('%s' % trace) fail_msg = 'error :: %s :: failed to get metrics_table meta for %s- %s' % ( function_str, base_name, e) current_logger.error('%s' % fail_msg) if engine: engine_disposal(current_skyline_app, engine) if current_skyline_app == 'webapp': # Raise to webapp raise return False, fail_msg, trace try: connection = engine.connect() stmt = select([metrics_table.c.metric ]).where(metrics_table.c.id == metric_id) result = connection.execute(stmt) for row in result: base_name = row['metric'] break connection.close() if log: current_logger.info( '%s :: determined metric with id %s base_name: %s' % (function_str, str(metric_id), base_name)) except Exception as e: if not log: current_logger = get_log(current_skyline_app) trace = traceback.format_exc() current_logger.error(trace) fail_msg = 'error :: %s :: could not determine id of metric from DB for %s - %s' % ( function_str, base_name, e) current_logger.error('%s' % fail_msg) if engine: engine_disposal(current_skyline_app, engine) if current_skyline_app == 'webapp': # Raise to webapp raise return False, fail_msg, trace if engine: engine_disposal(current_skyline_app, engine) if not base_name: current_logger.error( 'error :: %s :: no base_name for metric in the DB with id %s' % (function_str, str(metric_id))) return base_name
def get_correlations(current_skyline_app, anomaly_id): """ Get all the correlations for an anomaly from the database :param current_skyline_app: the Skyline app name calling the function :param anomaly_id: thee base_name of the metric :type current_skyline_app: str :type anomaly_id: int :return: list :return: [[metric_name, coefficient, shifted, shifted_coefficient],[metric_name, coefficient, ...]] :rtype: [[str, float, float, float]] """ current_skyline_app_logger = current_skyline_app + 'Log' current_logger = logging.getLogger(current_skyline_app_logger) func_name = 'get_correlations' correlations = [] current_logger.info('get_correlations :: getting MySQL engine') try: engine, fail_msg, trace = fp_create_get_an_engine(current_skyline_app) current_logger.info(fail_msg) except: trace = traceback.format_exc() current_logger.error(trace) fail_msg = 'error :: could not get a MySQL engine' current_logger.error('%s' % fail_msg) # return False, False, fail_msg, trace, False raise # to webapp to return in the UI metrics_table = None try: metrics_table, fail_msg, trace = metrics_table_meta(current_skyline_app, engine) current_logger.info(fail_msg) except: trace = traceback.format_exc() current_logger.error('%s' % trace) fail_msg = 'error :: %s :: failed to get metrics_table_meta' % func_name current_logger.error('%s' % fail_msg) luminosity_table = None try: luminosity_table, fail_msg, trace = luminosity_table_meta(current_skyline_app, engine) current_logger.info(fail_msg) except: trace = traceback.format_exc() current_logger.error('%s' % trace) fail_msg = 'error :: %s :: failed to get luminosity_table_meta' % func_name current_logger.error('%s' % fail_msg) metrics_list = [] try: connection = engine.connect() stmt = select([metrics_table]).where(metrics_table.c.id > 0) results = connection.execute(stmt) for row in results: metric_id = row['id'] metric_name = row['metric'] metrics_list.append([int(metric_id), str(metric_name)]) connection.close() except: current_logger.error(traceback.format_exc()) current_logger.error('error :: could not determine metrics from MySQL') if engine: fp_create_engine_disposal(current_skyline_app, engine) raise try: connection = engine.connect() stmt = select([luminosity_table]).where(luminosity_table.c.id == int(anomaly_id)) result = connection.execute(stmt) for row in result: metric_id = row['metric_id'] metric_name = None if metric_id: metric_name = [metrics_list_name for metrics_list_id, metrics_list_name in metrics_list if int(metric_id) == int(metrics_list_id)] coefficient = row['coefficient'] shifted = row['shifted'] shifted_coefficient = row['shifted_coefficient'] correlations.append([metric_name, coefficient, shifted, shifted_coefficient]) connection.close() except: current_logger.error(traceback.format_exc()) current_logger.error('error :: could not determine correlations for anomaly id - %s' % str(anomaly_id)) if engine: fp_create_engine_disposal(current_skyline_app, engine) raise if engine: fp_create_engine_disposal(current_skyline_app, engine) if correlations: sorted_correlations = sorted(correlations, key=lambda x: x[1], reverse=True) correlations = sorted_correlations return correlations, fail_msg, trace