def echo_get_metric_from_metrics(base_name, engine):
"""
Called by :func:`~echo` and returns the metric id and metric db object
:param timestamp: timestamp at which learn was called
:type timestamp: int
:return: tuple
:rtype: (int, object)
"""
logger = logging.getLogger(skyline_app_logger)
metrics_id = 0
metric_db_object = None
# Get the metrics_table metadata
metrics_table = None
try:
metrics_table, log_msg, trace = metrics_table_meta(skyline_app, engine)
logger.info('echo :: metrics_table OK for %s' % base_name)
except:
logger.error(traceback.format_exc())
logger.error(
'error :: echo :: failed to get metrics_table meta for %s' %
base_name)
return False
try:
connection = engine.connect()
stmt = select([metrics_table
]).where(metrics_table.c.metric == base_name)
result = connection.execute(stmt)
row = result.fetchone()
metric_db_object = row
metrics_id = row['id']
connection.close()
except:
logger.error(traceback.format_exc())
logger.error(
'error :: echo :: could not determine id from metrics table for - %s'
% base_name)
return False
return metrics_id, metric_db_object
result = connection.execute(stmt)
for row in result:
if row['enabled'] != 1:
continue
if row['deleted'] == 1:
continue
fp_id = row['id']
metric_id = row['metric_id']
anomaly_timestamp = row['anomaly_timestamp']
fps_data.append([int(fp_id), int(metric_id), int(anomaly_timestamp)])
connection.close()
except:
print('Failed to get ionosphere data from the database')
try:
metrics_table, log_msg, trace = metrics_table_meta('autobuild', engine)
except:
print(traceback.format_exc())
# get all fp ids and metric ids from ionosphere table
ionosphere_metrics_data = []
try:
connection = engine.connect()
stmt = select([metrics_table]).where(metrics_table.c.ionosphere_enabled > 0)
result = connection.execute(stmt)
for row in result:
metric_id = row['id']
metric = row['metric']
ionosphere_metrics_data.append([int(metric_id), str(metric)])
connection.close()
except:
def update_snab_result(snab_id, anomaly_id, snab_result):
"""
Update the relevant field in the snab table.
:param snab_id: the snab table id
:param anomaly_id: the anomaly id
:param snab_result: a selected result
:type snab_id: int
:type anomaly_id: int
:type result: str
:return: snab_result_updated, base_name, anomaly_timestamp
:rtype: tuple
"""
snab_result_updated = False
base_name = None
anomaly_timestamp = None
connection = None
logger.info(
'update_snab_result :: for snab id %s with anomaly id %s and result %s' % (
str(snab_id), str(anomaly_id), str(snab_result)))
logger.info('getting MySQL engine')
try:
engine, fail_msg, trace = get_snab_engine()
logger.info(fail_msg)
except:
trace = traceback.format_exc()
logger.error(trace)
logger.error('%s' % fail_msg)
logger.error('error :: update_snab_result :: could not get a MySQL engine to get update snab table')
raise # to webapp to return in the UI
if not engine:
trace = 'none'
fail_msg = 'error :: update_snab_result :: engine not obtained'
logger.error(fail_msg)
raise
try:
snab_table, log_msg, trace = snab_table_meta(skyline_app, engine)
logger.info(log_msg)
logger.info('snab_table OK')
except:
logger.error(traceback.format_exc())
logger.error('error :: update_snab_result :: failed to get snab_table meta')
if engine:
snab_engine_disposal(engine)
raise # to webapp to return in the UI
# @modified 20201004 - Task #3748: POC SNAB
# Branch #3068: SNAB
# Allow results to be changed
determined_result = False
existing_result = None
try:
if not connection:
connection = engine.connect()
stmt = select([snab_table]).\
where(snab_table.c.id == snab_id)
result = connection.execute(stmt)
for row in result:
tP_set = row['tP']
if tP_set:
existing_result = 'tP'
fP_set = row['fP']
if fP_set:
existing_result = 'fP'
tN_set = row['tN']
if tN_set:
existing_result = 'tN'
fN_set = row['fN']
if fN_set:
existing_result = 'fN'
unsure_set = row['unsure']
if unsure_set:
existing_result = 'unsure'
determined_result = True
break
logger.info('update_snab_result :: current result values to snab id %s - tP: %s, fP: %s, tN: %s, fN: %s, unsure: %s' % (
str(snab_id), str(tP_set), str(fP_set), str(tN_set), str(fN_set),
str(unsure_set)))
except:
trace = traceback.format_exc()
logger.error(trace)
logger.error('error :: update_snab_result :: could not determine current result values for snab id %s' % (
str(snab_id)))
fail_msg = 'error :: update_snab_result :: could not determine current result values for snab id %s' % (
str(snab_id))
if engine:
try:
connection.close()
except:
pass
snab_engine_disposal(engine)
raise
if not determined_result:
logger.error('error :: update_snab_result :: did not determine current result values for snab id %s' % (
str(snab_id)))
else:
logger.info('update_snab_result :: current existing result for snab id %s - %s' % (
str(snab_id), str(existing_result)))
try:
if not connection:
connection = engine.connect()
if snab_result == 'tP':
stmt = snab_table.update().\
values(tP=1, fP=None, tN=None, fN=None, unsure=None).\
where(snab_table.c.id == int(snab_id)).\
where(snab_table.c.anomaly_id == int(anomaly_id))
if snab_result == 'fP':
stmt = snab_table.update().\
values(tP=None, fP=1, tN=None, fN=None, unsure=None).\
where(snab_table.c.id == int(snab_id)).\
where(snab_table.c.anomaly_id == int(anomaly_id))
if snab_result == 'tN':
stmt = snab_table.update().\
values(tP=None, fP=None, tN=1, fN=None, unsure=None).\
where(snab_table.c.id == int(snab_id)).\
where(snab_table.c.anomaly_id == int(anomaly_id))
if snab_result == 'fN':
stmt = snab_table.update().\
values(tP=None, fP=None, tN=None, fN=1, unsure=None).\
where(snab_table.c.id == int(snab_id)).\
where(snab_table.c.anomaly_id == int(anomaly_id))
if snab_result == 'unsure':
stmt = snab_table.update().\
values(tP=None, fP=None, tN=None, fN=None, unsure=1).\
where(snab_table.c.id == int(snab_id)).\
where(snab_table.c.anomaly_id == int(anomaly_id))
if snab_result == 'NULL':
stmt = snab_table.update().\
values(tP=None, fP=None, tN=None, fN=None, unsure=None).\
where(snab_table.c.id == int(snab_id)).\
where(snab_table.c.anomaly_id == int(anomaly_id))
connection.execute(stmt)
snab_result_updated = True
logger.info('update_snab_result :: update result for snab id %s with anomaly id %s and result %s' % (
str(snab_id), str(anomaly_id), str(snab_result)))
except:
trace = traceback.format_exc()
logger.error(trace)
logger.error('error :: update_snab_result :: could not update result for snab id %s with anomaly id %s and result %s' % (
str(snab_id), str(anomaly_id), str(result)))
fail_msg = 'error :: update_snab_result :: could not update result for snab id %s with anomaly id %s and result %s' % (
str(snab_id), str(anomaly_id), str(result))
if engine:
try:
connection.close()
except:
pass
snab_engine_disposal(engine)
raise
try:
metrics_table, log_msg, trace = metrics_table_meta(skyline_app, engine)
logger.info(log_msg)
logger.info('update_snab_result :: metrics_table OK')
except:
logger.error(traceback.format_exc())
logger.error('error :: update_snab_result :: failed to get metrics_table meta for %s' % base_name)
if engine:
try:
connection.close()
except:
pass
snab_engine_disposal(engine)
raise # to webapp to return in the UI
try:
anomalies_table, log_msg, trace = anomalies_table_meta(skyline_app, engine)
logger.info(log_msg)
logger.info('update_snab_result :: anomalies_table OK')
except:
logger.error(traceback.format_exc())
logger.error('error :: update_snab_result :: failed to get anomalies_table meta')
if engine:
try:
connection.close()
except:
pass
snab_engine_disposal(engine)
raise # to webapp to return in the UI
metric_id = None
try:
if not connection:
connection = engine.connect()
stmt = select([anomalies_table]).\
where(anomalies_table.c.id == anomaly_id)
result = connection.execute(stmt)
for row in result:
metric_id = row['metric_id']
anomaly_timestamp = row['anomaly_timestamp']
break
logger.info('update_snab_result :: determined anomaly_timestamp %s for metric id %s for anomaly id %s' % (
str(anomaly_timestamp), str(metric_id), str(anomaly_id)))
except:
trace = traceback.format_exc()
logger.error(trace)
fail_msg = 'error :: update_snab_result :: could not determine anomaly timestamp or metric id from DB for anomaly id %s' % (
str(anomaly_id))
logger.error('%s' % fail_msg)
if engine:
try:
connection.close()
except:
pass
snab_engine_disposal(engine)
raise # to webapp to return in the UI
if metric_id:
try:
if not connection:
connection = engine.connect()
stmt = select([metrics_table]).where(metrics_table.c.id == int(metric_id))
result = connection.execute(stmt)
for row in result:
base_name = row['metric']
except:
trace = traceback.format_exc()
logger.error(trace)
fail_msg = 'error :: could not determine metric id from metrics table'
if engine:
try:
connection.close()
except:
pass
snab_engine_disposal(engine)
raise
if connection:
try:
connection.close()
except:
pass
if engine:
snab_engine_disposal(engine)
return snab_result_updated, base_name, anomaly_timestamp, existing_result
def metric_id_from_base_name(current_skyline_app, base_name):
"""
Given a base name, return the metric_id
"""
metric_id = 0
function_str = 'database_queries.metric_id_from_base_name'
current_skyline_app_logger = current_skyline_app + 'Log'
current_logger = logging.getLogger(current_skyline_app_logger)
try:
engine, fail_msg, trace = get_engine(current_skyline_app)
except Exception as e:
trace = traceback.format_exc()
current_logger.error(trace)
fail_msg = 'error :: %s :: could not get a MySQL engine - %s' % (
function_str, e)
current_logger.error('%s' % fail_msg)
return False, fail_msg, trace
try:
metrics_table, fail_msg, trace = metrics_table_meta(
current_skyline_app, engine)
current_logger.info(fail_msg)
except Exception as e:
trace = traceback.format_exc()
current_logger.error('%s' % trace)
fail_msg = 'error :: %s :: failed to get metrics_table meta for %s- %s' % (
function_str, base_name, e)
current_logger.error('%s' % fail_msg)
if engine:
engine_disposal(current_skyline_app, engine)
if current_skyline_app == 'webapp':
# Raise to webapp
raise
return False, fail_msg, trace
try:
connection = engine.connect()
stmt = select([metrics_table
]).where(metrics_table.c.metric == base_name)
result = connection.execute(stmt)
for row in result:
metric_id = int(row['id'])
break
connection.close()
current_logger.info('%s :: determined db metric id: %s' %
(function_str, str(metric_id)))
except Exception as e:
trace = traceback.format_exc()
current_logger.error(trace)
fail_msg = 'error :: %s :: could not determine id of metric from DB for %s - %s' % (
function_str, base_name, e)
current_logger.error('%s' % fail_msg)
if engine:
engine_disposal(current_skyline_app, engine)
if current_skyline_app == 'webapp':
# Raise to webapp
raise
return False, fail_msg, trace
if engine:
engine_disposal(current_skyline_app, engine)
if not metric_id:
current_logger.error('error :: %s :: no id for metric in the DB - %s' %
(function_str, base_name))
return metric_id
def get_metrics_db_object(base_name):
"""
Returns the data of a metric from metrics table as an object and populates
memcached with a dict of the object
:param base_name: the metric base_name
:type base_name: str
:return: metrics_db_object
:rtype: object
"""
def get_an_engine():
try:
engine, log_msg, trace = get_engine(skyline_app)
return engine, log_msg, trace
except:
logger.error(traceback.format_exc())
log_msg = 'error :: failed to get MySQL engine in spin_process'
logger.error('error :: failed to get MySQL engine in spin_process')
return None, log_msg, trace
def engine_disposal(engine):
if engine:
try:
engine.dispose()
except:
logger.error(traceback.format_exc())
logger.error('error :: calling engine.dispose()')
return
metrics_db_object = None
memcache_metrics_db_object = None
metrics_db_object_key = 'metrics_db_object.%s' % str(base_name)
memcache_metric_dict = None
if settings.MEMCACHE_ENABLED:
memcache_metric_dict = get_memcache_metric_object(
skyline_app, base_name)
query_metric_table = True
if memcache_metric_dict:
query_metric_table = False
metrics_db_object = memcache_metric_dict
logger.info('using %s key data from memcache' % metrics_db_object_key)
# @modified 20170825 - Task #2132: Optimise Ionosphere DB usage
# If no memcache data then MySQL query_metric_table
if query_metric_table:
try:
engine, log_msg, trace = get_an_engine()
logger.info(log_msg)
except:
logger.error(traceback.format_exc())
logger.error(
'error :: could not get a MySQL engine to determine ionosphere_enabled'
)
if not engine:
logger.error(
'error :: engine not obtained to determine ionosphere_enabled')
# Get the metrics_table metadata
metrics_table = None
try:
metrics_table, log_msg, trace = metrics_table_meta(
skyline_app, engine)
logger.info('metrics_table OK for %s' % base_name)
except:
logger.error(traceback.format_exc())
logger.error('error :: failed to get metrics_table meta for %s' %
base_name)
try:
connection = engine.connect()
stmt = select([metrics_table
]).where(metrics_table.c.metric == base_name)
result = connection.execute(stmt)
try:
result
except:
logger.error(traceback.format_exc())
logger.error(
'error :: got no result from MySQL from metrics table for - %s'
% base_name)
row = result.fetchone()
# @added 20170825 - Task #2132: Optimise Ionosphere DB usage
# @modified - 20180524 - Task #2132: Optimise Ionosphere DB usage
# Feature #2378: Add redis auth to Skyline and rebrow
# Wrapped memcache_metrics_db_object, metrics_id,
# metric_ionosphere_enabled and metrics_db_object in if row
# as if row is None it can fail with:
# TypeError: 'NoneType' object is not iterable
# memcache_metrics_db_object = dict(row)
if row:
memcache_metrics_db_object = dict(row)
# @added 20170115 - Feature #1854: Ionosphere learn - generations
# Create the metrics_db_object so it is available throughout
# Here we go! Learn!
metrics_db_object = row
else:
logger.info('could not determine metric id for %s' % base_name)
connection.close()
except:
logger.error(traceback.format_exc())
logger.error(
'error :: could not determine ionosphere_enabled from metrics table for - %s'
% base_name)
# @added 20170825 - Task #2132: Optimise Ionosphere DB usage
# Add the metric db object data to memcache
# @modified 20191031 - Branch #3262: py3
# Task #3304: py3 - handle pymemcache bytes not str
# if settings.MEMCACHE_ENABLED and query_metric_table:
if settings.MEMCACHE_ENABLED and query_metric_table and memcache_metrics_db_object:
try:
memcache_metric_dict = {}
# @modified 20191030 - Branch #3262: py3
# Task #3304: py3 - handle pymemcache bytes not str
# for k, v in memcache_metrics_db_object.iteritems():
if python_version == 2:
for k, v in memcache_metrics_db_object.iteritems():
key_name = str(k)
key_value = str(v)
memcache_metric_dict[key_name] = key_value
else:
for k, v in memcache_metrics_db_object.items():
key_name = str(k)
key_value = str(v)
memcache_metric_dict[key_name] = key_value
memcache_client.set(metrics_db_object_key,
memcache_metric_dict,
expire=3600)
logger.info('set the memcache key - %s' % metrics_db_object_key)
except:
logger.error(traceback.format_exc())
logger.error('error :: failed to set %s in memcache' %
metrics_db_object_key)
try:
memcache_client.close()
except:
# @modified 20170913 - Task #2160: Test skyline with bandit
# pass
logger.error('error :: failed to close memcache_client')
return metrics_db_object
def create_features_profile(current_skyline_app, requested_timestamp, data_for_metric, context, ionosphere_job, fp_parent_id, fp_generation, fp_learn):
"""
Add a features_profile to the Skyline ionosphere database table.
:param current_skyline_app: Skyline app name
:param requested_timestamp: The timestamp of the dir that the features
profile data is in
:param data_for_metric: The base_name of the metric
:param context: The context of the caller
:param ionosphere_job: The ionosphere_job name related to creation request
valid jobs are ``learn_fp_human``, ``learn_fp_generation``,
``learn_fp_learnt`` and ``learn_fp_automatic``.
:param fp_parent_id: The id of the parent features profile that this was
learnt from, 0 being an original human generated features profile
:param fp_generation: The number of generations away for the original
human generated features profile, 0 being an original human generated
features profile.
:param fp_learn: Whether Ionosphere should learn at use_full_duration_days
:type current_skyline_app: str
:type requested_timestamp: int
:type data_for_metric: str
:type context: str
:type ionosphere_job: str
:type fp_parent_id: int
:type fp_generation: int
:type fp_learn: boolean
:return: fp_id, fp_in_successful, fp_exists, fail_msg, traceback_format_exc
:rtype: str, boolean, boolean, str, str
"""
current_skyline_app_logger = current_skyline_app + 'Log'
current_logger = logging.getLogger(current_skyline_app_logger)
base_name = data_for_metric.replace(settings.FULL_NAMESPACE, '', 1)
if context == 'training_data':
log_context = 'training data'
ionosphere_learn_job = 'learn_fp_human'
if context == 'features_profiles':
log_context = 'features profile data'
# @added 20170113 - Feature #1854: Ionosphere learn
if context == 'ionosphere_learn':
log_context = 'learn'
current_logger.info('create_features_profile :: %s :: requested for %s at %s' % (
context, str(base_name), str(requested_timestamp)))
metric_timeseries_dir = base_name.replace('.', '/')
if context == 'training_data':
metric_training_data_dir = '%s/%s/%s' % (
settings.IONOSPHERE_DATA_FOLDER, str(requested_timestamp),
metric_timeseries_dir)
if context == 'features_profiles':
metric_training_data_dir = '%s/%s/%s' % (
settings.IONOSPHERE_PROFILES_FOLDER, metric_timeseries_dir,
str(requested_timestamp))
# @added 20170113 - Feature #1854: Ionosphere learn
if context == 'ionosphere_learn':
# @modified 20170116 - Feature #1854: Ionosphere learn
# Allowing ionosphere_learn to create a features profile for a training
# data set that it has learnt is not anomalous
if ionosphere_job != 'learn_fp_automatic':
metric_training_data_dir = '%s/%s/%s' % (
settings.IONOSPHERE_LEARN_FOLDER, str(requested_timestamp),
metric_timeseries_dir)
else:
metric_training_data_dir = '%s/%s/%s' % (
settings.IONOSPHERE_DATA_FOLDER, str(requested_timestamp),
metric_timeseries_dir)
features_file = '%s/%s.tsfresh.input.csv.features.transposed.csv' % (
metric_training_data_dir, base_name)
features_profile_dir = '%s/%s' % (
settings.IONOSPHERE_PROFILES_FOLDER, metric_timeseries_dir)
ts_features_profile_dir = '%s/%s/%s' % (
settings.IONOSPHERE_PROFILES_FOLDER, metric_timeseries_dir,
str(requested_timestamp))
features_profile_created_file = '%s/%s.%s.fp.created.txt' % (
metric_training_data_dir, str(requested_timestamp), base_name)
features_profile_details_file = '%s/%s.%s.fp.details.txt' % (
metric_training_data_dir, str(requested_timestamp), base_name)
anomaly_check_file = '%s/%s.txt' % (metric_training_data_dir, base_name)
trace = 'none'
fail_msg = 'none'
new_fp_id = False
calculated_with_tsfresh = False
calculated_time = False
fcount = None
fsum = None
# @added 20170104 - Feature #1842: Ionosphere - Graphite now graphs
# Added the ts_full_duration parameter so that the appropriate graphs can be
# embedded for the user in the training data page
ts_full_duration = '0'
if context == 'ionosphere_learn':
if not path.isfile(features_profile_details_file):
current_logger.error('error :: create_features_profile :: no features_profile_details_file - %s' % features_profile_details_file)
return 'none', False, False, fail_msg, trace
if path.isfile(features_profile_details_file):
current_logger.info('create_features_profile :: getting features profile details from from - %s' % features_profile_details_file)
# Read the details file
with open(features_profile_details_file, 'r') as f:
fp_details_str = f.read()
fp_details = literal_eval(fp_details_str)
calculated_with_tsfresh = fp_details[1]
calculated_time = str(fp_details[2])
fcount = str(fp_details[3])
fsum = str(fp_details[4])
try:
ts_full_duration = str(fp_details[5])
except:
current_logger.error('error :: create_features_profile :: could not determine the full duration from - %s' % features_profile_details_file)
ts_full_duration = '0'
if context != 'ionosphere_learn':
if ts_full_duration == '0':
if path.isfile(anomaly_check_file):
current_logger.info('create_features_profile :: determining the full duration from anomaly_check_file - %s' % anomaly_check_file)
# Read the details file
with open(anomaly_check_file, 'r') as f:
anomaly_details = f.readlines()
for i, line in enumerate(anomaly_details):
if 'full_duration' in line:
_ts_full_duration = '%s' % str(line).split("'", 2)
full_duration_array = literal_eval(_ts_full_duration)
ts_full_duration = str(int(full_duration_array[1]))
current_logger.info('create_features_profile :: determined the full duration as - %s' % str(ts_full_duration))
if path.isfile(features_profile_created_file):
# Read the created file
with open(features_profile_created_file, 'r') as f:
fp_created_str = f.read()
fp_created = literal_eval(fp_created_str)
new_fp_id = fp_created[0]
return str(new_fp_id), True, True, fail_msg, trace
# Have data
if path.isfile(features_file):
current_logger.info('create_features_profile :: features_file exists: %s' % features_file)
else:
trace = traceback.format_exc()
current_logger.error(trace)
fail_msg = 'error :: create_features_profile :: features_file does not exist: %s' % features_file
current_logger.error('%s' % fail_msg)
if context == 'training' or context == 'features_profile':
# Raise to webbapp I believe to provide traceback to user in UI
raise
else:
return False, False, False, fail_msg, trace
features_data = []
with open(features_file, 'rb') as fr:
reader = csv.reader(fr, delimiter=',')
for i, line in enumerate(reader):
feature_name_item = False
fname_id = False
f_value = False
feature_name = str(line[0])
feature_name_item = filter(
lambda x: x[1] == feature_name, TSFRESH_FEATURES)
if feature_name_item:
feature_name_id = feature_name_item[0]
if feature_name_item:
feature_name_list = feature_name_item[0]
fname_id = int(feature_name_list[0])
f_value = str(line[1])
if fname_id and f_value:
features_data.append([fname_id, f_value])
# @added 20170113 - Feature #1854: Ionosphere learn - generations
# Set the learn generations variables with the IONOSPHERE_LEARN_DEFAULT_ and any
# settings.IONOSPHERE_LEARN_NAMESPACE_CONFIG values. These will later be
# overridden by any database values determined for the specific metric if
# they exist.
# Set defaults
use_full_duration_days = int(settings.IONOSPHERE_LEARN_DEFAULT_FULL_DURATION_DAYS)
valid_learning_duration = int(settings.IONOSPHERE_LEARN_DEFAULT_VALID_TIMESERIES_OLDER_THAN_SECONDS)
max_generations = int(settings.IONOSPHERE_LEARN_DEFAULT_MAX_GENERATIONS)
max_percent_diff_from_origin = float(settings.IONOSPHERE_LEARN_DEFAULT_MAX_PERCENT_DIFF_FROM_ORIGIN)
try:
use_full_duration, valid_learning_duration, use_full_duration_days, max_generations, max_percent_diff_from_origin = get_ionosphere_learn_details(current_skyline_app, base_name)
learn_full_duration_days = use_full_duration_days
except:
current_logger.error(traceback.format_exc())
current_logger.error('error :: create_features_profile :: failed to get_ionosphere_learn_details')
current_logger.info('create_features_profile :: learn_full_duration_days :: %s days' % (str(learn_full_duration_days)))
current_logger.info('create_features_profile :: valid_learning_duration :: %s seconds' % (str(valid_learning_duration)))
current_logger.info('create_features_profile :: max_generations :: %s' % (str(max_generations)))
current_logger.info('create_features_profile :: max_percent_diff_from_origin :: %s' % (str(max_percent_diff_from_origin)))
current_logger.info('create_features_profile :: getting MySQL engine')
try:
engine, fail_msg, trace = fp_create_get_an_engine(current_skyline_app)
current_logger.info(fail_msg)
except:
trace = traceback.format_exc()
current_logger.error(trace)
fail_msg = 'error :: create_features_profile :: could not get a MySQL engine'
current_logger.error('%s' % fail_msg)
if context == 'training' or context == 'features_profile':
# Raise to webbapp I believe to provide traceback to user in UI
raise
else:
return False, False, False, fail_msg, trace
if not engine:
trace = 'none'
fail_msg = 'error :: create_features_profile :: engine not obtained'
current_logger.error(fail_msg)
if context == 'training' or context == 'features_profile':
# Raise to webbapp I believe to provide traceback to user in UI
raise
else:
return False, False, False, fail_msg, trace
# Get metric details from the database
metrics_id = False
# Use the learn details as per config
metric_learn_full_duration_days = int(use_full_duration_days)
metric_learn_valid_ts_older_than = int(valid_learning_duration)
metric_max_generations = int(max_generations)
metric_max_percent_diff_from_origin = int(max_percent_diff_from_origin)
metrics_table = None
try:
metrics_table, fail_msg, trace = metrics_table_meta(current_skyline_app, engine)
current_logger.info(fail_msg)
except:
trace = traceback.format_exc()
current_logger.error('%s' % trace)
fail_msg = 'error :: create_features_profile :: failed to get metrics_table meta for %s' % base_name
current_logger.error('%s' % fail_msg)
if context == 'training' or context == 'features_profile':
# @added 20170806 - Bug #2130: MySQL - Aborted_clients
# Added missing disposal
if engine:
fp_create_engine_disposal(current_skyline_app, engine)
# Raise to webbapp I believe to provide traceback to user in UI
raise
else:
current_logger.info('create_features_profile :: disposing of any engine')
fp_create_engine_disposal(current_skyline_app, engine)
return False, False, False, fail_msg, trace
current_logger.info('create_features_profile :: metrics_table OK')
metric_db_object = None
try:
connection = engine.connect()
# @modified 20161209 - - Branch #922: ionosphere
# Task #1658: Patterning Skyline Ionosphere
# result = connection.execute('select id from metrics where metric=\'%s\'' % base_name)
# for row in result:
# while not metrics_id:
# metrics_id = row['id']
stmt = select([metrics_table]).where(metrics_table.c.metric == base_name)
result = connection.execute(stmt)
for row in result:
metrics_id = row['id']
# @added 20170113 - Feature #1854: Ionosphere learn - generations
# Added Ionosphere LEARN generation related variables
try:
metric_learn_full_duration_days = int(row['learn_full_duration_days'])
metric_learn_valid_ts_older_than = int(row['learn_valid_ts_older_than'])
metric_max_generations = int(row['max_generations'])
metric_max_percent_diff_from_origin = float(row['max_percent_diff_from_origin'])
except:
current_logger.error('error :: create_features_profile :: failed to determine learn related values from DB for %s' % base_name)
row = result.fetchone()
# metric_db_object = row
connection.close()
current_logger.info('create_features_profile :: determined db metric id: %s' % str(metrics_id))
current_logger.info('create_features_profile :: determined db metric learn_full_duration_days: %s' % str(metric_learn_full_duration_days))
current_logger.info('create_features_profile :: determined db metric learn_valid_ts_older_than: %s' % str(metric_learn_valid_ts_older_than))
current_logger.info('create_features_profile :: determined db metric max_generations: %s' % str(metric_max_generations))
current_logger.info('create_features_profile :: determined db metric max_percent_diff_from_origin: %s' % str(metric_max_percent_diff_from_origin))
except:
trace = traceback.format_exc()
current_logger.error(trace)
fail_msg = 'error :: create_features_profile :: could not determine id of metric from DB: %s' % base_name
current_logger.error('%s' % fail_msg)
if metric_learn_full_duration_days:
learn_full_duration_days = metric_learn_full_duration_days
# learn_full_duration = int(learn_full_duration_days) * 86400
if metric_learn_valid_ts_older_than:
learn_valid_ts_older_than = metric_learn_valid_ts_older_than
if metric_max_generations:
max_generations = metric_max_generations
if metric_max_percent_diff_from_origin:
max_percent_diff_from_origin = metric_max_percent_diff_from_origin
current_logger.info('create_features_profile :: generation info - learn_full_duration_days :: %s' % (str(learn_full_duration_days)))
current_logger.info('create_features_profile :: generation info - learn_valid_ts_older_than :: %s' % (str(learn_valid_ts_older_than)))
current_logger.info('create_features_profile :: generation info - max_generations :: %s' % (str(max_generations)))
current_logger.info('create_features_profile :: generation info - max_percent_diff_from_origin :: %s' % (str(max_percent_diff_from_origin)))
# @added 20170120 - Feature #1854: Ionosphere learn
# Always use the timestamp from the anomaly file
use_anomaly_timestamp = int(requested_timestamp)
if context == 'ionosphere_learn':
if path.isfile(anomaly_check_file):
current_logger.info('create_features_profile :: determining the full duration from anomaly_check_file - %s' % anomaly_check_file)
# Read the details file
with open(anomaly_check_file, 'r') as f:
anomaly_details = f.readlines()
for i, line in enumerate(anomaly_details):
if 'metric_timestamp' in line:
_metric_timestamp = '%s' % str(line).split("'", 2)
metric_timestamp_array = literal_eval(_metric_timestamp)
use_anomaly_timestamp = (int(metric_timestamp_array[1]))
current_logger.info('create_features_profile :: determined the anomaly metric_timestamp as - %s' % str(use_anomaly_timestamp))
ionosphere_table = None
try:
ionosphere_table, fail_msg, trace = ionosphere_table_meta(current_skyline_app, engine)
current_logger.info(fail_msg)
except:
trace = traceback.format_exc()
current_logger.error('%s' % trace)
fail_msg = 'error :: create_features_profile :: failed to get ionosphere_table meta for %s' % base_name
current_logger.error('%s' % fail_msg)
if context == 'training' or context == 'features_profile':
# Raise to webbapp I believe to provide traceback to user in UI
# @added 20170806 - Bug #2130: MySQL - Aborted_clients
# Added missing disposal
if engine:
fp_create_engine_disposal(current_skyline_app, engine)
raise
else:
current_logger.info('create_features_profile :: disposing of any engine')
fp_create_engine_disposal(current_skyline_app, engine)
return False, False, False, fail_msg, trace
current_logger.info('create_features_profile :: ionosphere_table OK')
# @added 20170403 - Feature #2000: Ionosphere - validated
# Set all learn_fp_human features profiles to validated.
fp_validated = 0
if ionosphere_job == 'learn_fp_human':
fp_validated = 1
# @added 20170424 - Feature #2000: Ionosphere - validated
# Set all generation 0 and 1 as validated
if int(fp_generation) <= 1:
fp_validated = 1
new_fp_id = False
try:
connection = engine.connect()
# @added 20170113 - Feature #1854: Ionosphere learn
# Added learn values parent_id, generation
# @modified 20170120 - Feature #1854: Ionosphere learn
# Added anomaly_timestamp
# @modified 20170403 - Feature #2000: Ionosphere - validated
ins = ionosphere_table.insert().values(
metric_id=int(metrics_id), full_duration=int(ts_full_duration),
anomaly_timestamp=int(use_anomaly_timestamp),
enabled=1, tsfresh_version=str(tsfresh_version),
calc_time=calculated_time, features_count=fcount,
features_sum=fsum, parent_id=fp_parent_id,
generation=fp_generation, validated=fp_validated)
result = connection.execute(ins)
connection.close()
new_fp_id = result.inserted_primary_key[0]
current_logger.info('create_features_profile :: new ionosphere fp_id: %s' % str(new_fp_id))
except:
trace = traceback.format_exc()
current_logger.error('%s' % trace)
fail_msg = 'error :: create_features_profile :: failed to insert a new record into the ionosphere table for %s' % base_name
current_logger.error('%s' % fail_msg)
if context == 'training' or context == 'features_profile':
# @added 20170806 - Bug #2130: MySQL - Aborted_clients
# Added missing disposal
if engine:
fp_create_engine_disposal(current_skyline_app, engine)
# Raise to webbapp I believe to provide traceback to user in UI
raise
else:
current_logger.info('create_features_profile :: disposing of any engine')
fp_create_engine_disposal(current_skyline_app, engine)
return False, False, False, fail_msg, trace
if not RepresentsInt(new_fp_id):
trace = traceback.format_exc()
current_logger.error('%s' % trace)
fail_msg = 'error :: create_features_profile :: unknown new ionosphere new_fp_id for %s' % base_name
current_logger.error('%s' % fail_msg)
if context == 'training' or context == 'features_profile':
# @added 20170806 - Bug #2130: MySQL - Aborted_clients
# Added missing disposal
if engine:
fp_create_engine_disposal(current_skyline_app, engine)
# Raise to webbapp I believe to provide traceback to user in UI
raise
else:
current_logger.info('create_features_profile :: disposing of any engine')
fp_create_engine_disposal(current_skyline_app, engine)
return False, False, False, fail_msg, trace
# Create z_fp_<metric_id> table
fp_table_created = False
fp_table_name = 'z_fp_%s' % str(metrics_id)
try:
fp_meta = MetaData()
# @modified 20161222 - Task #1812: z_fp table type
# Changed to InnoDB from MyISAM as no files open issues and MyISAM clean
# up, there can be LOTS of file_per_table z_fp_ tables/files without
# the MyISAM issues. z_fp_ tables are mostly read and will be shuffled
# in the table cache as required.
fp_metric_table = Table(
fp_table_name, fp_meta,
Column('id', Integer, primary_key=True),
Column('fp_id', Integer, nullable=False, key='fp_id'),
Column('feature_id', Integer, nullable=False),
Column('value', DOUBLE(), nullable=True),
mysql_charset='utf8',
mysql_key_block_size='255',
mysql_engine='InnoDB')
fp_metric_table.create(engine, checkfirst=True)
fp_table_created = True
except:
trace = traceback.format_exc()
current_logger.error('%s' % trace)
fail_msg = 'error :: create_features_profile :: failed to create table - %s' % fp_table_name
current_logger.error('%s' % fail_msg)
if context == 'training' or context == 'features_profile':
# @added 20170806 - Bug #2130: MySQL - Aborted_clients
# Added missing disposal
if engine:
fp_create_engine_disposal(current_skyline_app, engine)
# Raise to webbapp I believe to provide traceback to user in UI
raise
else:
current_logger.info('create_features_profile :: %s - automated so the table should exists continuing' % context)
if not fp_table_created:
trace = traceback.format_exc()
current_logger.error('%s' % trace)
fail_msg = 'error :: create_features_profile :: failed to determine True for create table - %s' % fp_table_name
current_logger.error('%s' % fail_msg)
if context == 'training' or context == 'features_profile':
# @added 20170806 - Bug #2130: MySQL - Aborted_clients
# Added missing disposal
if engine:
fp_create_engine_disposal(current_skyline_app, engine)
# Raise to webbapp I believe to provide traceback to user in UI
raise
else:
current_logger.info('create_features_profile :: %s - automated so the table should exists continuing' % context)
# Insert features and values
insert_statement = []
for fname_id, f_value in features_data:
insert_statement.append({'fp_id': new_fp_id, 'feature_id': fname_id, 'value': f_value},)
if insert_statement == []:
trace = traceback.format_exc()
current_logger.error('%s' % trace)
fail_msg = 'error :: create_features_profile :: empty insert_statement for %s inserts' % fp_table_name
current_logger.error('%s' % fail_msg)
# raise
# else:
# feature_count = sum(1 for x in a if isinstance(x, insert_statement))
# current_logger.info(
# 'fp_id - %s - %s feature values in insert_statement for %s ' %
# (str(feature_count), str(new_fp_id), fp_table_name))
# feature_count = sum(1 for x in a if isinstance(x, insert_statement))
# current_logger.info(
# 'fp_id - %s - feature values in insert_statement for %s ' %
# (str(new_fp_id), fp_table_name))
try:
connection = engine.connect()
connection.execute(fp_metric_table.insert(), insert_statement)
connection.close()
current_logger.info('create_features_profile :: fp_id - %s - feature values inserted into %s' % (str(new_fp_id), fp_table_name))
except:
trace = traceback.format_exc()
current_logger.error('%s' % trace)
fail_msg = 'error :: create_features_profile :: failed to insert a feature values into %s' % fp_table_name
current_logger.error('%s' % fail_msg)
if context == 'training' or context == 'features_profile':
# @added 20170806 - Bug #2130: MySQL - Aborted_clients
# Added missing disposal
if engine:
fp_create_engine_disposal(current_skyline_app, engine)
# Raise to webbapp I believe to provide traceback to user in UI
raise
else:
current_logger.info('create_features_profile :: %s - automated so the table should exists continuing' % context)
# Create metric ts table if not exists ts_<metric_id>
# Create z_ts_<metric_id> table
# @modified 20170121 - Feature #1854: Ionosphere learn - generations
# TODO Adding the option to not save timeseries to DB, as default?
# ts_table_created = False
ts_table_name = 'z_ts_%s' % str(metrics_id)
try:
ts_meta = MetaData()
# @modified 20161222 - Task #1812: z_fp table type
# Changed to InnoDB from MyISAM as no files open issues and MyISAM clean
# up, there can be LOTS of file_per_table z_fp_ tables/files without
# the MyISAM issues. z_fp_ tables are mostly read and will be shuffled
# in the table cache as required.
ts_metric_table = Table(
ts_table_name, ts_meta,
Column('id', Integer, primary_key=True),
Column('fp_id', Integer, nullable=False, key='fp_id'),
Column('timestamp', Integer, nullable=False),
Column('value', DOUBLE(), nullable=True),
mysql_charset='utf8',
mysql_key_block_size='255',
mysql_engine='InnoDB')
ts_metric_table.create(engine, checkfirst=True)
# ts_table_created = True
current_logger.info('create_features_profile :: metric ts table created OK - %s' % (ts_table_name))
except:
trace = traceback.format_exc()
current_logger.error('%s' % trace)
fail_msg = 'error :: create_features_profile :: failed to create table - %s' % ts_table_name
current_logger.error('%s' % fail_msg)
if context == 'training' or context == 'features_profile':
# @added 20170806 - Bug #2130: MySQL - Aborted_clients
# Added missing disposal
if engine:
fp_create_engine_disposal(current_skyline_app, engine)
# Raise to webbapp I believe to provide traceback to user in UI
raise
else:
current_logger.info('create_features_profile :: %s - automated so the table should exists continuing' % context)
# Insert timeseries that the features profile was created from
raw_timeseries = []
anomaly_json = '%s/%s.json' % (metric_training_data_dir, base_name)
if path.isfile(anomaly_json):
current_logger.info('create_features_profile :: metric anomaly json found OK - %s' % (anomaly_json))
try:
# Read the timeseries json file
with open(anomaly_json, 'r') as f:
raw_timeseries = f.read()
except:
trace = traceback.format_exc()
current_logger.error(trace)
fail_msg = 'error :: create_features_profile :: failed to read timeseries data from %s' % anomaly_json
current_logger.error('%s' % (fail_msg))
fail_msg = 'error: failed to read timeseries data from %s' % anomaly_json
# end = timer()
if context == 'training' or context == 'features_profile':
# @added 20170806 - Bug #2130: MySQL - Aborted_clients
# Added missing disposal
if engine:
fp_create_engine_disposal(current_skyline_app, engine)
# Raise to webbapp I believe to provide traceback to user in UI
raise
else:
trace = 'none'
fail_msg = 'error: file not found - %s' % (anomaly_json)
current_logger.error(fail_msg)
# raise
# Convert the timeseries to csv
timeseries_array_str = str(raw_timeseries).replace('(', '[').replace(')', ']')
timeseries = literal_eval(timeseries_array_str)
datapoints = timeseries
validated_timeseries = []
for datapoint in datapoints:
try:
new_datapoint = [str(int(datapoint[0])), float(datapoint[1])]
validated_timeseries.append(new_datapoint)
# @modified 20170913 - Task #2160: Test skyline with bandit
# Added nosec to exclude from bandit tests
except: # nosec
continue
insert_statement = []
for ts, value in validated_timeseries:
insert_statement.append({'fp_id': new_fp_id, 'timestamp': ts, 'value': value},)
try:
connection = engine.connect()
connection.execute(ts_metric_table.insert(), insert_statement)
connection.close()
current_logger.info('create_features_profile :: fp_id - %s - timeseries inserted into %s' % (str(new_fp_id), ts_table_name))
except:
trace = traceback.format_exc()
current_logger.error('%s' % trace)
fail_msg = 'error :: create_features_profile :: failed to insert the timeseries into %s' % ts_table_name
current_logger.error('%s' % fail_msg)
if context == 'training' or context == 'features_profile':
# @added 20170806 - Bug #2130: MySQL - Aborted_clients
# Added missing disposal
if engine:
fp_create_engine_disposal(current_skyline_app, engine)
raise
else:
current_logger.info('create_features_profile :: %s - automated so the table should exists continuing' % context)
# Create a created features profile file
try:
# data = '[%s, %s, ]' % (new_fp_id, str(int(time.time())))
# write_data_to_file(skyline_app, features_profile_created_file, 'w', data)
# @modified 20170115 - Feature #1854: Ionosphere learn - generations
# Added parent_id and generation
data = '[%s, %s, \'%s\', %s, %s, %s, %s, %s, %s]' % (
new_fp_id, str(int(time.time())), str(tsfresh_version),
str(calculated_time), str(fcount), str(fsum), str(ts_full_duration),
str(fp_parent_id), str(fp_generation))
write_data_to_file(current_skyline_app, features_profile_created_file, 'w', data)
except:
trace = traceback.format_exc()
current_logger.error('%s' % trace)
fail_msg = 'error :: create_features_profile :: failed to write fp.created file'
current_logger.error('%s' % fail_msg)
# Set ionosphere_enabled for the metric
try:
# update_statement = 'UPDATE metrics SET ionosphere_enabled=1 WHERE id=%s' % str(metrics_id)
connection = engine.connect()
# result = connection.execute('UPDATE metrics SET ionosphere_enabled=1 WHERE id=%s' % str(metrics_id))
# connection.execute(ts_metric_table.insert(), insert_statement)
connection.execute(
metrics_table.update(
metrics_table.c.id == metrics_id).values(ionosphere_enabled=1))
connection.close()
current_logger.info('create_features_profile :: ionosphere_enabled set on metric id: %s' % str(metrics_id))
except:
trace = traceback.format_exc()
current_logger.error(trace)
fail_msg = 'error :: create_features_profile :: could not update metrics table and set ionosphere_enabled on id %s' % str(metrics_id)
current_logger.error('%s' % fail_msg)
# raise
# Copy data from training data dir to features_profiles dir
if not path.isdir(ts_features_profile_dir):
mkdir_p(ts_features_profile_dir)
if path.isdir(ts_features_profile_dir):
current_logger.info('create_features_profile :: fp_id - %s - features profile dir created - %s' % (str(new_fp_id), ts_features_profile_dir))
# src_files = os.listdir(src)
# for file_name in src_files:
# full_file_name = path.join(src, file_name)
# if (path.isfile(full_file_name)):
# shutil.copy(full_file_name, dest)
data_files = []
try:
glob_path = '%s/*.*' % metric_training_data_dir
data_files = glob.glob(glob_path)
except:
trace = traceback.format_exc()
current_logger.error('%s' % trace)
current_logger.error('error :: create_features_profile :: glob - fp_id - %s - training data not copied to %s' % (str(new_fp_id), ts_features_profile_dir))
for i_file in data_files:
try:
shutil.copy(i_file, ts_features_profile_dir)
current_logger.info('create_features_profile :: fp_id - %s - training data copied - %s' % (str(new_fp_id), i_file))
except shutil.Error as e:
trace = traceback.format_exc()
current_logger.error('%s' % trace)
current_logger.error('error :: create_features_profile :: shutil error - fp_id - %s - training data not copied to %s' % (str(new_fp_id), ts_features_profile_dir))
current_logger.error('error :: create_features_profile :: %s' % (e))
# Any error saying that the directory doesn't exist
except OSError as e:
trace = traceback.format_exc()
current_logger.error('%s' % trace)
current_logger.error('error :: create_features_profile :: OSError error - fp_id - %s - training data not copied to %s' % (str(new_fp_id), ts_features_profile_dir))
current_logger.error('error :: create_features_profile :: %s' % (e))
current_logger.info('create_features_profile :: fp_id - %s - training data copied to %s' % (str(new_fp_id), ts_features_profile_dir))
else:
current_logger.error('error :: create_features_profile :: fp_id - %s - training data not copied to %s' % (str(new_fp_id), ts_features_profile_dir))
current_logger.info('create_features_profile :: disposing of any engine')
try:
if engine:
fp_create_engine_disposal(current_skyline_app, engine)
else:
current_logger.info('create_features_profile :: no engine to dispose of' % (str(new_fp_id), ts_features_profile_dir))
except:
trace = traceback.format_exc()
current_logger.error('%s' % trace)
current_logger.error('error :: create_features_profile :: OSError error - fp_id - %s - training data not copied to %s' % (str(new_fp_id), ts_features_profile_dir))
# @added 20170113 - Feature #1854: Ionosphere learn - Redis ionosphere.learn.work namespace
# Ionosphere learn needs Redis works sets
# When a features profile is created there needs to be work added to a Redis
# set. When a human makes a features profile, we want Ionosphere to make a
# use_full_duration_days features profile valid_learning_duration (e.g.
# 3361) later.
if settings.IONOSPHERE_LEARN and new_fp_id:
create_redis_work_item = False
if context == 'training_data' and ionosphere_job == 'learn_fp_human':
create_redis_work_item = True
# @modified 20170120 - Feature #1854: Ionosphere learn - generations
# Added fp_learn parameter to allow the user to not learn the
# use_full_duration_days
if not fp_learn:
create_redis_work_item = False
current_logger.info('fp_learn is False not adding an item to Redis ionosphere.learn.work set')
if ionosphere_job == 'learn_fp_automatic':
create_redis_work_item = True
# @added 20170131 - Feature #1886 Ionosphere learn - child like parent with evolutionary maturity
# TODO: here a check may be required to evaluate whether the origin_fp_id
# had a use_full_duration features profile created, however
# due to the fact that it is in learn, suggests that it did
# have, not 100% sure.
origin_fp_id_was_allowed_to_learn = False
child_use_full_duration_count_of_origin_fp_id = 1
# TODO: Determine the state
# child_use_full_duration_count_of_origin_fp_id = SELECT COUNT(id) FROM ionosphere WHERE parent_id=origin_fp_id AND full_duration=use_full_duration
if child_use_full_duration_count_of_origin_fp_id == 0:
current_logger.info('the origin parent was not allowed to learn not adding to Redis ionosphere.learn.work set')
create_redis_work_item = False
if create_redis_work_item:
try:
current_logger.info(
'adding work to Redis ionosphere.learn.work set - [\'Soft\', \'%s\', %s, \'%s\', %s, %s] to make a learn features profile later' % (
str(ionosphere_job), str(requested_timestamp), base_name,
str(new_fp_id), str(fp_generation)))
redis_conn = StrictRedis(unix_socket_path=settings.REDIS_SOCKET_PATH)
redis_conn.sadd('ionosphere.learn.work', ['Soft', str(ionosphere_job), int(requested_timestamp), base_name, int(new_fp_id), int(fp_generation)])
except:
current_logger.error(traceback.format_exc())
current_logger.error(
'error :: failed adding work to Redis ionosphere.learn.work set - [\'Soft\', \'%s\', %s, \'%s\', %s, %s] to make a learn features profile later' % (
str(ionosphere_job), str(requested_timestamp), base_name,
str(new_fp_id), str(fp_generation)))
# @added 20170806 - Bug #2130: MySQL - Aborted_clients
# Added missing disposal
if engine:
fp_create_engine_disposal(current_skyline_app, engine)
return str(new_fp_id), True, False, fail_msg, trace
def submit_crucible_job(from_timestamp, until_timestamp, metrics_list,
namespaces_list, source, alert_interval, user_id, user,
add_to_panorama, pad_timeseries, training_data_json,
run_algorithms):
"""
Get a list of all the metrics passed and generate Crucible check files for
each
:param from_timestamp: the timestamp at which to start the time series
:param until_timestamp: the timestamp at which to end the time series
:param metrics_list: a list of metric names to analyse
:param namespaces_list: a list of metric namespaces to analyse
:param source: the source webapp making the request
:param alert_interval: the alert_interval at which Crucible should trigger
anomalies
:param user_id: the user id of the user making the request
:param user: the username making the request
:param add_to_panorama: whether Crucible should add Skyline CONSENSUS
anomalies to Panorama
:param pad_timeseries: the amount of data to pad the time series with
:param training_data_json: the full path to the training_data json file if
source is training_data
:param run_algorithms: list of algorithms to run
:type from_timestamp: int
:type until_timestamp: int
:type metrics_list: list
:type namespaces_list: list
:type source: str
:type alert_interval: int
:type user_id: int
:type user: str
:type add_to_panorama: boolean
:type pad_timeseries: str
:type training_data_json: str
:type run_algorithms: list
:return: tuple of lists
:rtype: (list, list, list, list)
Returns (crucible_job_id, metrics_submitted_to_process, fail_msg, trace)
"""
fail_msg = None
trace = None
crucible_job_id = None
metrics_submitted_to_process = 0
# Generate a job id based on the YMDHMS.user_id and a job directory
try:
jobid_timestamp = int(time())
jobid_datetimestamp = dt.datetime.fromtimestamp(
jobid_timestamp).strftime('%Y%m%d%H%M%S')
crucible_job_id = '%s.%s' % (str(jobid_datetimestamp), str(user_id))
except:
logger.error(traceback.format_exc())
logger.error('error :: failed to determine a crucible_job_id')
raise # to webapp to return in the UI
# Generate a job id based on the YMDHMS.user_id and a job directory
try:
crucible_path = os.path.dirname(settings.CRUCIBLE_DATA_FOLDER)
crucible_job_dir = '%s/jobs/%s' % (crucible_path, crucible_job_id)
if not path.exists(crucible_job_dir):
logger.info('creating crucible job directory - %s' %
(str(crucible_job_dir)))
mkdir_p(crucible_job_dir)
except:
trace = traceback.format_exc()
fail_msg = 'error :: failed to create the crucible job directory'
logger.error(trace)
logger.error(fail_msg)
raise # to webapp to return in the UI
# TODO added checks of metric names
metric_names = []
if metrics_list:
logger.info('submit_crucible_job :: %s metrics passed' %
str(len(metrics_list)))
for metric in metrics_list:
metric_names.append(metric)
# TODO added checks of metric namespaces, harder to do, but so that the UI
# errors to the usr rather than sending a bad or non-existent metric to
# Crucible
if namespaces_list:
logger.info('submit_crucible_job :: %s namespaces passed' %
str(len(namespaces_list)))
logger.info(
'submit_crucible_job :: determine metrics for submit_crucible_job between %s and %s'
% (str(from_timestamp), str(until_timestamp)))
logger.info('getting MySQL engine')
try:
engine, fail_msg, trace = get_an_engine()
logger.info(fail_msg)
except:
trace = traceback.format_exc()
logger.error(trace)
logger.error('%s' % fail_msg)
logger.error(
'error :: could not get a MySQL engine to get metric names')
raise # to webapp to return in the UI
if not engine:
trace = 'none'
fail_msg = 'error :: engine not obtained'
logger.error(fail_msg)
raise
try:
metrics_table, log_msg, trace = metrics_table_meta(
skyline_app, engine)
logger.info(log_msg)
logger.info('metrics_table OK')
except:
logger.error(traceback.format_exc())
logger.error('error :: failed to get metrics_table meta')
if engine:
engine_disposal(engine)
raise # to webapp to return in the UI
metrics_like_query = text(
"""SELECT metric FROM metrics WHERE metric LIKE :like_string""")
for namespace in namespaces_list:
try:
connection = engine.connect()
results = connection.execute(metrics_like_query,
like_string=str(namespace))
connection.close()
for row in results:
metric_name = str(row[0])
metric_names.append(metric_name)
except:
trace = traceback.format_exc()
logger.error(trace)
logger.error(
'error :: could not determine metrics from metrics table')
if engine:
engine_disposal(engine)
raise
logger.info(
'submit_crucible_job :: %s metrics determined from passed namespaces'
% str(len(metric_names)))
logger.info('submit_crucible_job :: %s metrics to process' %
str(len(metric_names)))
metrics_submitted_to_process = []
datapoint = 0
triggered_algorithms = [
'histogram_bins', 'first_hour_average', 'stddev_from_average',
'grubbs', 'ks_test', 'mean_subtraction_cumulation',
'median_absolute_deviation', 'stddev_from_moving_average',
'least_squares'
]
added_at = int(time())
for base_name in metric_names:
sane_metricname = filesafe_metricname(str(base_name))
derivative_metric = is_derivative_metric(skyline_app, base_name)
if derivative_metric:
target = 'nonNegativeDerivative(%s)' % base_name
else:
target = base_name
# Generate a metric job directory
crucible_anomaly_dir = '%s/%s' % (crucible_job_dir, sane_metricname)
try:
if not path.exists(crucible_anomaly_dir):
logger.info('creating crucible metric job directory - %s' %
(str(crucible_anomaly_dir)))
mkdir_p(crucible_anomaly_dir)
except:
trace = traceback.format_exc()
fail_msg = 'error :: failed to create the crucible metric job directory'
logger.error(trace)
logger.error(fail_msg)
raise # to webapp to return in the UI
if source == 'graphite':
graphite_metric = True
else:
graphite_metric = False
# @added 20200422 - Feature #3500: webapp - crucible_process_metrics
# Feature #1448: Crucible web UI
# In order for metrics to be analysed in Crucible like the Analyzer or
# Mirage analysis, the time series data needs to be padded
# Added pad_timeseries
graphite_override_uri_parameters = 'from=%s&until=%s&target=%s' % (
str(from_timestamp), str(until_timestamp), target)
timeseries_full_duration = int(until_timestamp) - int(from_timestamp)
pad_timeseries_with = 0
if pad_timeseries == 'auto':
if timeseries_full_duration > 3600:
pad_timeseries_with = 3600
if timeseries_full_duration > 86400:
pad_timeseries_with = 86400
if pad_timeseries == '86400':
pad_timeseries_with = 86400
if pad_timeseries == '604800':
pad_timeseries_with = 604800
if pad_timeseries == '0':
pad_timeseries_with = 0
if pad_timeseries_with:
try:
padded_from_timestamp = int(
from_timestamp) - pad_timeseries_with
graphite_override_uri_parameters = 'from=%s&until=%s&target=%s' % (
str(padded_from_timestamp), str(until_timestamp), target)
logger.info('padding time series with %s seconds - %s' %
(str(pad_timeseries_with),
str(graphite_override_uri_parameters)))
except:
logger.error(traceback.format_exc())
logger.error(
'error :: failed to construct graphite_override_uri_parameters with pad_timeseries_with %s'
% str(pad_timeseries_with))
# @added 20200817 - Feature #3682: SNAB - webapp - crucible_process - run_algorithms
# Allow the user to pass algorithms to run
algorithms = settings.ALGORITHMS
if run_algorithms:
algorithms = run_algorithms
# @modified 20200421 - Feature #3500: webapp - crucible_process_metrics
# Feature #1448: Crucible web UI
# Added add_to_panorama
# @added 20200607 - Feature #3630: webapp - crucible_process_training_data
# Added training_data_json
crucible_anomaly_data = 'metric = \'%s\'\n' \
'value = \'%s\'\n' \
'from_timestamp = \'%s\'\n' \
'metric_timestamp = \'%s\'\n' \
'algorithms = %s\n' \
'triggered_algorithms = %s\n' \
'anomaly_dir = \'%s\'\n' \
'graphite_metric = %s\n' \
'run_crucible_tests = True\n' \
'added_by = \'%s\'\n' \
'added_at = \'%s\'\n' \
'graphite_override_uri_parameters = \'%s\'\n' \
'alert_interval = \'%s\'\n' \
'add_to_panorama = %s\n' \
'training_data_json = %s\n' \
% (base_name, str(datapoint), str(from_timestamp),
# @modified 20200817 - Feature #3682: SNAB - webapp - crucible_process - run_algorithms
# str(until_timestamp), str(settings.ALGORITHMS),
str(until_timestamp), str(algorithms),
triggered_algorithms, crucible_anomaly_dir, str(graphite_metric),
skyline_app, str(added_at), str(graphite_override_uri_parameters),
str(alert_interval), str(add_to_panorama), str(training_data_json))
# Create an anomaly file with details about the anomaly
crucible_anomaly_file = '%s/%s.txt' % (crucible_anomaly_dir,
sane_metricname)
try:
write_data_to_file(skyline_app, crucible_anomaly_file, 'w',
crucible_anomaly_data)
logger.info('added crucible anomaly file :: %s' %
(crucible_anomaly_file))
except:
logger.error(traceback.format_exc())
logger.error('error :: failed to add crucible anomaly file :: %s' %
(crucible_anomaly_file))
# Create a crucible check file
crucible_check_file = '%s/%s.%s.txt' % (settings.CRUCIBLE_CHECK_PATH,
str(added_at), sane_metricname)
try:
write_data_to_file(skyline_app, crucible_check_file, 'w',
crucible_anomaly_data)
logger.info('added crucible check :: %s,%s' %
(base_name, str(added_at)))
metrics_submitted_to_process.append(base_name)
except:
logger.error(traceback.format_exc())
logger.error('error :: failed to add crucible check file :: %s' %
(crucible_check_file))
return (crucible_job_id, metrics_submitted_to_process, fail_msg, trace)
def get_all_db_metric_names(current_skyline_app, with_ids=False):
"""
Given return all metric names from the database as a list
"""
metric_names = []
metric_names_with_ids = {}
function_str = 'database_queries.get_all_db_metric_names'
current_skyline_app_logger = current_skyline_app + 'Log'
current_logger = logging.getLogger(current_skyline_app_logger)
try:
engine, fail_msg, trace = get_engine(current_skyline_app)
except Exception as e:
trace = traceback.format_exc()
current_logger.error(trace)
fail_msg = 'error :: %s :: could not get a MySQL engine - %s' % (function_str, e)
current_logger.error('%s' % fail_msg)
return False, fail_msg, trace
try:
metrics_table, fail_msg, trace = metrics_table_meta(current_skyline_app, engine)
current_logger.info(fail_msg)
except Exception as e:
trace = traceback.format_exc()
current_logger.error('%s' % trace)
fail_msg = 'error :: %s :: failed to get metrics_table meta - %s' % (
function_str, e)
current_logger.error('%s' % fail_msg)
if engine:
engine_disposal(current_skyline_app, engine)
if current_skyline_app == 'webapp':
# Raise to webapp
raise
return False, fail_msg, trace
try:
connection = engine.connect()
if with_ids:
stmt = select([metrics_table.c.id, metrics_table.c.metric])
else:
stmt = select([metrics_table.c.metric])
result = connection.execute(stmt)
for row in result:
base_name = row['metric']
metric_names.append(base_name)
if with_ids:
metric_names_with_ids[base_name] = row['id']
connection.close()
current_logger.info('%s :: determined metric names from the db: %s' % (
function_str, str(len(metric_names))))
except Exception as e:
trace = traceback.format_exc()
current_logger.error(trace)
fail_msg = 'error :: %s :: could not determine metric names from DB for - %s' % (
function_str, e)
current_logger.error('%s' % fail_msg)
if engine:
engine_disposal(current_skyline_app, engine)
if current_skyline_app == 'webapp':
# Raise to webapp
raise
return False, fail_msg, trace
if engine:
engine_disposal(current_skyline_app, engine)
if not metric_names:
current_logger.error('error :: %s :: no metric names returned from the DB' % (
function_str))
if with_ids:
return metric_names, metric_names_with_ids
return metric_names
def metric_ids_from_metric_like(current_skyline_app, metrics_like_str):
"""
Given a SQL metric name wildcard, return a list of metric_ids
"""
metric_ids = []
function_str = 'database_queries.metric_ids_from_metric_like'
current_skyline_app_logger = current_skyline_app + 'Log'
current_logger = logging.getLogger(current_skyline_app_logger)
try:
engine, fail_msg, trace = get_engine(current_skyline_app)
except Exception as e:
trace = traceback.format_exc()
current_logger.error(trace)
fail_msg = 'error :: %s :: could not get a MySQL engine - %s' % (
function_str, e)
current_logger.error('%s' % fail_msg)
if current_skyline_app == 'webapp':
# Raise to webapp
raise
return False
try:
metrics_table, fail_msg, trace = metrics_table_meta(
current_skyline_app, engine)
current_logger.info(fail_msg)
except Exception as e:
trace = traceback.format_exc()
current_logger.error('%s' % trace)
fail_msg = 'error :: %s :: failed to get metrics_table meta - %s' % (
function_str, e)
current_logger.error('%s' % fail_msg)
if engine:
engine_disposal(current_skyline_app, engine)
if current_skyline_app == 'webapp':
# Raise to webapp
raise
return False, fail_msg, trace
try:
connection = engine.connect()
stmt = select([metrics_table.c.id
]).where(metrics_table.c.metric.ilike(metrics_like_str))
result = connection.execute(stmt)
for row in result:
metric_ids.append(int(row['id']))
connection.close()
current_logger.info(
'%s :: determined %s metric_ids for metric_like_str: %s' %
(function_str, str(len(metric_ids)), metrics_like_str))
except Exception as e:
trace = traceback.format_exc()
current_logger.error(trace)
fail_msg = 'error :: %s :: could not determine ids of metrics_like_str: %s -%s' % (
function_str, str(metrics_like_str), e)
current_logger.error('%s' % fail_msg)
if engine:
engine_disposal(current_skyline_app, engine)
if current_skyline_app == 'webapp':
# Raise to webapp
raise
return False, fail_msg, trace
if engine:
engine_disposal(current_skyline_app, engine)
if not metric_ids:
current_logger.error('error :: %s :: no ids for metrics_like_str: %s' %
str(metrics_like_str))
return metric_ids
def base_name_from_metric_id(current_skyline_app, metric_id, log=False):
"""
Given a metric id, return the base_name
"""
base_name = None
function_str = 'database_queries.base_name_from_metric_id'
def get_log(current_skyline_app):
current_skyline_app_logger = current_skyline_app + 'Log'
current_logger = logging.getLogger(current_skyline_app_logger)
return current_logger
if log:
current_logger = get_log(current_skyline_app)
try:
engine, fail_msg, trace = get_engine(current_skyline_app)
except Exception as e:
if not log:
current_logger = get_log(current_skyline_app)
trace = traceback.format_exc()
current_logger.error(trace)
fail_msg = 'error :: %s :: could not get a MySQL engine - %s' % (
function_str, e)
current_logger.error('%s' % fail_msg)
return False, fail_msg, trace
try:
metrics_table, fail_msg, trace = metrics_table_meta(
current_skyline_app, engine)
if log:
current_logger.info(fail_msg)
except Exception as e:
if not log:
current_logger = get_log(current_skyline_app)
trace = traceback.format_exc()
current_logger.error('%s' % trace)
fail_msg = 'error :: %s :: failed to get metrics_table meta for %s- %s' % (
function_str, base_name, e)
current_logger.error('%s' % fail_msg)
if engine:
engine_disposal(current_skyline_app, engine)
if current_skyline_app == 'webapp':
# Raise to webapp
raise
return False, fail_msg, trace
try:
connection = engine.connect()
stmt = select([metrics_table.c.metric
]).where(metrics_table.c.id == metric_id)
result = connection.execute(stmt)
for row in result:
base_name = row['metric']
break
connection.close()
if log:
current_logger.info(
'%s :: determined metric with id %s base_name: %s' %
(function_str, str(metric_id), base_name))
except Exception as e:
if not log:
current_logger = get_log(current_skyline_app)
trace = traceback.format_exc()
current_logger.error(trace)
fail_msg = 'error :: %s :: could not determine id of metric from DB for %s - %s' % (
function_str, base_name, e)
current_logger.error('%s' % fail_msg)
if engine:
engine_disposal(current_skyline_app, engine)
if current_skyline_app == 'webapp':
# Raise to webapp
raise
return False, fail_msg, trace
if engine:
engine_disposal(current_skyline_app, engine)
if not base_name:
current_logger.error(
'error :: %s :: no base_name for metric in the DB with id %s' %
(function_str, str(metric_id)))
return base_name
def get_correlations(current_skyline_app, anomaly_id):
"""
Get all the correlations for an anomaly from the database
:param current_skyline_app: the Skyline app name calling the function
:param anomaly_id: thee base_name of the metric
:type current_skyline_app: str
:type anomaly_id: int
:return: list
:return: [[metric_name, coefficient, shifted, shifted_coefficient],[metric_name, coefficient, ...]]
:rtype: [[str, float, float, float]]
"""
current_skyline_app_logger = current_skyline_app + 'Log'
current_logger = logging.getLogger(current_skyline_app_logger)
func_name = 'get_correlations'
correlations = []
current_logger.info('get_correlations :: getting MySQL engine')
try:
engine, fail_msg, trace = fp_create_get_an_engine(current_skyline_app)
current_logger.info(fail_msg)
except:
trace = traceback.format_exc()
current_logger.error(trace)
fail_msg = 'error :: could not get a MySQL engine'
current_logger.error('%s' % fail_msg)
# return False, False, fail_msg, trace, False
raise # to webapp to return in the UI
metrics_table = None
try:
metrics_table, fail_msg, trace = metrics_table_meta(current_skyline_app, engine)
current_logger.info(fail_msg)
except:
trace = traceback.format_exc()
current_logger.error('%s' % trace)
fail_msg = 'error :: %s :: failed to get metrics_table_meta' % func_name
current_logger.error('%s' % fail_msg)
luminosity_table = None
try:
luminosity_table, fail_msg, trace = luminosity_table_meta(current_skyline_app, engine)
current_logger.info(fail_msg)
except:
trace = traceback.format_exc()
current_logger.error('%s' % trace)
fail_msg = 'error :: %s :: failed to get luminosity_table_meta' % func_name
current_logger.error('%s' % fail_msg)
metrics_list = []
try:
connection = engine.connect()
stmt = select([metrics_table]).where(metrics_table.c.id > 0)
results = connection.execute(stmt)
for row in results:
metric_id = row['id']
metric_name = row['metric']
metrics_list.append([int(metric_id), str(metric_name)])
connection.close()
except:
current_logger.error(traceback.format_exc())
current_logger.error('error :: could not determine metrics from MySQL')
if engine:
fp_create_engine_disposal(current_skyline_app, engine)
raise
try:
connection = engine.connect()
stmt = select([luminosity_table]).where(luminosity_table.c.id == int(anomaly_id))
result = connection.execute(stmt)
for row in result:
metric_id = row['metric_id']
metric_name = None
if metric_id:
metric_name = [metrics_list_name for metrics_list_id, metrics_list_name in metrics_list if int(metric_id) == int(metrics_list_id)]
coefficient = row['coefficient']
shifted = row['shifted']
shifted_coefficient = row['shifted_coefficient']
correlations.append([metric_name, coefficient, shifted, shifted_coefficient])
connection.close()
except:
current_logger.error(traceback.format_exc())
current_logger.error('error :: could not determine correlations for anomaly id - %s' % str(anomaly_id))
if engine:
fp_create_engine_disposal(current_skyline_app, engine)
raise
if engine:
fp_create_engine_disposal(current_skyline_app, engine)
if correlations:
sorted_correlations = sorted(correlations, key=lambda x: x[1], reverse=True)
correlations = sorted_correlations
return correlations, fail_msg, trace
