class Configuration:
"""docstring for Configuration."""
prometheus_url = os.getenv(
"PROMETEUS_URL",
"http://prometheus-k8s-monitoring.192.168.99.104.nip.io")
prometheus_headers = None
if os.getenv("PROMETEUS_ACCESS_TOKEN"):
prom_connect_headers = {
"Authorization": "bearer " + os.getenv("PROMETEUS_ACCESS_TOKEN")
}
metrics_list = str(
os.getenv(
"METRICS_LIST",
"go_memstats_heap_objects{endpoint='web',instance='172.17.0.17:9090',job='prometheus-k8s',namespace='monitoring',pod='prometheus-k8s-1',service='prometheus-k8s'}",
)).split(";")
rolling_training_window_size = parse_timedelta(
"now", os.getenv("ROLLING_TRAINING_WINDOW_SIZE", "120m"))
retraining_interval_minutes = int(
os.getenv("RETRAINING_INTERVAL_MINUTES", "60"))
metric_chunk_size = parse_timedelta("now",
str(retraining_interval_minutes) + "m")
deviations = int(os.getenv("DEVIATIONS", "3"))
anomaly_border = str(os.getenv("ANOMALY_BORDER", "both"))
algorithm_name = str(os.getenv("ALGORITHM", "agile"))
algorithm_resolver = {
"robust": prophet_model.MetricPredictor,
"agile": sarima_model.MetricPredictor,
"basic": fourier_model.MetricPredictor
}
algorithm = algorithm_resolver.get(algorithm_name)
seasonality = str(os.getenv("SEASONALITY", "daily"))
mlflow_tracking_uri = "http://localhost:5000"
metric_start_time = parse_datetime(
os.getenv("DATA_START_TIME", "2020-02-05 13:00:00"))
metric_end_time = parse_datetime(
os.getenv("DATA_END_TIME", "2020-02-05 13:36:00"))
metric_train_data_end_time = metric_start_time + rolling_training_window_size
_LOGGER.info("Metric train data start time: %s", metric_start_time)
_LOGGER.info("Metric train data end time/test data start time: %s",
metric_train_data_end_time)
_LOGGER.info("Metric test end time: %s", metric_end_time)
_LOGGER.info("Metric data rolling training window size: %s",
rolling_training_window_size)
_LOGGER.info("Model retraining interval: %s minutes",
retraining_interval_minutes)
class Configuration:
"""docstring for Configuration."""
# url for the prometheus host
prometheus_url = os.getenv("FLT_PROM_URL")
# any headers that need to be passed while connecting to the prometheus host
prom_connect_headers = None
# example oath token passed as a header
if os.getenv("FLT_PROM_ACCESS_TOKEN"):
prom_connect_headers = {
"Authorization": "bearer " + os.getenv("FLT_PROM_ACCESS_TOKEN")
}
# example basic passed as a header
if os.getenv("FLT_PROM_ACCESS_BASIC"):
prom_connect_headers = {
"Authorization": "Basic " + os.getenv("FLT_PROM_ACCESS_BASIC")
}
# list of metrics that need to be scraped and predicted
# multiple metrics can be separated with a ";"
# if a metric configuration matches more than one timeseries,
# it will scrape all the timeseries that match the config.
metrics_list = str(
os.getenv(
"FLT_METRICS_LIST",
"up{app='openshift-web-console', instance='172.44.0.18:8443'}",
)).split(";")
# this will create a rolling data window on which the model will be trained
# example: if set to 15d will train the model on past 15 days of data,
# every time new data is added, it will truncate the data that is out of this range.
rolling_training_window_size = parse_timedelta(
"now", os.getenv("FLT_ROLLING_TRAINING_WINDOW_SIZE", "3d"))
# current data window size is the time window from now where the current data value
# will be found
# default is 5 minutes
current_data_window_size = parse_timedelta(
"now", os.getenv("FLT_CURRENT_DATA_WINDOW_SIZE", "5m"))
# How often should the anomaly detector retrain the model (in minutes)
retraining_interval_minutes = int(
os.getenv("FLT_RETRAINING_INTERVAL_MINUTES", "120"))
port = int(os.getenv("PORT", "8088"))
check_perf = bool(int(os.getenv("FLT_CHECK_PERF", "0")))
model_name = os.getenv("FLT_MODEL", list(model_module_map.keys())[0])
model_module = model_module_map[model_name]
_LOGGER.info("Using model: \"%s\"", model_name)
_LOGGER.info("Metric data rolling training window size: %s",
rolling_training_window_size)
_LOGGER.info("Model retraining interval: %s minutes",
retraining_interval_minutes)
class Configuration:
"""docstring for Configuration."""
# url for the prometheus host
prometheus_url = os.getenv("FLT_PROM_URL")
# any headers that need to be passed while connecting to the prometheus host
prom_connect_headers = None
# example oath token passed as a header
if os.getenv("FLT_PROM_ACCESS_TOKEN"):
prom_connect_headers = {
"Authorization": "bearer " + os.getenv("FLT_PROM_ACCESS_TOKEN")
}
# list of metrics that need to be scraped and predicted
# multiple metrics can be separated with a ";"
# if a metric configuration matches more than one timeseries,
# it will scrape all the timeseries that match the config.
metrics_list = str(
os.getenv(
"FLT_METRICS_LIST",
"up{app='openshift-web-console', instance='172.44.0.18:8443'}",
)).split(";")
# this will create a rolling data window on which the model will be trained
# example: if set to 15d will train the model on past 15 days of data,
# every time new data is added, it will truncate the data that is out of this range.
rolling_training_window_size = parse_timedelta(
"now", os.getenv("FLT_ROLLING_TRAINING_WINDOW_SIZE", "3d"))
# How often should the anomaly detector retrain the model (in minutes)
retraining_interval_minutes = int(
os.getenv("FLT_RETRAINING_INTERVAL_MINUTES", "120"))
metric_chunk_size = parse_timedelta("now",
str(retraining_interval_minutes) + "m")
_LOGGER.info("Metric data rolling training window size: %s",
rolling_training_window_size)
_LOGGER.info("Model retraining interval: %s minutes",
retraining_interval_minutes)
# An option for Parallelism.
# An Integer specifying the number of metrics to be trained in parallel.
# Default: 1.
# Note: The upper limit to this will be decided by the number of CPU cores
# available to the container.
parallelism = int(os.getenv("FLT_PARALLELISM", "1"))
def get_holidays(self):
if not 'holidays' in self.template:
return pd.DataFrame([], columns=['holiday', 'ds'])
now = datetime.now()
delta_past = parse_timedelta(
'now', self.template['params']['training_window'])
delta_future = timedelta(minutes=self.get_forecast_minutes())
start_date = now - delta_past
end_date = now + delta_future
return get_holidays_between(get_instance(self.template['holidays']),
start_date.date(), end_date.date())
def query_range(expr, time_range, resolution, fill_na):
fill_method = {
'zeros': fill_na_zeros,
'default': None
}[fill_na or 'default']
delta = parse_timedelta('now', time_range)
start_time = datetime.now() - delta
end_time = datetime.now()
data = pc.custom_query_range(query=expr,
start_time=start_time,
end_time=end_time,
step=resolution)
if fill_method:
for item in data:
item['values'] = fill_method(start_time, end_time, resolution,
item['values'])
return withHash(expr, data)
def fetch(self, expression, number_of_days):
start_time = parse_datetime('%dd' % number_of_days)
end_time = parse_datetime('now')
chunk_size = parse_timedelta('now', '1d')
metric_data = self.prom.get_metric_range_data(
expression,
start_time=start_time,
end_time=end_time,
chunk_size=chunk_size,
)
# MetricsList combines the chunks into a single metric
metric = MetricsList(metric_data)[0]
# Yield tuples of timestamp, value
for value in metric.metric_values.values:
ts, val = value.tolist()
# The timestamp is delivered in UTC, convert to local
ts = ts.to_pydatetime().replace(tzinfo=tz.tzutc())
ts = ts.astimezone(tz.tzlocal())
yield ts, val
class Configuration:
"""docstring for Configuration."""
# url for the prometheus host
prometheus_url = os.getenv("FLT_PROM_URL")
# any headers that need to be passed while connecting to the prometheus host
prom_connect_headers = None
# example oath token passed as a header
if os.getenv("FLT_PROM_ACCESS_TOKEN"):
prom_connect_headers = {
"Authorization": "bearer " + os.getenv("FLT_PROM_ACCESS_TOKEN")
}
# list of metrics that need to be scraped and predicted
# multiple metrics can be separated with a ";"
# if a metric configuration matches more than one timeseries,
# it will scrape all the timeseries that match the config.
metrics_list = str(
os.getenv(
"FLT_METRICS_LIST",
"up{app='openshift-web-console', instance='172.44.0.18:8443'}")
).split(";")
# uri for the mlflow tracking server
mlflow_tracking_uri = str(os.getenv("MLFLOW_TRACKING_URI"))
# threshold value to calculate true anomalies using a linear function
true_anomaly_threshold = float(
os.getenv("FLT_TRUE_ANOMALY_THRESHOLD", "0.001"))
metric_start_time = parse_datetime(
os.getenv("FLT_DATA_START_TIME", "2019-08-05 18:00:00"))
metric_end_time = parse_datetime(
os.getenv("FLT_DATA_END_TIME", "2019-08-08 18:00:00"))
# this will create a rolling data window on which the model will be trained
# example: if set to 15d will train the model on past 15 days of data,
# every time new data is added, it will truncate the data that is out of this range.
rolling_training_window_size = parse_timedelta(
"now", os.getenv("FLT_ROLLING_TRAINING_WINDOW_SIZE", "2d"))
metric_train_data_end_time = metric_start_time + rolling_training_window_size
# How often should the anomaly detector retrain the model (in minutes)
retraining_interval_minutes = int(
os.getenv("FLT_RETRAINING_INTERVAL_MINUTES", "120"))
metric_chunk_size = parse_timedelta("now",
str(retraining_interval_minutes) + "m")
_LOGGER.info("Metric train data start time: %s", metric_start_time)
_LOGGER.info("Metric train data end time/test data start time: %s",
metric_train_data_end_time)
_LOGGER.info("Metric test end time: %s", metric_end_time)
_LOGGER.info("Metric data rolling training window size: %s",
rolling_training_window_size)
_LOGGER.info("Model retraining interval: %s minutes",
retraining_interval_minutes)
_LOGGER.info("True anomaly threshold: %s", true_anomaly_threshold)
_LOGGER.info("MLflow server url: %s", mlflow_tracking_uri)
from prometheus_api_client import Metric, MetricsList, PrometheusConnect
from prometheus_api_client.utils import parse_datetime, parse_timedelta
import matplotlib.pyplot as plt
import pandas as pd
pc = PrometheusConnect(
url="https://prometheus-k8s-openshift-monitoring.apps-crc.testing",
headers={
"Authorization": "bearer 7lmyVwWaTrWZYwiM0KRN30fBw5W70OkcqOMnizZ-cr0"
},
disable_ssl=True)
start_time = parse_datetime("7d")
end_time = parse_datetime("now")
chunk_size = parse_timedelta("now", "1d")
def get_data(metrics, timestamp_filenames, datafile):
def _getTimestamps(timestamp_filename):
print(timestamp_filename)
file_name = open(timestamp_filename, "r")
ts = file_name.readlines()
return ts
def _getMetricsData(metric):
metric_data = pc.get_metric_range_data(
metric,
start_time=start_time,
end_time=end_time,
chunk_size=chunk_size,
def get_interval_minutes(interval):
delta = parse_timedelta('now', interval)
return round(delta.seconds / 60) + round(delta.days * 24 * 60)
