def getNodeList(self):
'''
:return: -> returns the list of registered nodes from dmon
'''
nUrl = "http://%s:%s/dmon/v1/observer/nodes" % (self.esEndpoint,
self.dmonPort)
logger.info(
'[%s] : [INFO] dmon get node url -> %s',
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'),
nUrl)
try:
rdmonNode = requests.get(nUrl)
except Exception as inst:
logger.error(
'[%s] : [ERROR] Exception has occured while connecting to dmon with type %s at arguments %s',
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), type(inst),
inst.args)
sys.exit(2)
rdata = rdmonNode.json()
nodes = []
for e in rdata['Nodes']:
for k in e:
nodes.append(k)
return nodes
def pushAnomalyKafka(self, body):
if self.producer is None:
logger.warning(
'[{}] : [WARN] Kafka reporter not defined, skipping reporting'.
format(
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S')))
else:
try:
self.producer.send(self.prKafkaTopic, body)
# self.producer.flush()
logger.info(
'[{}] : [INFO] Anomalies reported to kafka topic {}'.
format(
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'),
self.prKafkaTopic))
except Exception as inst:
logger.error(
'[{}] : [ERROR] Failed to report anomalies to kafka topic {} with {} and {}'
.format(
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'),
self.prKafkaTopic, type(inst), inst.args))
return 0
def sdbscanTrain(self, settings,
mname,
data):
'''
:param data: -> dataframe with data
:param settings: -> settings dictionary
:param mname: -> name of serialized clusterer
:return: -> clusterer
:example settings: -> {eps:0.9, min_samples:10, metric:'euclidean' ,
algorithm:'auto, leaf_size:30, p:0.2, n_jobs:1}
'''
for k, v in settings.items():
logger.info('[%s] : [INFO] SDBSCAN %s set to %s',
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), k, v)
sdata = StandardScaler().fit_transform(data)
try:
db = DBSCAN(eps=float(settings['eps']), min_samples=int(settings['min_samples']), metric=settings['metric'],
algorithm=settings['algorithm'], leaf_size=int(settings['leaf_size']), p=float(settings['p']),
n_jobs=int(settings['n_jobs'])).fit(sdata)
except Exception as inst:
logger.error('[%s] : [ERROR] Cannot instanciate sDBSCAN with %s and %s',
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), type(inst), inst.args)
print("Error while instanciating sDBSCAN with %s and %s" % (type(inst), inst.args))
sys.exit(1)
labels = db.labels_
print(labels)
n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)
print('Estimated number of clusters: %d' % n_clusters_)
self.__serializemodel(db, 'sdbscan', mname)
return db
def chainMerge(self, lFiles, colNames, iterStart=1):
'''
:param lFiles: -> list of files to be opened
:param colNames: -> dict with master column names
:param iterStart: -> start of iteration default is 1
:return: -> merged dataframe
'''
#Parsing colNames
slaveCol = {}
for k, v in colNames.items():
slaveCol[k] = '_'.join([v.split('_')[0], 'slave'])
dfList = []
if all(isinstance(x, str) for x in lFiles):
for f in lFiles:
df = pd.read_csv(f)
dfList.append(df)
elif all(isinstance(x, pd.DataFrame) for x in lFiles):
dfList = lFiles
else:
logger.error('[%s] : [ERROR] Cannot merge type %s ',
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), str(type(dfList[0])))
sys.exit(1)
# Get first df and set as master
current = dfList[0].rename(columns=colNames)
for i, frame in enumerate(dfList[1:], iterStart):
iterSlave = {}
for k, v in slaveCol.items():
iterSlave[k] = v+str(i)
current = current.merge(frame).rename(columns=iterSlave)
#current.to_csv(mergedFile)
# current.set_index('key', inplace=True)
return current
def pr_status(self, type=None):
"""
Get status of prometheus
TODO: check runtimeinfo and flags
:param type: suported types
:return:
"""
suported = ['runtimeinfo', 'config', 'flags']
if type is None:
pr_target_string = '/api/v1/status/config'
elif type in suported:
pr_target_string = '/api/v1/status/{}'.format(type)
else:
logger.error(
'[{}] : [ERROR] unsupported status type {}, supported types are {}'
.format(
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), type,
suported))
sys.exit(1)
try:
resp = requests.get("http://{}:{}{}".format(
self.prEndpoint, self.MInstancePort, pr_target_string))
except Exception as inst:
logger.error(
'[{}] : [ERROR] Exception has occured while connecting to PR endpoint with type {} at arguments {}'
.format(
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), type(inst),
inst.args))
sys.exit(2)
return resp.json()
def dask_clusterMethod(self, cluster_method,
mname,
data
):
try:
logger.info('[{}] : [INFO] Loading Clustering method {}'.format(
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), type(cluster_method)))
# delattr(cluster_method, 'behaviour')
# del cluster_method.__dict__['behaviour']
for k, v in cluster_method.get_params().items():
logger.info('[{}] : [INFO] Method parameter {} set to {}'.format(
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), k, v))
try:
with joblib.parallel_backend('dask'):
logger.info('[{}] : [INFO] Using Dask backend for user defined method'.format(
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')))
clf = cluster_method.fit(data)
except Exception as inst:
logger.error('[{}] : [ERROR] Failed to fit user defined method with dask backedn with {} and {}'.format(
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), type(inst), inst.args))
logger.warning('[{}] : [WARN] using default process based backedn for user defined method'.format(
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')))
clf = cluster_method.fit(data)
except Exception as inst:
logger.error('[{}] : [ERROR] Failed to fit {} with {} and {}'.format(
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), type(cluster_method),
type(inst), inst.args))
sys.exit(1)
predictions = clf.predict(data)
logger.debug('[{}] : [DEBUG] Predicted Anomaly Array {}'.format(
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), predictions))
fname = str(clf).split('(')[0]
self.__serializemodel(clf, fname, mname)
return clf
def scale(self, data,
scaler_type=None,
rindex='time'): # todo, integrate
if not scaler_type:
logger.warning('[{}] : [WARN] No data scaling used!'.format(
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')))
return data
if scaler_type is None:
scaler_type = {"StandardScaler": {"copy": True, "with_mean": True, "with_std": True}}
logger.warning('[{}] : [WARN] No user defined scaler using default'.format(datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), scaler_type))
scaler_name = list(scaler_type.keys())[-1]
scaler_attr = list(scaler_type.values())[-1]
logger.info('[{}] : [INFO] Scaler set to {} with parameters {}.'.format(datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), scaler_name, scaler_attr))
try:
sc_mod = importlib.import_module(self.scaler_mod)
scaler_instance = getattr(sc_mod, scaler_name)
scaler = scaler_instance(**scaler_attr)
except Exception as inst:
logger.error('[{}] : [ERROR] Error while initializing scaler {}'.format(
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), scaler_name))
sys.exit(2)
# Fit and transform data
logger.info('[{}] : [INFO] Scaling data ...'.format(
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')))
scaled_data = scaler.fit_transform(data)
# Transform numpy array into dataframe, re-add columns to scaled numpyarray
df_scaled = pd.DataFrame(scaled_data, columns=data.columns)
df_scaled[rindex] = list(data.index)
df_scaled.set_index(rindex, inplace=True)
scaler_file = '{}.scaler'.format(scaler_name)
logger.info('[{}] : [INFO] Saving scaler instance {} ...'.format(
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), scaler_file))
scale_file_location = os.path.join(self.dataDir, scaler_file)
joblib.dump(scaler, filename=scale_file_location)
return df_scaled
def isolationForest(self, settings, mname, data):
'''
:param settings: -> settings dictionary
:param mname: -> name of serialized clusterer
:return: -> isolation forest instance
:example settings: -> {n_estimators:100, max_samples:100, contamination:0.1, bootstrap:False,
max_features:1.0, n_jobs:1, random_state:None, verbose:0}
'''
# rng = np.random.RandomState(42)
if settings['random_state'] == 'None':
settings['random_state'] = None
if isinstance(settings['bootstrap'], str):
settings['bootstrap'] = str2Bool(settings['bootstrap'])
if isinstance(settings['verbose'], str):
settings['verbose'] = str2Bool(settings['verbose'])
if settings['max_samples'] != 'auto':
settings['max_samples'] = int(settings['max_samples'])
# print type(settings['max_samples'])
for k, v in settings.items():
logger.info(
'[%s] : [INFO] IsolationForest %s set to %s',
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), k, v)
print("IsolationForest %s set to %s" % (k, v))
try:
clf = IsolationForest(n_estimators=int(settings['n_estimators']),
max_samples=settings['max_samples'],
contamination=float(
settings['contamination']),
bootstrap=settings['bootstrap'],
max_features=float(settings['max_features']),
n_jobs=int(settings['n_jobs']),
random_state=settings['random_state'],
verbose=settings['verbose'])
except Exception as inst:
logger.error(
'[%s] : [ERROR] Cannot instanciate isolation forest with %s and %s',
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), type(inst),
inst.args)
sys.exit(1)
# clf = IsolationForest(max_samples=100, random_state=rng)
# print "*&*&*&& %s" % type(data)
try:
clf.fit(data)
except Exception as inst:
logger.error(
'[%s] : [ERROR] Cannot fit isolation forest model with %s and %s',
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), type(inst),
inst.args)
sys.exit(1)
predict = clf.predict(data)
print("Anomaly Array:")
print(predict)
self.__serializemodel(clf, 'isoforest', mname)
return clf
def inner(func):
if kwargs['endpont'] is None:
logger.error(
'[{}] : [ERROR] Endpoint not defined in config'.format(
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S')))
sys.exit(1)
else:
return func
def dask_sdbscanTrain(self, settings, mname, data, scaler=None):
'''
:param data: -> dataframe with data
:param settings: -> settings dictionary
:param mname: -> name of serialized clusterer
:param scaler: -> scaler to use on data
:return: -> clusterer
:example settings: -> {eps:0.9, min_samples:10, metric:'euclidean' ,
algorithm:'auto, leaf_size:30, p:0.2, n_jobs:1}
'''
if scaler is None:
logger.warning('[{}] : [WARN] Scaler not defined'.format(
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S')))
else:
logger.info('[{}] : [INFO] Scaling data ...'.format(
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S')))
data = scaler.fit_transform(data)
if not settings or settings is None:
logger.warning(
'[{}] : [WARN] No DBScan parameters defined using default'.
format(
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S')))
settings = {}
else:
for k, v in settings.items():
logger.info(
'[{}] : [INFO] DBScan parameter {} set to {}'.format(
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), k, v))
try:
db = DBSCAN(**settings).fit(data)
except Exception as inst:
logger.error(
'[{}] : [INFO] Failed to instanciate DBScan with {} and {}'.
format(
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), type(inst),
inst.args))
sys.exit(1)
labels = db.labels_
logger.info('[{}] : [INFO] DBScan labels: {} '.format(
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'),
labels))
n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)
logger.info(
'[{}] : [INFO] DBScan estimated number of clusters {} '.format(
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), n_clusters_))
self.__serializemodel(db, 'sdbscan', mname)
return db
def prtoDF(self, data,
checkpoint=False,
verbose=False,
index=None,
detect=False):
"""
From PR backend to dataframe
:param data: PR response JSON
:return: dataframe
"""
if not data:
logger.error('[{}] : [ERROR] PR query response is empty, exiting.'.format(
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')))
sys.exit(2)
df = pd.DataFrame()
df_time = pd.DataFrame()
if verbose:
dr = tqdm.tqdm(data['data']['result'])
else:
dr = data['data']['result']
for el in dr:
metric_name = el['metric']['__name__']
instance_name = el['metric']['instance']
new_metric = "{}_{}".format(metric_name, instance_name)
values = el['values']
proc_val = []
proc_time = []
for val in values:
proc_val.append(val[1])
proc_time.append(val[0])
df[new_metric] = proc_val
time_new_metric = "time_{}".format(new_metric)
df_time[time_new_metric] = proc_time
# Calculate the meant time for all metrics
df_time['mean'] = df_time.mean(axis=1)
# Round to np.ceil all metrics
df_time['mean'] = df_time['mean'].apply(np.ceil)
# Add the meant time to rest of metrics
df['time'] = df_time['mean']
logger.info('[{}] : [INFO] PR query resulted in dataframe of size: {}'.format(
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), df.shape))
if index is not None:
df.set_index(index, inplace=True)
logger.warning('[{}] : [WARN] PR query dataframe index set to {}'.format(
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), index))
if checkpoint:
if detect:
pr = "pr_data_detect.csv"
else:
pr = "pr_data.csv"
pr_csv_loc = os.path.join(self.dataDir, pr)
df.to_csv(pr_csv_loc, index=True)
logger.info('[{}] : [INFO] PR query dataframe persisted to {}'.format(
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), self.dataDir))
return df
def info(self):
# self.__check_valid_es()
try:
res = self.esInstance.info()
except Exception as inst:
logger.error(
'[%s] : [ERROR] Exception has occured while connecting to ES dmon with type %s at arguments %s',
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), type(inst),
inst.args)
sys.exit(2)
return res
def toDF(self, fileName):
'''
:param fileName: absolute path to file
:return: dataframe
'''
if not os.path.isfile(fileName):
print("File %s does not exist, cannot load data! Exiting ..." % str(fileName))
logger.error('[%s] : [ERROR] File %s does not exist',
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), str(fileName))
sys.exit(1)
df = pd.read_csv(fileName)
return df
def deleteWatch(self, watch_id):
try:
self.esInstance.watcher.delete_watch(id=watch_id, force=True)
except Exception as inst:
logger.error(
'[%s] : [ERROR] Could not delete watch %s with %s and %s!',
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), watch_id,
type(inst), inst.args)
logger.info(
'[%s] : [INFO] Watch %s succesfully deleted!',
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'))
def varianceSelection(self, df, threashold=.8):
if not isinstance(df, pandas.core.frame.DataFrame):
logger.error(
'[%s] : [ERROR] Variance selection only possible on Dataframe not %s',
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), type(df))
sys.exit(1)
sel = VarianceThreshold(threshold=(threashold * (1 - threashold)))
sel.fit_transform(df)
return df[[
c for (s, c) in zip(sel.get_support(), df.columns.values) if s
]]
def __operationMethod(self, method,
data):
try:
logger.info('[{}] : [INFO] Loading user defined operation'.format(
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')))
data_op = method(data)
except Exception as inst:
logger.error('[{}] : [ERROR] Failed to load user operation with {} and {}'.format(
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), type(inst), inst.args))
return data
logger.info('[{}] : [INFO] Finished user operation'.format(
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')))
return data_op
def closeIndex(self, indexName):
try:
self.esInstance.close(index=indexName)
logger.info(
'[%s] : [INFO] Closed index %s',
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), indexName)
except Exception as inst:
logger.error(
'[%s] : [ERROR] Failed to close index %s with %s and %s',
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), indexName,
type(inst), inst.args)
def pushAnomalyES(self, anomalyIndex, doc_type, body):
try:
res = self.esInstance.index(index=anomalyIndex,
doc_type=doc_type,
body=body)
except Exception as inst:
logger.error(
'[%s] : [ERROR] Exception has occured while pushing anomaly with type %s at arguments %s',
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), type(inst),
inst.args)
sys.exit(2)
return res
def createIndex(self, indexName):
# self.__check_valid_es()
try:
self.esInstance.create(index=indexName, ignore=400)
logger.info(
'[%s] : [INFO] Created index %s',
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), indexName)
except Exception as inst:
logger.error(
'[%s] : [ERROR] Failed to created index %s with %s and %s',
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), indexName,
type(inst), inst.args)
def watcherInfo(self):
try:
ver = self.esInstance.watcher.info()['version']['number']
except Exception as inst:
logger.error(
'[%s] : [ERROR] Could not find ES watcher with %s and %s!',
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), type(inst),
inst.args)
return 1
logger.info(
'[%s] : [INFO] Watcher version %s detected',
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'),
ver)
return ver
def dask_isolationForest(self, settings,
mname,
data
):
'''
:param settings: -> settings dictionary
:param mname: -> name of serialized clusterer
:param scaler: -> scaler to use on data
:return: -> isolation forest instance
:example settings: -> {n_estimators:100, max_samples:100, contamination:0.1, bootstrap:False,
max_features:1.0, n_jobs:1, random_state:None, verbose:0}
'''
if not settings or settings is None:
logger.warning('[{}] : [WARN] No IsolationForest parameters defined using defaults'.format(
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')))
# print(settings)
settings = {}
else:
for k, v in settings.items():
logger.info('[{}] : [INFO] IsolationForest parameter {} set to {}'.format(
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), k, v))
try:
clf = IsolationForest(**settings)
# print(clf)
except Exception as inst:
logger.error('[{}] : [INFO] Failed to instanciate IsolationForest with {} and {}'.format(
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), type(inst), inst.args))
sys.exit(1)
try:
with joblib.parallel_backend('dask'):
logger.info('[{}] : [INFO] Using Dask backend for IsolationForest'.format(
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')))
clf.fit(data)
except Exception as inst:
logger.error('[{}] : [ERROR] Failed to fit IsolationForest with {} and {}'.format(
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), type(inst), inst.args))
sys.exit(1)
predict = clf.predict(data)
anoOnly = np.argwhere(predict == -1)
logger.info('[{}] : [INFO] Found {} anomalies in training dataset of shape {}.'.format(
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), len(anoOnly), data.shape))
logger.info('[{}] : [DEBUG] Predicted Anomaly Array {}'.format(
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'), predict))
self.__serializemodel(clf, 'isoforest', mname)
self.__appendPredictions(method='isoforest', mname=mname, data=data, pred=predict)
def deleteIndex(self, indexName):
try:
res = self.esInstance.indices.delete(index=indexName,
ignore=[400, 404])
logger.info(
'[%s] : [INFO] Deleted index %s',
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), indexName)
except Exception as inst:
logger.error(
'[%s] : [ERROR] Failed to delete index %s with %s and %s',
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), indexName,
type(inst), inst.args)
return 0
return res
def pr_labels(self, label=None):
if label is None:
pr_target_string = '/api/v1/labels'
else:
pr_target_string = '/api/v1/label/{}/values'.format(label)
try:
resp = requests.get("http://{}:{}{}".format(
self.prEndpoint, self.MInstancePort, pr_target_string))
except Exception as inst:
logger.error(
'[{}] : [ERROR] Exception has occured while connecting to PR endpoint with type {} at arguments {}'
.format(
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), type(inst),
inst.args))
sys.exit(2)
return resp.json()
def getDmonStatus(self):
nUrl = "http://%s:%s/dmon/v1/overlord/core/status" % (self.esEndpoint,
self.dmonPort)
logger.info(
'[%s] : [INFO] dmon get core status url -> %s',
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'),
nUrl)
try:
rdmonStatus = requests.get(nUrl)
except Exception as inst:
logger.error(
'[%s] : [ERROR] Exception has occured while connecting to dmon with type %s at arguments %s',
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), type(inst),
inst.args)
sys.exit(2)
return rdmonStatus.json()
def aggQuery(self, queryBody):
adt_timeout = os.environ['ADP_TIMEOUT'] = os.getenv(
'ADP_TIMEOUT', str(60)
) # Set timeout as env variable ADT_TIMEOUT, if not set use default 60
# print "QueryString -> {}".format(queryBody)
try:
res = self.esInstance.search(index=self.myIndex,
body=queryBody,
request_timeout=float(adt_timeout))
except Exception as inst:
logger.error(
'[%s] : [ERROR] Exception while executing ES query with %s and %s',
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), type(inst),
inst.args)
sys.exit(2)
return res
def localData(self, data):
data_loc = os.path.join(self.dataDir, data)
try:
df = pd.read_csv(data_loc)
except Exception as inst:
logger.error(
'[{}] : [ERROR] Cannot load local data with {} and {}'.format(
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), type(inst),
inst.args))
sys.exit(2)
logger.info(
'[{}] : [INFO] Loading local data from {} with shape {}'.format(
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), data_loc,
df.shape))
return df
def pr_targets(self):
"""
Get Monitored Target Info
:return: Targets Dict
"""
pr_target_string = '/api/v1/targets'
try:
resp = requests.get("http://{}:{}{}".format(
self.prEndpoint, self.MInstancePort, pr_target_string))
except Exception as inst:
logger.error(
'[{}] : [ERROR] Exception has occured while connecting to PR endpoint with type {} at arguments {}'
.format(
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), type(inst),
inst.args))
sys.exit(2)
return resp.json()
def roles(self):
# self.__check_valid_es()
nUrl = "http://%s:%s/dmon/v1/overlord/nodes/roles" % (self.esEndpoint,
self.dmonPort)
logger.info(
'[%s] : [INFO] dmon get roles url -> %s',
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'),
nUrl)
try:
rRoles = requests.get(nUrl)
except Exception as inst:
logger.error(
'[%s] : [ERROR] Exception has occured while connecting to dmon with type %s at arguments %s',
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), type(inst),
inst.args)
sys.exit(2)
rData = rRoles.json()
return rData
def pr_query(self, query):
"""
QUery Monitoring Data From PR backend
:param query: Query string for PR backend
:return: Monitoring Data
"""
try:
url = '/api/v1/query'
resp = requests.get('http://{}:{}{}'.format(
self.prEndpoint, self.MInstancePort, url),
params=query)
except Exception as inst:
logger.error(
'[{}] : [ERROR] Exception has occured while connecting to PR endpoint with type {} at arguments {}'
.format(
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), type(inst),
inst.args))
sys.exit(2)
return resp.json()
def getStormTopology(self):
nUrl = "http://%s:%s/dmon/v1/overlord/detect/storm" % (self.esEndpoint,
self.dmonPort)
logger.info(
'[%s] : [INFO] dmon get storm topology url -> %s',
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'),
nUrl)
try:
rStormTopology = requests.get(nUrl)
except Exception as inst:
logger.error(
'[%s] : [ERROR] Exception has occured while connecting to dmon with type %s at arguments %s',
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), type(inst),
inst.args)
print("Can't connect to dmon at %s port %s" %
(self.esEndpoint, self.dmonPort))
sys.exit(2)
rData = rStormTopology.json()
return rData