def test_avg_same(self):
testMetric = Metric("A")
testMetric.addDataPoint(4)
testMetric.addDataPoint(4)
testMetric.addDataPoint(4)
testMetric.addDataPoint(4)
self.assertEqual(4, testMetric.Avg)
def test_sum(self):
testMetric = Metric("A")
testMetric.addDataPoint(1)
testMetric.addDataPoint(2)
testMetric.addDataPoint(3)
testMetric.addDataPoint(4)
self.assertEqual(10, testMetric.Sum)
def test_avg_different(self):
testMetric = Metric("A")
testMetric.addDataPoint(16)
testMetric.addDataPoint(8)
testMetric.addDataPoint(4)
testMetric.addDataPoint(4)
self.assertEqual(8, testMetric.Avg)
def test_n(self):
testMetric = Metric("A")
testMetric.addDataPoint(1)
testMetric.addDataPoint(1)
testMetric.addDataPoint(1)
testMetric.addDataPoint(1)
testMetric.addDataPoint(1)
testMetric.addDataPoint(1)
self.assertEqual(6, testMetric.N)
def __init__(self):
print "the log class"
self.metric = Metric()
self.mach_learn = MachineLearning()
self.storage_nodes = []
self.normal_value_count = []
self.metrics = []
self.anomaly_value_count = []
self.thresholds_values = []
self.thresholds_lines = []
self.no_of_lines = 200
def __init__(self, function, env_points, min_metric_threshold,
max_metric_threshold, metric_is_objective):
"""
@arguments
function -- see class description
env_points -- see class description
min_metric_threshold -- float/ int -- lower bound; helps define metric
max_metric_threshold -- float/ int -- upper bound; helps define metric
metric_is_objective -- bool -- is metric an objective (vs.
constraint(s) ?)
@return
FunctionAnalysis
"""
Analysis.__init__(self, env_points)
self.function = function
metric_name = 'metric_' + function.func_name
self.metric = Metric(metric_name, min_metric_threshold,
max_metric_threshold, metric_is_objective)
def __init__(self, data, mode, sampled_data_index=None):
self.data = data
self.mode = mode
self.metric = Metric()
# Original Data
self.X = self.data.iloc[:, 0:9].values.tolist()
self.Y = self.data.iloc[:, -1].values.tolist()
(self.sampled_X, self.sampled_Y) = ([_ for _ in xrange(len(self.X))],
[_ for _ in xrange(len(self.Y))])
for i in xrange(len(self.sampled_X)):
self.sampled_X[i] = self.X[i]
self.sampled_Y[i] = self.Y[i]
self.RF()
self.NN()
self.RT()
self.BRT()
self.SVR()
self.MARS()
def __init__(self):
# Reading the configuration file of HMS
self.metrics = [
"account-auditor failures", "account-auditor timing",
"account-reaper timing", "account-reaper container_failures",
"account-reaper object-failures", "account-server GET timing",
"account-server HEAD timing", "account-server PUT timing",
"account-server POST timing", "account-server DELETE timing",
"account-server REPLICATE timing", "account-replicator failures",
"account-replicator timing ", "container-auditor failures",
"container-auditor timing", "container-replicator failures",
"container-replicator timing", "container-server GET timing",
"container-server HEAD timing", "container-server PUT timing",
"container-server POST timing", "container-server DELETE timing",
"container-server REPLICATE timing",
"container-sync deletes timing", "container-sync puts timing",
"container-updater failures", "container-updater timing",
"object-auditor timing", "object-expirer timing",
"object-replicator partition delete timing",
"object-replicator partition update timing",
"object-server GET timing", "object-server HEAD timing",
"object-server PUT timing", "object-server POST timing",
"object-server DELETE timing", "object-server REPLICATE timing",
"object-updater timing"
]
self.thresholds_lines = []
self.thresholds_values = []
self.local_time = time.time()
self.line_data = ""
self.no_of_lines = 200
self.warning_dynamic_metric_name = []
self.warning_dynamic_thresholds_values = []
self.warning_dynamic_thresholds_lines = []
self.threshold_count = 0
self.metric_name = ""
self.prefix = " "
self.metric_value = 0.0
self.normal_value_count = []
self.anomaly_value_count = []
self.storage_nodes = []
self.storage_nodes_count = 0
self.now1 = datetime.now()
self.join_line = ""
# Creating objects for Log.py, MachineLearning.py and Metric.py
log = Log()
metric = Metric()
# Reading the file
log.create_new_file()
self.fo = open(os.path.join("log", "logfile.txt"), "a+")
# Read the configuration file
self.conf = ConfigParser.ConfigParser()
self.conf.read('tulsiclient.conf')
try:
print "Reading conf parameters "
self.host = self.conf.get('tulsistatsd', 'host')
log_duration = self.conf.get('tulsistatsd', 'log_duration')
self.port = int(self.conf.get('tulsistatsd', 'port'))
self.log_duration = log_duration
except:
print "Conf not able to read"
self.host = gethostbyname(gethostname())
self.log_duration = 9
self.port = 8125
self.buf = 105600
self.addr = (self.host, self.port)
self.UDPSock = socket(AF_INET, SOCK_DGRAM)
self.UDPSock.bind((self.host, self.port))
self.now = int(time.time())
# Starting the Infinite while loop
while True:
data = self.recv_msg()
self.line_data = str(data)
print data
self.later = int(time.time())
if (self.fo.tell()) > 524288000:
self.fo.close()
self.fo = log.create_new_file()
self.now = int(time.time())
else:
self.fo.close()
self.fo = open(os.path.join("log", "logfile.txt"), "a+")
# Condition to check the number of lines for logfiles
# If they exceed beyond 10000 a new file will be created
# backing up the exisiting file with timestamp
if len(list(self.fo)) > 10000:
timestr = time.strftime("%Y_%m_%d-%H_%M_%S")
filename = "logfile_" + timestr + ".txt"
self.fo.close()
os.rename(os.path.join("log", "logfile.txt"),
os.path.join("log", filename))
self.fo = log.create_new_file()
else:
self.fo = log.create_new_file()
self.local_time = time.strftime("%d %a %H:%M:%S - ",
time.localtime(time.time()))
self.test_time = datetime.now().strftime("%H:%M:%S.%f")
str_data = self.local_time, 'I: ', self.line_data
self.join_line = ''.join(str_data)
(self.prefix, self.metric_value,
self.metric_name) = metric.parsing_line(self.join_line)
print self.storage_nodes
metric.node_files(self.metric_value)
self.log_files = log.create_log_files(self.prefix)
(self.storage_nodes, self.normal_value_count,
self.anomaly_value_count, self.thresholds_values,
self.thresholds_lines) = log.setmessagelevel(
self.line_data, self.local_time, self.storage_nodes,
self.normal_value_count, self.metrics,
self.anomaly_value_count, self.thresholds_values,
self.thresholds_lines, self.fo, self.log_files,
self.metric_name, self.prefix)
print self.storage_nodes
print(sys.argv)
_input_mat = np.load(sys.argv[1])
gt_array = np.load(sys.argv[2])
# define hyper-parameters
lib_path = "./ProjL1InfPyAPI.so"
_lambda = 3e-3
mu = .1
maxIter = 50
errThr = 1e-6
# predict
pred_label = run(_input_mat, lib_path, _lambda, mu, errThr, maxIter)
# evaluate
performance = Metric(pred_label, gt_array)
print("\n*****************Performance************************")
print("------------------Metrics1----------------------------")
print("| TP: %d" % performance[5])
print("| FP: %d" % performance[6])
print("| TN: %d" % performance[7])
print("| FN: %d" % performance[8])
print("| TPR: %.4f" % performance[9])
print("| FPR: %.4f" % performance[10])
print("|-----------------Metrics2----------------------------")
print("| PRE: %.4f" % performance[0])
print("| REC: %.4f" % performance[1])
print("| F1 : %.4f" % performance[2])
print("| RI : %.4f" % performance[3])
print("| ARI: %.4f" % performance[4])
print("|--------------Categories Number----------------------")
def __init__(self, name, value):
Metric.__init__(self, name)
self.value = value
class Log(object):
def __init__(self):
print "the log class"
self.metric = Metric()
self.mach_learn = MachineLearning()
self.storage_nodes = []
self.normal_value_count = []
self.metrics = []
self.anomaly_value_count = []
self.thresholds_values = []
self.thresholds_lines = []
self.no_of_lines = 200
def write_data_to_file(self, file_name, line):
file_name.write(line)
file_name.write("\n")
file_name.flush()
def create_log_files(self, prefix):
self.prefix = prefix
if not os.path.exists("Metric\\" + self.prefix):
os.makedirs("Metric\\" + self.prefix)
return open(os.path.join("Metric\\" + self.prefix, "log.txt"), "a+")
def check_file(self, file_name, prefix):
self.prefix = prefix
self.filename = file_name + ".txt"
filepath = os.path.join("Metric\\" + self.prefix, self.filename)
if ((os.path.exists(filepath)) == False):
open(os.path.join("Metric\\" + self.prefix, self.filename), "a")
else:
pass
def setmessagelevel(
self,
line_data,
local_time,
storage_nodes,
normal_value_count,
metrics,
anomaly_value_count,
thresholds_values,
thresholds_lines,
fo,
lo,
metric_name,
prefix,
):
self.metric_name = metric_name
self.fo = fo
self.log_files = lo
self.anomaly_value_count = anomaly_value_count
self.line_data = line_data
self.local_time = local_time
self.storage_nodes = storage_nodes
self.normal_value_count = normal_value_count
self.thresholds_values = thresholds_values
self.thresholds_lines = thresholds_lines
self.metrics = metrics
self.prefix = prefix
if self.line_data.find("errors.timing") > 0:
str_data = self.local_time, 'E: ', self.line_data
self.join_line = ''.join(str_data)
self.metric.write_data_to_file(self.fo, self.join_line)
self.metric.write_data_to_file(self.log_files, self.join_line)
elif (((self.line_data.find("errors")) > 0) |
((self.line_data.find("quarantines")) > 0)):
str_data = self.local_time, 'C: ', self.line_data
self.join_line = ''.join(str_data)
self.metric.write_data_to_file(self.fo, self.join_line)
self.metric.write_data_to_file(self.log_files, self.join_line)
elif self.line_data.find("timing") > 0:
str_data = self.local_time, 'I: ', self.line_data
self.join_line = ''.join(str_data)
self.metric.parsing_line(self.join_line)
if len(self.storage_nodes) == 0:
self.storage_nodes.append(self.prefix)
self.normal_value_count.append(
[0 * x for x in range(0, len(self.metrics))])
self.anomaly_value_count.append(
[0 * x for x in range(0, len(self.metrics))])
self.thresholds_values.append(
[0 for x in range(0, len(self.metrics))])
self.thresholds_lines.append(
[self.no_of_lines for x in range(0, len(self.metrics))])
self.mach_learn.anomaly_detection(
0, self.metrics, self.fo, self.log_files, self.metric_name,
self.prefix, self.local_time, self.line_data,
self.normal_value_count, self.thresholds_lines,
self.thresholds_values
) # calling function code to detect anomaly values
self.storage_nodes_count = 1
else:
for i in range(0, len(self.storage_nodes)):
if self.prefix == self.storage_nodes[i]:
self.mach_learn.anomaly_detection(
i, self.metrics, self.fo, self.log_files,
self.metric_name, self.prefix, self.local_time,
self.line_data, self.normal_value_count,
self.thresholds_lines, self.thresholds_values
) # calling function code to detect anomaly values
self.storage_nodes_count = 1
if self.storage_nodes_count == 0:
self.storage_nodes.append(self.prefix)
self.normal_value_count.append(
[0 * x for x in range(0, len(self.metrics))])
self.anomaly_value_count.append(
[0 * x for x in range(0, len(self.metrics))])
self.thresholds_values.append(
[0 for x in range(0, len(self.metrics))])
self.thresholds_lines.append(
[self.no_of_lines for x in range(0, len(self.metrics))])
self.mach_learn.anomaly_detection(
len(self.storage_nodes) - 1, self.metrics, self.fo,
self.log_files, self.metric_name, self.prefix,
self.local_time, self.line_data, self.normal_value_count,
self.thresholds_lines, self.thresholds_values
) # calling function code to detect anomaly values
self.storage_nodes_count = 0
else:
str_data = self.local_time, 'I: ', self.line_data
self.join_line = ''.join(str_data)
self.metric.write_data_to_file(self.fo, self.join_line)
self.metric.write_data_to_file(self.log_files, self.join_line)
return self.storage_nodes, self.normal_value_count, self.anomaly_value_count, self.thresholds_values, self.thresholds_lines
def create_new_file(self):
self.timestr = time.strftime("%Y%m%d-%H%M%S")
self.filename = "logfile.txt"
if not os.path.exists("log"):
os.makedirs("log")
return open(os.path.join("log", self.filename), "a")
def run(self, tempDistanceMetric=None, iDims=2, strDistanceMatrixFile=None, istrmTree=None, istrmEnvr=None):
"""
Runs analysis on loaded data.
:param tempDistanceMetric: The name of the distance metric to use when performing PCoA.
None indicates a distance matrix was already given when loading and will be used.
Supports "braycurtis","canberra","chebyshev","cityblock","correlation",
"cosine","euclidean","hamming","sqeuclidean",unifrac_unweighted","unifrac_weighted"
:type: String Distance matrix name
:param iDims: How many dimension to plot the PCoA graphs.
(This can be minimally 2; all combinations of dimensions are plotted).
iDims start with 1 (not index-based).
:type: Integer Positive integer 2 or greater.
:param strDistanceMatrixFile: If the underlying distance matrix should be output, this is the file to output to.
:type: String Output file for distances of None for indicating it shoudl not be done.
:param istrmTree: One of two files needed for unifrac calculations, this is the phylogeny of the features.
:type: String Path to file
:param istrmEnvr: One of two files needed for unifrac calculations, this is the environment file for the features.
:type: String Path to file
:return boolean: Indicator of success (True)
"""
if iDims > 1:
self._iDimensions = iDims
#If distance metric is none, check to see if the matrix is a distance matrix
#If so, run NMDS on the distance matrix
#Otherwise return a false and do not run
if(tempDistanceMetric==None):
if(ValidateData.funcIsTrue(self.isRawData)):
print("PCoA:run::Error, no distance metric was specified but the previous load was not of a distance matrix.")
return False
elif(ValidateData.funcIsFalse(self.isRawData)):
self.pcoa = NMDS(dataMatrix, verbosity=0)
return True
#Make sure the distance metric was a valid string type
if(not ValidateData.funcIsValidString(tempDistanceMetric)):
print("PCoA:run::Error, distance metric was not a valid string type.")
return False
#Supported distances
distanceMatrix = None
if(tempDistanceMetric==self.c_SPEARMAN):
distanceMatrix = Metric().funcGetDissimilarity(ldSampleTaxaAbundancies=self.dataMatrix, funcDistanceFunction=lambda u,v: spearmanr(u,v)[0])
if(tempDistanceMetric in [Metric.c_strUnifracUnweighted,Metric.c_strUnifracWeighted]):
distanceMatrix,lsLabels = Metric().funcGetBetaMetric(sMetric=tempDistanceMetric, istrmTree=istrmTree, istrmEnvr=istrmEnvr)
self.lsIDs = lsLabels
else:
distanceMatrix = Metric().funcGetBetaMetric(npadAbundancies=self.dataMatrix, sMetric=tempDistanceMetric)
if(ValidateData.funcIsFalse(distanceMatrix)):
print "PCoA:run::Error, when generating distance matrix."
return False
# Make squareform
distanceMatrix = squareform(distanceMatrix)
# Writes distance measures if needed.
if strDistanceMatrixFile:
csvrDistance = csv.writer(open(strDistanceMatrixFile, 'w'))
if self.lsIDs:
csvrDistance.writerow(["ID"]+self.lsIDs)
for x in xrange(distanceMatrix.shape[0]):
strId = [self.lsIDs[x]] if self.lsIDs else []
csvrDistance.writerow(strId+distanceMatrix[x].tolist())
self.pcoa = NMDS(distanceMatrix, dimension=max(self._iDimensions,2), verbosity=0)
self.strRecentMetric = tempDistanceMetric
return True
class Regression(object):
"""This class is used to implement 6 different regression techniques
1. Regression Tree (RT)
2. Random Forest (RF)
3. Boosted Regression Tree (BRT)
4. Support Vector Regressor (SVR)
5. Neural Network (NN)
6. Multivariate Adaptive Regression Splines (MARS)
"""
def __init__(self, data, mode, sampled_data_index=None):
self.data = data
self.mode = mode
self.metric = Metric()
# Original Data
self.X = self.data.iloc[:, 0:9].values.tolist()
self.Y = self.data.iloc[:, -1].values.tolist()
if sampled_data_index is not None:
# Sampled Data
self.sampled_data_index = sampled_data_index
(self.sampled_X, self.sampled_Y) = ([
_ for _ in xrange(len(self.sampled_data_index))
], [_ for _ in xrange(len(self.sampled_data_index))])
for i in xrange(len(self.sampled_X)):
self.sampled_X[i] = self.X[i]
self.sampled_Y[i] = self.Y[i]
def get_sampled_data(self):
"""This function is used to return sampled data
@returns sampled_X, sampled_Y
"""
return (self.sampled_X, self.sampled_Y)
def RT(self, X=None, Y=None):
"""This function is used to implement a regression tree
"""
from sklearn.tree import DecisionTreeRegressor
rgr = DecisionTreeRegressor(random_state=20)
if (X is not None and Y is not None):
(self.sampled_X, self.sampled_Y) = (X, Y)
# train
rgr.fit(self.sampled_X, self.sampled_Y)
# test
Y_pred = rgr.predict(self.X)
# compute metric
m_nmse = self.metric.normalized_mean_square_error(Y_pred, self.Y)
m_mape = self.metric.mean_absolute_percentage_error(Y_pred, self.Y)
return (m_nmse, m_mape)
def RF(self, X=None, Y=None):
"""This function is used to implement a random forest
"""
from sklearn.ensemble import RandomForestRegressor
rgr = RandomForestRegressor(n_estimators=512,
max_depth=None,
max_features=None,
max_leaf_nodes=None)
if (X is not None and Y is not None):
(self.sampled_X, self.sampled_Y) = (X, Y)
# train
rgr.fit(self.sampled_X, self.sampled_Y)
# test
Y_pred = rgr.predict(self.X)
# compute metric
m_nmse = self.metric.normalized_mean_square_error(Y_pred, self.Y)
m_mape = self.metric.mean_absolute_percentage_error(Y_pred, self.Y)
return (m_nmse, m_mape)
def SVR(self, X=None, Y=None):
"""This function is used to implement a Support Vector Regressor
"""
from sklearn.svm import SVR
rgr = SVR(gamma='scale', C=1.0, epsilon=0.1)
if (X is not None and Y is not None):
(self.sampled_X, self.sampled_Y) = (X, Y)
# train
rgr.fit(self.sampled_X, self.sampled_Y)
# test
Y_pred = rgr.predict(self.X)
# compute metric
m_nmse = self.metric.normalized_mean_square_error(Y_pred, self.Y)
m_mape = self.metric.mean_absolute_percentage_error(Y_pred, self.Y)
return (m_nmse, m_mape)
def BRT(self, X=None, Y=None):
"""This function is used to implement boosted regression tree
"""
from sklearn.ensemble import GradientBoostingRegressor
rgr = GradientBoostingRegressor()
if (X is not None and Y is not None):
(self.sampled_X, self.sampled_Y) = (X, Y)
# train
rgr.fit(self.sampled_X, self.sampled_Y)
print(len(self.sampled_Y))
# test
Y_pred = rgr.predict(self.X)
# compute metric
m_nmse = self.metric.normalized_mean_square_error(Y_pred, self.Y)
m_mape = self.metric.mean_absolute_percentage_error(Y_pred, self.Y)
return (m_nmse, m_mape)
def NN(self, X=None, Y=None):
"""This function is used to implement Neural Network
"""
from sklearn.neural_network import MLPRegressor
import matplotlib.pyplot as plt
rgr = MLPRegressor(hidden_layer_sizes=(8),
activation="tanh",
solver="lbfgs")
if (X is not None and Y is not None):
(self.sampled_X, self.sampled_Y) = (X, Y)
# train
rgr.fit(self.sampled_X, self.sampled_Y)
# test
Y_pred = rgr.predict(self.X)
# compute metric
m_nmse = self.metric.normalized_mean_square_error(Y_pred, self.Y)
m_mape = self.metric.mean_absolute_percentage_error(Y_pred, self.Y)
return (m_nmse, m_mape)
def MARS(self, X=None, Y=None):
"""This function is used to imeplement Multivariate Adadptive Regression Splines
"""
from pyearth import Earth
rgr = Earth()
if (X is not None and Y is not None):
(self.sampled_X, self.sampled_Y) = (X, Y)
# train
rgr.fit(self.sampled_X, self.sampled_Y)
# test
Y_pred = rgr.predict(self.X)
# compute metric
m_nmse = self.metric.normalized_mean_square_error(Y_pred, self.Y)
m_mape = self.metric.mean_absolute_percentage_error(Y_pred, self.Y)
return (m_nmse, m_mape)
class Log(object):
def __init__(self):
print "the log class"
self.metric = Metric()
self.mach_learn = MachineLearning()
self.storage_nodes = []
self.normal_value_count = []
self.metrics = []
self.anomaly_value_count = []
self.thresholds_values = []
self.thresholds_lines = []
self.no_of_lines = 200
def write_data_to_file(self, file_name, line):
file_name.write(line)
file_name.write("\n")
file_name.flush()
def create_log_files(self, prefix):
self.prefix = prefix
if not os.path.exists("Metric\\" + self.prefix):
os.makedirs("Metric\\"+self.prefix)
return open(os.path.join("Metric\\"+self.prefix, "log.txt"), "a+")
def check_file(self, file_name, prefix):
self.prefix = prefix
self.filename = file_name+".txt"
filepath = os.path.join("Metric\\"+self.prefix, self.filename)
if ((os.path.exists(filepath)) == False):
open(os.path.join("Metric\\"+self.prefix, self.filename),"a")
else:
pass
def setmessagelevel(self,
line_data,
local_time,
storage_nodes,
normal_value_count,
metrics, anomaly_value_count,
thresholds_values, thresholds_lines, fo, lo,
metric_name,
prefix,
):
self.metric_name = metric_name
self.fo = fo
self.log_files = lo
self.anomaly_value_count = anomaly_value_count
self.line_data = line_data
self.local_time = local_time
self.storage_nodes = storage_nodes
self.normal_value_count = normal_value_count
self.thresholds_values = thresholds_values
self.thresholds_lines = thresholds_lines
self.metrics = metrics
self.prefix = prefix
if self.line_data.find("errors.timing") > 0:
str_data = self.local_time, 'E: ', self.line_data
self.join_line = ''.join(str_data)
self.metric.write_data_to_file(self.fo, self.join_line)
self.metric.write_data_to_file(self.log_files, self.join_line)
elif (((self.line_data.find("errors")) > 0) | ((self.line_data.find("quarantines")) > 0)):
str_data = self.local_time, 'C: ', self.line_data
self.join_line = ''.join(str_data)
self.metric.write_data_to_file(self.fo, self.join_line)
self.metric.write_data_to_file(self.log_files,
self.join_line)
elif self.line_data.find("timing") > 0:
str_data = self.local_time, 'I: ', self.line_data
self.join_line = ''.join(str_data)
self.metric.parsing_line(self.join_line)
if len(self.storage_nodes) == 0:
self.storage_nodes.append(self.prefix)
self.normal_value_count.append([0*x for x in range(
0, len(self.metrics))])
self.anomaly_value_count.append([0*x for x in range(0, len(self.metrics))])
self.thresholds_values.append([0 for x in range(0, len(self.metrics))])
self.thresholds_lines.append([self.no_of_lines for x in range(0,len(self.metrics))])
self.mach_learn.anomaly_detection(0, self.metrics, self.fo, self.log_files, self.metric_name, self.prefix, self.local_time, self.line_data, self.normal_value_count, self.thresholds_lines, self.thresholds_values) # calling function code to detect anomaly values
self.storage_nodes_count = 1
else:
for i in range(0, len(self.storage_nodes)):
if self.prefix == self.storage_nodes[i]:
self.mach_learn.anomaly_detection(i, self.metrics, self.fo, self.log_files, self.metric_name, self.prefix, self.local_time,self.line_data, self.normal_value_count, self.thresholds_lines, self.thresholds_values) # calling function code to detect anomaly values
self.storage_nodes_count = 1
if self.storage_nodes_count == 0:
self.storage_nodes.append(self.prefix)
self.normal_value_count.append([0*x for x in range(0,len(self.metrics))])
self.anomaly_value_count.append([0*x for x in range(0,len(self.metrics))])
self.thresholds_values.append([0 for x in range(0,len(self.metrics))])
self.thresholds_lines.append([self.no_of_lines for x in range(0,len(self.metrics))])
self.mach_learn.anomaly_detection(len(self.storage_nodes)-1, self.metrics, self.fo, self.log_files, self.metric_name, self.prefix, self.local_time,self.line_data, self.normal_value_count, self.thresholds_lines, self.thresholds_values) # calling function code to detect anomaly values
self.storage_nodes_count = 0
else:
str_data = self.local_time, 'I: ', self.line_data
self.join_line = ''.join(str_data)
self.metric.write_data_to_file(self.fo, self.join_line)
self.metric.write_data_to_file(self.log_files, self.join_line)
return self.storage_nodes, self.normal_value_count, self.anomaly_value_count, self.thresholds_values, self.thresholds_lines
def create_new_file(self):
self.timestr = time.strftime("%Y%m%d-%H%M%S")
self.filename = "logfile.txt"
if not os.path.exists("log"):
os.makedirs("log")
return open(os.path.join("log", self.filename), "a")
def __init__(self):
# Reading the configuration file of HMS
self.metrics = ["account-auditor failures",
"account-auditor timing",
"account-reaper timing",
"account-reaper container_failures",
"account-reaper object-failures",
"account-server GET timing",
"account-server HEAD timing",
"account-server PUT timing",
"account-server POST timing",
"account-server DELETE timing",
"account-server REPLICATE timing",
"account-replicator failures",
"account-replicator timing ",
"container-auditor failures",
"container-auditor timing",
"container-replicator failures",
"container-replicator timing",
"container-server GET timing",
"container-server HEAD timing",
"container-server PUT timing",
"container-server POST timing",
"container-server DELETE timing",
"container-server REPLICATE timing",
"container-sync deletes timing",
"container-sync puts timing",
"container-updater failures",
"container-updater timing",
"object-auditor timing",
"object-expirer timing",
"object-replicator partition delete timing",
"object-replicator partition update timing",
"object-server GET timing",
"object-server HEAD timing",
"object-server PUT timing",
"object-server POST timing",
"object-server DELETE timing",
"object-server REPLICATE timing",
"object-updater timing"]
self.thresholds_lines = []
self.thresholds_values = []
self.local_time = time.time()
self.line_data = ""
self.no_of_lines = 200
self.warning_dynamic_metric_name = []
self.warning_dynamic_thresholds_values = []
self.warning_dynamic_thresholds_lines = []
self.threshold_count = 0
self.metric_name = ""
self.prefix = " "
self.metric_value = 0.0
self.normal_value_count = []
self.anomaly_value_count = []
self.storage_nodes = []
self.storage_nodes_count = 0
self.now1 = datetime.now()
self.join_line = ""
# Creating objects for Log.py, MachineLearning.py and Metric.py
log = Log()
metric = Metric()
# Reading the file
log.create_new_file()
self.fo = open(os.path.join("log", "logfile.txt"), "a+")
# Read the configuration file
self.conf = ConfigParser.ConfigParser()
self.conf.read('tulsiclient.conf')
try:
print "Reading conf parameters "
self.host = self.conf.get('tulsistatsd', 'host')
log_duration = self.conf.get('tulsistatsd' , 'log_duration')
self.port = int( self.conf.get('tulsistatsd', 'port'))
self.log_duration = log_duration
except:
print "Conf not able to read"
self.host = gethostbyname(gethostname())
self.log_duration = 9
self.port = 8125
self.buf = 105600
self.addr = (self.host, self.port)
self.UDPSock = socket(AF_INET, SOCK_DGRAM)
self.UDPSock.bind((self.host,self.port))
self.now = int(time.time())
# Starting the Infinite while loop
while True:
data = self.recv_msg()
self.line_data = str(data)
print data
self.later = int(time.time())
if(self.fo.tell()) > 524288000:
self.fo.close()
self.fo = log.create_new_file()
self.now = int(time.time())
else:
self.fo.close()
self.fo = open(os.path.join("log", "logfile.txt"), "a+")
# Condition to check the number of lines for logfiles
# If they exceed beyond 10000 a new file will be created
# backing up the exisiting file with timestamp
if len(list(self.fo)) > 10000:
timestr = time.strftime("%Y_%m_%d-%H_%M_%S")
filename = "logfile_"+timestr+".txt"
self.fo.close()
os.rename(os.path.join("log", "logfile.txt"),
os.path.join("log", filename))
self.fo = log.create_new_file()
else:
self.fo = log.create_new_file()
self.local_time = time.strftime("%d %a %H:%M:%S - ",
time.localtime(time.time()))
self.test_time = datetime.now().strftime("%H:%M:%S.%f")
str_data = self.local_time, 'I: ', self.line_data
self.join_line = ''.join(str_data)
(self.prefix, self.metric_value,
self.metric_name) = metric.parsing_line(
self.join_line)
print self.storage_nodes
metric.node_files(self.metric_value)
self.log_files = log.create_log_files(self.prefix)
(self.storage_nodes,
self.normal_value_count,
self.anomaly_value_count,
self.thresholds_values,
self.thresholds_lines) = log.setmessagelevel(
self.line_data,
self.local_time,
self.storage_nodes,
self.normal_value_count,
self.metrics,
self.anomaly_value_count,
self.thresholds_values,
self.thresholds_lines,
self.fo,
self.log_files,
self.metric_name,
self.prefix)
print self.storage_nodes
import os
import discord
import nltk
from RedBlackTree import RedBlackTree
from discord.ext import commands
from dotenv import load_dotenv
from LanguageBot import LanguageBot
from Metric import Metric
load_dotenv("bot.env") # name of env file goes here
TOKEN = os.getenv('DISCORD_TOKEN') # bot token is in env file
GUILD = os.getenv('DISCORD_GUILD') # guild token is in env file
# prefix used before all commands
bot = LanguageBot(command_prefix='-')
met = Metric()
print(f'Connecting...')
# Checks if bot connected to discord
@bot.listen()
async def on_ready():
print(f'{bot} has connected to {bot.guilds}')
# sends an embed to discord with the users nouns sorted by count
@bot.command(name='nouns', help='prints users nouns')
async def nouns(ctx):
userLogs = await bot.getLogs(ctx)
nouns = bot.getNouns(userLogs, ctx)
def test_add_metric(self):
testMetric = Metric("A")
testMetric.addDataPoint(0)
self.assertEqual(1, testMetric.N)
def test_max(self):
testMetric = Metric("A")
testMetric.addDataPoint(0)
testMetric.addDataPoint(5)
self.assertEqual(5, testMetric.Max)
def test_min(self):
testMetric = Metric("A")
testMetric.addDataPoint(2)
testMetric.addDataPoint(8)
self.assertEqual(2, testMetric.Min)
tablefmt='orgtbl'))
##
# kernels => Kernel('linear') || Kernel('gaussian') || Kernel('polynomial')
# metrics => Metric('pearson') || Metric('spearman') || Metric('ig')
# svm_C => Float || None
# keys_limit => Number
# show_plot => False || True
# logs => False || True
##
if __name__ == "__main__":
keys_limit = 1000
svm_C = 1
show_plot = True
create_table([Kernel('linear')],
[Metric('pearson'),
Metric('spearman'),
Metric('ig')], keys_limit, svm_C)
if show_plot:
pearson_keys, m, f = build(Kernel('linear'), Metric('pearson'),
keys_limit, svm_C, False)
spearman_keys, m, f = build(Kernel('linear'), Metric('spearman'),
keys_limit, svm_C, False)
ig_keys, m, f = build(Kernel('linear'), Metric('ig'), keys_limit,
svm_C, False)
Plot().euler(pearson_keys, spearman_keys, ig_keys)
def test_RoundToSecondDecimal(self):
testMetric = Metric("Round to two decimal")
testMetric.addDataPoint(1.851)
testMetric.addDataPoint(1.85)
testMetric.addDataPoint(1.85)
testMetric.addDataPoint(1.85)
testMetric.addDataPoint(1.85)
testMetric.addDataPoint(1.85)
testMetric.addDataPoint(1.85)
testMetric.addDataPoint(1.849)
self.assertEqual(1.85, testMetric.Avg)
self.assertEqual(1.85, testMetric.Max)
self.assertEqual(1.85, testMetric.Min)
def run_master(num_split):
print("run master")
# put the two model parts on worker1 and worker2 respectively
model = DistResNet50(
num_split, ["worker1", "worker2"])
opt = DistributedOptimizer(
optim.SGD,
model.parameter_rrefs(),
lr=lr, momentum=0.9
)
transform = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
train_dataset = datasets.CIFAR100(root='./data', train=True,
download=True, transform=transform)
train_loader = torch.utils.data.DataLoader(
train_dataset, batch_size=batch_size)
val_dataset = datasets.CIFAR100(root='./data', train=False,
download=True, transform=transform)
val_loader = torch.utils.data.DataLoader(
val_dataset, batch_size=128)
for e in range(epoch):
model.train()
train_loss = Metric("train_loss")
train_accuracy = Metric("train_accuracy")
with tqdm(
total=len(train_loader),
desc="Train Epoch #{}".format(e + 1),
) as t:
for idx, (data, target) in enumerate(train_loader):
with dist_autograd.context() as context_id:
outputs = model(data)
loss = F.cross_entropy(outputs, target)
dist_autograd.backward(context_id, [loss])
opt.step(context_id)
train_loss.update(loss)
train_accuracy.update(accuracy(outputs, target))
t.set_postfix(
{
"loss": train_loss.avg.item(),
"accuracy": 100.0 * train_accuracy.avg.item(),
}
)
t.update(1)
model.eval()
with tqdm(
total=len(val_loader),
desc="Valid Epoch #{}".format(e + 1),
) as t:
with torch.no_grad():
val_loss = Metric("val_loss")
val_accuracy = Metric("val_accuracy")
for data, target in val_loader:
output = model(data)
val_loss.update(F.cross_entropy(output, target))
val_accuracy.update(accuracy(output, target))
t.set_postfix(
{
"loss": val_loss.avg.item(),
"accuracy": 100.0 * val_accuracy.avg.item(),
}
)
t.update(1)
def printMetric():
metric = Metric()
print("MAE")
print(metric.mae(actual, hybrid))
print(metric.mae(actual, syntatic))
print(metric.mae(actual, sematic))
print(metric.mae(actual, dynamic))
# print(metric.mae(actual, two_semantic))
# print(metric.mae(actual, two_syntatic))
print("MAPE")
print(metric.mape(actual, hybrid))
print(metric.mape(actual, syntatic))
print(metric.mape(actual, sematic))
print(metric.mape(actual, dynamic))
# print(metric.mape(actual, two_semantic))
# print(metric.mape(actual, two_syntatic))
print("SMAPE")
print(metric.smape(actual, hybrid))
print(metric.smape(actual, syntatic))
print(metric.smape(actual, sematic))
print(metric.smape(actual, dynamic))
# print(metric.smape(actual, two_semantic))
# print(metric.smape(actual, two_syntatic))
print("RMSE")
print(metric.rmse(actual, hybrid))
print(metric.rmse(actual, syntatic))
print(metric.rmse(actual, sematic))
print(metric.rmse(actual, dynamic))
