def GET(self):
params = web.input()
filename = params.machine + "-" + params.metric + ".data"
path = os.path.join(config.TS_DIR, filename)
try:
if params.method=='MA':
anomalies= get_anomalies(path, "naive", "deviance")
#anomalies = detect_SMA(path, int(params.window), float(params.threshold))
return json.dumps(anomalies)
elif params.method=='HMMX':
anomalies= hmm_interface.get_anomalies(path, int(params.n_states), float(params.percentage)/100)
return json.dumps(anomalies)
elif params.method=='HMM':
#anomalies= get_anomalies(path, "hmm", None, percent= float(params.percentage))
#anomalies= get_anomalies(path, "mv", None, percent= float(params.percentage))
anomalies= get_anomalies(path, "optimal", None, percent= float(params.percentage))
#anomalies= get_anomalies(path, "combined_hmm", None, percent= float(params.percentage))
#anomalies= get_anomalies(path, "var_based", None, percent= float(params.percentage))
print anomalies
return json.dumps(anomalies)
elif params.method=='CASCADE':
anomalies= get_anomalies(path, "cascade", None, base=int(params.base), levels=int(params.levels))
return json.dumps(anomalies)
elif params.method== 'NAIVE':
anomalies= get_anomalies(path, "naive", "var", window_size= 30)
return json.dumps(anomalies)
except Exception, e:
print "Exception:", e
traceback.print_exc()
return str(e)
def GET(self):
params = web.input()
filename = params.machine + "-" + params.metric + ".data"
path = os.path.join(config.TS_DIR, filename)
try:
if params.method == 'MA':
anomalies = get_anomalies(path, "naive", "deviance")
#anomalies = detect_SMA(path, int(params.window), float(params.threshold))
return json.dumps(anomalies)
elif params.method == 'HMMX':
anomalies = hmm_interface.get_anomalies(
path, int(params.n_states),
float(params.percentage) / 100)
return json.dumps(anomalies)
elif params.method == 'HMM':
#anomalies= get_anomalies(path, "hmm", None, percent= float(params.percentage))
#anomalies= get_anomalies(path, "mv", None, percent= float(params.percentage))
anomalies = get_anomalies(path,
"optimal",
None,
percent=float(params.percentage))
#anomalies= get_anomalies(path, "combined_hmm", None, percent= float(params.percentage))
#anomalies= get_anomalies(path, "var_based", None, percent= float(params.percentage))
print anomalies
return json.dumps(anomalies)
elif params.method == 'CASCADE':
anomalies = get_anomalies(path,
"cascade",
None,
base=int(params.base),
levels=int(params.levels))
return json.dumps(anomalies)
elif params.method == 'NAIVE':
anomalies = get_anomalies(path, "naive", "var", window_size=30)
return json.dumps(anomalies)
except Exception, e:
print "Exception:", e
traceback.print_exc()
return str(e)
def cluster_anomalies(paths, n_clusters=4, n_iter=75):
weight_lists = list()
time_lists = list()
index = 0
for path in paths:
anomaly_list = list()
for method, feature, w_size in gateway.algo_iter(methods=["naive"]):
anomaly_list.append(
gateway.get_anomalies(path,
method,
feature,
percent=2,
mul_dev=3,
window_size=w_size))
print method, feature, w_size
weights = anomalies.anomalies_to_weights(anomaly_list)
#times, values= read_lists(path)
if len(weights) == 0:
weights = [0]
weight_lists.append(weights)
#time_lists.append(times)
#del values
print index, len(paths), len(weights), len(weight_lists[-1])
index += 1
# pad all weight_lists with 0s to make them of equal length
max_length = max([len(weights) for weights in weight_lists])
for i, weights in enumerate(weight_lists):
increase_length = max_length - len(weights)
#if increase_length > 0:
weights.extend([0 for x in range(0, increase_length)])
# find distance matrix
# mat= [[cluster.non_normalized_distance(weights1, weights2) for weights2 in weight_lists] for weights1 in weight_lists]
# pprint.pprint(mat)
# return cluster.cluster(weight_lists, n_clusters, n_iter, distance_func= cluster.non_normalized_distance)
#cluster using numpy/ scipy
np_wts = numpy.array([numpy.array(weights) for weights in weight_lists])
centroids, _ = kmeans(np_wts, n_clusters)
idx, _ = vq(np_wts, centroids)
print idx
# convert idx to clusters
clusters = [list() for i in range(0, n_clusters)]
for i, c_no in enumerate(idx):
clusters[c_no].append(i)
return clusters
def find_corr_matrix(dataset):
paths = config.paths(dataset)
anomaly_dict = dict()
for index, path in enumerate([os.path.join(paths.TS_DIR, f) for f in os.listdir(paths.TS_DIR)]):
anomaly_dict[path] = gateway.get_anomalies(path, "combined_hmm", None, percent=0.5)
# print(path)
paths = list()
cor_mat = list()
for i, path in enumerate(anomaly_dict):
print i, anomaly_dict[path]
paths.append(path)
cor_mat.append(list())
weights = anomalies_to_onesided_expweights(anomaly_dict[path])
for j, otherpath in enumerate(anomaly_dict):
cor_mat[i].append(anomaly_weight_overlap(anomaly_dict[otherpath], weights))
return paths, cor_mat
def cluster_anomalies(paths, n_clusters= 4, n_iter= 75):
weight_lists= list()
time_lists= list()
index= 0
for path in paths:
anomaly_list= list()
for method, feature, w_size in gateway.algo_iter(methods=["naive"]):
anomaly_list.append(gateway.get_anomalies(path, method, feature, percent=2, mul_dev= 3, window_size= w_size))
print method, feature, w_size
weights= anomalies.anomalies_to_weights(anomaly_list)
#times, values= read_lists(path)
if len(weights)== 0:
weights= [0]
weight_lists.append(weights)
#time_lists.append(times)
#del values
print index, len(paths), len(weights), len(weight_lists[-1])
index+= 1
# pad all weight_lists with 0s to make them of equal length
max_length= max([len(weights) for weights in weight_lists])
for i, weights in enumerate(weight_lists):
increase_length= max_length - len(weights)
#if increase_length > 0:
weights.extend([0 for x in range(0, increase_length)])
# find distance matrix
# mat= [[cluster.non_normalized_distance(weights1, weights2) for weights2 in weight_lists] for weights1 in weight_lists]
# pprint.pprint(mat)
# return cluster.cluster(weight_lists, n_clusters, n_iter, distance_func= cluster.non_normalized_distance)
#cluster using numpy/ scipy
np_wts= numpy.array([numpy.array(weights) for weights in weight_lists])
centroids, _ = kmeans(np_wts, n_clusters)
idx, _= vq(np_wts, centroids)
print idx
# convert idx to clusters
clusters= [list() for i in range(0, n_clusters)]
for i, c_no in enumerate(idx):
clusters[c_no].append(i)
return clusters
def find_corr_matrix(dataset):
paths = config.paths(dataset)
anomaly_dict = dict()
for index, path in enumerate(
[os.path.join(paths.TS_DIR, f) for f in os.listdir(paths.TS_DIR)]):
anomaly_dict[path] = gateway.get_anomalies(path,
"combined_hmm",
None,
percent=0.5)
#print(path)
paths = list()
cor_mat = list()
for i, path in enumerate(anomaly_dict):
print i, anomaly_dict[path]
paths.append(path)
cor_mat.append(list())
weights = anomalies_to_onesided_expweights(anomaly_dict[path])
for j, otherpath in enumerate(anomaly_dict):
cor_mat[i].append(
anomaly_weight_overlap(anomaly_dict[otherpath], weights))
return paths, cor_mat
