def GET(self): params = web.input() filename = params.machine + "-" + params.metric + ".data" path = os.path.join(config.TS_DIR, filename) try: if params.method=='MA': anomalies= get_anomalies(path, "naive", "deviance") #anomalies = detect_SMA(path, int(params.window), float(params.threshold)) return json.dumps(anomalies) elif params.method=='HMMX': anomalies= hmm_interface.get_anomalies(path, int(params.n_states), float(params.percentage)/100) return json.dumps(anomalies) elif params.method=='HMM': #anomalies= get_anomalies(path, "hmm", None, percent= float(params.percentage)) #anomalies= get_anomalies(path, "mv", None, percent= float(params.percentage)) anomalies= get_anomalies(path, "optimal", None, percent= float(params.percentage)) #anomalies= get_anomalies(path, "combined_hmm", None, percent= float(params.percentage)) #anomalies= get_anomalies(path, "var_based", None, percent= float(params.percentage)) print anomalies return json.dumps(anomalies) elif params.method=='CASCADE': anomalies= get_anomalies(path, "cascade", None, base=int(params.base), levels=int(params.levels)) return json.dumps(anomalies) elif params.method== 'NAIVE': anomalies= get_anomalies(path, "naive", "var", window_size= 30) return json.dumps(anomalies) except Exception, e: print "Exception:", e traceback.print_exc() return str(e)
def GET(self): params = web.input() filename = params.machine + "-" + params.metric + ".data" path = os.path.join(config.TS_DIR, filename) try: if params.method == 'MA': anomalies = get_anomalies(path, "naive", "deviance") #anomalies = detect_SMA(path, int(params.window), float(params.threshold)) return json.dumps(anomalies) elif params.method == 'HMMX': anomalies = hmm_interface.get_anomalies( path, int(params.n_states), float(params.percentage) / 100) return json.dumps(anomalies) elif params.method == 'HMM': #anomalies= get_anomalies(path, "hmm", None, percent= float(params.percentage)) #anomalies= get_anomalies(path, "mv", None, percent= float(params.percentage)) anomalies = get_anomalies(path, "optimal", None, percent=float(params.percentage)) #anomalies= get_anomalies(path, "combined_hmm", None, percent= float(params.percentage)) #anomalies= get_anomalies(path, "var_based", None, percent= float(params.percentage)) print anomalies return json.dumps(anomalies) elif params.method == 'CASCADE': anomalies = get_anomalies(path, "cascade", None, base=int(params.base), levels=int(params.levels)) return json.dumps(anomalies) elif params.method == 'NAIVE': anomalies = get_anomalies(path, "naive", "var", window_size=30) return json.dumps(anomalies) except Exception, e: print "Exception:", e traceback.print_exc() return str(e)
def cluster_anomalies(paths, n_clusters=4, n_iter=75): weight_lists = list() time_lists = list() index = 0 for path in paths: anomaly_list = list() for method, feature, w_size in gateway.algo_iter(methods=["naive"]): anomaly_list.append( gateway.get_anomalies(path, method, feature, percent=2, mul_dev=3, window_size=w_size)) print method, feature, w_size weights = anomalies.anomalies_to_weights(anomaly_list) #times, values= read_lists(path) if len(weights) == 0: weights = [0] weight_lists.append(weights) #time_lists.append(times) #del values print index, len(paths), len(weights), len(weight_lists[-1]) index += 1 # pad all weight_lists with 0s to make them of equal length max_length = max([len(weights) for weights in weight_lists]) for i, weights in enumerate(weight_lists): increase_length = max_length - len(weights) #if increase_length > 0: weights.extend([0 for x in range(0, increase_length)]) # find distance matrix # mat= [[cluster.non_normalized_distance(weights1, weights2) for weights2 in weight_lists] for weights1 in weight_lists] # pprint.pprint(mat) # return cluster.cluster(weight_lists, n_clusters, n_iter, distance_func= cluster.non_normalized_distance) #cluster using numpy/ scipy np_wts = numpy.array([numpy.array(weights) for weights in weight_lists]) centroids, _ = kmeans(np_wts, n_clusters) idx, _ = vq(np_wts, centroids) print idx # convert idx to clusters clusters = [list() for i in range(0, n_clusters)] for i, c_no in enumerate(idx): clusters[c_no].append(i) return clusters
def find_corr_matrix(dataset): paths = config.paths(dataset) anomaly_dict = dict() for index, path in enumerate([os.path.join(paths.TS_DIR, f) for f in os.listdir(paths.TS_DIR)]): anomaly_dict[path] = gateway.get_anomalies(path, "combined_hmm", None, percent=0.5) # print(path) paths = list() cor_mat = list() for i, path in enumerate(anomaly_dict): print i, anomaly_dict[path] paths.append(path) cor_mat.append(list()) weights = anomalies_to_onesided_expweights(anomaly_dict[path]) for j, otherpath in enumerate(anomaly_dict): cor_mat[i].append(anomaly_weight_overlap(anomaly_dict[otherpath], weights)) return paths, cor_mat
def cluster_anomalies(paths, n_clusters= 4, n_iter= 75): weight_lists= list() time_lists= list() index= 0 for path in paths: anomaly_list= list() for method, feature, w_size in gateway.algo_iter(methods=["naive"]): anomaly_list.append(gateway.get_anomalies(path, method, feature, percent=2, mul_dev= 3, window_size= w_size)) print method, feature, w_size weights= anomalies.anomalies_to_weights(anomaly_list) #times, values= read_lists(path) if len(weights)== 0: weights= [0] weight_lists.append(weights) #time_lists.append(times) #del values print index, len(paths), len(weights), len(weight_lists[-1]) index+= 1 # pad all weight_lists with 0s to make them of equal length max_length= max([len(weights) for weights in weight_lists]) for i, weights in enumerate(weight_lists): increase_length= max_length - len(weights) #if increase_length > 0: weights.extend([0 for x in range(0, increase_length)]) # find distance matrix # mat= [[cluster.non_normalized_distance(weights1, weights2) for weights2 in weight_lists] for weights1 in weight_lists] # pprint.pprint(mat) # return cluster.cluster(weight_lists, n_clusters, n_iter, distance_func= cluster.non_normalized_distance) #cluster using numpy/ scipy np_wts= numpy.array([numpy.array(weights) for weights in weight_lists]) centroids, _ = kmeans(np_wts, n_clusters) idx, _= vq(np_wts, centroids) print idx # convert idx to clusters clusters= [list() for i in range(0, n_clusters)] for i, c_no in enumerate(idx): clusters[c_no].append(i) return clusters
def find_corr_matrix(dataset): paths = config.paths(dataset) anomaly_dict = dict() for index, path in enumerate( [os.path.join(paths.TS_DIR, f) for f in os.listdir(paths.TS_DIR)]): anomaly_dict[path] = gateway.get_anomalies(path, "combined_hmm", None, percent=0.5) #print(path) paths = list() cor_mat = list() for i, path in enumerate(anomaly_dict): print i, anomaly_dict[path] paths.append(path) cor_mat.append(list()) weights = anomalies_to_onesided_expweights(anomaly_dict[path]) for j, otherpath in enumerate(anomaly_dict): cor_mat[i].append( anomaly_weight_overlap(anomaly_dict[otherpath], weights)) return paths, cor_mat