def generateClusteringVisualization(self):
if self.families_analysis:
clustering = Clustering(self.categories.instances,
self.categories.assigned_categories)
clustering.generateClustering(None, None)
else:
self.clustering_exp = None
def generateClusteringVisualization(self):
if self.families_analysis:
self.clustering_exp = self.createClusteringExperiment()
clustering = Clustering(self.categories.instances,
self.categories.assigned_categories)
clustering.generateClustering(self.clustering_exp.getOutputDirectory(),
None, None)
else:
self.clustering_exp = None
def getClusterPredictedLabel(project, dataset, experiment_id, selected_cluster):
selected_cluster = int(selected_cluster)
experiment = ExperimentFactory.getFactory().fromJson(project, dataset, experiment_id,
db, cursor)
clustering = Clustering.fromJson(experiment)
predicted_label = clustering.getClusterLabel(selected_cluster)
return predicted_label
def getClusterLabelsFamilies(project, dataset, experiment_id, selected_cluster):
selected_cluster = int(selected_cluster)
experiment = ExperimentFactory.getFactory().fromJson(project, dataset, experiment_id,
db, cursor)
clustering = Clustering.fromJson(experiment)
labels_families = clustering.getClusterLabelsFamilies(selected_cluster)
return jsonify(labels_families)
def removeClusterLabel(experiment_id, selected_cluster, num_results):
selected_cluster = int(selected_cluster)
num_results = int(num_results)
experiment = updateCurrentExperiment(experiment_id)
clustering = Clustering.fromJson(experiment)
clustering.removeClusterLabel(selected_cluster, num_results)
return ''
def getClusterLabelFamilyIds(experiment_id, selected_cluster, label, family, num_results):
selected_cluster = int(selected_cluster)
num_results = int(num_results)
experiment = updateCurrentExperiment(experiment_id)
clustering = Clustering.fromJson(experiment)
ids = clustering.getClusterLabelFamilyIds(selected_cluster, label, family)
res = listResultWebFormat(ids, num_results)
return jsonify(res)
def getNumElements(experiment_id, selected_cluster):
selected_cluster = int(selected_cluster)
experiment = updateCurrentExperiment(experiment_id)
clustering = Clustering.fromJson(experiment)
cluster = clustering.clusters[selected_cluster]
res = {}
res['num_elements'] = cluster.numInstances()
return jsonify(res)
def alertsClassification(self, alerts_ids):
multiclass_model = self.buildMulticlassClassifier(alerts_ids)
num_families = len(
self.datasets.train_instances.getFamiliesValues(label='malicious'))
clustering_experiment = self.createClusteringExperiment(
num_clusters=num_families)
self.grouping_exp_id = clustering_experiment.experiment_id
all_families = list(multiclass_model.class_labels)
predicted_families = multiclass_model.testing_monitoring.getPredictedLabels(
)
predicted_families = [
all_families.index(x) for x in predicted_families
]
clustering = Clustering(clustering_experiment,
multiclass_model.datasets.test_instances,
predicted_families)
clustering.generateClustering(None, None, cluster_labels=all_families)
def getClustersLabels(experiment_id):
experiment = updateCurrentExperiment(experiment_id)
clustering = Clustering.fromJson(experiment)
# Do not consider empty clusters for visualization
clusters = []
for c in range(clustering.num_clusters):
#if clustering.clusters[c].numInstances() > 0:
clusters.append({'id': c, 'label': clustering.clusters[c].label})
return jsonify({'clusters': clusters})
def removeClusterLabel(project, dataset, experiment_id, selected_cluster, num_results):
selected_cluster = int(selected_cluster)
num_results = int(num_results)
experiment = ExperimentFactory.getFactory().fromJson(project, dataset, experiment_id,
db, cursor)
clustering = Clustering.fromJson(experiment)
clustering.removeClusterLabel(selected_cluster, num_results)
db.commit()
return ''
def addClusterLabel(experiment_id, selected_cluster, num_results, label,
family, label_iteration, label_method):
selected_cluster = int(selected_cluster)
num_results = int(num_results)
experiment = updateCurrentExperiment(experiment_id)
clustering = Clustering.fromJson(experiment)
clustering.addClusterLabel(selected_cluster, num_results, label, family,
label_iteration, label_method)
return ''
def getNumElements(project, dataset, experiment_id, selected_cluster):
selected_cluster = int(selected_cluster)
experiment = ExperimentFactory.getFactory().fromJson(project, dataset, experiment_id,
db, cursor)
clustering = Clustering.fromJson(experiment)
cluster = clustering.clusters[selected_cluster]
res = {}
res['num_elements'] = cluster.numInstances()
return jsonify(res)
def getClustersLabels(project, dataset, experiment_id):
experiment = ExperimentFactory.getFactory().fromJson(
project, dataset, experiment_id, db, cursor)
clustering = Clustering.fromJson(experiment)
# Do not consider empty clusters for visualization
clusters = []
for c in range(clustering.num_clusters):
#if clustering.clusters[c].numInstances() > 0:
clusters.append({'id': c, 'label': clustering.clusters[c].label})
return jsonify({'clusters': clusters})
def getClustersLabels(project, dataset, experiment_id):
experiment = ExperimentFactory.getFactory().fromJson(project, dataset, experiment_id,
db, cursor)
clustering = Clustering.fromJson(experiment)
# Do not consider empty clusters for visualization
labels = []
for c in range(clustering.num_clusters):
if clustering.clusters[c].numInstances() > 0:
labels.append('c_' + str(c))
return jsonify({'labels': labels})
def getClusterInstancesVisu(experiment_id, selected_cluster, c_e_r, num_results):
num_results = int(num_results)
selected_cluster = int(selected_cluster)
experiment = updateCurrentExperiment(experiment_id)
clustering = Clustering.fromJson(experiment)
selected_cluster_ids = {}
selected_cluster_ids[selected_cluster] = \
clustering.getClusterInstancesVisu(
selected_cluster, num_results, random = True)[c_e_r]
return jsonify(selected_cluster_ids)
def getClusterLabelFamilyIds(project, dataset, experiment_id, selected_cluster, label, family, num_results):
selected_cluster = int(selected_cluster)
num_results = int(num_results)
experiment = ExperimentFactory.getFactory().fromJson(project, dataset, experiment_id,
db, cursor)
clustering = Clustering.fromJson(experiment)
ids = clustering.getClusterLabelFamilyIds(selected_cluster,
label, family)
res = web_tools.listResultWebFormat(ids, num_results)
return jsonify(res)
def getClusterInstancesVisu(project, dataset, experiment_id, selected_cluster, c_e_r, num_results):
num_results = int(num_results)
selected_cluster = int(selected_cluster)
experiment = ExperimentFactory.getFactory().fromJson(project, dataset, experiment_id,
db, cursor)
clustering = Clustering.fromJson(experiment)
selected_cluster_ids = {}
selected_cluster_ids[selected_cluster] = \
clustering.getClusterInstancesVisu(
selected_cluster, num_results, random = True)[c_e_r]
return jsonify(selected_cluster_ids)
def addClusterLabel(project, dataset, experiment_id, selected_cluster, num_results, label, family,
label_iteration, label_method):
selected_cluster = int(selected_cluster)
num_results = int(num_results)
experiment = ExperimentFactory.getFactory().fromJson(project, dataset, experiment_id,
db, cursor)
clustering = Clustering.fromJson(experiment)
clustering.addClusterLabel(selected_cluster, num_results,
label, family, label_iteration, label_method)
db.commit()
return ''
def getClusterStats(project, dataset, experiment_id):
experiment = ExperimentFactory.getFactory().fromJson(project, dataset, experiment_id,
db, cursor)
clustering = Clustering.fromJson(experiment)
num_clusters = clustering.num_clusters
num_instances_v = []
labels = []
mysql_tools.useDatabase(cursor, project, dataset)
for c in range(num_clusters):
instances_in_cluster = clustering.clusters[c].instances_ids
num_instances = len(instances_in_cluster)
# the empty clusters are not displayed
if num_instances > 0:
num_instances_v.append(num_instances)
labels.append('c_' + str(c))
barplot = BarPlot(labels)
barplot.addDataset(num_instances_v, colors_tools.blue, 'Num. Instances')
return jsonify(barplot.barplot)
def getClusterStats(experiment_id):
experiment = updateCurrentExperiment(experiment_id)
clustering = Clustering.fromJson(experiment)
num_clusters = clustering.num_clusters
num_instances_v = []
labels = []
for c in range(num_clusters):
instances_in_cluster = clustering.clusters[c].instances_ids
num_instances = len(instances_in_cluster)
# the empty clusters are not displayed
#if num_instances > 0:
num_instances_v.append(num_instances)
#labels.append('c_' + str(c))
labels.append(clustering.clusters[c].label)
barplot = BarPlot(labels)
dataset = PlotDataset(num_instances_v, 'Num. Instances')
barplot.addDataset(dataset)
return jsonify(barplot.toJson())
def getClusterPredictedLabel(experiment_id, selected_cluster):
selected_cluster = int(selected_cluster)
experiment = updateCurrentExperiment(experiment_id)
clustering = Clustering.fromJson(experiment)
predicted_label = clustering.getClusterLabel(selected_cluster)
return predicted_label
def getClusterLabelsFamilies(experiment_id, selected_cluster):
selected_cluster = int(selected_cluster)
experiment = updateCurrentExperiment(experiment_id)
clustering = Clustering.fromJson(experiment)
labels_families = clustering.getClusterLabelsFamilies(selected_cluster)
return jsonify(labels_families)
