def getConf(project, dataset, experiment_id):
experiment = ExperimentFactory.getFactory().fromJson(
project, dataset, experiment_id, db, cursor)
conf = experiment.toJson()
mysql_tools.useDatabase(cursor, project, dataset)
conf['has_true_labels'] = labels_tools.hasTrueLabels(cursor)
return jsonify(conf)
def mergeFamilies(project, dataset, experiment_label_id, label, families,
new_family_name):
families = families.split(',')
mysql_tools.useDatabase(cursor, project, dataset)
labels_tools.mergeFamilies(cursor, label, families, new_family_name,
experiment_label_id)
return ''
def addLabel(project, dataset, experiment_label_id, iteration_number,
instance_id, label, family, method, annotation):
annotation = annotation == 'true'
mysql_tools.useDatabase(cursor, project, dataset)
labels_tools.addLabel(cursor, experiment_label_id, instance_id, label,
family, iteration_number, method, annotation)
db.commit()
return ''
def getFeatures(project, dataset, experiment, instance_dataset, instance_id):
instance_id = int(instance_id)
mysql_tools.useDatabase(cursor, project, dataset)
experiment_obj = ExperimentFactory.getFactory().fromJson(
project, instance_dataset, experiment, db, cursor)
features_names, features_values = experiment_obj.getFeatures(instance_id)
features = zip(features_names, features_values)
return jsonify(features)
def getExperimentsNames(project, dataset, exp_kind):
db.commit()
mysql_tools.useDatabase(cursor, project, dataset)
experiments = experiment_db_tools.getExperiments(cursor, exp_kind)
experience_dict = {}
for e in experiments:
experience_dict[e] = experiment_db_tools.getExperimentId(cursor, e)
return jsonify(experience_dict)
def getLabel(project, dataset, experiment_label_id, instance_id):
mysql_tools.useDatabase(cursor, project, dataset)
label = labels_tools.getLabel(cursor,
instance_id,
experiment_label_id=experiment_label_id)
if label is None:
label = {}
else:
label = {'label': label[0], 'family': label[1]}
return jsonify(label)
def datasetHasFamilies(project, dataset, experiment_label_id):
mysql_tools.useDatabase(cursor, project, dataset)
families = labels_tools.getDatasetFamilies(cursor, project, dataset,
experiment_label_id)
if (len(families) == 0):
return str(False)
if (len(families) == 1):
if families[0] == 'other':
return str(False)
return str(True)
def getLabelsFamilies(project, dataset, experiment_label_id, iteration_max):
if iteration_max == 'None':
iteration_max = None
else:
iteration_max = int(iteration_max)
mysql_tools.useDatabase(cursor, project, dataset)
db.commit()
labels_families = labels_tools.getLabelsFamilies(
cursor, experiment_label_id, iteration_max=iteration_max)
return jsonify(labels_families)
def getLabeledInstances(project, dataset, experiment_label_id):
mysql_tools.useDatabase(cursor, project, dataset)
res = {}
res['malicious'] = labels_tools.getLabelIds(cursor,
'malicious',
experiment_label_id,
annotation=True)
res['benign'] = labels_tools.getLabelIds(cursor,
'benign',
experiment_label_id,
annotation=True)
return jsonify(res)
def getAlertsClusteringExperimentId(project, dataset, experiment_id):
experiment = ExperimentFactory.getFactory().fromJson(project, dataset, experiment_id,
db, cursor)
test_conf = experiment.classification_conf.test_conf
if test_conf.method == 'random_split':
test_dataset = dataset
test_exp_id = experiment_id
elif test_conf.method == 'test_dataset':
test_dataset = test_conf.test_exp.dataset
test_exp_id = test_conf.test_exp.experiment_id
mysql_tools.useDatabase(cursor, project, test_dataset)
clustering_experiment_id = AlertsMonitoring.AlertsMonitoring.getAlertsClusteringExperimentId(
cursor, test_exp_id)
return str(clustering_experiment_id)
def getClusterStats(project, dataset, experiment_id):
experiment = ExperimentFactory.getFactory().fromJson(project, dataset, experiment_id,
db, cursor)
clustering = Clustering.fromJson(experiment)
num_clusters = clustering.num_clusters
num_instances_v = []
labels = []
mysql_tools.useDatabase(cursor, project, dataset)
for c in range(num_clusters):
instances_in_cluster = clustering.clusters[c].instances_ids
num_instances = len(instances_in_cluster)
# the empty clusters are not displayed
if num_instances > 0:
num_instances_v.append(num_instances)
labels.append('c_' + str(c))
barplot = BarPlot(labels)
barplot.addDataset(num_instances_v, colors_tools.blue, 'Num. Instances')
return jsonify(barplot.barplot)
def getFamiliesInstances(project, dataset, experiment_label_id, label,
iteration_max):
if iteration_max == 'None':
iteration_max = None
else:
iteration_max = int(iteration_max)
mysql_tools.useDatabase(cursor, project, dataset)
db.commit()
families = labels_tools.getLabelsFamilies(
cursor, experiment_label_id, iteration_max=iteration_max)[label]
families_instances = {}
for f in families:
families_instances[f] = labels_tools.getLabelFamilyIds(
cursor,
experiment_label_id,
label,
family=f,
iteration_max=iteration_max)
return jsonify(families_instances)
def changeFamilyLabel(project, dataset, experiment_id, experiment_label_id,
label, family):
mysql_tools.useDatabase(cursor, project, dataset)
labels_tools.changeFamilyLabel(cursor, label, family, experiment_label_id)
if user_exp:
experiment = ExperimentFactory.getFactory().fromJson(
project, dataset, experiment_id, db, cursor)
filename = dir_tools.getExperimentOutputDirectory(experiment)
filename += 'user_actions.log'
file_exists = dir_tools.checkFileExists(filename)
mode = 'a' if file_exists else 'w'
to_print = [
datetime.datetime.now(), 'changeFamilyLabel', family, label
]
to_print = map(str, to_print)
to_print = ','.join(to_print)
with open(filename, mode) as f:
print >> f, to_print
return ''
def mergeFamilies(project, dataset, experiment_id, experiment_label_id, label,
families, new_family_name):
families = families.split(',')
mysql_tools.useDatabase(cursor, project, dataset)
labels_tools.mergeFamilies(cursor, label, families, new_family_name,
experiment_label_id)
if user_exp:
experiment = ExperimentFactory.getFactory().fromJson(
project, dataset, experiment_id, db, cursor)
filename = dir_tools.getExperimentOutputDirectory(experiment)
filename += 'user_actions.log'
file_exists = dir_tools.checkFileExists(filename)
mode = 'a' if file_exists else 'w'
to_print = [datetime.datetime.now(), 'mergeFamilies', new_family_name]
to_print += map(str, families)
to_print = map(str, to_print)
to_print = ','.join(to_print)
with open(filename, mode) as f:
print >> f, to_print
return ''
def removeLabel(project, dataset, experiment_id, inst_dataset,
inst_experiment_label_id, iteration_number, instance_id):
mysql_tools.useDatabase(cursor, project, inst_dataset)
labels_tools.removeLabel(cursor, inst_experiment_label_id, instance_id)
db.commit()
if user_exp:
experiment = ExperimentFactory.getFactory().fromJson(
project, dataset, experiment_id, db, cursor)
filename = dir_tools.getExperimentOutputDirectory(experiment)
filename += 'user_actions.log'
file_exists = dir_tools.checkFileExists(filename)
mode = 'a' if file_exists else 'w'
to_print = [
datetime.datetime.now(), 'removeLabel', project, dataset,
instance_id
]
to_print = map(str, to_print)
to_print = ','.join(to_print)
with open(filename, mode) as f:
print >> f, to_print
return ''
def setDbConnection(self, db, cursor):
self.db = db
self.cursor = cursor
mysql_tools.useDatabase(self.cursor, self.project, self.dataset)
def datasetHasFamilies(project, dataset, experiment_label_id):
mysql_tools.useDatabase(cursor, project, dataset)
has_families = labels_tools.datasetHasFamilies(cursor, project, dataset,
experiment_label_id)
return str(has_families)
def hasTrueLabels(project, dataset):
mysql_tools.useDatabase(cursor, project, dataset)
has_true_labels = labels_tools.hasTrueLabels(cursor)
return str(has_true_labels)
def getChildren(project, dataset, experiment_id):
db.commit()
mysql_tools.useDatabase(cursor, project, dataset)
return ' '.join(
map(str, experiment_db_tools.getChildren(cursor, experiment_id)))
def getExperimentId(project, dataset, experiment_name):
db.commit()
mysql_tools.useDatabase(cursor, project, dataset)
return str(experiment_db_tools.getExperimentId(cursor, experiment_name))
def getExperimentName(project, dataset, experiment_id):
db.commit()
mysql_tools.useDatabase(cursor, project, dataset)
return experiment_db_tools.getExperimentName(cursor, experiment_id)
def removeLabel(project, dataset, experiment, instance_id):
mysql_tools.useDatabase(cursor, project, dataset)
labels_tools.removeLabel(cursor, experiment, instance_id)
db.commit()
return ''
def getDescriptiveStatsExp(project, dataset, features_filenames):
mysql_tools.useDatabase(cursor, project, dataset)
return str(
experiment_db_tools.getDescriptiveStatsExp(cursor, features_filenames))
def getIdent(project, dataset, instance_id):
mysql_tools.useDatabase(cursor, project, dataset)
ident = idents_tools.getIdent(cursor, instance_id)
return ident
def createDatabase(self):
database_name = self.project + '_' + self.dataset
mysql_tools.dropDatabaseIfExists(self.cursor, database_name)
mysql_tools.createDatabase(self.cursor, database_name)
mysql_tools.useDatabase(self.cursor, self.project, self.dataset)
def changeFamilyName(project, dataset, experiment_label_id, label, family,
new_family_name):
mysql_tools.useDatabase(cursor, project, dataset)
labels_tools.changeFamilyName(cursor, label, family, new_family_name,
experiment_label_id)
return ''
def getDatasetFamilies(cursor, project, dataset, experiment_label_id):
mysql_tools.useDatabase(cursor, project, dataset)
query = 'SELECT DISTINCT family FROM Labels '
query += 'WHERE experiment_label_id = ' + str(experiment_label_id) + ';'
cursor.execute(query)
return [s[0] for s in cursor.fetchall()]
def changeFamilyLabel(project, dataset, experiment_label_id, label, family):
mysql_tools.useDatabase(cursor, project, dataset)
labels_tools.changeFamilyLabel(cursor, label, family, experiment_label_id)
return ''
