def getConf(project, dataset, experiment_id): experiment = ExperimentFactory.getFactory().fromJson( project, dataset, experiment_id, db, cursor) conf = experiment.toJson() mysql_tools.useDatabase(cursor, project, dataset) conf['has_true_labels'] = labels_tools.hasTrueLabels(cursor) return jsonify(conf)
def mergeFamilies(project, dataset, experiment_label_id, label, families, new_family_name): families = families.split(',') mysql_tools.useDatabase(cursor, project, dataset) labels_tools.mergeFamilies(cursor, label, families, new_family_name, experiment_label_id) return ''
def addLabel(project, dataset, experiment_label_id, iteration_number, instance_id, label, family, method, annotation): annotation = annotation == 'true' mysql_tools.useDatabase(cursor, project, dataset) labels_tools.addLabel(cursor, experiment_label_id, instance_id, label, family, iteration_number, method, annotation) db.commit() return ''
def getFeatures(project, dataset, experiment, instance_dataset, instance_id): instance_id = int(instance_id) mysql_tools.useDatabase(cursor, project, dataset) experiment_obj = ExperimentFactory.getFactory().fromJson( project, instance_dataset, experiment, db, cursor) features_names, features_values = experiment_obj.getFeatures(instance_id) features = zip(features_names, features_values) return jsonify(features)
def getExperimentsNames(project, dataset, exp_kind): db.commit() mysql_tools.useDatabase(cursor, project, dataset) experiments = experiment_db_tools.getExperiments(cursor, exp_kind) experience_dict = {} for e in experiments: experience_dict[e] = experiment_db_tools.getExperimentId(cursor, e) return jsonify(experience_dict)
def getLabel(project, dataset, experiment_label_id, instance_id): mysql_tools.useDatabase(cursor, project, dataset) label = labels_tools.getLabel(cursor, instance_id, experiment_label_id=experiment_label_id) if label is None: label = {} else: label = {'label': label[0], 'family': label[1]} return jsonify(label)
def datasetHasFamilies(project, dataset, experiment_label_id): mysql_tools.useDatabase(cursor, project, dataset) families = labels_tools.getDatasetFamilies(cursor, project, dataset, experiment_label_id) if (len(families) == 0): return str(False) if (len(families) == 1): if families[0] == 'other': return str(False) return str(True)
def getLabelsFamilies(project, dataset, experiment_label_id, iteration_max): if iteration_max == 'None': iteration_max = None else: iteration_max = int(iteration_max) mysql_tools.useDatabase(cursor, project, dataset) db.commit() labels_families = labels_tools.getLabelsFamilies( cursor, experiment_label_id, iteration_max=iteration_max) return jsonify(labels_families)
def getLabeledInstances(project, dataset, experiment_label_id): mysql_tools.useDatabase(cursor, project, dataset) res = {} res['malicious'] = labels_tools.getLabelIds(cursor, 'malicious', experiment_label_id, annotation=True) res['benign'] = labels_tools.getLabelIds(cursor, 'benign', experiment_label_id, annotation=True) return jsonify(res)
def getAlertsClusteringExperimentId(project, dataset, experiment_id): experiment = ExperimentFactory.getFactory().fromJson(project, dataset, experiment_id, db, cursor) test_conf = experiment.classification_conf.test_conf if test_conf.method == 'random_split': test_dataset = dataset test_exp_id = experiment_id elif test_conf.method == 'test_dataset': test_dataset = test_conf.test_exp.dataset test_exp_id = test_conf.test_exp.experiment_id mysql_tools.useDatabase(cursor, project, test_dataset) clustering_experiment_id = AlertsMonitoring.AlertsMonitoring.getAlertsClusteringExperimentId( cursor, test_exp_id) return str(clustering_experiment_id)
def getClusterStats(project, dataset, experiment_id): experiment = ExperimentFactory.getFactory().fromJson(project, dataset, experiment_id, db, cursor) clustering = Clustering.fromJson(experiment) num_clusters = clustering.num_clusters num_instances_v = [] labels = [] mysql_tools.useDatabase(cursor, project, dataset) for c in range(num_clusters): instances_in_cluster = clustering.clusters[c].instances_ids num_instances = len(instances_in_cluster) # the empty clusters are not displayed if num_instances > 0: num_instances_v.append(num_instances) labels.append('c_' + str(c)) barplot = BarPlot(labels) barplot.addDataset(num_instances_v, colors_tools.blue, 'Num. Instances') return jsonify(barplot.barplot)
def getFamiliesInstances(project, dataset, experiment_label_id, label, iteration_max): if iteration_max == 'None': iteration_max = None else: iteration_max = int(iteration_max) mysql_tools.useDatabase(cursor, project, dataset) db.commit() families = labels_tools.getLabelsFamilies( cursor, experiment_label_id, iteration_max=iteration_max)[label] families_instances = {} for f in families: families_instances[f] = labels_tools.getLabelFamilyIds( cursor, experiment_label_id, label, family=f, iteration_max=iteration_max) return jsonify(families_instances)
def changeFamilyLabel(project, dataset, experiment_id, experiment_label_id, label, family): mysql_tools.useDatabase(cursor, project, dataset) labels_tools.changeFamilyLabel(cursor, label, family, experiment_label_id) if user_exp: experiment = ExperimentFactory.getFactory().fromJson( project, dataset, experiment_id, db, cursor) filename = dir_tools.getExperimentOutputDirectory(experiment) filename += 'user_actions.log' file_exists = dir_tools.checkFileExists(filename) mode = 'a' if file_exists else 'w' to_print = [ datetime.datetime.now(), 'changeFamilyLabel', family, label ] to_print = map(str, to_print) to_print = ','.join(to_print) with open(filename, mode) as f: print >> f, to_print return ''
def mergeFamilies(project, dataset, experiment_id, experiment_label_id, label, families, new_family_name): families = families.split(',') mysql_tools.useDatabase(cursor, project, dataset) labels_tools.mergeFamilies(cursor, label, families, new_family_name, experiment_label_id) if user_exp: experiment = ExperimentFactory.getFactory().fromJson( project, dataset, experiment_id, db, cursor) filename = dir_tools.getExperimentOutputDirectory(experiment) filename += 'user_actions.log' file_exists = dir_tools.checkFileExists(filename) mode = 'a' if file_exists else 'w' to_print = [datetime.datetime.now(), 'mergeFamilies', new_family_name] to_print += map(str, families) to_print = map(str, to_print) to_print = ','.join(to_print) with open(filename, mode) as f: print >> f, to_print return ''
def removeLabel(project, dataset, experiment_id, inst_dataset, inst_experiment_label_id, iteration_number, instance_id): mysql_tools.useDatabase(cursor, project, inst_dataset) labels_tools.removeLabel(cursor, inst_experiment_label_id, instance_id) db.commit() if user_exp: experiment = ExperimentFactory.getFactory().fromJson( project, dataset, experiment_id, db, cursor) filename = dir_tools.getExperimentOutputDirectory(experiment) filename += 'user_actions.log' file_exists = dir_tools.checkFileExists(filename) mode = 'a' if file_exists else 'w' to_print = [ datetime.datetime.now(), 'removeLabel', project, dataset, instance_id ] to_print = map(str, to_print) to_print = ','.join(to_print) with open(filename, mode) as f: print >> f, to_print return ''
def setDbConnection(self, db, cursor): self.db = db self.cursor = cursor mysql_tools.useDatabase(self.cursor, self.project, self.dataset)
def datasetHasFamilies(project, dataset, experiment_label_id): mysql_tools.useDatabase(cursor, project, dataset) has_families = labels_tools.datasetHasFamilies(cursor, project, dataset, experiment_label_id) return str(has_families)
def hasTrueLabels(project, dataset): mysql_tools.useDatabase(cursor, project, dataset) has_true_labels = labels_tools.hasTrueLabels(cursor) return str(has_true_labels)
def getChildren(project, dataset, experiment_id): db.commit() mysql_tools.useDatabase(cursor, project, dataset) return ' '.join( map(str, experiment_db_tools.getChildren(cursor, experiment_id)))
def getExperimentId(project, dataset, experiment_name): db.commit() mysql_tools.useDatabase(cursor, project, dataset) return str(experiment_db_tools.getExperimentId(cursor, experiment_name))
def getExperimentName(project, dataset, experiment_id): db.commit() mysql_tools.useDatabase(cursor, project, dataset) return experiment_db_tools.getExperimentName(cursor, experiment_id)
def removeLabel(project, dataset, experiment, instance_id): mysql_tools.useDatabase(cursor, project, dataset) labels_tools.removeLabel(cursor, experiment, instance_id) db.commit() return ''
def getDescriptiveStatsExp(project, dataset, features_filenames): mysql_tools.useDatabase(cursor, project, dataset) return str( experiment_db_tools.getDescriptiveStatsExp(cursor, features_filenames))
def getIdent(project, dataset, instance_id): mysql_tools.useDatabase(cursor, project, dataset) ident = idents_tools.getIdent(cursor, instance_id) return ident
def createDatabase(self): database_name = self.project + '_' + self.dataset mysql_tools.dropDatabaseIfExists(self.cursor, database_name) mysql_tools.createDatabase(self.cursor, database_name) mysql_tools.useDatabase(self.cursor, self.project, self.dataset)
def changeFamilyName(project, dataset, experiment_label_id, label, family, new_family_name): mysql_tools.useDatabase(cursor, project, dataset) labels_tools.changeFamilyName(cursor, label, family, new_family_name, experiment_label_id) return ''
def getDatasetFamilies(cursor, project, dataset, experiment_label_id): mysql_tools.useDatabase(cursor, project, dataset) query = 'SELECT DISTINCT family FROM Labels ' query += 'WHERE experiment_label_id = ' + str(experiment_label_id) + ';' cursor.execute(query) return [s[0] for s in cursor.fetchall()]
def changeFamilyLabel(project, dataset, experiment_label_id, label, family): mysql_tools.useDatabase(cursor, project, dataset) labels_tools.changeFamilyLabel(cursor, label, family, experiment_label_id) return ''