def getTopWeightedFeatures(experiment_id, inst_exp_id, instance_id, size):
instance_id = int(instance_id)
exp = ExperimentFactory.getFactory().fromJson(experiment_id, session)
validation_experiment = ExperimentFactory.getFactory().fromJson(
inst_exp_id, session)
#get the features
features_names, features_values = validation_experiment.getFeatures(
instance_id)
features_values = [float(value) for value in features_values]
#get the pipeline with scaler and logistic model
pipeline = exp.getModelPipeline()
#scale the features
scaled_values = pipeline.named_steps['scaler'].transform(
np.reshape(features_values, (1, -1)))
weighted_values = np.multiply(scaled_values,
pipeline.named_steps['model'].coef_)
features = map(lambda name, value, w_value: (name, value, w_value),
features_names, features_values, weighted_values[0])
features.sort(key=lambda tup: abs(tup[2]))
features = features[:-int(size) - 1:-1]
tooltips = [x[1] for x in features]
barplot = BarPlot([x[0] for x in features])
dataset = PlotDataset([x[2] for x in features], None)
dataset.setColor(colors_tools.red)
barplot.addDataset(dataset)
return jsonify(barplot.toJson(tooltip_data=tooltips))
def executionTimeDisplay(self):
uncertain = PlotDataset(None, 'Uncertain Queries')
malicious = PlotDataset(None, 'Malicious Queries')
malicious.setLinestyle('dotted')
malicious.setColor(colors_tools.getLabelColor('malicious'))
benign = PlotDataset(None, 'Benign Queries')
benign.setLinestyle('dashed')
benign.setColor(colors_tools.getLabelColor('benign'))
return [malicious, uncertain, benign]
def executionTimeDisplay(self):
binary_model = PlotDataset(None, 'Binary model')
malicious = PlotDataset(None, 'Malicious Analysis')
malicious.setLinestyle('dotted')
malicious.setColor(colors_tools.getLabelColor('malicious'))
benign = PlotDataset(None, 'Benign Analysis')
benign.setLinestyle('dashed')
benign.setColor(colors_tools.getLabelColor('benign'))
return [binary_model, malicious, benign
] + QueryStrategy.executionTimeDisplay(self)
def generateBinaryHistogram(self):
barplot = BarPlot(['0', '1'])
for label, dataset in self.plot_datasets.iteritems():
num_0 = sum(dataset.values == 0)
num_1 = sum(dataset.values == 1)
hist_dataset = PlotDataset([num_0, num_1], dataset.label)
hist_dataset.setColor(dataset.color)
barplot.addDataset(hist_dataset)
output_filename = self.output_directory + 'binary_histogram.json'
with open(output_filename, 'w') as f:
barplot.exportJson(f)
def generatePlotDatasets(self, instances):
self.plot_datasets = {}
if self.has_true_labels:
malicious_instances = instances.getInstancesFromIds(instances.getMaliciousIds(true_labels = True))
malicious_dataset = PlotDataset(malicious_instances.getFeatureValues(self.feature), 'malicious')
malicious_dataset.setColor(colors_tools.getLabelColor('malicious'))
self.plot_datasets['malicious'] = malicious_dataset
benign_instances = instances.getInstancesFromIds(instances.getBenignIds(true_labels = True))
benign_dataset = PlotDataset(benign_instances.getFeatureValues(self.feature), 'benign')
benign_dataset.setColor(colors_tools.getLabelColor('benign'))
self.plot_datasets['benign'] = benign_dataset
else:
self.plot_datasets['all'] = PlotDataset(instances.getFeatureValues(self.feature), 'all')
self.plot_datasets['all'].setColor(colors_tools.getLabelColor('all'))
def getTopModelFeatures(experiment_id, size):
size = int(size)
exp = ExperimentFactory.getFactory().fromJson(experiment_id, session)
model_coefficients = exp.getTopFeatures()
features_names = exp.getFeaturesNames()
coefficients = map(lambda name, coef: (name, coef), features_names,
model_coefficients)
coefficients.sort(key=lambda tup: abs(tup[1]))
coefficients = coefficients[:-size - 1:-1]
barplot = BarPlot([x[0] for x in coefficients])
dataset = PlotDataset([x[1] for x in coefficients], None)
if (exp.classification_conf.featureImportance() == 'weight'):
dataset.setColor(colors_tools.red)
barplot.addDataset(dataset)
return jsonify(barplot.toJson())
def generateHistogram(self):
# 10 equal-width bins computed on all the data
if not self.has_true_labels:
hist, bin_edges = np.histogram(self.plot_datasets['all'].values, bins = 10, density = False)
else:
hist, bin_edges = np.histogram(self.plot_datasets['malicious'].values, bins = 10, density = False)
x_labels = [str(bin_edges[e]) + ' - ' + str(bin_edges[e+1]) for e in range(len(bin_edges)-1)]
barplot = BarPlot(x_labels)
for label, dataset in self.plot_datasets.iteritems():
hist, bin_edges = np.histogram(dataset.values, bins = bin_edges, density = False)
hist_dataset = PlotDataset(hist, dataset.label)
hist_dataset.setColor(dataset.color)
barplot.addDataset(hist_dataset)
output_filename = self.output_directory + 'histogram.json'
with open(output_filename, 'w') as f:
barplot.exportJson(f)
def getFamiliesBarplot(experiment_id, iteration, label):
experiment = updateCurrentExperiment(experiment_id)
experiment_label_id = experiment.labels_id
if iteration == 'None':
iteration = None
family_counts = labels_tools.getFamiliesCounts(experiment.session,
experiment_label_id,
iteration_max=iteration,
label=label)
df = pd.DataFrame({
'families':
family_counts.keys(),
'counts': [family_counts[k] for k in family_counts.keys()]
})
matrix_tools.sortDataFrame(df, 'families', ascending=True, inplace=True)
barplot = BarPlot(list(df['families']))
dataset = PlotDataset(list(df['counts']), 'Num. Instances')
dataset.setColor(colors_tools.getLabelColor(label))
barplot.addDataset(dataset)
return jsonify(barplot.toJson())
def display(self, directory):
labels = ['0-10%', '10-20%', '20-30%', '30-40%', '40-50%', '50-60%', '60-70%', '70-80%', '80-90%', '90-100%']
barplot = BarPlot(labels)
dataset = PlotDataset(map(len, self.ranges), 'numInstances')
dataset.setColor(colors_tools.getLabelColor('all'))
barplot.addDataset(dataset)
filename = directory + 'predictions_barplot.json'
with open(filename, 'w') as f:
barplot.exportJson(f)
barplot = BarPlot(labels)
malicious_ranges = map(
lambda l: filter(lambda x: x['true_label'], l),
self.ranges)
benign_ranges = map(
lambda l: filter(lambda x: not x['true_label'], l),
self.ranges)
malicious_dataset = PlotDataset(map(len, malicious_ranges), 'malicious')
malicious_dataset.setColor(colors_tools.getLabelColor('malicious'))
barplot.addDataset(malicious_dataset)
benign_dataset = PlotDataset(map(len, benign_ranges), 'benign')
benign_dataset.setColor(colors_tools.getLabelColor('benign'))
barplot.addDataset(benign_dataset)
filename = directory
filename += 'predictions_barplot_labels.json'
with open(filename, 'w') as f:
barplot.exportJson(f)
def executionTimeDisplay(self):
generate_queries = PlotDataset(None, 'Queries generation')
generate_queries.setColor('purple')
return [generate_queries]