def build_leaf(self, sample_indice):
if (self.y[sample_indice].std() == 0) | (self.y[sample_indice].sum() < 5) | ((1 - self.y[sample_indice]).sum() < 5):
best_estimator = None
predict_func = lambda x: np.ones(x.shape[0]) * self.y[sample_indice].mean()
best_impurity = self.get_loss(self.y[sample_indice], predict_func(self.x[sample_indice]))
else:
best_estimator = LogisticRegressionCV(Cs=self.reg_lambda, penalty="l1", solver="liblinear", scoring="roc_auc",
cv=5, random_state=self.random_state)
mx = self.x[sample_indice].mean(0)
sx = self.x[sample_indice].std(0) + self.EPSILON
nx = (self.x[sample_indice] - mx) / sx
best_estimator.fit(nx, self.y[sample_indice])
best_estimator.coef_ = best_estimator.coef_ / sx
best_estimator.intercept_ = best_estimator.intercept_ - np.dot(mx, best_estimator.coef_.T)
xmin = np.min(np.dot(self.x[sample_indice], best_estimator.coef_.ravel()))
xmax = np.max(np.dot(self.x[sample_indice], best_estimator.coef_.ravel()))
predict_func = lambda x: 1 / (1 + np.exp(- np.clip(np.dot(x, best_estimator.coef_.ravel()), xmin, xmax) - best_estimator.intercept_))
best_impurity = self.get_loss(self.y[sample_indice], best_estimator.predict_proba(self.x[sample_indice])[:, 1])
return predict_func, best_estimator, best_impurity
def create_reduced_linear_classifier(clf, x, transformable_feature_idxs):
r"""Construct a reduced-dimension classifier based on the original one for a given example.
The reduced-dimension classifier should behave the same way as the original one, but operate in
a smaller feature space. This is done by fixing the score of the classifier on a static part of
``x``, and integrating it into the bias parameter of the reduced classifier.
For example, let $$x = [1, 2, 3]$$, weights of the classifier $$w = [1, 1, 1]$$ and bias term $$b =
0$$, and the only transformable feature index is 0. Then the reduced classifier has weights $$w' =
[1]$$, and the bias term incorporates the non-transformable part of $$x$$: $$b' = -1 \cdot 2 + 1
\cdot 3$$.
:param clf: Original logistic regression classifier
:param x: An example
:param transformable_feature_idxs: List of features that can be changed in the given example.
"""
if not isinstance(clf, LogisticRegressionCV) or not isinstance(
clf, LogisticRegression):
raise ValueError(
"Only logistic regression classifiers can be reduced.")
# Establish non-transformable feature indexes.
feature_idxs = np.arange(x.size)
non_transformable_feature_idxs = np.setdiff1d(feature_idxs,
transformable_feature_idxs)
# Create the reduced classifier.
clf_reduced = LogisticRegressionCV()
clf_reduced.coef_ = clf.coef_[:, transformable_feature_idxs]
clf_reduced.intercept_ = np.dot(
clf.coef_[0, non_transformable_feature_idxs],
x[non_transformable_feature_idxs])
clf_reduced.intercept_ += clf.intercept_
assert np.allclose(
clf.predict_proba([x]),
clf_reduced.predict_proba([x[transformable_feature_idxs]]),
)
return clf_reduced
def find_adv_examples(
X,
clf,
target_confidence,
confidence_margin,
feat_count,
epsilon,
p_norm=1,
q_norm=np.inf,
):
# Define the file location to store results for given epsilon and feature count.
file_path = "results/malware_{}_{}.pickle".format(epsilon, feat_count)
# List for storing the results.
results = []
# Indices of examples classified in the (target_confidence, target_confidence+0.1) range.
neg_indices, = np.where((clf.predict_proba(X)[:, 1] > target_confidence)
& (clf.predict_proba(X)[:, 1] <=
target_confidence + confidence_margin))
# Specify how many different subsets of features to choose.
sampling_count = 25
for i in range(sampling_count):
batch_msg = "(Batch: {}; Feats: {}; Epsilon: {})".format(
i, feat_count, epsilon)
logger.info(
">> {} Using JSMA to find adversarial examples for {} samples.".
format(batch_msg, len(neg_indices)))
# Choose randomly 'feat_count' features to perturb.
manifest_subset = np.random.choice((list(MANIFEST_SET)),
size=feat_count,
replace=False)
assert set(manifest_subset).issubset(MANIFEST_SET)
# Oracle required by the JSMA algorithm.
oracle = LogisticRegressionScikitSaliencyOracle(clf)
# Start by finding adversarial examples using JSMA and record their costs.
jsma_results = jsma_wrapper(
X,
neg_indices,
clf,
oracle,
manifest_subset,
target_confidence,
k=1000,
debug=batch_msg,
)
logger.info(
">> {} JSMA found adversarial examples for {} samples.".format(
batch_msg, len(jsma_results)))
# Skip this batch if no results are found by JSMA.
if not len(jsma_results):
logger.warning(
">> {} WARN! Insufficient adversarial examples returned by JSMA. Skipping..."
.format(batch_msg))
continue
# Keep only those results that have path_costs > 2.
jsma_results = {k: v for k, v in jsma_results.items() if v[1] > 2}
if not len(jsma_results):
logger.warning(
">> {} WARN! JSMA did not find adversarial examples with required path cost. Skipping..."
.format(batch_msg))
continue
# Now only look at the malware samples with lowest path cost according to JSMA.
jsma_results_sorted = sorted(jsma_results.items(),
key=lambda d: d[1][1])[:10]
logger.info(
">> {} Using IDA* search with heuristic to find adversarial examples for {} samples."
.format(batch_msg, len(jsma_results_sorted)))
for idx, (x_adv_jsma, cost_jsma,
runtime_jsma) in tqdm(jsma_results_sorted, ascii=True):
x = X[idx].toarray()[0]
# Instantiate a counter for expanded nodes, and a profiler.
expanded_counter = Counter()
per_example_profiler = Profiler()
logger.debug(
">> {} Locating adversarial example for sample at index: {}..."
.format(batch_msg, idx))
with expanded_counter.as_default(
), per_example_profiler.as_default():
try:
x_adv, cost = find_adversarial(
x,
clf,
ida_star_search,
manifest_subset,
epsilon,
p_norm=1,
q_norm=np.inf,
target_confidence=target_confidence,
)
except Exception as e:
logger.debug(
">> {} WARN! IDA* search failed for sample at index {} with the following message:\n{}"
.format(batch_msg, idx, e))
continue
nodes_expanded = expanded_counter.count()
runtime = per_example_profiler.compute_stats(
)["find_adversarial"]["tot"]
print("RUNTIME (normal CLF) IS {}".format(runtime))
# Construct a new lightweight classifier.
clf_light = LogisticRegressionCV()
non_feature_subset = np.setdiff1d(np.arange(X.shape[1]),
manifest_subset)
clf_light.coef_ = clf.coef_[:, manifest_subset]
clf_light.intercept_ = clf.intercept_ + np.dot(
clf.coef_[0, non_feature_subset], x[non_feature_subset])
import pdb
pdb.set_trace()
with expanded_counter.as_default(
), per_example_profiler.as_default():
try:
x_adv, cost = find_adversarial(
x[manifest_subset],
clf_light,
ida_star_search,
manifest_subset,
epsilon,
p_norm=1,
q_norm=np.inf,
target_confidence=target_confidence,
)
except Exception as e:
logger.debug(
">> {} WARN! IDA* search failed for sample at index {} with the following message:\n{}"
.format(batch_msg, idx, e))
continue
nodes_expanded = expanded_counter.count()
runtime = per_example_profiler.compute_stats(
)["find_adversarial"]["tot"]
print("RUNTIME (reduced CLF) IS {}".format(runtime))
pdb.set_trace()
confidence_jsma = clf.predict_proba([x_adv_jsma])[0, 1]
confidence = clf.predict_proba([x_adv])[0, 1]
result = {
"index": idx,
"feat_count": feat_count,
"manifest_subset": manifest_subset,
"x_adv_jsma": x_adv_jsma,
"path_cost_jsma": cost_jsma,
"confidence_jsma": confidence_jsma,
"runtime_jsma": runtime_jsma,
"x_adv": x_adv,
"path_cost": cost,
"confidence": confidence,
"nodes_expanded": nodes_expanded,
"epsilon": epsilon,
"runtime": runtime,
"sampling_count": i,
}
results.append(result)
logger.debug(">> {} Saving intermediary results to '{}'.".format(
batch_msg, file_path))
with open(file_path, "wb") as f:
pickle.dump(results, f)
logger.debug(">> {} Intermediary results saved to '{}'.".format(
batch_msg, file_path))
return results
else:
assert False, 'Not supported model arch'
eval_ood_detector(args.base_dir, args.in_dataset, out_datasets,
args.batch_size, args.method, method_args, args.name,
args.epochs, args.adv, args.corrupt,
args.adv_corrupt, adv_args, mode_args)
elif args.method == 'mahalanobis':
sample_mean, precision, lr_weights, lr_bias, magnitude = np.load(
os.path.join('output/hyperparams/', args.in_dataset, args.name,
'results.npy'),
allow_pickle=True)
regressor = LogisticRegressionCV(cv=2).fit(
[[0, 0, 0, 0], [0, 0, 0, 0], [1, 1, 1, 1], [1, 1, 1, 1]],
[0, 0, 1, 1])
regressor.coef_ = lr_weights
regressor.intercept_ = lr_bias
method_args['sample_mean'] = sample_mean
method_args['precision'] = precision
method_args['magnitude'] = magnitude
method_args['regressor'] = regressor
eval_ood_detector(args.base_dir, args.in_dataset, out_datasets,
args.batch_size, args.method, method_args, args.name,
args.epochs, args.adv, args.corrupt,
args.adv_corrupt, adv_args, mode_args)
elif args.method == 'sofl':
eval_ood_detector(args.base_dir, args.in_dataset, out_datasets,
args.batch_size, args.method, method_args, args.name,
args.epochs, args.adv, args.corrupt,
def run_eval(model, in_loader, out_loader, logger, args, num_classes):
# switch to evaluate mode
model.eval()
logger.info("Running test...")
logger.flush()
if args.score == 'MSP':
logger.info("Processing in-distribution data...")
in_scores = iterate_data_msp(in_loader, model)
logger.info("Processing out-of-distribution data...")
out_scores = iterate_data_msp(out_loader, model)
elif args.score == 'ODIN':
logger.info("Processing in-distribution data...")
in_scores = iterate_data_odin(in_loader, model, args.epsilon_odin,
args.temperature_odin, logger)
logger.info("Processing out-of-distribution data...")
out_scores = iterate_data_odin(out_loader, model, args.epsilon_odin,
args.temperature_odin, logger)
elif args.score == 'Energy':
logger.info("Processing in-distribution data...")
in_scores = iterate_data_energy(in_loader, model,
args.temperature_energy)
logger.info("Processing out-of-distribution data...")
out_scores = iterate_data_energy(out_loader, model,
args.temperature_energy)
elif args.score == 'Mahalanobis':
sample_mean, precision, lr_weights, lr_bias, magnitude = np.load(
os.path.join(args.mahalanobis_param_path, 'results.npy'),
allow_pickle=True)
sample_mean = [s.cuda() for s in sample_mean]
precision = [p.cuda() for p in precision]
regressor = LogisticRegressionCV(cv=2).fit(
[[0, 0, 0, 0], [0, 0, 0, 0], [1, 1, 1, 1], [1, 1, 1, 1]],
[0, 0, 1, 1])
regressor.coef_ = lr_weights
regressor.intercept_ = lr_bias
temp_x = torch.rand(2, 3, 480, 480)
temp_x = Variable(temp_x).cuda()
temp_list = model(x=temp_x, layer_index='all')[1]
num_output = len(temp_list)
logger.info("Processing in-distribution data...")
in_scores = iterate_data_mahalanobis(in_loader, model, num_classes,
sample_mean, precision,
num_output, magnitude, regressor,
logger)
logger.info("Processing out-of-distribution data...")
out_scores = iterate_data_mahalanobis(out_loader, model, num_classes,
sample_mean, precision,
num_output, magnitude, regressor,
logger)
elif args.score == 'KL_Div':
logger.info("Processing in-distribution data...")
in_dist_logits, in_labels = iterate_data_kl_div(in_loader, model)
logger.info("Processing out-of-distribution data...")
out_dist_logits, _ = iterate_data_kl_div(out_loader, model)
class_mean_logits = []
for c in range(num_classes):
selected_idx = (in_labels == c)
selected_logits = in_dist_logits[selected_idx]
class_mean_logits.append(np.mean(selected_logits, axis=0))
class_mean_logits = np.array(class_mean_logits)
logger.info("Compute distance for in-distribution data...")
in_scores = []
for i, logit in enumerate(in_dist_logits):
if i % 100 == 0:
logger.info('{} samples processed...'.format(i))
min_div = float('inf')
for c_mean in class_mean_logits:
cur_div = kl(logit, c_mean)
if cur_div < min_div:
min_div = cur_div
in_scores.append(-min_div)
in_scores = np.array(in_scores)
logger.info("Compute distance for out-of-distribution data...")
out_scores = []
for i, logit in enumerate(out_dist_logits):
if i % 100 == 0:
logger.info('{} samples processed...'.format(i))
min_div = float('inf')
for c_mean in class_mean_logits:
cur_div = kl(logit, c_mean)
if cur_div < min_div:
min_div = cur_div
out_scores.append(-min_div)
out_scores = np.array(out_scores)
else:
raise ValueError("Unknown score type {}".format(args.score))
in_examples = in_scores.reshape((-1, 1))
out_examples = out_scores.reshape((-1, 1))
auroc, aupr_in, aupr_out, fpr95 = get_measures(in_examples, out_examples)
logger.info('============Results for {}============'.format(args.score))
logger.info('AUROC: {}'.format(auroc))
logger.info('AUPR (In): {}'.format(aupr_in))
logger.info('AUPR (Out): {}'.format(aupr_out))
logger.info('FPR95: {}'.format(fpr95))
logger.flush()
