def fig10(data, labels):
'''
Reproduction of results published in Table 12 of "Malicious PDF
Detection Using Metadata and Structural Features" by Charles Smutz
and Angelos Stavrou, ACSAC 2012.
'''
ben_means, ben_devs = common.get_benign_mean_stddev(data, labels)
mim_data, mim_labels = common.get_FTC_mimicry()
TRIALS = 5
nCV = 10
subsets = [0, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1]
pool = multiprocessing.Pool(processes=None)
pool_args = [(data, labels, mim_data, mim_labels, ben_means, ben_devs,
subset, TRIALS, nCV) for subset in subsets]
print '\n % {:>15}{:>15}{:>15}'.format('ORIGINAL', 'MIMICRY',
'OUR MIMICRY'),
norm = TRIALS * nCV
res = []
for accs, subset in pool.imap(perturbate_CV_parallel, pool_args):
print '\n{:>6.2f}'.format(subset * 100),
for acc in accs:
sys.stdout.write('{:>15.3f}'.format(acc / norm))
res.append(tuple([acc / norm for acc in accs]))
return res
def fig10(data, labels):
"""
Reproduction of results published in Table 12 of "Malicious PDF
Detection Using Metadata and Structural Features" by Charles Smutz
and Angelos Stavrou, ACSAC 2012.
"""
ben_means, ben_devs = common.get_benign_mean_stddev(data, labels)
mim_data, mim_labels = common.get_FTC_mimicry()
TRIALS = 5
nCV = 10
subsets = [0, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1]
pool = multiprocessing.Pool(processes=None)
pool_args = [(data, labels, mim_data, mim_labels, ben_means, ben_devs, subset, TRIALS, nCV) for subset in subsets]
print "\n % {:>15}{:>15}{:>15}".format("ORIGINAL", "MIMICRY", "OUR MIMICRY"),
norm = TRIALS * nCV
res = []
for accs, subset in pool.imap(perturbate_CV_parallel, pool_args):
print "\n{:>6.2f}".format(subset * 100),
for acc in accs:
sys.stdout.write("{:>15.3f}".format(acc / norm))
res.append(tuple([acc / norm for acc in accs]))
return res
def fig9(tr_vec, tr_labels, te_vec, te_labels, fnames):
'''
Reproduction of results published in Table 10 of "Malicious PDF Detection
Using Metadata and Structural Features" by Charles Smutz and
Angelos Stavrou, ACSAC 2012.
'''
print 'Loading random forest classifier...'
rf = RandomForest()
rf.load_model(config.get('experiments', 'FTC_model'))
ben_means, ben_devs = common.get_benign_mean_stddev(tr_vec, tr_labels)
res = []
# te_vec will be randomly modified in feature space.
# f_vec will be randomly modified in feature space but the
# randomly generated variables will be adjusted to be
# valid for the given feature
f_vec = te_vec.copy()
print 'Got {} samples. Modifying them for attack...'.format(len(te_vec))
print '{:>25s} {:>15s} {:>15s}'.format('Feature name', 'Feature space',
'Problem space')
pool = multiprocessing.Pool(processes=None)
# Modify top features one by one
for f_name in common.top_feats:
f_i = FeatureDescriptor.get_feature_names().index(f_name)
f_desc = FeatureDescriptor.get_feature_description(f_name)
print '{:>25s}'.format(f_name),
# For all files
for i in range(len(te_vec)):
if te_labels[i] != 1:
# Modify only malicious files
continue
first_val = True
while True:
# Keep randomly generating a new value
# Stop when it becomes valid for the current feature
new_val = random.gauss(ben_means[f_i], ben_devs[f_i])
if first_val:
# Make sure we generate random values for te_vec
te_vec[i][f_i] = new_val
first_val = False
# If not valid, retry
if f_desc['type'] == bool:
new_val = False if new_val < 0.5 else True
elif f_desc['type'] == int:
new_val = int(round(new_val))
if f_desc['range'][0] == FileDefined and new_val < 0:
continue
elif (f_desc['range'][0] != FileDefined and
new_val < f_desc['range'][0]):
continue
if f_desc['type'] != bool and f_desc['range'][1] < new_val:
continue
# Valid, win!
f_vec[i][f_i] = new_val
break
# mod_data has feature values read from the problem space,
# i.e., by converting feature vectors to files and back
mod_data = f_vec.copy()
pargs = [(fnames[i], f_vec[i], i)
for i, l in enumerate(te_labels) if l == 1]
for mimic, m_id in pool.imap(mimicry_wrap, pargs):
mod_data[m_id] = mimic
pred = rf.predict(te_vec)
fspace = accuracy_score(te_labels, pred)
print '{:>15.3f}'.format(fspace),
pred = rf.predict(mod_data)
pspace = accuracy_score(te_labels, pred)
print '{:>15.3f}'.format(pspace)
res.append((fspace, pspace))
return res
def fig9(tr_vec, tr_labels, te_vec, te_labels, fnames):
'''
Reproduction of results published in Table 10 of "Malicious PDF Detection
Using Metadata and Structural Features" by Charles Smutz and
Angelos Stavrou, ACSAC 2012.
'''
print 'Loading random forest classifier...'
rf = RandomForest()
rf.load_model(config.get('experiments', 'FTC_model'))
ben_means, ben_devs = common.get_benign_mean_stddev(tr_vec, tr_labels)
res = []
# te_vec will be randomly modified in feature space.
# f_vec will be randomly modified in feature space but the
# randomly generated variables will be adjusted to be
# valid for the given feature
f_vec = te_vec.copy()
print 'Got {} samples. Modifying them for attack...'.format(len(te_vec))
print '{:>25s} {:>15s} {:>15s}'.format('Feature name', 'Feature space',
'Problem space')
pool = multiprocessing.Pool(processes=None)
# Modify top features one by one
for f_name in common.top_feats:
f_i = FeatureDescriptor.get_feature_names().index(f_name)
f_desc = FeatureDescriptor.get_feature_description(f_name)
print '{:>25s}'.format(f_name),
# For all files
for i in range(len(te_vec)):
if te_labels[i] != 1:
# Modify only malicious files
continue
first_val = True
while True:
# Keep randomly generating a new value
# Stop when it becomes valid for the current feature
new_val = random.gauss(ben_means[f_i], ben_devs[f_i])
if first_val:
# Make sure we generate random values for te_vec
te_vec[i][f_i] = new_val
first_val = False
# If not valid, retry
if f_desc['type'] == bool:
new_val = False if new_val < 0.5 else True
elif f_desc['type'] == int:
new_val = int(round(new_val))
if f_desc['range'][0] == FileDefined and new_val < 0:
continue
elif (f_desc['range'][0] != FileDefined
and new_val < f_desc['range'][0]):
continue
if f_desc['type'] != bool and f_desc['range'][1] < new_val:
continue
# Valid, win!
f_vec[i][f_i] = new_val
break
# mod_data has feature values read from the problem space,
# i.e., by converting feature vectors to files and back
mod_data = f_vec.copy()
pargs = [(fnames[i], f_vec[i], i) for i, l in enumerate(te_labels)
if l == 1]
for mimic, m_id in pool.imap(mimicry_wrap, pargs):
mod_data[m_id] = mimic
pred = rf.predict(te_vec)
fspace = accuracy_score(te_labels, pred)
print '{:>15.3f}'.format(fspace),
pred = rf.predict(mod_data)
pspace = accuracy_score(te_labels, pred)
print '{:>15.3f}'.format(pspace)
res.append((fspace, pspace))
return res
