def fig10(data, labels): ''' Reproduction of results published in Table 12 of "Malicious PDF Detection Using Metadata and Structural Features" by Charles Smutz and Angelos Stavrou, ACSAC 2012. ''' ben_means, ben_devs = common.get_benign_mean_stddev(data, labels) mim_data, mim_labels = common.get_FTC_mimicry() TRIALS = 5 nCV = 10 subsets = [0, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1] pool = multiprocessing.Pool(processes=None) pool_args = [(data, labels, mim_data, mim_labels, ben_means, ben_devs, subset, TRIALS, nCV) for subset in subsets] print '\n % {:>15}{:>15}{:>15}'.format('ORIGINAL', 'MIMICRY', 'OUR MIMICRY'), norm = TRIALS * nCV res = [] for accs, subset in pool.imap(perturbate_CV_parallel, pool_args): print '\n{:>6.2f}'.format(subset * 100), for acc in accs: sys.stdout.write('{:>15.3f}'.format(acc / norm)) res.append(tuple([acc / norm for acc in accs])) return res
def fig10(data, labels): """ Reproduction of results published in Table 12 of "Malicious PDF Detection Using Metadata and Structural Features" by Charles Smutz and Angelos Stavrou, ACSAC 2012. """ ben_means, ben_devs = common.get_benign_mean_stddev(data, labels) mim_data, mim_labels = common.get_FTC_mimicry() TRIALS = 5 nCV = 10 subsets = [0, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1] pool = multiprocessing.Pool(processes=None) pool_args = [(data, labels, mim_data, mim_labels, ben_means, ben_devs, subset, TRIALS, nCV) for subset in subsets] print "\n % {:>15}{:>15}{:>15}".format("ORIGINAL", "MIMICRY", "OUR MIMICRY"), norm = TRIALS * nCV res = [] for accs, subset in pool.imap(perturbate_CV_parallel, pool_args): print "\n{:>6.2f}".format(subset * 100), for acc in accs: sys.stdout.write("{:>15.3f}".format(acc / norm)) res.append(tuple([acc / norm for acc in accs])) return res
def fig9(tr_vec, tr_labels, te_vec, te_labels, fnames): ''' Reproduction of results published in Table 10 of "Malicious PDF Detection Using Metadata and Structural Features" by Charles Smutz and Angelos Stavrou, ACSAC 2012. ''' print 'Loading random forest classifier...' rf = RandomForest() rf.load_model(config.get('experiments', 'FTC_model')) ben_means, ben_devs = common.get_benign_mean_stddev(tr_vec, tr_labels) res = [] # te_vec will be randomly modified in feature space. # f_vec will be randomly modified in feature space but the # randomly generated variables will be adjusted to be # valid for the given feature f_vec = te_vec.copy() print 'Got {} samples. Modifying them for attack...'.format(len(te_vec)) print '{:>25s} {:>15s} {:>15s}'.format('Feature name', 'Feature space', 'Problem space') pool = multiprocessing.Pool(processes=None) # Modify top features one by one for f_name in common.top_feats: f_i = FeatureDescriptor.get_feature_names().index(f_name) f_desc = FeatureDescriptor.get_feature_description(f_name) print '{:>25s}'.format(f_name), # For all files for i in range(len(te_vec)): if te_labels[i] != 1: # Modify only malicious files continue first_val = True while True: # Keep randomly generating a new value # Stop when it becomes valid for the current feature new_val = random.gauss(ben_means[f_i], ben_devs[f_i]) if first_val: # Make sure we generate random values for te_vec te_vec[i][f_i] = new_val first_val = False # If not valid, retry if f_desc['type'] == bool: new_val = False if new_val < 0.5 else True elif f_desc['type'] == int: new_val = int(round(new_val)) if f_desc['range'][0] == FileDefined and new_val < 0: continue elif (f_desc['range'][0] != FileDefined and new_val < f_desc['range'][0]): continue if f_desc['type'] != bool and f_desc['range'][1] < new_val: continue # Valid, win! f_vec[i][f_i] = new_val break # mod_data has feature values read from the problem space, # i.e., by converting feature vectors to files and back mod_data = f_vec.copy() pargs = [(fnames[i], f_vec[i], i) for i, l in enumerate(te_labels) if l == 1] for mimic, m_id in pool.imap(mimicry_wrap, pargs): mod_data[m_id] = mimic pred = rf.predict(te_vec) fspace = accuracy_score(te_labels, pred) print '{:>15.3f}'.format(fspace), pred = rf.predict(mod_data) pspace = accuracy_score(te_labels, pred) print '{:>15.3f}'.format(pspace) res.append((fspace, pspace)) return res