def PCA_on_training_model(): file_list = interface.get_available_sha256() ex_list = np.array([ pefeatures.PEFeatureExtractor().extract(interface.fetch_file(b)) for b in file_list ]) print("all_samples: ", ex_list.shape) # nor_list = normalize(ex_list, axis=0) # nor_list = MinMaxScaler().fit_transform(ex_list) nor_list, data_min, data_max, scale_, min_ = MinMaxImp(ex_list) pca = PCA(n_components=0.99).fit(nor_list) U, S, V = pca._fit(nor_list) # dic_elements = {"n_component":pca.n_components_, "scale_":scale_, "min_":min_} dic_elements = {"n_component": pca.n_components_} np.save("pca_models/features.npy", ex_list) np.save("pca_models/nor_features.npy", nor_list) np.save("pca_models/U.npy", U) np.save("pca_models/S.npy", S) np.save("pca_models/V.npy", V) np.save("pca_models/scale.npy", scale_) np.save("pca_models/min.npy", min_) createDictCSV("pca_models/dic_elements.csv", dic_elements) print("reduced dimension: ", pca.n_components_) return ex_list, nor_list, U, S, V
from gym_malware.envs.utils import interface from gym_malware.envs.utils.pefeatures import PEFeatureExtractor # bytez = interface.fetch_file("Backdoor.Win32.Hupigon.zay") # features = PEFeatureExtractor().extract2(bytez) # features2 = PEFeatureExtractor().extract(bytez) # print(features.__len__()) # print(features2.__len__()) # bytez = interface.fetch_file("VirusShare_0b3c009aa4e461a00c0b3755976b485e") # # print(bytez) # features = PEFeatureExtractor().extract(bytez) # print(features.__len__()) # print(features) file_list = interface.get_available_sha256() np.set_printoptions(threshold=1e6) # run the tests extractor = PEFeatureExtractor() index = 0 for sha256 in file_list: print("{}:[file]:{}".format(index + 1, sha256)) bytez = interface.fetch_file(sha256) index = index + 1 print(extractor.extract2(bytez)) print(extractor.extract2(bytez).shape) if index > 1: break
from gym.envs.registration import register # get samples for environment from gym_malware.envs.utils import interface, pefeatures sha256 = interface.get_available_sha256() # create a holdout set from sklearn.model_selection import train_test_split import numpy as np np.random.seed(123) sha256_train, sha256_holdout = train_test_split(sha256, test_size=200) MAXTURNS = 80 register(id='malware-v0', entry_point='gym_malware.envs:MalwareEnv', kwargs={ 'random_sample': True, 'maxturns': MAXTURNS, 'sha256list': sha256_train }) register(id='malware-test-v0', entry_point='gym_malware.envs:MalwareEnv', kwargs={ 'random_sample': False, 'maxturns': MAXTURNS, 'sha256list': sha256_holdout }) register(id='malware-score-v0', entry_point='gym_malware.envs:MalwareScoreEnv',
from gym.envs.registration import register # get samples for environment from gym_malware.envs.utils import interface, pefeatures sha256 = interface.get_available_sha256() # create a holdout set from sklearn.model_selection import train_test_split import numpy as np np.random.seed(123) sha256_train, sha256_holdout = train_test_split(sha256, test_size=200) MAXTURNS = 10 register( id='malware-v0', entry_point='gym_malware.envs:MalwareEnv', kwargs={'random_sample': True, 'maxturns': MAXTURNS, 'sha256list': sha256_train} ) register( id='malware-test-v0', entry_point='gym_malware.envs:MalwareEnv', kwargs={'random_sample': False, 'maxturns': MAXTURNS, 'sha256list': sha256_holdout} ) register( id='malware-score-v0', entry_point='gym_malware.envs:MalwareScoreEnv', kwargs={'random_sample': True, 'maxturns': MAXTURNS, 'sha256list': sha256_train} )