def main(): parser = argparse.ArgumentParser(description=\ "An APK Analysis SandBox.") parser.add_argument('-r, --runHeadless', action='store_true',default=False, dest='runHeadless', help='Run emulator without window.') parser.add_argument('-v, --version', action='version', version='SandDroid v0.1beta') #args = parser.parse_args([-p','santoku']) args = parser.parse_args() # SandDroid sandDroid = SandDroid(SandDroidConst.CONFIG_FILE_PATH, theLogger=Logger()) # Set SandDroid sandDroid.runHeadless = args.runHeadless sandDroid.startTime = datetime.datetime.now() # Build logger sandDroid._createLogDir(sandDroid._getLogDir()) logLevel = LogLevel.INFO logger = Logger(theLevel=logLevel, theMode=LogMode.FILE, theLogFile='%s/%s-SandDroid-run.log' % (sandDroid._getLogDir(), Utils.getTimeAsString(sandDroid.startTime)), thePrintAlwaysFlag=True) sandDroid.log = logger sandDroid.run() sandDroid.log.log.close()
def __init__(self, theConfigFilePath, theLogger=Logger()): # parse config file self.configParser = ConfigParser() # self.configParser.parseFile(theConfigFilePath) self.configParser.generateDirectories() self.log = theLogger # keytool path to parse apk's signature self.keytoolPath = None # sanddroid directories self.mainDir = os.path.dirname(__file__) self.appList = [] # list to store apk file - full path self.runningApps = [] # list to store apk file which in being analyzed self.runHeadless = False self.emulatorStartPort = 5554 self.numThreads = 1 self.maxThreadRuntime = 600 # control running threads self.threadLogFileList = [] # list to store thread log file path self.numFinishedApps = 0 # number of analyzed apps self.numRunningThreads = 0 # number of running threads self.threadList = [] # list of threads, size=numThreads self.threadActiveMask = [ ] # bitmask to determine if thread is active, size=numThreads self.avdsheartbeat = (0, 0, 0, 0) # list avds' times used in one cycle self.avdheartbeat = 0 self.startTime = datetime.datetime.now()
def __init__(self, theApkObj, theAvdName, decompressDir, runHeadless, theLogger=Logger()): Thread.__init__(self) # configParser self.configParser = ConfigParser() self.apkObj = theApkObj self.log = theLogger self.curDir = os.path.dirname(__file__) self.staticAnalyzer = None self.dynamicAnalyzer = None self.logcatAnalyzer = None self.startTimeStr = None self.endTimeStr = None self.emulator = None self.emulatorPort = 5554 self.avdName = theAvdName self.runHeadless = runHeadless self.decompressPath = decompressDir self.logcatFile = None self.session = None self.cancelFlag = False # Flag for canceling run
def main(): """ get run args """ parser = argparse.ArgumentParser() #parser.add_argument('--o_dir', type=str, default='options', # help='dir containing options files') parser.add_argument('--o', type=str, help='options file to use') parser.add_argument('--id', type=str, help='ID of current experiment') parser.add_argument('--log_dir', type=str, default='logs', help='ID of current experiment') parser.add_argument('--use_gpu', type=str, default=None, help='list gpus you do not want to use') run_args = parser.parse_args() """ get exp args and merge args""" args_file = run_args.o args = parse_exp_args(args_file) for k,v in vars(run_args).items(): vars(args)[k] = v """ set seeds """ random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) """ multi GPU """ CPU_COUNT = multiprocessing.cpu_count() GPU_COUNT = torch.cuda.device_count() print('Using device {}'.format(args.device)) if 'cuda' in args.device.type: if args.use_gpu is not None: os.environ['CUDA_VISIBLE_DEVICES'] = args.use_gpu gpu_list = args.use_gpu.split(',') args.device_ids = ['cuda:'+str(x) for x in gpu_list] GPU_COUNT = len(gpu_list) else: args.device_ids = ['cuda:'+str(x) for x in range(GPU_COUNT)] torch.backends.cudnn.benchmark = True if args.multi_gpu: #_DEVICE = torch.device(args.device) args.device = args.device_ids[0] args.batch *= GPU_COUNT print('Total batch size per iteration is now {}'.format(args.batch)) args.num_cpu = CPU_COUNT args.num_gpu = GPU_COUNT """ make logger """ print('Creating logger for log dir {}/{}'.format(args.log_dir, args.id)) logger = Logger(args.id, args_file, args.log_dir, args) """ train """ import pdb; pdb.set_trace() train(args, logger)
def __createThreadLogFile(self, logFileName): """ Create log file for each thread """ logFile = os.path.join(self._getLogDir(), logFileName) threadLogger = Logger(theLevel=self.log.level, theMode=LogMode.FILE, theLogFile=logFile) return threadLogger
def __init__(self, smaliDir, theLogger=Logger()): self.smaliDir = smaliDir self.smaliFiles = [] self.sensitiveAPIs = {} self.sensitiveStrs = {} self.adModules = {} self.urls = [] self.log = theLogger
def __init__(self, decompressPath, avdName, curDir, theLogger=Logger()): self.decompressPath = decompressPath self.avdName = avdName self.mainDir = curDir self.log = theLogger self.emulator = None self.logcatRedirectFile = None
def __init__(self, thePermissions, theWekaPath, theArffFile, theArffTemplate, theModelsDir, theOutFile, theLogger=Logger()): self.permissions = thePermissions self.wekaPath = theWekaPath self.arffFile = theArffFile self.arffTemplate = theArffTemplate self.outFile = theOutFile self.modelsDir = theModelsDir self.log = theLogger
def __init__(self, theLogger=Logger()): self.log = theLogger self.dexClassLoders = [] # DexClassLoader Used self.startedServices = [] # Service Started # Network Operation: Open, Close, Receive, Send self.openedNets = [] self.closedNets = [] self.receivedNets = [] self.sentNets = [] self.accessedFiles = {} # 'FdAccess': to get file path self.fileRWs = [] # File Read and Write # Data leakage self.leakedDatas = [] self.phoneCalls = [] # Phone Calls self.sentSMSs = [] # SMS Send self.cryptOperations = [] # Crypto Operation
def __init__(self, apkObj, decompressPath, curDir, theLogger=Logger()): self.apkObj = apkObj self.decompressPath = decompressPath self.mainDir = curDir self.log = theLogger self.basicInfo = {} self.permissions = {} self.sensitiveAPIs = {} self.sensitiveStrs = {} self.adModules = {} self.urls = [] self.mainActivity = None self.activities = [] self.services = [] self.receivers = [] self.providers = [] self.exposedActivities = [] self.exposedServices = [] self.exposedReceivers = [] self.classifyInfo = {} # Sensitive Codes: Native, dynamic, crypto, refelection self.sensitiveCodes = {} # Sensitive Files: file suffix doesn't match magic code self.sensitiveFiles = {} self.riskValue = 0 self.gexfOut = None self.malware = None self.isRepackaged = False self.orgAPKUrl = None
def main(): """ 1. get run args """ run_args = parse_run_args() if bool(run_args.debug): import pudb pudb.set_trace() """ 2. get exp args and merge args""" #args_file = run_args.o #use_gpu = run_args.use_gpu #use_signal = run_args.sig args = parse_exp_args(run_args) #args_file, use_gpu, use_signal) for k, v in vars(run_args).items(): vars(args)[k] = v """ 3. set seeds """ set_seeds(args.seed) """ 4. GPU args """ args = parse_gpu_options(args) """ 5. print model type """ if args.model_type in ['vanilla', 'pnn']: print_model_specs(args) """ check args """ print(DIV) print(args) # import pdb; pdb.set_trace() """ make model """ if args.load_model: model = load_model(args) else: model = create_single_model(args) """ check before launch """ #import pdb; pdb.set_trace() """ make logger """ print(DIV) print('Creating logger for log dir {}/{}'.format(args.log_dir, args.id)) logger = Logger(args.id, args.o, args.log_dir, args) """ launch training """ train(args, model, logger)
def __init__(self, theSdkPath='', thePort=5554, theImageDir='', thePcapFile='', theRunHeadless=False, theAvdName=None, theLogger=Logger()): self.sdkPath = theSdkPath self.port = thePort self.imageDir = theImageDir self.pcapFile = thePcapFile self.runHeadless = theRunHeadless self.log = theLogger self.avdName = theAvdName self.emulator = None self.logcatRedirectFile = '' self.logcatRedirectProcess = None self.adbProcess = None
class Trainer(object): def __init__(self, gen, discr, dataset, lr, vis_screen, save_path, l1_coef, l2_coef, pre_trained_gen, pre_trained_disc, batch_size, num_workers, epochs, pretrained=False): self.device = torch.device("cuda") # self.generator = gen.apply(weights_init) # self.generator = self.generator.apply(weights_init_last_gen) # self.generator = self.generator.to(self.device) self.generator = torch.nn.DataParallel( gen.to(self.device )) #torch.nn.DataParallel(self.init_net(gen, self.device)) self.discriminator = torch.nn.DataParallel( discr.to(self.device) ) #torch.nn.DataParallel(self.init_net(discr, self.device)) if pre_trained_disc: self.discriminator.load_state_dict(torch.load(pre_trained_disc)) else: self.discriminator.apply(Utils.weights_init) if pre_trained_gen: self.generator.load_state_dict(torch.load(pre_trained_gen)) else: self.generator.apply(Utils.weights_init) self.dataset = dataset self.noise_dim = 100 self.batch_size = batch_size self.num_workers = num_workers self.lr = lr self.beta1 = 0.5 self.num_epochs = epochs self.l1_coef = l1_coef self.l2_coef = l2_coef self.data_loader = DataLoader(self.dataset, batch_size=self.batch_size, shuffle=True, num_workers=self.num_workers) self.optimD = torch.optim.Adam(self.discriminator.parameters(), lr=self.lr, betas=(self.beta1, 0.999)) self.optimG = torch.optim.Adam(self.generator.parameters(), lr=self.lr, betas=(self.beta1, 0.999)) self.logger = Logger(vis_screen) self.checkpoints_path = 'checkpoints' self.save_path = save_path def train(self, cls=False, spe=10): self._train_gan(cls, spe) def init_net(self, net, device): mdl = net.to(device) mdl.apply(weights_init) return mdl def _normalize(self, img_tensor, mean, std): img_tensor = img_tensor.numpy() num_samples = img_tensor.shape[0] tensor_normed = np.concatenate([[ np.stack([(img_tensor[i][j, :, :] - mean[j]) / std[j] for j in range(3)], axis=0) ] for i in range(num_samples)], axis=0) tensor_normed = torch.from_numpy(tensor_normed) return tensor_normed def _train_gan(self, cls, steps_p_e): criterion = nn.BCELoss() l2_loss = nn.MSELoss() l1_loss = nn.L1Loss() #teration = 0 for epoch in range(self.num_epochs): for i in range(steps_p_e): for j, sample in enumerate(self.data_loader, 0): sample = sample sample = sample iteration = i print(iteration) right_images = sample['right_images'] * 255 right_embed = sample['right_embed'] wrong_images = sample['wrong_images'] * 255 right_images = right_images.sub_(127.5).div_(127.5) wrong_images = wrong_images.sub_(127.5).div_(127.5) right_images = Variable(right_images.float()).cuda() right_embed = Variable(right_embed.float()).cuda() wrong_images = Variable(wrong_images.float()).cuda() real_labels = torch.ones(right_images.size(0)) fake_labels = torch.zeros(right_images.size(0)) # ======== One sided label smoothing ========== # Helps preventing the discriminator from overpowering the # generator adding penalty when the discriminator is too confident # ============================================= smoothed_real_labels = torch.FloatTensor( Utils.smooth_label(real_labels.numpy(), -0.1)) real_labels = Variable(real_labels).cuda() smoothed_real_labels = Variable(smoothed_real_labels).cuda() fake_labels = Variable(fake_labels).cuda() # Train the discriminator self.discriminator.zero_grad() outputs, activation_real = self.discriminator( right_images, right_embed) real_loss = criterion(outputs, smoothed_real_labels) real_score = outputs if cls: outputs, _ = self.discriminator(wrong_images, right_embed) wrong_loss = criterion(outputs, fake_labels) wrong_score = outputs noise = Variable(torch.randn(right_images.size(0), 100)).cuda() noise = noise.view(noise.size(0), 100, 1, 1) fake_images = self.generator(right_embed, noise) outputs, _ = self.discriminator(fake_images, right_embed) fake_loss = criterion(outputs, fake_labels) fake_score = outputs d_loss = real_loss + fake_loss if cls: d_loss = d_loss + wrong_loss d_loss.backward() self.optimD.step() # Train the generator self.generator.zero_grad() noise = Variable(torch.randn(right_images.size(0), 100)).cuda() noise = noise.view(noise.size(0), 100, 1, 1) fake_images = self.generator(right_embed, noise) outputs, activation_fake = self.discriminator( fake_images, right_embed) _, activation_real = self.discriminator( right_images, right_embed) activation_fake = torch.mean(activation_fake, 0) activation_real = torch.mean(activation_real, 0) # ======= Generator Loss function============ # This is a customized loss function, the first term is the regular cross entropy loss # The second term is feature matching loss, this measure the distance between the real and generated # images statistics by comparing intermediate layers activations # The third term is L1 distance between the generated and real images, this is helpful for the conditional case # because it links the embedding feature vector directly to certain pixel values. # =========================================== g_loss = criterion(outputs, real_labels) \ + self.l2_coef * l2_loss(activation_fake, activation_real.detach()) \ + self.l1_coef * l1_loss(fake_images, right_images) g_loss.backward() self.optimG.step() self.logger.log_iteration_gan(iteration, d_loss, g_loss, real_score, fake_score) if iteration % 5 == 0: self.logger.draw(right_images, fake_images) #self.logger.plot_epoch_w_scores(epoch) if (epoch) % 10 == 0: Utils.save_checkpoint(self.discriminator, self.generator, self.checkpoints_path, self.save_path, epoch) def predict(self): for sample in self.data_loader: right_images = sample['right_images'] right_embed = sample['right_embed'] #txt = sample['txt'] if not os.path.exists('results/{0}'.format(self.save_path)): os.makedirs('results/{0}'.format(self.save_path)) right_images = Variable(right_images.float()).cuda() right_embed = Variable(right_embed.float()).cuda() # Train the generator noise = Variable(torch.randn(right_images.size(0), 100)).cuda() noise = noise.view(noise.size(0), 100, 1, 1) fake_images = self.generator(right_embed, noise) self.logger.draw(right_images, fake_images)
def __init__(self, theLogger=Logger()): self.log = theLogger self.logLines = [] self.logEntryList = [] self.logcatParser = LogcatParser()
FP += val_fp FN += val_fn logger.log('k:{} val{}'.format(k, val)) P = TP * 1.0 / (TP + FP) R = TP * 1.0 / (TP + FN) F = 2 * P * R / (P + R) logger.log('F:{}'.format(F)) return F if __name__ == "__main__": torch.backends.cudnn.benchmark = True args = parse_args() with open(args.params) as f: cfg = yaml.load(f, Loader=yaml.FullLoader) # data dict logger = Logger(args.local_rank, cfg['log_path']) logger.log('start training') assert cfg['network']['backbone'] in ['resnet_18', '34', 'mobilenetv2'] distributed = False if 'WORLD_SIZE' in os.environ: distributed = int(os.environ['WORLD_SIZE']) > 1 if distributed: torch.cuda.set_device(args.local_rank) torch.distributed.init_process_group(backend='nccl', init_method='env://') w, h = cfg['dataset']['w'], cfg['dataset']['h'] net = parsingNet(network=cfg['network'], datasets=cfg['dataset']).cuda() if distributed: net = torch.nn.parallel.DistributedDataParallel( net, device_ids=[args.local_rank])
import xlsxwriter import os import sys from utils.db import DB from utils.db_search import DbSearch from utils.jira import JiraSta from utils.redmine import RedmineSta from sheet.write_proj_sheet import write_Proj_sheet from sheet.write_summaryChart_sheet01 import write_summaryChart_sheet01 from sheet.write_summary_sheet02 import write_summary_sheet02 from utils.common import Logger import config filename = "log_test" + time.strftime('%Y-%m-%d_%H_%M_%S') + ".txt" filename_path = os.path.join(config.root_dir, "log", filename) sys.stdout = Logger(filename_path) start_time = time.time() print( time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) + "---loading completed!") class Statistics(object): # options = {"server": "http://192.168.1.212:8088"} # auth = ("lig", "lig") # user_name:test user_passwd:test connect = DB().conn() """ type:缺陷管理工具类型 redmine_project_id:redmine接口识别的项目代号(查看redmine项目配置获取) redmine_tracker_id:redmine项目缺陷的跟踪代号(通过打印redmine单个issue的tracker_id获取) jira_tag:jira接口识别的项目代号(通过选择JIRA高级查询查看)
val_data_path = "./sample_data/news.dev" test_data_path = "./sample_data/news.test" bv = BertVec() num_epochs = 1 max_text_length = 512 # 主要为了Batch训练 vec_length = 768 num_classes = 2 labels = ["股票", "体育"] lr_init = 0.01 GPU_id = "cuda:0" os.makedirs(work_dir, exist_ok=True) device = torch.device(GPU_id if torch.cuda.is_available() else "cpu") logger = Logger(file_path="{}log.log".format(work_dir)) # --------------------data pipeline------------------- datasets_train = BaselineDataSet(train_data_path, labels, bv, max_text_length, vec_length) datasets_val = BaselineDataSet(val_data_path, labels, bv, max_text_length, vec_length) # num_workers不为0的话会死锁(原因见BaselineDataSet) dataLoader_train = torch.utils.data.DataLoader(datasets_train, batch_size=32, shuffle=True, num_workers=0) dataLoader_val = torch.utils.data.DataLoader(datasets_val, batch_size=32, shuffle=False,
def __init__(self, gen, discr, dataset, lr, vis_screen, save_path, l1_coef, l2_coef, pre_trained_gen, pre_trained_disc, batch_size, num_workers, epochs, pretrained=False): self.device = torch.device("cuda") # self.generator = gen.apply(weights_init) # self.generator = self.generator.apply(weights_init_last_gen) # self.generator = self.generator.to(self.device) self.generator = torch.nn.DataParallel( gen.to(self.device )) #torch.nn.DataParallel(self.init_net(gen, self.device)) self.discriminator = torch.nn.DataParallel( discr.to(self.device) ) #torch.nn.DataParallel(self.init_net(discr, self.device)) if pre_trained_disc: self.discriminator.load_state_dict(torch.load(pre_trained_disc)) else: self.discriminator.apply(Utils.weights_init) if pre_trained_gen: self.generator.load_state_dict(torch.load(pre_trained_gen)) else: self.generator.apply(Utils.weights_init) self.dataset = dataset self.noise_dim = 100 self.batch_size = batch_size self.num_workers = num_workers self.lr = lr self.beta1 = 0.5 self.num_epochs = epochs self.l1_coef = l1_coef self.l2_coef = l2_coef self.data_loader = DataLoader(self.dataset, batch_size=self.batch_size, shuffle=True, num_workers=self.num_workers) self.optimD = torch.optim.Adam(self.discriminator.parameters(), lr=self.lr, betas=(self.beta1, 0.999)) self.optimG = torch.optim.Adam(self.generator.parameters(), lr=self.lr, betas=(self.beta1, 0.999)) self.logger = Logger(vis_screen) self.checkpoints_path = 'checkpoints' self.save_path = save_path