def extract(self, task): target = fs.join(task.directory, task.name) basicstring = printer.format('extracter', printer.Color.CAN) extractstring = printer.format('extracting', printer.Color.YEL) print('[{0}] {1} {2}'.format(basicstring, extractstring, task.name)) if self.is_tar(task.url): ttar = tarfile.open(target, 'r') ttar.extractall(path=task.directory) elif self.is_zip(task.url): tzip = zipfile.ZipFile(target, 'r') tzip.extractall(task.directory) tzip.close() elif self.is_gzip(task.url): with gzip.open(target, 'rb') as f_in: with open(task.directory, 'wb') as f_out: fs.cp(f_in, f_out) else: return finishedstring = printer.format('extracted', printer.Color.GRN) print('[{0}] {1} {2}'.format(basicstring, finishedstring, task.name)) fs.rm(fs.join(task.directory, task.name)) dircontents = fs.ls(task.directory) if len(dircontents) == 1 and fs.isdir(task.directory, dircontents[0]): subdircontents = fs.ls(task.directory, dircontents[0]) for file in subdircontents: path = fs.join(task.directory, dircontents[0]) fs.mv(fs.join(path, file), task.directory) fs.rm(task.directory, dircontents[0], ignore_errors=True)
def analysis_submenu(self): if not fs.ls(self.current_path): print('Nothing to analyse here') return while True: print('Results for which method do you want to analyse?') options = self.get_result_sub_dirs() chosenopts, result = menu.standard_menu(options, lambda x: str(x)) if result == menu.MenuResults.CHOSEN: if len(chosenopts) == 1: self.current_path = fs.join(self.current_path, chosenopts[0]) self.analysis_sub_submenu() return elif len(chosenopts) > 1: self.analyse_sub_all( [fs.join(self.current_path, x) for x in chosenopts]) return elif result == menu.MenuResults.EVERYTHING: self.analyse_sub_all( [fs.join(self.current_path, x) for x in chosenopts]) return elif result == menu.MenuResults.BACK: self.current_path = fs.dirname(self.current_path) return
def get_analysis_component(self, path): path_array = fs.split(path) datetime = path_array[-3] method_name = path_array[-2] apk_name = path_array[-1] method = comp.Component.get_component_for_name(self.components, method_name) results = fs.join(s.resultsdir, datetime, 'results.csv') analysis_result_loc = fs.join(s.analyseddir, datetime, method_name, apk_name) return a_comp.AnalysisComponent(path, analysis_result_loc, method, results)
def analysis_menu(self): if not fs.isdir(s.resultsdir) or not fs.ls(s.resultsdir): print('Nothing to analyse.') return while True: print('Results for which run do you want to analyse?') options = self.get_result_dirs() chosenopts, result = menu.standard_menu( options, lambda x: fs.basename(str(x))) if result == menu.MenuResults.CHOSEN: if len(chosenopts) == 1: self.current_path = fs.join(self.current_path, chosenopts[0]) self.analysis_submenu() return elif len(chosenopts) > 1: self.analyse_all(chosenopts) return elif result == menu.MenuResults.EVERYTHING: self.analyse_all(chosenopts) return elif result == menu.MenuResults.BACK: return
def init_from_components(components): dldcmp = DownloadComponent() dldcmp.taskset = set() for component in components: config = cnf.DownloadConfiguration(fs.join(component.installerloc,'conf.conf')) dldcmp.taskset.update(DownloadComponent.get_downloads(component,config)) return dldcmp
def get_config_files(): configs = [] for item in fs.ls(s.execconfdir): if not fs.isdir(s.execconfdir, item) and item.endswith('.conf'): configs.append(fs.join(s.execconfdir,item)) configs.sort() return configs
def generate_tasks(self): self.tasks = [] for apk in self.apks: apkname = fs.split(apk.path)[-1] for component in self.methodcomponents: resultpath = fs.join(self.resultdir, str(component), apkname) self.tasks.append( task.ExecutionTask(component, apk, resultpath))
def get_finished_apknames(execomponent): if not fs.isfile(execomponent.resultdir, 'results.csv'): return [] returnlist = [] with open(fs.join(execomponent.resultdir, 'results.csv')) as csv: for line in csv: returnlist.append(line.split(',', 2)[1]) return returnlist
def __init__(self, methodcomponents, path=None): if path: self.from_file(path, methodcomponents) else: self.resultdir = fs.join(s.resultsdir, self.make_result_dirname()) self.methodcomponents = methodcomponents self.apks = cnf.get_apks() self.generate_tasks()
def get_execution_components(components): execomponents = [] if fs.isdir(s.resultsdir): for item in fs.lsonlydir(s.resultsdir, full_paths=True): if fs.isfile(item, s.runconf): execomponents.append( comp.ExecutionComponent(components, fs.join(item, s.runconf))) return execomponents
def download_android_linux(self, versions): if type(versions) is int: versions = [versions] path_to_java = fs.join(s.dependencydir, 'jdk8') env = os.environ.copy() env['JAVA_HOME'] = path_to_java env['PATH'] = '{0}:{1}'.format(fs.join(path_to_java, 'bin'), env['PATH']) print('Downloading android platforms {0}'.format(versions)) if not fs.exists(s.dependencydir,'Android-SDK') \ or not fs.exists(s.dependencydir,'Android-SDK','tools','bin','sdkmanager'): print('Error: Android-SDK not found. Cannot download platforms') return downloadversions = [] for version in versions: if not fs.exists(s.dependencydir, 'Android-SDK', 'platforms', 'android-{0}'.format(str(version))): downloadversions.append(version) if len(downloadversions) == 0: return downloadarray = [ 'sh', fs.join(s.dependencydir, 'Android-SDK', 'tools', 'bin', 'sdkmanager') ] for version in downloadversions: downloadarray.append('platforms;android-{0}'.format(str(version))) ps = subprocess.Popen(['yes'], stdout=subprocess.PIPE) subprocess.call(downloadarray, stdin=ps.stdout, stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, env=env) ps.terminate()
def get_java_env(self): if not fs.isdir(s.dependencydir, 'jdk8'): raise RuntimeError( 'Error: Could not find jdk8 directory in {0}'.format( s.dependencydir)) path_to_java = fs.join(s.dependencydir, 'jdk8') env = os.environ.copy() env['JAVA_HOME'] = path_to_java env['PATH'] = '{0}:{1}'.format(os.path.join(path_to_java, 'bin'), env['PATH']) return env
def make_apkpath(path): if path.endswith('.apk'): return [path] elif fs.isdir(path): result=[] for item in fs.ls(path): if not fs.isdir(path, item) and item.endswith('.apk'): result.append(fs.join(path,item)) print('Found {0} apk\'s in directory'.format(len(result))) return result return []
def download(self, task): basicstring = printer.format('downloader', printer.Color.CAN) downloadstring = printer.format('downloading', printer.Color.YEL) print('[{0}] {1} {2}'.format(basicstring, downloadstring, task.name)) u = urllib.request.urlopen(task.url) with open(fs.join(task.directory, task.name), 'wb') as out_file: out_file.write(u.read()) finishedstring = printer.format('downloaded', printer.Color.GRN) print('[{0}] {1} {2}'.format(basicstring, finishedstring, task.name))
def get_downloads(component, config): taskset = set() for name in config.deps: deploc = fs.join(s.dependencydir,config.deps[name].directory) depurl = config.deps[name].url current_task = task.DownloadTask(name) current_task.directory = deploc current_task.url = depurl if DownloadComponent.should_download(deploc): taskset.add(current_task) for name in config.pkgs: pkgloc = fs.join(component.installationloc,config.pkgs[name].directory) pkgurl = config.pkgs[name].url current_task = task.DownloadTask(name) current_task.directory = pkgloc current_task.url = pkgurl if DownloadComponent.should_download(pkgloc): taskset.add(current_task) return taskset
def test_apkpath(testpath): try: if fs.isfile(testpath) and testpath.endswith('.apk'): return True elif fs.isdir(testpath): for item in fs.ls(testpath): if fs.isfile(fs.join(testpath,item)) and item.endswith('.apk'): return True print('Specified directory has no apk files in it') return False except Exception as e: print('{0} does not exist'.format(testpath)) return False
def execute(execomponent): prepare_execute(execomponent) logger = lgr.Logger(fs.join(execomponent.resultdir, 'results.csv')) firepool = fpool.Firepool() timeout = ask_timeout() args = arg_generator(execomponent, firepool, logger, timeout) logger.start() firepool.fire(parallel_execute, args) logger.stop() post_execute(execomponent)
def restart(execomponents): firepool = fpool.Firepool() for execomponent in execomponents: remove_finished_tasks(execomponent) e.prepare_execute(execomponent) timeout = e.ask_timeout() logger = lgr.Logger(fs.join(execomponent.resultdir, 'results.csv')) args = e.arg_generator(execomponent, firepool, logger, timeout) logger.start() firepool.fire(e.parallel_execute, args) logger.stop() e.post_execute(execomponent)
def get_components(): components = [] installers = get_installer_dirs() for item in installers: if not fs.isfile(s.installersdir,item,'install.py'): print('Error in "{0}": No install.py found!'.format(item)) continue elif not fs.isfile(s.installersdir,item,'run.py'): print('Error in "{0}": No run.py found!'.format(item)) continue elif not fs.isfile(s.installersdir,item,'analyse.py'): print('Error in "{0}": No analyse.py found!'.format(item)) continue installmod = 'installers.{0}.install'.format(item) runmod = 'installers.{0}.run'.format(item) analysemod = 'installers.{0}.analyse'.format(item) installerloc = fs.join(s.installersdir,item) installloc = fs.join(s.installdir,item) component = cmpnt.Component(installmod, runmod, analysemod, installerloc, installloc) components.append(component) components.sort() return components
def init(_path): global root root = _path global installersdir installersdir = fs.join(_path, 'installers') global installdir installdir = fs.join(_path, 'installed') global dependencydir dependencydir = fs.join(installdir, 'dependencies') global resultsdir resultsdir = fs.join(_path, 'results') global analyseddir analyseddir = fs.join(_path, 'analysed') global execconfdir execconfdir = fs.join(_path, 'executionconfigs') global runconf runconf = 'run.conf' global errorlog errorlog = 'errors.log' global outlog outlog = 'out.log'
def get_logger(analysecomponent): default_name = fs.split(analysecomponent.runresult_path)[-3] default_path = fs.join(s.analyseddir, default_name, 'out.csv') return lgr.Logger(default_path)
def prepare_execute(self): for task in self.tasks: fs.mkdir(task.resultpath, exist_ok=True) for component in self.methodcomponents: component.prepare_run(self.tasks) self.to_file(fs.join(self.resultdir, s.runconf))
def to_file(name, apks): filepath = fs.join(s.execconfdir,name+'.conf') with open(filepath, 'w') as conf: for apk in apks: conf.write(apkfile.Apk.get_string(apk)+'\n')
def is_complete_execution(execomponent): finished_names_list = get_finished_apknames( fs.join(execomponent.resultdir, 'results.csv')) return len(execomponent.tasks) / len( execomponent.methodcomponents) == len(finished_names_list)