def extract(self, task):
target = fs.join(task.directory, task.name)
basicstring = printer.format('extracter', printer.Color.CAN)
extractstring = printer.format('extracting', printer.Color.YEL)
print('[{0}] {1} {2}'.format(basicstring, extractstring, task.name))
if self.is_tar(task.url):
ttar = tarfile.open(target, 'r')
ttar.extractall(path=task.directory)
elif self.is_zip(task.url):
tzip = zipfile.ZipFile(target, 'r')
tzip.extractall(task.directory)
tzip.close()
elif self.is_gzip(task.url):
with gzip.open(target, 'rb') as f_in:
with open(task.directory, 'wb') as f_out:
fs.cp(f_in, f_out)
else:
return
finishedstring = printer.format('extracted', printer.Color.GRN)
print('[{0}] {1} {2}'.format(basicstring, finishedstring, task.name))
fs.rm(fs.join(task.directory, task.name))
dircontents = fs.ls(task.directory)
if len(dircontents) == 1 and fs.isdir(task.directory, dircontents[0]):
subdircontents = fs.ls(task.directory, dircontents[0])
for file in subdircontents:
path = fs.join(task.directory, dircontents[0])
fs.mv(fs.join(path, file), task.directory)
fs.rm(task.directory, dircontents[0], ignore_errors=True)
def analysis_submenu(self):
if not fs.ls(self.current_path):
print('Nothing to analyse here')
return
while True:
print('Results for which method do you want to analyse?')
options = self.get_result_sub_dirs()
chosenopts, result = menu.standard_menu(options, lambda x: str(x))
if result == menu.MenuResults.CHOSEN:
if len(chosenopts) == 1:
self.current_path = fs.join(self.current_path,
chosenopts[0])
self.analysis_sub_submenu()
return
elif len(chosenopts) > 1:
self.analyse_sub_all(
[fs.join(self.current_path, x) for x in chosenopts])
return
elif result == menu.MenuResults.EVERYTHING:
self.analyse_sub_all(
[fs.join(self.current_path, x) for x in chosenopts])
return
elif result == menu.MenuResults.BACK:
self.current_path = fs.dirname(self.current_path)
return
def get_analysis_component(self, path):
path_array = fs.split(path)
datetime = path_array[-3]
method_name = path_array[-2]
apk_name = path_array[-1]
method = comp.Component.get_component_for_name(self.components,
method_name)
results = fs.join(s.resultsdir, datetime, 'results.csv')
analysis_result_loc = fs.join(s.analyseddir, datetime, method_name,
apk_name)
return a_comp.AnalysisComponent(path, analysis_result_loc, method,
results)
def analysis_menu(self):
if not fs.isdir(s.resultsdir) or not fs.ls(s.resultsdir):
print('Nothing to analyse.')
return
while True:
print('Results for which run do you want to analyse?')
options = self.get_result_dirs()
chosenopts, result = menu.standard_menu(
options, lambda x: fs.basename(str(x)))
if result == menu.MenuResults.CHOSEN:
if len(chosenopts) == 1:
self.current_path = fs.join(self.current_path,
chosenopts[0])
self.analysis_submenu()
return
elif len(chosenopts) > 1:
self.analyse_all(chosenopts)
return
elif result == menu.MenuResults.EVERYTHING:
self.analyse_all(chosenopts)
return
elif result == menu.MenuResults.BACK:
return
def init_from_components(components):
dldcmp = DownloadComponent()
dldcmp.taskset = set()
for component in components:
config = cnf.DownloadConfiguration(fs.join(component.installerloc,'conf.conf'))
dldcmp.taskset.update(DownloadComponent.get_downloads(component,config))
return dldcmp
def get_config_files():
configs = []
for item in fs.ls(s.execconfdir):
if not fs.isdir(s.execconfdir, item) and item.endswith('.conf'):
configs.append(fs.join(s.execconfdir,item))
configs.sort()
return configs
def generate_tasks(self):
self.tasks = []
for apk in self.apks:
apkname = fs.split(apk.path)[-1]
for component in self.methodcomponents:
resultpath = fs.join(self.resultdir, str(component), apkname)
self.tasks.append(
task.ExecutionTask(component, apk, resultpath))
def get_finished_apknames(execomponent):
if not fs.isfile(execomponent.resultdir, 'results.csv'):
return []
returnlist = []
with open(fs.join(execomponent.resultdir, 'results.csv')) as csv:
for line in csv:
returnlist.append(line.split(',', 2)[1])
return returnlist
def __init__(self, methodcomponents, path=None):
if path:
self.from_file(path, methodcomponents)
else:
self.resultdir = fs.join(s.resultsdir, self.make_result_dirname())
self.methodcomponents = methodcomponents
self.apks = cnf.get_apks()
self.generate_tasks()
def get_execution_components(components):
execomponents = []
if fs.isdir(s.resultsdir):
for item in fs.lsonlydir(s.resultsdir, full_paths=True):
if fs.isfile(item, s.runconf):
execomponents.append(
comp.ExecutionComponent(components,
fs.join(item, s.runconf)))
return execomponents
def download_android_linux(self, versions):
if type(versions) is int:
versions = [versions]
path_to_java = fs.join(s.dependencydir, 'jdk8')
env = os.environ.copy()
env['JAVA_HOME'] = path_to_java
env['PATH'] = '{0}:{1}'.format(fs.join(path_to_java, 'bin'),
env['PATH'])
print('Downloading android platforms {0}'.format(versions))
if not fs.exists(s.dependencydir,'Android-SDK') \
or not fs.exists(s.dependencydir,'Android-SDK','tools','bin','sdkmanager'):
print('Error: Android-SDK not found. Cannot download platforms')
return
downloadversions = []
for version in versions:
if not fs.exists(s.dependencydir, 'Android-SDK', 'platforms',
'android-{0}'.format(str(version))):
downloadversions.append(version)
if len(downloadversions) == 0:
return
downloadarray = [
'sh',
fs.join(s.dependencydir, 'Android-SDK', 'tools', 'bin',
'sdkmanager')
]
for version in downloadversions:
downloadarray.append('platforms;android-{0}'.format(str(version)))
ps = subprocess.Popen(['yes'], stdout=subprocess.PIPE)
subprocess.call(downloadarray,
stdin=ps.stdout,
stderr=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
env=env)
ps.terminate()
def get_java_env(self):
if not fs.isdir(s.dependencydir, 'jdk8'):
raise RuntimeError(
'Error: Could not find jdk8 directory in {0}'.format(
s.dependencydir))
path_to_java = fs.join(s.dependencydir, 'jdk8')
env = os.environ.copy()
env['JAVA_HOME'] = path_to_java
env['PATH'] = '{0}:{1}'.format(os.path.join(path_to_java, 'bin'),
env['PATH'])
return env
def make_apkpath(path):
if path.endswith('.apk'):
return [path]
elif fs.isdir(path):
result=[]
for item in fs.ls(path):
if not fs.isdir(path, item) and item.endswith('.apk'):
result.append(fs.join(path,item))
print('Found {0} apk\'s in directory'.format(len(result)))
return result
return []
def download(self, task):
basicstring = printer.format('downloader', printer.Color.CAN)
downloadstring = printer.format('downloading', printer.Color.YEL)
print('[{0}] {1} {2}'.format(basicstring, downloadstring, task.name))
u = urllib.request.urlopen(task.url)
with open(fs.join(task.directory, task.name), 'wb') as out_file:
out_file.write(u.read())
finishedstring = printer.format('downloaded', printer.Color.GRN)
print('[{0}] {1} {2}'.format(basicstring, finishedstring, task.name))
def get_downloads(component, config):
taskset = set()
for name in config.deps:
deploc = fs.join(s.dependencydir,config.deps[name].directory)
depurl = config.deps[name].url
current_task = task.DownloadTask(name)
current_task.directory = deploc
current_task.url = depurl
if DownloadComponent.should_download(deploc):
taskset.add(current_task)
for name in config.pkgs:
pkgloc = fs.join(component.installationloc,config.pkgs[name].directory)
pkgurl = config.pkgs[name].url
current_task = task.DownloadTask(name)
current_task.directory = pkgloc
current_task.url = pkgurl
if DownloadComponent.should_download(pkgloc):
taskset.add(current_task)
return taskset
def test_apkpath(testpath):
try:
if fs.isfile(testpath) and testpath.endswith('.apk'):
return True
elif fs.isdir(testpath):
for item in fs.ls(testpath):
if fs.isfile(fs.join(testpath,item)) and item.endswith('.apk'):
return True
print('Specified directory has no apk files in it')
return False
except Exception as e:
print('{0} does not exist'.format(testpath))
return False
def execute(execomponent):
prepare_execute(execomponent)
logger = lgr.Logger(fs.join(execomponent.resultdir, 'results.csv'))
firepool = fpool.Firepool()
timeout = ask_timeout()
args = arg_generator(execomponent, firepool, logger, timeout)
logger.start()
firepool.fire(parallel_execute, args)
logger.stop()
post_execute(execomponent)
def restart(execomponents):
firepool = fpool.Firepool()
for execomponent in execomponents:
remove_finished_tasks(execomponent)
e.prepare_execute(execomponent)
timeout = e.ask_timeout()
logger = lgr.Logger(fs.join(execomponent.resultdir, 'results.csv'))
args = e.arg_generator(execomponent, firepool, logger, timeout)
logger.start()
firepool.fire(e.parallel_execute, args)
logger.stop()
e.post_execute(execomponent)
def get_components():
components = []
installers = get_installer_dirs()
for item in installers:
if not fs.isfile(s.installersdir,item,'install.py'):
print('Error in "{0}": No install.py found!'.format(item))
continue
elif not fs.isfile(s.installersdir,item,'run.py'):
print('Error in "{0}": No run.py found!'.format(item))
continue
elif not fs.isfile(s.installersdir,item,'analyse.py'):
print('Error in "{0}": No analyse.py found!'.format(item))
continue
installmod = 'installers.{0}.install'.format(item)
runmod = 'installers.{0}.run'.format(item)
analysemod = 'installers.{0}.analyse'.format(item)
installerloc = fs.join(s.installersdir,item)
installloc = fs.join(s.installdir,item)
component = cmpnt.Component(installmod, runmod, analysemod, installerloc, installloc)
components.append(component)
components.sort()
return components
def init(_path):
global root
root = _path
global installersdir
installersdir = fs.join(_path, 'installers')
global installdir
installdir = fs.join(_path, 'installed')
global dependencydir
dependencydir = fs.join(installdir, 'dependencies')
global resultsdir
resultsdir = fs.join(_path, 'results')
global analyseddir
analyseddir = fs.join(_path, 'analysed')
global execconfdir
execconfdir = fs.join(_path, 'executionconfigs')
global runconf
runconf = 'run.conf'
global errorlog
errorlog = 'errors.log'
global outlog
outlog = 'out.log'
def get_logger(analysecomponent):
default_name = fs.split(analysecomponent.runresult_path)[-3]
default_path = fs.join(s.analyseddir, default_name, 'out.csv')
return lgr.Logger(default_path)
def prepare_execute(self):
for task in self.tasks:
fs.mkdir(task.resultpath, exist_ok=True)
for component in self.methodcomponents:
component.prepare_run(self.tasks)
self.to_file(fs.join(self.resultdir, s.runconf))
def to_file(name, apks):
filepath = fs.join(s.execconfdir,name+'.conf')
with open(filepath, 'w') as conf:
for apk in apks:
conf.write(apkfile.Apk.get_string(apk)+'\n')
def is_complete_execution(execomponent):
finished_names_list = get_finished_apknames(
fs.join(execomponent.resultdir, 'results.csv'))
return len(execomponent.tasks) / len(
execomponent.methodcomponents) == len(finished_names_list)
