def analyze_apk(self, eandro_apk):
''' Analyze the `eandro_apk` and return the analysis results.
Parameters
----------
eandro_apk : EAndroApk
The apk to analyze.
Returns
-------
list<FastApk, AndroScript>
None
If error happened.
'''
if eandro_apk is not None:
# analysis
res = AnalyzeUtil.analyze_apk(eandro_apk, self.androscripts, self.min_script_needs, reset_scripts = True)
if res is not None:
fastapk, scripts = res
# we need to backup the scripts cause they will be reused for a new analysis
res[1] = deepcopy(scripts)
clilog.debug("analyzed %s", fastapk.short_description())
return res
def analyze_apk(self, eandro_apk):
''' Analyze the `eandro_apk` and return the analysis results.
Parameters
----------
eandro_apk : EAndroApk
The apk to analyze.
Returns
-------
list<FastApk, AndroScript>
None
If error happened.
'''
if eandro_apk is not None:
# analysis
res = AnalyzeUtil.analyze_apk(eandro_apk,
self.androscripts,
self.min_script_needs,
reset_scripts=True)
if res is not None:
fastapk, scripts = res
# we need to backup the scripts cause they will be reused for a new analysis
res[1] = deepcopy(scripts)
clilog.debug("analyzed %s", fastapk.short_description())
return res
def run(self,
androscripts,
min_script_needs,
script_hashes,
apk_zipfile_or_hash,
is_hash=True,
fast_apk=None):
'''
Do the analysis on the apk with the given scripts.
Parameters
----------
androscripts : list<str>
List of package names.
script_hashes : list<str>
If given, set the hash for the `AndroScript`s
min_script_needs : tuple<bool>
See :py:method:`ScriptUtil.get_maximal_script_options`.
apk_zipfile_or_hash : str
The raw contents of the .apk file or the hash (sha256).
The raw content of the .apk file (zipfile) or the hash of it (id in db).
is_hash : bool, optional (default is True)
Determines if `apk_zipfile_or_hash` is a hash (id).
fast_apk : FastApk, optional (default is None)
Holds the meta infos for the apk.
Returns
-------
tuple<tuple<str, bool>>
First component is the id of the entry
and the second a boolean indication if the result has been stored in gridfs.
()
If an error occurred.
'''
try:
# method retry_arguments
self.__retry_arguments = androscripts, min_script_needs, script_hashes, apk_zipfile_or_hash, is_hash, fast_apk
eandro_apk = None
do_script_hash_validation = settings.script_hash_validation_enabled(
)
# open database/apk storage if not already done
# reschedule job if connection/open error
self.__open_db()
self.__open_apk_storage()
# setup scripts
if do_script_hash_validation:
# validate sent hashes with local script hashes
self.__setup_scripts_hash_validation(androscripts,
script_hashes)
else:
# reuse if possible
self.__setup_scripts_reuse(androscripts, script_hashes)
# open apk
if not is_hash:
log.info("opening apk via raw data ... ")
eandro_apk = AnalyzeUtil.open_apk(
apk_or_path=apk_zipfile_or_hash, apk=fast_apk, raw=True)
else:
# get apk from prefetched apk pool
eandro_apk = apk_prefetch_pool.get(apk_zipfile_or_hash, None)
# could not prefetch
if eandro_apk is None:
eandro_apk = self.__get_apk_from_storage_retry(
apk_zipfile_or_hash, apk=fast_apk)
# if None, could not be opened and error has been logged
if eandro_apk is not None:
result = AnalyzeUtil.analyze_apk(
eandro_apk,
self.androscripts,
min_script_needs,
propagate_error=False,
reset_scripts=not do_script_hash_validation)
if result is not None:
fastapk, script_results = result
log.info("analyzed %s", fastapk.short_description())
storage_results = self.__store_results(
fastapk, script_results)
# can be None if errorr occurred
if storage_results:
return tuple(storage_results)
return ()
except SoftTimeLimitExceeded:
log.warn("Task %s exceeded it's soft time limit!", self)
raise
except ScriptHashValidationError:
raise
finally:
# delete from pool -> we don't need it anymore in the pool
if is_hash and apk_zipfile_or_hash in apk_prefetch_pool:
del apk_prefetch_pool[apk_zipfile_or_hash]
def _analyze(self, test = False):
'''
Start the analysis and store the results in the predefined place.
Parameters
----------
test : bool, optional (default is False)
Use for testing. Will not store any result !
Returns
-------
int
Number of analyzed apks
list<ResultObject>
List of the results (only if `test`)
'''
androscripts = self.script_list
# collect results for test mode
test_results = []
# get minimum options for all scripts -> boost performance
# use only as much options as needed!
# run over apks
for apk_path, _apk, _ in apk_gen(self.apks_or_paths):
eandro_apk = open_apk(apk_path, apk=_apk)
# if is None error happened and has been logged
# otherwise proceed with analysis
if eandro_apk is not None:
# tuple<FastApk, AndroScript>
res = AnalyzeUtil.analyze_apk(eandro_apk, androscripts, self.min_script_needs, reset_scripts = True)
if res:
# unpack results
fastapk, script_results = res
# store results if not in test mode
if not test:
for script in script_results:
try:
storage_result = AnalyzeUtil.store_script_res(self.storage, script, fastapk)
# keep storage results
self.add_storage_result(storage_result)
except StorageException as e:
log.warn(e)
else:
# deliver result object in testing mode
test_results += [s.res for s in script_results]
clilog.info("analyzed %s", fastapk.short_description())
# increment counter, no lock needed, nobody else is writing to this value
self.cnt_analyzed_apks.value += 1
if test:
return test_results
return self.cnt_analyzed_apks.value
def _analyze(self, test=False):
'''
Start the analysis and store the results in the predefined place.
Parameters
----------
test : bool, optional (default is False)
Use for testing. Will not store any result !
Returns
-------
int
Number of analyzed apks
list<ResultObject>
List of the results (only if `test`)
'''
androscripts = self.script_list
# collect results for test mode
test_results = []
# get minimum options for all scripts -> boost performance
# use only as much options as needed!
# run over apks
for apk_path, _apk, _ in apk_gen(self.apks_or_paths):
eandro_apk = open_apk(apk_path, apk=_apk)
# if is None error happened and has been logged
# otherwise proceed with analysis
if eandro_apk is not None:
# tuple<FastApk, AndroScript>
res = AnalyzeUtil.analyze_apk(eandro_apk,
androscripts,
self.min_script_needs,
reset_scripts=True)
if res:
# unpack results
fastapk, script_results = res
# store results if not in test mode
if not test:
for script in script_results:
try:
storage_result = AnalyzeUtil.store_script_res(
self.storage, script, fastapk)
# keep storage results
self.add_storage_result(storage_result)
except StorageException as e:
log.warn(e)
else:
# deliver result object in testing mode
test_results += [s.res for s in script_results]
clilog.info("analyzed %s", fastapk.short_description())
# increment counter, no lock needed, nobody else is writing to this value
self.cnt_analyzed_apks.value += 1
if test:
return test_results
return self.cnt_analyzed_apks.value
def run(self, androscripts, min_script_needs, script_hashes, apk_zipfile_or_hash, is_hash = True, fast_apk = None):
'''
Do the analysis on the apk with the given scripts.
Parameters
----------
androscripts : list<str>
List of package names.
script_hashes : list<str>
If given, set the hash for the `AndroScript`s
min_script_needs : tuple<bool>
See :py:method:`ScriptUtil.get_maximal_script_options`.
apk_zipfile_or_hash : str
The raw contents of the .apk file or the hash (sha256).
The raw content of the .apk file (zipfile) or the hash of it (id in db).
is_hash : bool, optional (default is True)
Determines if `apk_zipfile_or_hash` is a hash (id).
fast_apk : FastApk, optional (default is None)
Holds the meta infos for the apk.
Returns
-------
tuple<tuple<str, bool>>
First component is the id of the entry
and the second a boolean indication if the result has been stored in gridfs.
()
If an error occurred.
'''
try:
# method retry_arguments
self.__retry_arguments = androscripts, min_script_needs, script_hashes, apk_zipfile_or_hash, is_hash, fast_apk
eandro_apk = None
do_script_hash_validation = settings.script_hash_validation_enabled()
# open database/apk storage if not already done
# reschedule job if connection/open error
self.__open_db()
self.__open_apk_storage()
# setup scripts
if do_script_hash_validation:
# validate sent hashes with local script hashes
self.__setup_scripts_hash_validation(androscripts, script_hashes)
else:
# reuse if possible
self.__setup_scripts_reuse(androscripts, script_hashes)
# open apk
if not is_hash:
log.info("opening apk via raw data ... ")
eandro_apk = AnalyzeUtil.open_apk(apk_or_path = apk_zipfile_or_hash, apk = fast_apk, raw = True)
else:
# get apk from prefetched apk pool
eandro_apk = apk_prefetch_pool.get(apk_zipfile_or_hash, None)
# could not prefetch
if eandro_apk is None:
eandro_apk = self.__get_apk_from_storage_retry(apk_zipfile_or_hash, apk = fast_apk)
# if None, could not be opened and error has been logged
if eandro_apk is not None:
result = AnalyzeUtil.analyze_apk(eandro_apk, self.androscripts, min_script_needs, propagate_error = False, reset_scripts = not do_script_hash_validation)
if result is not None:
fastapk, script_results = result
log.info("analyzed %s", fastapk.short_description())
storage_results = self.__store_results(fastapk, script_results)
# can be None if errorr occurred
if storage_results:
return tuple(storage_results)
return ()
except SoftTimeLimitExceeded:
log.warn("Task %s exceeded it's soft time limit!", self)
raise
except ScriptHashValidationError:
raise
finally:
# delete from pool -> we don't need it anymore in the pool
if is_hash and apk_zipfile_or_hash in apk_prefetch_pool:
del apk_prefetch_pool[apk_zipfile_or_hash]
