def analyze_apk(self, eandro_apk):
''' Analyze the `eandro_apk` and return the analysis results.
Parameters
----------
eandro_apk : EAndroApk
The apk to analyze.
Returns
-------
list<FastApk, AndroScript>
None
If error happened.
'''
if eandro_apk is not None:
# analysis
res = AnalyzeUtil.analyze_apk(eandro_apk, self.androscripts, self.min_script_needs, reset_scripts = True)
if res is not None:
fastapk, scripts = res
# we need to backup the scripts cause they will be reused for a new analysis
res[1] = deepcopy(scripts)
clilog.debug("analyzed %s", fastapk.short_description())
return res
def analyze_apk(self, eandro_apk):
''' Analyze the `eandro_apk` and return the analysis results.
Parameters
----------
eandro_apk : EAndroApk
The apk to analyze.
Returns
-------
list<FastApk, AndroScript>
None
If error happened.
'''
if eandro_apk is not None:
# analysis
res = AnalyzeUtil.analyze_apk(eandro_apk,
self.androscripts,
self.min_script_needs,
reset_scripts=True)
if res is not None:
fastapk, scripts = res
# we need to backup the scripts cause they will be reused for a new analysis
res[1] = deepcopy(scripts)
clilog.debug("analyzed %s", fastapk.short_description())
return res
def fetch_results_from_mongodb(self, rds, results, wait_for_db = True,
# progress
nice_progess = False, synced_entries = None, total_sync_entries = None):
'''
Fetch some results from the result database and write them to disk.
If data cannot be loaded from db, try until it can be.
Parameters
----------
rds : ResultDatabaseStorage
The database to query for the results.
results : list< tuple<id, gridfs (bool)> >
Define which results shall be fetched.
wait_for_db : bool, optional (default is True)
Wait until data could be fetched from db.
nice_progess : bool, optional (default is False)
If enabled update show some nice progress bar on the cli.
synced_entries : multiprocessing.Value<int>, optional (default is None)
If supplied store number of already synces entries.
total_sync_entries : multiprocessing.Value<int>, optional (default is None)
If supplied store number of total entries to sync.
Raises
------
DatabaseLoadException
If `wait_for_db` is False and an error occurred.
'''
# retry in ... seconds
DATABASE_RETRY_TIME = 5
# if true assume both counts are shared memory (Value)
use_shared_memory = synced_entries is not None and total_sync_entries is not None
if results is not None:
results_stored = False
while not results_stored:
try:
# get ids
non_gridfs_ids, gridfs_ids = MongoUtil.split_result_ids(results)
# counts
cnt_non_gridfs_ids = len(non_gridfs_ids)
cnt_gridfs_ids = len(gridfs_ids)
if use_shared_memory:
total_sync_entries.value = cnt_gridfs_ids + cnt_non_gridfs_ids
# gridfs raw data as well as metadata
gridfs_entries_raw = []
if gridfs_ids:
gridfs_entries_raw = rds.get_results_for_ids(gridfs_ids, non_document = True, non_document_raw = True)
# regular documents (non gridfs)
non_gridfs_entries = []
if non_gridfs_ids:
non_gridfs_entries = rds.get_results_for_ids(non_gridfs_ids, non_document = False, non_document_raw = True)
if not nice_progess:
log.debug("fetching %d non-documents (gridfs) ... ", cnt_gridfs_ids)
for i, gridfs_entry_raw in enumerate(gridfs_entries_raw, 1):
# get our stored metadata (for script and apk)
gridfs_entry_meta = gridfs_entry_raw.metadata
if not nice_progess:
log.debug("getting results for %s", gridfs_entry_meta[RESOBJ_APK_META][RESOBJ_APK_META_PACKAGE_NAME])
else:
Util.print_dyn_progress(Util.format_progress(i, cnt_gridfs_ids))
# use apk to extract data from dict
fastapk = FastApk.load_from_result_dict(gridfs_entry_meta)
# get filename
file_name = gridfs_entry_raw.filename
# write results to disk
try:
self.store_custom_data(fastapk.package_name, fastapk.version_name, fastapk.hash, file_name, gridfs_entry_raw.read())
except FileSysStoreException as e:
log.exception(e)
# update shared memory progress indicitor
if use_shared_memory:
with synced_entries.get_lock():
synced_entries.value += 1
if not nice_progess:
log.debug("fetching %d documents (non-gridfs) ... ", cnt_non_gridfs_ids)
for i, non_gridfs_entry in enumerate(non_gridfs_entries, 1):
if not nice_progess:
clilog.debug("getting results for %s" % non_gridfs_entry[RESOBJ_APK_META][RESOBJ_APK_META_PACKAGE_NAME])
else:
Util.print_dyn_progress(Util.format_progress(i, cnt_non_gridfs_ids))
# write results to disk
self.store_result_dict(non_gridfs_entry)
# update shared memory progress indicitor
if use_shared_memory:
with synced_entries.get_lock():
synced_entries.value += 1
# if not wait for db wanted stop here
results_stored = True or not wait_for_db
except (DatabaseLoadException, PyMongoError) as e:
if not wait_for_db:
raise
log.warn(e)
Util.log_will_retry(DATABASE_RETRY_TIME, exc = e)
sleep(DATABASE_RETRY_TIME)
