def __init__(self, *args, **kwargs):
'''
See :py:method`.BaseAnalyzer.__init__` for details.
Parameters
----------
serialize_apks : bool, optional (default is True)
If true, serialize .apk .
Otherwise id (hash) of the apk will be send and fetched by the worker from the result db.
Be sure to import the apks to the result db first!
'''
serialize_apks = kwargs.get("serialize_apks", True)
super(DistributedAnalyzer, self).__init__(*args, **kwargs)
# list(apk_path, _apk, is_apk)
self.__apks = list(AnalyzeUtil.apk_gen(self.apks_or_paths))
# result group
self.group_result = None
# serialize .apk data
self.__serialize_apks = serialize_apks
if serialize_apks:
clilog.info("Will serialize .apk data!")
else:
clilog.info("Will send id of apks!")
self.analyze_stats_view = None
# stats view for cli
self.analyze_stats_view = AnalysisStatsView(self._cnt_apks)
self.analyze_stats_view.daemon = True
# the `TaskCollection` for the analysis tasks
self.task_collection = TaskCollection(self._cnt_apks)
# register celery signals
self.register_signals()
self.lock = Lock()
def __init__(self, *args, **kwargs):
'''
See :py:method`.BaseAnalyzer.__init__` for details.
Parameters
----------
serialize_apks : bool, optional (default is True)
If true, serialize .apk .
Otherwise id (hash) of the apk will be send and fetched by the worker from the result db.
Be sure to import the apks to the result db first!
'''
serialize_apks = kwargs.get("serialize_apks", True)
super(DistributedAnalyzer, self).__init__(*args, **kwargs)
# list(apk_path, _apk, is_apk)
self.__apks = list(AnalyzeUtil.apk_gen(self.apks_or_paths))
# result group
self.group_result = None
# serialize .apk data
self.__serialize_apks = serialize_apks
if serialize_apks:
clilog.info("Will serialize .apk data!")
else:
clilog.info("Will send id of apks!")
self.analyze_stats_view = None
# stats view for cli
self.analyze_stats_view = AnalysisStatsView(self._cnt_apks)
self.analyze_stats_view.daemon = True
# the `TaskCollection` for the analysis tasks
self.task_collection = TaskCollection(self._cnt_apks)
# register celery signals
self.register_signals()
self.lock = Lock()
def _analyze(self):
''' See doc of :py:method:BaseAnalyzer.analyze`. '''
try:
work_queue = self.work_queue
# create worker pool
log.debug("starting %s workers ...", self.concurrency)
for _ in range(self.concurrency):
p = Worker(self.script_list, self.script_hashes, self.min_script_needs,
work_queue, self.storage,
self.cnt_analyzed_apks, self.analyzed_apks, self.storage_results)
self.workers.append(p)
p.daemon = True
# start workers
for p in self.workers:
p.start()
# queue has size limit -> start workers first then enqueue items
log.info("Loading apk paths into work queue ...")
for apk_stuff in AnalyzeUtil.apk_gen(self.apks_or_paths):
# task is apk with all scripts
work_queue.put(apk_stuff)
for _ in range(self.concurrency):
# signal end-of-work
work_queue.put(STOP_SENTINEL)
# progress view for cli
av = AnalysisStatsView(self.cnt_analyzed_apks, self._cnt_apks, self.analyzed_apks)
av.daemon = True
av.start()
# block until workers finished
work_queue.join()
av.terminate()
log.debug("joined on work queue ...")
return self.cnt_analyzed_apks.value
# try hot shutdown first
except KeyboardInterrupt:
log.warn("Hot shutdown ... ")
try:
log.warn("clearing work queue ... ")
Util.clear_queue(work_queue)
log.warn("cleared work queue ... ")
for _ in range(self.concurrency):
# signal end-of-work
work_queue.put(STOP_SENTINEL)
for worker in self.workers:
worker.join()
log.warn("waited for all workers ... ")
return self.cnt_analyzed_apks.value
# if user really wants make a cold shutdown -> kill processes
except KeyboardInterrupt:
log.warn("Cold shutdown ... ")
log.warn("Hard shutdown wanted! Killing all workers!")
# kill processes via SIGINT -> send CTRL-C
for w in self.workers:
try:
os.kill(w.pid, signal.SIGINT)
except:
pass
return self.cnt_analyzed_apks.value
def _analyze(self):
''' See doc of :py:method:BaseAnalyzer.analyze`. '''
try:
work_queue = self.work_queue
# create worker pool
log.debug("starting %s workers ...", self.concurrency)
for _ in range(self.concurrency):
p = Worker(self.script_list, self.script_hashes,
self.min_script_needs, work_queue, self.storage,
self.cnt_analyzed_apks, self.analyzed_apks,
self.storage_results)
self.workers.append(p)
p.daemon = True
# start workers
for p in self.workers:
p.start()
# queue has size limit -> start workers first then enqueue items
log.info("Loading apk paths into work queue ...")
for apk_stuff in AnalyzeUtil.apk_gen(self.apks_or_paths):
# task is apk with all scripts
work_queue.put(apk_stuff)
for _ in range(self.concurrency):
# signal end-of-work
work_queue.put(STOP_SENTINEL)
# progress view for cli
av = AnalysisStatsView(self.cnt_analyzed_apks, self._cnt_apks,
self.analyzed_apks)
av.daemon = True
av.start()
# block until workers finished
work_queue.join()
av.terminate()
log.debug("joined on work queue ...")
return self.cnt_analyzed_apks.value
# try hot shutdown first
except KeyboardInterrupt:
log.warn("Hot shutdown ... ")
try:
log.warn("clearing work queue ... ")
Util.clear_queue(work_queue)
log.warn("cleared work queue ... ")
for _ in range(self.concurrency):
# signal end-of-work
work_queue.put(STOP_SENTINEL)
for worker in self.workers:
worker.join()
log.warn("waited for all workers ... ")
return self.cnt_analyzed_apks.value
# if user really wants make a cold shutdown -> kill processes
except KeyboardInterrupt:
log.warn("Cold shutdown ... ")
log.warn("Hard shutdown wanted! Killing all workers!")
# kill processes via SIGINT -> send CTRL-C
for w in self.workers:
try:
os.kill(w.pid, signal.SIGINT)
except:
pass
return self.cnt_analyzed_apks.value
