def __init__(self, *args, **kwargs): ''' See :py:method`.BaseAnalyzer.__init__` for details. Parameters ---------- serialize_apks : bool, optional (default is True) If true, serialize .apk . Otherwise id (hash) of the apk will be send and fetched by the worker from the result db. Be sure to import the apks to the result db first! ''' serialize_apks = kwargs.get("serialize_apks", True) super(DistributedAnalyzer, self).__init__(*args, **kwargs) # list(apk_path, _apk, is_apk) self.__apks = list(AnalyzeUtil.apk_gen(self.apks_or_paths)) # result group self.group_result = None # serialize .apk data self.__serialize_apks = serialize_apks if serialize_apks: clilog.info("Will serialize .apk data!") else: clilog.info("Will send id of apks!") self.analyze_stats_view = None # stats view for cli self.analyze_stats_view = AnalysisStatsView(self._cnt_apks) self.analyze_stats_view.daemon = True # the `TaskCollection` for the analysis tasks self.task_collection = TaskCollection(self._cnt_apks) # register celery signals self.register_signals() self.lock = Lock()
def _analyze(self): ''' See doc of :py:method:BaseAnalyzer.analyze`. ''' try: work_queue = self.work_queue # create worker pool log.debug("starting %s workers ...", self.concurrency) for _ in range(self.concurrency): p = Worker(self.script_list, self.script_hashes, self.min_script_needs, work_queue, self.storage, self.cnt_analyzed_apks, self.analyzed_apks, self.storage_results) self.workers.append(p) p.daemon = True # start workers for p in self.workers: p.start() # queue has size limit -> start workers first then enqueue items log.info("Loading apk paths into work queue ...") for apk_stuff in AnalyzeUtil.apk_gen(self.apks_or_paths): # task is apk with all scripts work_queue.put(apk_stuff) for _ in range(self.concurrency): # signal end-of-work work_queue.put(STOP_SENTINEL) # progress view for cli av = AnalysisStatsView(self.cnt_analyzed_apks, self._cnt_apks, self.analyzed_apks) av.daemon = True av.start() # block until workers finished work_queue.join() av.terminate() log.debug("joined on work queue ...") return self.cnt_analyzed_apks.value # try hot shutdown first except KeyboardInterrupt: log.warn("Hot shutdown ... ") try: log.warn("clearing work queue ... ") Util.clear_queue(work_queue) log.warn("cleared work queue ... ") for _ in range(self.concurrency): # signal end-of-work work_queue.put(STOP_SENTINEL) for worker in self.workers: worker.join() log.warn("waited for all workers ... ") return self.cnt_analyzed_apks.value # if user really wants make a cold shutdown -> kill processes except KeyboardInterrupt: log.warn("Cold shutdown ... ") log.warn("Hard shutdown wanted! Killing all workers!") # kill processes via SIGINT -> send CTRL-C for w in self.workers: try: os.kill(w.pid, signal.SIGINT) except: pass return self.cnt_analyzed_apks.value