def cli_check_n_exec(func, prompt_prefix = "", circumvent_check = False, args=(), kwargs={}): ''' Check if the user really want's to continue and exec `func`. Otherwise quit the program with exit code 1 and log some message. Returns the result of the `func` or None. ''' BASE_PROMPT = 'Continue and execute? (y/n): ' prompt = "%s\n\t\t%s" % (prompt_prefix, BASE_PROMPT) if prompt_prefix else BASE_PROMPT if circumvent_check or raw_input(prompt).lower() == 'y': return func(*args, **kwargs) else: log.critical("aborted ...") sys.exit(1)
def cli_check_n_exec(func, prompt_prefix="", circumvent_check=False, args=(), kwargs={}): ''' Check if the user really want's to continue and exec `func`. Otherwise quit the program with exit code 1 and log some message. Returns the result of the `func` or None. ''' BASE_PROMPT = 'Continue and execute? (y/n): ' prompt = "%s\n\t\t%s" % (prompt_prefix, BASE_PROMPT) if prompt_prefix else BASE_PROMPT if circumvent_check or raw_input(prompt).lower() == 'y': return func(*args, **kwargs) else: log.critical("aborted ...") sys.exit(1)
def __recreate_collections(self, gridfs = False, res_collection = False): ''' Drop and recreate collections. Parameters ---------- gridfs : bool, optional (default is False) Recreate gridfs. res_collection, bool, optional (default is False) Recreate results collection. ''' try: if gridfs: log.debug("dropping collection %s", GRIDFS_COLLS_PREFIX) log.debug("dropping collection %s", FILES_COLL_NAME) self.db.drop_collection(FILES_COLL_NAME) log.debug("dropping collection %s", CHUNKS_COLL_NAME) self.db.drop_collection(CHUNKS_COLL_NAME) log.debug("recreating collection %s", GRIDFS_COLLS_PREFIX) self._open_gridfs() self._create_idx_for_colls() except PyMongoError as e: log.critical(e) try: if res_collection: log.debug("dropping collection %s", RESULT_DOCUMENTS_COLLECTION_NAME) self.db.drop_collection(RESULT_DOCUMENTS_COLLECTION_NAME) self._open_res_coll() log.debug("recreating collection %s", RESULT_DOCUMENTS_COLLECTION_NAME) except PyMongoError as e: log.critical(e)
def _analyze(self): ''' See doc of :py:method:`.BaseAnalyzer.analyze`. ''' # try to get registered workers # it network fails at this point -> stop analysis try: clilog.info(CeleryUtil.get_workers_and_check_network()) except NetworkError as e: log.critical(e) return 0 # storage objects storage = self.storage clilog.info("Number of apks to analyze: %d", self._cnt_apks) try: # get analyze task analyze_task = tasks[CeleryConstants.get_analyze_task_name()] # create storage storage.create_or_open_sub_storages() # send tasks start = time() # apk generator over .apk or apk hashes apk_gen = AnalyzeUtil.apk_id_or_raw_data_gen( self.apks, force_raw_data=self.serialize_apks) clilog.info("Task publishing progress:") # send and serialize .apks # if analysis via path serialize them! if self.serialize_apks: log.info("sending .apks to message broker") self.group_result = group_result = GroupResult(results=[]) for args in self.send_apk_args_generator(apk_gen): task = analyze_task.delay(*args) group_result.add(task) # send only apk id and let fetch via mongodb else: log.info("sending ids of apks") task_group = group( (analyze_task.s(*args) for args in self.send_id_args_generator(apk_gen))) # publish tasks self.group_result = task_group() log.info("sending took %ss", (time() - start)) sys.stderr.write("\nAnalysis progress:\n") # start showing analysis progress self.analyze_stats_view.start() # wait for results log.debug("joining on ResultGroup ... ") # setup callback callback_func = self.get_callback_func(self.success_handler, self.error_handler) CeleryUtil.join_native(self.group_result, propagate=False, callback=callback_func) clilog.info("\nanalysis done ... ") log.info("distributed analysis took %ss", (time() - start)) return self.stop_analysis_view() except DatabaseOpenError as e: log.critical(e) return 0 except (KeyboardInterrupt, Exception) as e: if not isinstance(e, KeyboardInterrupt): log.exception(e) log.warn( "Interrupting distributed analysis ... Please wait a moment!") log.warn("revoking tasks on all workers ...") if celerysettings.CELERY_TASK_REVOCATION_ENABLED: # revoke tasks if self.group_result is None: # revoke via task ids log.debug("revoking while publishing tasks ...") self.task_collection.revoke_all(terminate=True, signal='SIGKILL') else: # revoke via GroupResult if yet available/created # first available after all tasks have been send self.group_result.revoke(terminate=True, signal='SIGKILL') log.warn("revoked tasks and killed workers ...") #return number of analyzed apks return self.stop_analysis_view()
def _analyze(self): ''' See doc of :py:method:`.BaseAnalyzer.analyze`. ''' # try to get registered workers # it network fails at this point -> stop analysis try: clilog.info(CeleryUtil.get_workers_and_check_network()) except NetworkError as e: log.critical(e) return 0 # storage objects storage = self.storage clilog.info("Number of apks to analyze: %d", self._cnt_apks) try: # get analyze task analyze_task = tasks[CeleryConstants.get_analyze_task_name()] # create storage storage.create_or_open_sub_storages() # send tasks start = time() # apk generator over .apk or apk hashes apk_gen = AnalyzeUtil.apk_id_or_raw_data_gen(self.apks, force_raw_data = self.serialize_apks) clilog.info("Task publishing progress:") # send and serialize .apks # if analysis via path serialize them! if self.serialize_apks: log.info("sending .apks to message broker") self.group_result = group_result = GroupResult(results = []) for args in self.send_apk_args_generator(apk_gen): task = analyze_task.delay(*args) group_result.add(task) # send only apk id and let fetch via mongodb else: log.info("sending ids of apks") task_group = group((analyze_task.s(*args) for args in self.send_id_args_generator(apk_gen))) # publish tasks self.group_result = task_group() log.info("sending took %ss", (time() - start)) sys.stderr.write("\nAnalysis progress:\n") # start showing analysis progress self.analyze_stats_view.start() # wait for results log.debug("joining on ResultGroup ... ") # setup callback callback_func = self.get_callback_func(self.success_handler, self.error_handler) CeleryUtil.join_native(self.group_result, propagate = False, callback = callback_func) clilog.info("\nanalysis done ... ") log.info("distributed analysis took %ss", (time() - start)) return self.stop_analysis_view() except DatabaseOpenError as e: log.critical(e) return 0 except (KeyboardInterrupt, Exception) as e: if not isinstance(e, KeyboardInterrupt): log.exception(e) log.warn("Interrupting distributed analysis ... Please wait a moment!") log.warn("revoking tasks on all workers ...") if celerysettings.CELERY_TASK_REVOCATION_ENABLED: # revoke tasks if self.group_result is None: # revoke via task ids log.debug("revoking while publishing tasks ...") self.task_collection.revoke_all(terminate = True, signal = 'SIGKILL') else: # revoke via GroupResult if yet available/created # first available after all tasks have been send self.group_result.revoke(terminate = True, signal = 'SIGKILL') log.warn("revoked tasks and killed workers ...") #return number of analyzed apks return self.stop_analysis_view()