Beispiel #1
0
def cli_check_n_exec(func, prompt_prefix = "", circumvent_check = False, args=(), kwargs={}):
    ''' Check if the user really want's to continue and exec `func`.
    Otherwise quit the program with exit code 1 and log some message.

    Returns the result of the `func` or None.
    '''

    BASE_PROMPT = 'Continue and execute? (y/n): '
    prompt = "%s\n\t\t%s" % (prompt_prefix, BASE_PROMPT) if prompt_prefix else BASE_PROMPT
    if circumvent_check or raw_input(prompt).lower() == 'y':
        return func(*args, **kwargs)
    else:
        log.critical("aborted ...")
        sys.exit(1)
Beispiel #2
0
def cli_check_n_exec(func,
                     prompt_prefix="",
                     circumvent_check=False,
                     args=(),
                     kwargs={}):
    ''' Check if the user really want's to continue and exec `func`.
    Otherwise quit the program with exit code 1 and log some message.

    Returns the result of the `func` or None.
    '''

    BASE_PROMPT = 'Continue and execute? (y/n): '
    prompt = "%s\n\t\t%s" % (prompt_prefix,
                             BASE_PROMPT) if prompt_prefix else BASE_PROMPT
    if circumvent_check or raw_input(prompt).lower() == 'y':
        return func(*args, **kwargs)
    else:
        log.critical("aborted ...")
        sys.exit(1)
    def __recreate_collections(self, gridfs = False, res_collection = False):
        '''
        Drop and recreate collections.

        Parameters
        ----------
        gridfs : bool, optional (default is False)
            Recreate gridfs.
        res_collection, bool, optional (default is False)
            Recreate results collection.
        '''
        try:
            if gridfs:
                log.debug("dropping collection %s", GRIDFS_COLLS_PREFIX)

                log.debug("dropping collection %s", FILES_COLL_NAME)
                self.db.drop_collection(FILES_COLL_NAME)

                log.debug("dropping collection %s", CHUNKS_COLL_NAME)
                self.db.drop_collection(CHUNKS_COLL_NAME)

                log.debug("recreating collection %s", GRIDFS_COLLS_PREFIX)
                self._open_gridfs()

                self._create_idx_for_colls()
        except PyMongoError as e:
            log.critical(e)

        try:
            if res_collection:
                log.debug("dropping collection %s", RESULT_DOCUMENTS_COLLECTION_NAME)
                self.db.drop_collection(RESULT_DOCUMENTS_COLLECTION_NAME)
                self._open_res_coll()
                log.debug("recreating collection %s", RESULT_DOCUMENTS_COLLECTION_NAME)
        except PyMongoError as e:
            log.critical(e)
    def _analyze(self):
        ''' See doc of :py:method:`.BaseAnalyzer.analyze`. '''

        # try to get registered workers
        # it network fails at this point -> stop analysis
        try:
            clilog.info(CeleryUtil.get_workers_and_check_network())
        except NetworkError as e:
            log.critical(e)
            return 0

        # storage objects
        storage = self.storage

        clilog.info("Number of apks to analyze: %d", self._cnt_apks)

        try:
            # get analyze task
            analyze_task = tasks[CeleryConstants.get_analyze_task_name()]

            # create storage
            storage.create_or_open_sub_storages()

            # send tasks
            start = time()

            # apk generator over .apk or apk hashes
            apk_gen = AnalyzeUtil.apk_id_or_raw_data_gen(
                self.apks, force_raw_data=self.serialize_apks)

            clilog.info("Task publishing progress:")

            # send and serialize .apks
            # if analysis via path serialize them!
            if self.serialize_apks:
                log.info("sending .apks to message broker")
                self.group_result = group_result = GroupResult(results=[])

                for args in self.send_apk_args_generator(apk_gen):
                    task = analyze_task.delay(*args)
                    group_result.add(task)

            # send only apk id and let fetch via mongodb
            else:
                log.info("sending ids of apks")

                task_group = group(
                    (analyze_task.s(*args)
                     for args in self.send_id_args_generator(apk_gen)))

                # publish tasks
                self.group_result = task_group()

            log.info("sending took %ss", (time() - start))
            sys.stderr.write("\nAnalysis progress:\n")

            # start showing analysis progress
            self.analyze_stats_view.start()

            # wait for results
            log.debug("joining on ResultGroup ... ")

            # setup callback
            callback_func = self.get_callback_func(self.success_handler,
                                                   self.error_handler)
            CeleryUtil.join_native(self.group_result,
                                   propagate=False,
                                   callback=callback_func)

            clilog.info("\nanalysis done ... ")
            log.info("distributed analysis took %ss", (time() - start))

            return self.stop_analysis_view()
        except DatabaseOpenError as e:
            log.critical(e)
            return 0

        except (KeyboardInterrupt, Exception) as e:
            if not isinstance(e, KeyboardInterrupt):
                log.exception(e)
            log.warn(
                "Interrupting distributed analysis ... Please wait a moment!")
            log.warn("revoking tasks on all workers ...")

            if celerysettings.CELERY_TASK_REVOCATION_ENABLED:
                # revoke tasks
                if self.group_result is None:
                    # revoke via task ids
                    log.debug("revoking while publishing tasks ...")

                    self.task_collection.revoke_all(terminate=True,
                                                    signal='SIGKILL')
                else:
                    # revoke via GroupResult if yet available/created
                    # first available after all tasks have been send
                    self.group_result.revoke(terminate=True, signal='SIGKILL')
                log.warn("revoked tasks and killed workers ...")

            #return number of analyzed apks
            return self.stop_analysis_view()
    def _analyze(self):
        ''' See doc of :py:method:`.BaseAnalyzer.analyze`. '''

        # try to get registered workers
        # it network fails at this point -> stop analysis
        try:
            clilog.info(CeleryUtil.get_workers_and_check_network())
        except NetworkError as e:
            log.critical(e)
            return 0

        # storage objects
        storage = self.storage

        clilog.info("Number of apks to analyze: %d", self._cnt_apks)

        try:
            # get analyze task
            analyze_task = tasks[CeleryConstants.get_analyze_task_name()]

            # create storage
            storage.create_or_open_sub_storages()

            # send tasks
            start = time()

            # apk generator over .apk or apk hashes
            apk_gen = AnalyzeUtil.apk_id_or_raw_data_gen(self.apks, force_raw_data = self.serialize_apks)

            clilog.info("Task publishing progress:")

            # send and serialize .apks
            # if analysis via path serialize them!
            if self.serialize_apks:
                log.info("sending .apks to message broker")
                self.group_result = group_result = GroupResult(results = [])

                for args in self.send_apk_args_generator(apk_gen):
                    task = analyze_task.delay(*args)
                    group_result.add(task)

            # send only apk id and let fetch via mongodb
            else:
                log.info("sending ids of apks")

                task_group = group((analyze_task.s(*args) for args in self.send_id_args_generator(apk_gen)))

                # publish tasks
                self.group_result = task_group()

            log.info("sending took %ss", (time() - start))
            sys.stderr.write("\nAnalysis progress:\n")

            # start showing analysis progress
            self.analyze_stats_view.start()

            # wait for results
            log.debug("joining on ResultGroup ... ")

            # setup callback
            callback_func = self.get_callback_func(self.success_handler, self.error_handler)
            CeleryUtil.join_native(self.group_result, propagate = False, callback = callback_func)

            clilog.info("\nanalysis done ... ")
            log.info("distributed analysis took %ss", (time() - start))

            return self.stop_analysis_view()
        except DatabaseOpenError as e:
            log.critical(e)
            return 0

        except (KeyboardInterrupt, Exception) as e:
            if not isinstance(e, KeyboardInterrupt):
                log.exception(e)
            log.warn("Interrupting distributed analysis ... Please wait a moment!")
            log.warn("revoking tasks on all workers ...")

            if celerysettings.CELERY_TASK_REVOCATION_ENABLED:
                # revoke tasks
                if self.group_result is None:
                    # revoke via task ids
                    log.debug("revoking while publishing tasks ...")

                    self.task_collection.revoke_all(terminate = True, signal = 'SIGKILL')
                else:
                    # revoke via GroupResult if yet available/created
                    # first available after all tasks have been send
                    self.group_result.revoke(terminate = True, signal = 'SIGKILL')
                log.warn("revoked tasks and killed workers ...")

            #return number of analyzed apks
            return self.stop_analysis_view()