def __init__(self, storage, script_list, script_hashes, *args, **kwargs): ''' See :py:method`.BaseAnalyzer.__init__` for details on the first attributes ''' super(Analyzer, self).__init__(storage, script_list, script_hashes, *args, **kwargs) # instantiate scripts self.script_list = sorted( ScriptUtil.instantiate_scripts(script_list, script_hashes=script_hashes))
def __init__(self, script_list, script_hashes, min_script_needs, work_queue, storage, sm_analyzed_apks, analyzed_apks, storage_results=None): ''' Parameters ---------- script_list: list<type<AndroScript>> List of `AndroScript`s references (not instantiated class!) script_hashes : list<str>, optional (default is None) If given, set the hash for the `AndroScript`s min_script_needs : tuple<bool> See :py:method:`ScriptUtil.get_maximal_script_options`. work_queue : Queue<str> Queue with paths to apks which shall be analyzed. storage: RedundantStorage The storage to store the results. sm_analyzed_apks : Value Shared memory to add number of analyzed apks. analyzed_apks : Queue<FastAPK> Holds the analyzed APKs. storage_results : Queue<tuple<str, bool>>, optional (default is None) Storage results. First component is the id of the entry and the second a boolean indication if the result has been stored in gridfs. Raises ------ AndroScriptError If an error happened while initializing some `AndroScript` ''' super(Worker, self).__init__() # instantiate scripts self.androscripts = sorted( ScriptUtil.instantiate_scripts(script_list, script_hashes=script_hashes)) self.min_script_needs = min_script_needs # queues self.work_queue = work_queue self.analyzed_apks = analyzed_apks self.analyzed_apks.cancel_join_thread() self.work_queue.cancel_join_thread() self.storage = storage self.__sm_analyzed_apks = sm_analyzed_apks self.__storage_results = storage_results self.__storage_results.cancel_join_thread()
def __setup_scripts_reuse(self, androscripts, script_hashes): ''' Setup scripts but first try to reuse them. This is done by comparing the hashes. If they equal -> reuse them! Otherwise reload from disk. Parameters ---------- androscripts : list<str> List of package names. script_hashes : list<str> If given, set the hash for the `AndroScript`s Raises ------ AnalyzeError If an NoAndroScriptSubclass, IOError or ModuleNotSameClassNameException has been raised. ImportError ''' # need tuple to compare script_hashes = tuple(script_hashes) script_reload_needed = script_hashes != self.script_hashes # script can be reused -> simply reset them # stupid comparison cause same scripts in different order are not reused # but reusing is rather intended for a reuse in the same analysis (where the order is kept) if not script_reload_needed: log.info("reusing scripts ... ") for s in self.androscripts: s.reset() # cannot be reused else: log.info("reloading scripts cause hashes changed ... ") # (re)import script modules script_types = ScriptUtil.import_scripts(androscripts, via_package=True, _reload=True) # instantiate scripts and get classes self.androscripts = ScriptUtil.instantiate_scripts( script_types, script_hashes=script_hashes) # set hashes for next comparison self.script_hashes = script_hashes
def __setup_scripts_reuse(self, androscripts, script_hashes): ''' Setup scripts but first try to reuse them. This is done by comparing the hashes. If they equal -> reuse them! Otherwise reload from disk. Parameters ---------- androscripts : list<str> List of package names. script_hashes : list<str> If given, set the hash for the `AndroScript`s Raises ------ AnalyzeError If an NoAndroScriptSubclass, IOError or ModuleNotSameClassNameException has been raised. ImportError ''' # need tuple to compare script_hashes = tuple(script_hashes) script_reload_needed = script_hashes != self.script_hashes # script can be reused -> simply reset them # stupid comparison cause same scripts in different order are not reused # but reusing is rather intended for a reuse in the same analysis (where the order is kept) if not script_reload_needed: log.info("reusing scripts ... ") for s in self.androscripts: s.reset() # cannot be reused else: log.info("reloading scripts cause hashes changed ... ") # (re)import script modules script_types = ScriptUtil.import_scripts(androscripts, via_package = True, _reload = True) # instantiate scripts and get classes self.androscripts = ScriptUtil.instantiate_scripts(script_types, script_hashes = script_hashes) # set hashes for next comparison self.script_hashes = script_hashes
def __init__(self, script_list, script_hashes, min_script_needs, work_queue, storage, sm_analyzed_apks, analyzed_apks, storage_results = None): ''' Parameters ---------- script_list: list<type<AndroScript>> List of `AndroScript`s references (not instantiated class!) script_hashes : list<str>, optional (default is None) If given, set the hash for the `AndroScript`s min_script_needs : tuple<bool> See :py:method:`ScriptUtil.get_maximal_script_options`. work_queue : Queue<str> Queue with paths to apks which shall be analyzed. storage: RedundantStorage The storage to store the results. sm_analyzed_apks : Value Shared memory to add number of analyzed apks. analyzed_apks : Queue<FastAPK> Holds the analyzed APKs. storage_results : Queue<tuple<str, bool>>, optional (default is None) Storage results. First component is the id of the entry and the second a boolean indication if the result has been stored in gridfs. Raises ------ AndroScriptError If an error happened while initializing some `AndroScript` ''' super(Worker, self).__init__() # instantiate scripts self.androscripts = sorted(ScriptUtil.instantiate_scripts(script_list, script_hashes = script_hashes)) self.min_script_needs = min_script_needs # queues self.work_queue = work_queue self.analyzed_apks = analyzed_apks self.analyzed_apks.cancel_join_thread() self.work_queue.cancel_join_thread() self.storage = storage self.__sm_analyzed_apks = sm_analyzed_apks self.__storage_results = storage_results self.__storage_results.cancel_join_thread()
def test(script, apk_paths): ''' Use this function to develop and test your script. E.g. find unregistered keys and other errors. Parameters ---------- script : type The reference to the script which shall be tested (not instantiated!) apk_paths : iterable<str> Paths to apks Examples -------- >>> for res in AndroScript.test(ClassDetails, ["../../../testenv/apks/a2dp.Vol.apk"]): ... # get result object ... print res ... # get json ... print res.write_to_json() Returns ------- list<ResultObject> The `ResultObject` for every analyzed apk ''' # no circular import from androlyze.analyze.Analyzer import Analyzer res = [] try: # init scripts to get options inst_script_list = ScriptUtil.instantiate_scripts([script]) script_options = ScriptUtil.get_minimum_script_options( inst_script_list) script_list = [script] # options: storage, script_list, script_hashes, min_script_needs, apks_or_paths # but the analyzer needs the scripts uninitialized! ana = Analyzer(None, script_list, None, script_options, apk_paths) res = ana.analyze(test=True) except AndroScriptError as e: log.exception(e) return res
def test(script, apk_paths): ''' Use this function to develop and test your script. E.g. find unregistered keys and other errors. Parameters ---------- script : type The reference to the script which shall be tested (not instantiated!) apk_paths : iterable<str> Paths to apks Examples -------- >>> for res in AndroScript.test(ClassDetails, ["../../../testenv/apks/a2dp.Vol.apk"]): ... # get result object ... print res ... # get json ... print res.write_to_json() Returns ------- list<ResultObject> The `ResultObject` for every analyzed apk ''' # no circular import from androlyze.analyze.Analyzer import Analyzer res = [] try: # init scripts to get options inst_script_list = ScriptUtil.instantiate_scripts([script]) script_options = ScriptUtil.get_minimum_script_options(inst_script_list) script_list = [script] # options: storage, script_list, script_hashes, min_script_needs, apks_or_paths # but the analyzer needs the scripts uninitialized! ana = Analyzer(None, script_list, None, script_options, apk_paths) res = ana.analyze(test = True) except AndroScriptError as e: log.exception(e) return res
def __setup_scripts_hash_validation(self, androscripts, script_hashes): ''' Setup scripts. Also validate submitted script hashes if script reload is needed! Parameters ---------- androscripts : list<str> List of package names. script_hashes : list<str> If given, set the hash for the `AndroScript`s Raises ------ AnalyzeError If an NoAndroScriptSubclass, IOError or ModuleNotSameClassNameException has been raised. ImportError ScriptHashValidationError If the validation of script hashes fails after reloading scripts from disk. ''' # need tuple to compare script_hashes = tuple(script_hashes) # import script modules script_types = ScriptUtil.import_scripts(androscripts, via_package=True, _reload=True) # instantiate scripts and get classes self.androscripts = ScriptUtil.instantiate_scripts( script_types, # needed for path calculation script_paths=[Util.package_name_2_path(s) for s in androscripts]) actual_hashes = tuple([s.hash for s in self.androscripts]) if sorted(actual_hashes) != sorted(script_hashes): raise ScriptHashValidationError(script_hashes, actual_hashes)
def __setup_scripts_hash_validation(self, androscripts, script_hashes): ''' Setup scripts. Also validate submitted script hashes if script reload is needed! Parameters ---------- androscripts : list<str> List of package names. script_hashes : list<str> If given, set the hash for the `AndroScript`s Raises ------ AnalyzeError If an NoAndroScriptSubclass, IOError or ModuleNotSameClassNameException has been raised. ImportError ScriptHashValidationError If the validation of script hashes fails after reloading scripts from disk. ''' # need tuple to compare script_hashes = tuple(script_hashes) # import script modules script_types = ScriptUtil.import_scripts(androscripts, via_package = True, _reload = True) # instantiate scripts and get classes self.androscripts = ScriptUtil.instantiate_scripts(script_types, # needed for path calculation script_paths = [Util.package_name_2_path(s) for s in androscripts]) actual_hashes = tuple([s.hash for s in self.androscripts]) if sorted(actual_hashes) != sorted(script_hashes): raise ScriptHashValidationError(script_hashes, actual_hashes)
def create_analyzer(storage, script_list, apks_or_paths = None, mode = ANALYZE_MODE_PARALLEL, concurrency = None, serialize_apks = True ): ''' Create the analyzer only. Parameters ---------- storage : RedundantStorage The store to use. script_list : list<str> List of paths to scripts (complete filename with extension). apks_or_paths: list<str> or list<Apk>, optional (default is None) List of `Apk` or paths to the apks which shall be analyzed with the given scripts If you analyze from paths the `import_date` is not set! mode : str, optional (default is `ANALYZE_MODE_PARALLEL`) Do an parallel analysis by default. Choose between : , , . concurrency : int, optional (default is number of cpu cores) Number of workers to spawn. serialize_apks : bool, optional (default is True) If true, serialize .apk . Otherwise id (hash) of the apk will be send and fetched by the worker from the result db. Be sure to import the apks to the result db first! ''' from androlyze.model.script import ScriptUtil from androlyze.analyze.exception import AndroScriptError try: # list<type<AndroScript>> androscript_list = ScriptUtil.import_scripts(script_list) instantiated_scripts = sorted(ScriptUtil.instantiate_scripts(androscript_list, script_paths = script_list)) if len(instantiated_scripts) == 0: log.warn("No scripts supplied!") return # get hashes for `AndroScript`s so that we can set the hash directly next time we instantiate the script script_hashes = [s.hash for s in instantiated_scripts] min_script_needs = ScriptUtil.get_minimum_script_options(instantiated_scripts) # log infos about scripts clilog.info('Loaded scripts:\n%s', '\n'.join((str(s) for s in instantiated_scripts))) log.info(ScriptUtil.androscript_options_descr(instantiated_scripts)) if apks_or_paths: def create_analyzer(): analyzer = None # argument for BaseAnalyzer args = storage, androscript_list, script_hashes, min_script_needs, apks_or_paths log.info("Mode: %s", mode) # normal analyzer if mode == ANALYZE_MODE_NON_PARALLEL: from androlyze.analyze.Analyzer import Analyzer analyzer = Analyzer(*args) # use parallel analyzer elif mode == ANALYZE_MODE_PARALLEL: from androlyze.analyze.parallel.ParallelAnalyzer import ParallelAnalyzer analyzer = ParallelAnalyzer(*args, concurrency = concurrency) # use distributed one elif mode == ANALYZE_MODE_DISTRIBUTED: from androlyze.analyze.distributed.DistributedAnalyzer import DistributedAnalyzer analyzer = DistributedAnalyzer(*args, concurrency = concurrency, serialize_apks = serialize_apks) return analyzer return create_analyzer() except ApkImportError as e: log.warn(e) except IOError as e: log.warn(AndroScriptError(e.filename, caused_by = e)) sys.exit(1) except ImportError as e: log.exception(e) except Exception as e: log.exception(e)
def __init__(self, storage, script_list, script_hashes, *args, **kwargs): ''' See :py:method`.BaseAnalyzer.__init__` for details on the first attributes ''' super(Analyzer, self).__init__(storage, script_list, script_hashes, *args, **kwargs) # instantiate scripts self.script_list = sorted(ScriptUtil.instantiate_scripts(script_list, script_hashes = script_hashes))