def run(self) -> None: self.apk_preparer.prepare() emulators_number = RequiredFeature('emulators_number').request() if (emulators_number < 1): return logger.log_progress("\nSetting up devices.") minimum_api = RequiredFeature('minimum_api').request() mapper = MapperOnDevices(DeviceSetupThread.setup, minimum_api=minimum_api) try: mapper.run() except Exception as e: logger.log_progress(f"An error happened setting up devices: {str(traceback.format_exc())}") return devices = self.device_manager.get_devices() for device in devices: if device.state == State.setting_up: raise Exception(f"An error occurred setting up devices before starting Evolutiz run") # run the strategy population = self.strategy.run() self.write_summary_files()
def get_suite_coverage(self, scripts: List[str], device: Device, generation: int, individual_index: int) -> CoverageResult: self.verbose_level = RequiredFeature('verbose_level').request() compiled_package_name: str = RequiredFeature( 'compiled_package_name').request() unique_crashes: Set[str] = set() scripts_crash_status: Dict[str, bool] = {} accumulated_output = "" accumulated_errors = "" # stop target app in the device (just in case) adb.shell_command(device, f"am force-stop {compiled_package_name}") coverage_folder_local_path = self.prepare_coverage_folder( generation, individual_index) # upload the test scripts to device adb.push_all(device, scripts, "/mnt/sdcard") # run scripts for test_case_index, script_path in enumerate(scripts): self.generate_test_coverage(device, coverage_folder_local_path, accumulated_output, accumulated_errors, script_path, generation, individual_index, test_case_index, unique_crashes, scripts_crash_status) # collect coverage data coverage = self.get_coverage(device, coverage_folder_local_path, accumulated_output, accumulated_errors) return coverage, unique_crashes, scripts_crash_status
def instrument(self) -> None: self.app_path: str = RequiredFeature('app_path').request() self.result_dir: str = RequiredFeature('result_dir').request() self.instrumented_subjects_path: str = RequiredFeature('instrumented_subjects_path').request() self.emma_instrument_path: str = RequiredFeature('emma_instrument_path').request() # first, check if we should assume apps are already instrumented assume_subjects_instrumented = RequiredFeature('assume_subjects_instrumented').request() if assume_subjects_instrumented: features.provide('instrumented_app_path', self.app_path) output, errors, result_code = run_cmd(f"aapt dump badging {self.app_path} | grep package:\\ name") package_name = output.split("package: name=\'")[1].split("\'")[0] features.provide('package_name', package_name) self.prepare_files_for_coverage() return logger.log_progress(f"\nInstrumenting app: {os.path.basename(self.app_path)}") # copy sources and instrument application instrumented_app_path, package_name = self.prepare_app_for_instrumentation() features.provide('package_name', package_name) features.provide('instrumented_app_path', instrumented_app_path) self.instrument_gradle_file(instrumented_app_path, package_name) result_code = os.system(f"./gradlew assembleDebug 2>&1 >{self.result_dir}/build.log") if result_code != 0: raise Exception("Unable run assembleDebug") os.chdir(settings.WORKING_DIR) self.prepare_files_for_coverage()
def run(self) -> List[Individual]: gen = 0 while self.budget_manager.is_budget_available(): logger.log_progress( f"\n---> Starting generation {str(gen)} " f"at {str(self.budget_manager.get_time_budget_used())}") new_individuals = self.population_generator.generate( self.sampling_size, gen=gen) if new_individuals is None: # Timeout occurred break success = self.parallel_evaluator.evaluate(new_individuals) if not success: # Timeout occurred break # update logbook self.parallel_evaluator.test_suite_evaluator.update_logbook( gen, new_individuals) history = RequiredFeature('history').request() history.update(self.population) # select best individuals between current population and new one self.population[:] = self.toolbox.selectBest( self.population + new_individuals, self.population_size) gen += 1 return self.population
def dump_logbook_to_file(self) -> None: self.logbook = RequiredFeature('logbook').request() self.result_dir = RequiredFeature('result_dir').request() logbook_file = open(f"{self.result_dir}/logbook.pickle", 'wb') pickle.dump(self.logbook, logbook_file) logbook_file.close()
def dump_hall_of_fame_to_file(self) -> None: self.result_dir = RequiredFeature('result_dir').request() hall_of_fame = RequiredFeature('hall_of_fame').request() hof_file = open(f"{self.result_dir}/hall_of_fame.pickle", 'wb') pickle.dump(hall_of_fame, hof_file) hof_file.close()
def dump_individual_to_files(self, individual: Individual) -> Tuple[List[str], Dict[str, int]]: self.result_dir = RequiredFeature('result_dir').request() main_activity = RequiredFeature('main_activity').request() package_name = RequiredFeature('compiled_package_name').request() script_path: List[str] = [] suite_lengths: Dict[str, int] = {} for test_case_index, test_case in enumerate(individual): length = len(test_case) # generate script file list filename = f"{self.result_dir}/intermediate/script." \ f"{str(individual.generation)}.{str(individual.index_in_generation)}.{str(test_case_index)}" with open(filename, "w") as script_file: script_file.write(settings.SCRIPT_HEADER) script_file.write(f"LaunchActivity({package_name},{main_activity})\n") script_file.write("\n".join(map(lambda t: str(t), test_case))) script_file.write("\n") script = os.path.abspath(filename) suite_lengths[script] = length script_path.append(script) return script_path, suite_lengths
class ApkPreparer(object): def __init__(self) -> None: self.app_instrumentator = RequiredFeature( 'app_instrumentator').request() self.apk_analyser = ApkAnalyser() def prepare(self) -> None: self.app_instrumentator.instrument() self.apk_analyser.analyse() def install_on_device(self, device: Device) -> None: package_name: str = RequiredFeature('compiled_package_name').request() apk_path: str = RequiredFeature('apk_path').request() successful = False for i in range(3): try: self.apk_analyser.upload_string_xml(device) adb.shell_command(device, "rm /mnt/sdcard/bugreport.crash") adb.uninstall(device, package_name) adb.install(device, package_name, apk_path) successful = self.app_instrumentator.instrument_device(device) if successful: break except Exception as e: logger.log_progress( f"\nThere was an error installing apk on device: {str(e)}") time.sleep(5) if not successful: raise Exception(f"Unable to setup device: {device.name}")
def set_empty_fitness(self, individual: IndividualMultiObjective) -> None: individual.fitness.values = (0, sys.maxsize, 0) individual.evaluation_finish_timestamp = time.time() individual.evaluation_elapsed_time = 0 hall_of_fame = RequiredFeature('hall_of_fame').request() hall_of_fame.update([individual])
def __init__(self) -> None: self.device_manager = RequiredFeature('device_manager').request() self.budget_manager = RequiredFeature('budget_manager').request() self.population_generator = RequiredFeature( 'population_generator').request() self.parallel_evaluator = ParallelEvaluator() self.toolbox = RequiredFeature('toolbox').request() self.result_dir = RequiredFeature('result_dir').request()
def __init__(self) -> None: super(RandomSearch, self).__init__() # use expected number of devices as sampling size to leverage parallelism when using more than one emulator. self.device_manager = RequiredFeature('device_manager').request() self.sampling_size = self.device_manager.get_total_number_of_devices_expected( ) self.population_size = 1 self.population: List[Individual] = []
def instrument(self) -> None: self.app_path: str = RequiredFeature('app_path').request() self.result_dir: str = RequiredFeature('result_dir').request() self.instrumented_subjects_path: str = RequiredFeature( 'instrumented_subjects_path').request() self.emma_instrument_path: str = RequiredFeature( 'emma_instrument_path').request() # first, check if we should assume apps are already instrumented assume_subjects_instrumented = RequiredFeature( 'assume_subjects_instrumented').request() if assume_subjects_instrumented: features.provide('instrumented_app_path', self.app_path) # copy emma generated file result_code = os.system( f"cp bin/coverage.em {self.result_dir}/{logger.redirect_string()}" ) if result_code != 0: raise Exception("Unable to copy coverage.em file") output, errors, result_code = run_cmd( f"aapt dump badging {self.app_path} | grep package:\\ name") package_name = output.split("package: name=\'")[1].split("\'")[0] features.provide('package_name', package_name) features.provide('compiled_package_name', package_name) return logger.log_progress( f"\nInstrumenting app: {os.path.basename(self.app_path)}") # copy sources and instrument application instrumented_app_path, package_name = self.prepare_app_for_instrumentation( ) # compile with emma data result_code = os.system( f"ant clean emma debug 2>&1 >{self.result_dir}/build.log") if result_code != 0: raise Exception("Unable run ant clean emma debug") # copy emma generated file result_code = os.system( f"cp bin/coverage.em {self.result_dir}/{logger.redirect_string()}") if result_code != 0: raise Exception("Unable to copy coverage.em file") os.chdir(settings.WORKING_DIR) features.provide('package_name', package_name) features.provide('instrumented_app_path', instrumented_app_path) # assume same compiled package name as the one declard in AndroidManifest.xml file features.provide('compiled_package_name', package_name)
def write_summary_files(self) -> None: if RequiredFeature('write_hall_of_fame').request(): self.test_suite_evaluator.dump_hall_of_fame_to_file() if RequiredFeature('write_history').request(): history_file = open(f"{self.result_dir}/history.pickle", 'wb') pickle.dump(self.history, history_file) history_file.close() if RequiredFeature('write_logbook').request(): self.test_suite_evaluator.dump_logbook_to_file()
def setup(device: 'Device') -> None: device.state = State.setting_up # give test runner opportunity to install on devices test_runner = RequiredFeature('test_runner').request() test_runner.test_runner_installer.install(device) apk_preparer = RequiredFeature('apk_preparer').request() apk_preparer.install_on_device(device) device.needs_setup = False device.state = State.ready_idle
def get_suite_coverage( self, scripts: List[str], device: Device, generation: int, individual_index: int ) -> CoverageResult: self.verbose_level = RequiredFeature('verbose_level').request() compiled_package_name: str = RequiredFeature('compiled_package_name').request() result_dir: str = RequiredFeature('result_dir').request() unique_crashes: Set[str] = set() scripts_crash_status: Dict[str, bool] = {} accumulated_output = "" accumulated_errors = "" self.clean_coverage_files_in_device(device) coverage_folder_local_path = self.prepare_coverage_folder(generation, individual_index) # copy coverage.em file to local folder coverage_em_local_path = f"{coverage_folder_local_path}/coverage.em" os.system(f"cp {result_dir}/coverage.em {coverage_em_local_path}{logger.redirect_string()}") adb.shell_command(device, f"am force-stop {compiled_package_name}") adb.push_all(device, scripts, "/mnt/sdcard") # run scripts there_is_coverage = False for test_case_index, script_path in enumerate(scripts): there_is_coverage = self.generate_test_coverage( device, coverage_folder_local_path, accumulated_output, accumulated_errors, script_path, generation, individual_index, test_case_index, unique_crashes, scripts_crash_status) # collect coverage data coverage = 0 if there_is_coverage: coverage = self.get_coverage(device, coverage_folder_local_path, accumulated_output, accumulated_errors) return coverage, unique_crashes, scripts_crash_status
def show_best_historic_fitness(self) -> None: self.logbook = RequiredFeature('logbook').request() fitness_by_gen = self.logbook.select("fitness") # best independent (i.e., from different individuals) historic values for each objective max_coverage = 0.0 min_length = float(sys.maxsize) max_crashes = 0.0 # the fitness of the best multi-objective individual best_individual_fitness = (max_coverage, min_length, max_crashes) for gen, population in enumerate(fitness_by_gen): for fitness in population: individual_coverage = fitness['coverage'] individual_length = fitness['length'] individual_crashes = fitness['crashes'] # is this a better individual than the one found so far? at_least_as_good = individual_coverage >= max_coverage \ and individual_length <= min_length \ and individual_crashes >= max_crashes partially_better = individual_coverage > max_coverage \ or individual_length < min_length \ or individual_crashes > max_crashes if at_least_as_good and partially_better: best_individual_fitness = (individual_coverage, individual_length, individual_crashes) if individual_coverage > max_coverage: max_coverage = individual_coverage if individual_length < min_length: min_length = individual_length if individual_crashes > max_crashes: max_crashes = individual_crashes logger.log_progress( f"\n- Best multi-objective individual: {best_individual_fitness}") # CAUTION: the following best values are from different individuals logger.log_progress(f"\n- Best historic coverage: {str(max_coverage)}") logger.log_progress(f"\n- Best historic crashes: {str(max_crashes)}") if max_crashes > 0: logger.log_progress(f"\n- Best historic length: {str(min_length)}") else: logger.log_progress("\n- Best historic length: --")
def run(strategy_name: str, app_paths: List[str]) -> None: compress = RequiredFeature('compress').request() continue_on_subject_failure = RequiredFeature( 'continue_on_subject_failure').request() for i in range(0, len(app_paths)): features.provide('app_path', app_paths[i]) success = run_one_app(strategy_name) if not success and not continue_on_subject_failure: break if compress: compress_results(strategy_name)
def __init__(self) -> None: self.emulators_number: int = RequiredFeature( 'emulators_number').request() self.real_devices_number: int = RequiredFeature( 'real_devices_number').request() self.next_available_emulator_port: int = 5554 self.next_available_adb_server_port: int = 5038 # all devices that can be used self.devices: List[Device] = [] # init available devices self.refresh_reachable_devices()
def analyse(self) -> None: self.result_dir: str = RequiredFeature('result_dir').request() self.package_name: str = RequiredFeature( 'compiled_package_name').request() self.instrumented_app_path: str = RequiredFeature( 'instrumented_app_path').request() self.get_apk_path() print("### Working on apk:", self.package_name) # static analysis self.decoded_dir = f"{self.result_dir}/decoded-apk" if settings.ENABLE_STRING_SEEDING: logger.log_progress("\nRunning static analysis on apk") self.decode_apk()
def run(self, device: Device, package_name: str, script_name: str) -> None: verbose_level = RequiredFeature('verbose_level').request() start_time = time.time() self.prepare_device_for_run(device) string_seeding_flag = "" if self.use_motifgene: string_seeding_flag = f"--string-seeding /mnt/sdcard/{package_name}_strings.xml" motifcore_cmd = f"motifcore -p {package_name} --ignore-crashes --ignore-security-exceptions --ignore-timeouts" \ f" --bugreport {string_seeding_flag} -f /mnt/sdcard/{script_name} 1" output, errors, result_code = adb.shell_command( device, motifcore_cmd, timeout=settings.TEST_CASE_EVAL_TIMEOUT) if verbose_level > 1: print(f"Test case running finished with output:\n{output}") if "Exception" in errors: device_stacktrace = errors.split("** Error: ")[1] raise Exception( f"An error occurred when running test case: {device_stacktrace}" ) # need to manually kill motifcore when timeout adb.pkill(device, "motifcore") self.clean_device_after_run(device) if verbose_level > 0: logger.log_progress( f'\nMotifcore test run took: {time.time() - start_time:.2f} seconds' )
def mutation(self, individual: EvolutizTestSuite) -> Tuple[EvolutizTestSuite]: """Implements a mutation function for test suites. It consists of randomly choosing a test case of the test suite and mutating it. """ device_manager = RequiredFeature('device_manager').request() devices = device_manager.get_idle_devices() device = random.choice(devices) device.mark_work_start() random_index = random.randint(0, len(individual)-1) individual[random_index] = self.mutate_test_case(device, individual[random_index]) device.mark_work_stop() return individual,
def log_evaluation_result(device: 'Device', result_dir: str, script: str, success: bool) -> None: verbose_level = RequiredFeature('verbose_level').request() if verbose_level > 0: device_adb_log_file = f"{result_dir}/{device.name}-evaluations.log" os.system( f"echo \"{str(success)} -> {script}\" >> {device_adb_log_file}")
def evolve(self) -> List[Individual]: verbose_level = RequiredFeature('verbose_level').request() for gen in range(1, self.max_generations): if not self.budget_manager.is_budget_available(): print("Budget ran out, exiting evolve") break logger.log_progress(f"\n---> Starting generation {str(gen)} " f"at {str(self.budget_manager.get_time_budget_used())}") # create and evaluate offspring offspring = self.generate_offspring(self.population, gen) success = self.parallel_evaluator.evaluate(offspring) if not success: print("Budget ran out during parallel evaluation, exiting evolve") break self.population[:] = self.toolbox.selectBest(self.population + offspring, self.population_size) self.parallel_evaluator.test_suite_evaluator.update_logbook(gen, self.population) if verbose_level > 0: logger.log_progress(f"\nFinished generation {str(gen)} " f"at {str(self.budget_manager.get_time_budget_used())}") return self.population
def dump_script_coverage(self, device: Device, coverage_folder_local_path: str, accumulated_output: str, accumulated_errors: str, script_path: str, generation: int, individual_index: int, test_case_index: int, unique_crashes: Set[str], scripts_crash_status: Dict[str, bool]) -> bool: result_dir: str = RequiredFeature('result_dir').request() if crash_handler.handle(device, script_path, generation, individual_index, test_case_index, unique_crashes): scripts_crash_status[script_path] = True return False else: # no crash, collect coverage scripts_crash_status[script_path] = False adb.log_evaluation_result(device, result_dir, script_path, True) # save the coverage.dat file of this test script output, errors, result_code = run_cmd( f"mv {self.get_coverage_dat_path(device)} {coverage_folder_local_path}/" ) if result_code != 0: raise Exception( f"Unable to move the coverage dat file for test script, path is: {self.get_coverage_dat_path(device)}" ) return True
def execute(self, device: Device, evolutiz_connector: EvolutizConnector) -> bool: package_name = RequiredFeature('compiled_package_name').request() result = evolutiz_connector.send_command( device, package_name, f"performview id {self.id()} {self.action_type()}") return result.startswith('OK')
def generate(self, device: Device, package_name: str, destination_file_name: str) -> TestCase: assert device.api_level() >= self.minimum_api verbose_level = RequiredFeature('verbose_level').request() start_time = time.time() self.prepare_device_for_run(device) evolutiz_events = settings.SEQUENCE_LENGTH_MAX test_case = [] launch_result = self.evolutiz_connector.send_command( device, package_name, f"performview launch-app") for i in range(0, evolutiz_events): widget_action_result = WidgetAction.random(device, self.evolutiz_connector) test_event: TestEvent = widget_action_result test_case.append(test_event) if widget_action_result.is_outbound(): break if verbose_level > 0: logger.log_progress( f'\nEvolutiz test generation took: {time.time() - start_time:.2f} ' f'seconds for {len(test_case):d} events') self.clean_device_after_run(device) return test_case
def run(self, device: Device, package_name: str, script_name: str) -> None: assert device.api_level() >= self.minimum_api verbose_level = RequiredFeature('verbose_level').request() start_time = time.time() self.prepare_device_for_run(device) evolutiz_cmd = f"evolutiz -p {package_name} -v -v -v --throttle 200 --ignore-crashes " \ f"--ignore-security-exceptions --ignore-timeouts --bugreport -f /mnt/sdcard/{script_name} 1" output, errors, result_code = adb.shell_command( device, evolutiz_cmd, timeout=settings.TEST_CASE_EVAL_TIMEOUT) if verbose_level > 1: print(f"Test case running finished with output:\n{output}") if "Exception" in errors: device_stacktrace = errors.split("** Error: ")[1] raise Exception( f"An error occurred when running test case: {device_stacktrace}" ) # need to manually kill evolutiz when timeout adb.pkill(device, "evolutiz") self.clean_device_after_run(device) if verbose_level > 0: logger.log_progress( f'\nEvolutiz test run took: {time.time() - start_time:.2f} seconds' )
def log_exception(self, e: Exception, stack_trace: str, device: Optional[Device] = None) -> None: verbose_level = RequiredFeature('verbose_level').request() if verbose_level == 0: return template_base = "\nAn error occurred when calling func in MultipleQueueConsumerThread" if device is None: if verbose_level > 0: formatted_string = f"{template_base}: \n{stack_trace}" logger.log_progress(formatted_string) else: logger.log_progress(f"{template_base}.\n") else: if verbose_level > 0: formatted_string = f"{template_base} on device {device}: \n{traceback.format_exc()}" logger.log_progress(formatted_string) else: formatted_string = f"{template_base} on device {device}.\n" logger.log_progress(formatted_string)
def update_logbook(self, gen: int, population: List[IndividualSingleObjective]) -> None: self.result_dir = RequiredFeature('result_dir').request() self.logbook = RequiredFeature('logbook').request() record = {} fitness = [] evaluation = [] creation = [] for individual in population: fitness.append({ 'evaluation': 'single-objective', 'generation': individual.generation, 'index_in_generation': individual.index_in_generation, 'coverage': individual.fitness.values[0], # The following objective values are taken directly from the Individual Python's object, since for a # single-objective algorithm the fitness values only contain the coverage data. 'length': individual.length, 'crashes': individual.crashes, }) evaluation.append({ 'generation': individual.generation, 'index_in_generation': individual.index_in_generation, 'evaluation_finish_timestamp': individual.evaluation_finish_timestamp, 'evaluation_elapsed_time': individual.evaluation_elapsed_time, }) creation.append({ 'generation': individual.generation, 'index_in_generation': individual.index_in_generation, 'creation_finish_timestamp': individual.creation_finish_timestamp, 'creation_elapsed_time': individual.creation_elapsed_time, }) record['fitness'] = numpy.array(fitness) record['evaluation'] = numpy.array(evaluation) record['creation'] = numpy.array(creation) self.logbook.record(gen=gen, **record) self.show_best_historic_fitness()
def instrument_device(self, device: Device) -> bool: package_name = RequiredFeature('compiled_package_name').request() instrumentation_cmd = f"am instrument {package_name}/{package_name}.EmmaInstrument.EmmaInstrumentation" output, errors, result_code = adb.shell_command( device, instrumentation_cmd) if result_code != 0: raise Exception(f"Unable to instrument {package_name}") return True