def run(self, device: Device, package_name: str, script_name: str) -> None: assert device.api_level() >= self.minimum_api verbose_level = RequiredFeature('verbose_level').request() start_time = time.time() self.prepare_device_for_run(device) evolutiz_cmd = f"evolutiz -p {package_name} -v -v -v --throttle 200 --ignore-crashes " \ f"--ignore-security-exceptions --ignore-timeouts --bugreport -f /mnt/sdcard/{script_name} 1" output, errors, result_code = adb.shell_command( device, evolutiz_cmd, timeout=settings.TEST_CASE_EVAL_TIMEOUT) if verbose_level > 1: print(f"Test case running finished with output:\n{output}") if "Exception" in errors: device_stacktrace = errors.split("** Error: ")[1] raise Exception( f"An error occurred when running test case: {device_stacktrace}" ) # need to manually kill evolutiz when timeout adb.pkill(device, "evolutiz") self.clean_device_after_run(device) if verbose_level > 0: logger.log_progress( f'\nEvolutiz test run took: {time.time() - start_time:.2f} seconds' )
def evolve(self) -> List[Individual]: verbose_level = RequiredFeature('verbose_level').request() for gen in range(1, self.max_generations): if not self.budget_manager.is_budget_available(): print("Budget ran out, exiting evolve") break logger.log_progress(f"\n---> Starting generation {str(gen)} " f"at {str(self.budget_manager.get_time_budget_used())}") # create and evaluate offspring offspring = self.generate_offspring(self.population, gen) success = self.parallel_evaluator.evaluate(offspring) if not success: print("Budget ran out during parallel evaluation, exiting evolve") break self.population[:] = self.toolbox.selectBest(self.population + offspring, self.population_size) self.parallel_evaluator.test_suite_evaluator.update_logbook(gen, self.population) if verbose_level > 0: logger.log_progress(f"\nFinished generation {str(gen)} " f"at {str(self.budget_manager.get_time_budget_used())}") return self.population
def generate(self, device: Device, package_name: str, destination_file_name: str) -> TestCase: assert device.api_level() >= self.minimum_api verbose_level = RequiredFeature('verbose_level').request() start_time = time.time() self.prepare_device_for_run(device) evolutiz_events = settings.SEQUENCE_LENGTH_MAX test_case = [] launch_result = self.evolutiz_connector.send_command( device, package_name, f"performview launch-app") for i in range(0, evolutiz_events): widget_action_result = WidgetAction.random(device, self.evolutiz_connector) test_event: TestEvent = widget_action_result test_case.append(test_event) if widget_action_result.is_outbound(): break if verbose_level > 0: logger.log_progress( f'\nEvolutiz test generation took: {time.time() - start_time:.2f} ' f'seconds for {len(test_case):d} events') self.clean_device_after_run(device) return test_case
def instrument(self) -> None: self.app_path: str = RequiredFeature('app_path').request() self.result_dir: str = RequiredFeature('result_dir').request() self.instrumented_subjects_path: str = RequiredFeature('instrumented_subjects_path').request() self.emma_instrument_path: str = RequiredFeature('emma_instrument_path').request() # first, check if we should assume apps are already instrumented assume_subjects_instrumented = RequiredFeature('assume_subjects_instrumented').request() if assume_subjects_instrumented: features.provide('instrumented_app_path', self.app_path) output, errors, result_code = run_cmd(f"aapt dump badging {self.app_path} | grep package:\\ name") package_name = output.split("package: name=\'")[1].split("\'")[0] features.provide('package_name', package_name) self.prepare_files_for_coverage() return logger.log_progress(f"\nInstrumenting app: {os.path.basename(self.app_path)}") # copy sources and instrument application instrumented_app_path, package_name = self.prepare_app_for_instrumentation() features.provide('package_name', package_name) features.provide('instrumented_app_path', instrumented_app_path) self.instrument_gradle_file(instrumented_app_path, package_name) result_code = os.system(f"./gradlew assembleDebug 2>&1 >{self.result_dir}/build.log") if result_code != 0: raise Exception("Unable run assembleDebug") os.chdir(settings.WORKING_DIR) self.prepare_files_for_coverage()
def run(self) -> None: self.apk_preparer.prepare() emulators_number = RequiredFeature('emulators_number').request() if (emulators_number < 1): return logger.log_progress("\nSetting up devices.") minimum_api = RequiredFeature('minimum_api').request() mapper = MapperOnDevices(DeviceSetupThread.setup, minimum_api=minimum_api) try: mapper.run() except Exception as e: logger.log_progress(f"An error happened setting up devices: {str(traceback.format_exc())}") return devices = self.device_manager.get_devices() for device in devices: if device.state == State.setting_up: raise Exception(f"An error occurred setting up devices before starting Evolutiz run") # run the strategy population = self.strategy.run() self.write_summary_files()
def run(self, device: Device, package_name: str, script_name: str) -> None: verbose_level = RequiredFeature('verbose_level').request() start_time = time.time() self.prepare_device_for_run(device) string_seeding_flag = "" if self.use_motifgene: string_seeding_flag = f"--string-seeding /mnt/sdcard/{package_name}_strings.xml" motifcore_cmd = f"motifcore -p {package_name} --ignore-crashes --ignore-security-exceptions --ignore-timeouts" \ f" --bugreport {string_seeding_flag} -f /mnt/sdcard/{script_name} 1" output, errors, result_code = adb.shell_command( device, motifcore_cmd, timeout=settings.TEST_CASE_EVAL_TIMEOUT) if verbose_level > 1: print(f"Test case running finished with output:\n{output}") if "Exception" in errors: device_stacktrace = errors.split("** Error: ")[1] raise Exception( f"An error occurred when running test case: {device_stacktrace}" ) # need to manually kill motifcore when timeout adb.pkill(device, "motifcore") self.clean_device_after_run(device) if verbose_level > 0: logger.log_progress( f'\nMotifcore test run took: {time.time() - start_time:.2f} seconds' )
def run(self) -> List[Individual]: gen = 0 while self.budget_manager.is_budget_available(): logger.log_progress( f"\n---> Starting generation {str(gen)} " f"at {str(self.budget_manager.get_time_budget_used())}") new_individuals = self.population_generator.generate( self.sampling_size, gen=gen) if new_individuals is None: # Timeout occurred break success = self.parallel_evaluator.evaluate(new_individuals) if not success: # Timeout occurred break # update logbook self.parallel_evaluator.test_suite_evaluator.update_logbook( gen, new_individuals) history = RequiredFeature('history').request() history.update(self.population) # select best individuals between current population and new one self.population[:] = self.toolbox.selectBest( self.population + new_individuals, self.population_size) gen += 1 return self.population
def instrument_apk(folder_name, app_path, result_dir): logger.log_progress("\nInstrumenting app: " + folder_name) result_code = os.system("mkdir -p " + result_dir) if result_code != 0: raise Exception("Unable to create result dir") result_code = os.system("ant clean emma debug 2>&1 >" + result_dir + "/build.log") if result_code != 0: raise Exception("Unable run ant clean emma debug") result_code = os.system("cp bin/coverage.em " + result_dir + "/" + logger.redirect_string()) if result_code != 0: raise Exception("Unable to copy coverage.em file") p = sub.Popen("ls bin/*-debug.apk", stdout=sub.PIPE, stderr=sub.PIPE, shell=True) apk_path, errors = p.communicate() apk_path = apk_path.rstrip('\n') p = sub.Popen( settings.AAPT + " d xmltree " + apk_path + " AndroidManifest.xml | grep package | awk 'BEGIN {FS=\"\\\"\"}{print $2}'", stdout=sub.PIPE, stderr=sub.PIPE, shell=True) package_name, errors = p.communicate() package_name = package_name.rstrip('\n') return apk_path, package_name
def reboot_devices(self, wait_to_be_ready: bool = False) -> None: logger.log_progress("\nRebooting devices.") for device in self.get_devices(): device.reboot() if wait_to_be_ready: self.wait_devices_to_be_ready()
def check_virtualbox_is_not_running() -> None: if is_command_available("vboxmanage"): output, errors, result_code = run_cmd("vboxmanage list runningvms") if output.strip() != "": cause = "Android emulators can't be run while VirtualBox is running.\n" \ "Shutdown VirtualBox before running Evolutiz." logger.log_progress(cause) raise Exception(cause)
def reboot(self) -> None: Device.reboot(self) output, errors, result_code = adb.adb_command( self, "reboot", timeout=settings.ADB_REGULAR_COMMAND_TIMEOUT) if result_code != 0: logger.log_progress(f"\nUnable to reboot device: {self.name}") logger.log_progress("\nPlease, turn it off and on manually.") raise Exception(f"Unable to reboot device: {self.name}")
def get_subject_paths(arguments: argparse.Namespace) -> List[str]: features.provide('assume_subjects_instrumented', arguments.assume_subjects_instrumented) subject_path = arguments.subject_path if subject_path is not None: absolute_path = os.path.abspath(subject_path) absolute_path = absolute_path.rstrip('/') features.provide('subjects_path', [absolute_path]) return [absolute_path] else: subjects_path = arguments.subjects_path.rstrip('/') + '/' features.provide('subjects_path', subjects_path) app_paths = [] if arguments.assume_subjects_instrumented: output, errors, result_code = run_cmd( f"find -L {subjects_path} -name \"*.apk\"") for line in output.strip().split('\n'): app_paths.append( line.rstrip('/')) # remove trailing forward slash else: output, errors, result_code = run_cmd( f"ls -1 \"{subjects_path}\"*") for relative_path in output.strip().split('\n'): # convert relative path to absolute path absolute_path = os.path.abspath(relative_path) absolute_path = absolute_path.rstrip('/') if os.path.isdir(absolute_path): app_paths.append(absolute_path) elif os.path.isfile(absolute_path): if absolute_path.endswith(".apk"): if "hydrate" not in relative_path: # hydrate app doesn't compile yet, so don't bother app_paths.append( absolute_path) # remove trailing forward slash else: logger.log_progress( f"Ignoring non-APK file {absolute_path} in subjects path" ) else: # special file (e.g., socket) logger.log_progress( f"Ignoring special file {absolute_path} in subjects path" ) continue if arguments.randomize_subjects: random.shuffle(app_paths) if arguments.limit_subjects_number != -1: app_paths = app_paths[0:arguments.limit_subjects_number] # return list(filter(lambda p: 'com.zhiliaoapp.musically' in p, app_paths)) return app_paths
def evolve(self) -> List[Individual]: verbose_level: bool = RequiredFeature('verbose_level').request() for gen in range(1, self.max_generations): if not self.budget_manager.is_budget_available(): print("Budget ran out, exiting evolve") break logger.log_progress( f"\n---> Starting generation {str(gen)}" f" at {str(self.budget_manager.get_time_budget_used())}") # create new population, starting with elitism new_population: List[Individual] = self.toolbox.selectBest( self.population, self.elitism_size) while len(new_population) < self.population_size: # select parents parents: List[Individual] = self.toolbox.select( self.population, 2) # generate offspring needed_offspring = min( self.population_size - len(new_population), 2) offspring: List[Individual] = self.crossover( parents, gen, needed_offspring, base_index_in_generation=len(new_population)) self.mutation(offspring) success = self.parallel_evaluator.evaluate(offspring) if not success: print( "Budget ran out during parallel evaluation, exiting evolve" ) return self.population # extend new population with offspring or parents, depending which ones have the best individual best_ind, = self.toolbox.selectBest(offspring + parents, 1) if best_ind in offspring: new_population.extend(offspring) else: new_population.extend(parents) self.population = new_population.copy() self.parallel_evaluator.test_suite_evaluator.update_logbook( gen, self.population) if verbose_level > 0: logger.log_progress( f"\nFinished generation {str(gen)} " f"at {str(self.budget_manager.get_time_budget_used())}") return self.population
def install_in_all_devices(self, minimum_api: Optional[int] = None) -> None: logger.log_progress(f"\nPreparing {self.test_runner_name} test runner in devices.") mapper = MapperOnDevices(self.install, minimum_api=minimum_api) try: mapper.run() except Exception as e: str = traceback.format_exc() print(str)
def instrument(self) -> None: self.app_path: str = RequiredFeature('app_path').request() self.result_dir: str = RequiredFeature('result_dir').request() self.instrumented_subjects_path: str = RequiredFeature( 'instrumented_subjects_path').request() self.emma_instrument_path: str = RequiredFeature( 'emma_instrument_path').request() # first, check if we should assume apps are already instrumented assume_subjects_instrumented = RequiredFeature( 'assume_subjects_instrumented').request() if assume_subjects_instrumented: features.provide('instrumented_app_path', self.app_path) # copy emma generated file result_code = os.system( f"cp bin/coverage.em {self.result_dir}/{logger.redirect_string()}" ) if result_code != 0: raise Exception("Unable to copy coverage.em file") output, errors, result_code = run_cmd( f"aapt dump badging {self.app_path} | grep package:\\ name") package_name = output.split("package: name=\'")[1].split("\'")[0] features.provide('package_name', package_name) features.provide('compiled_package_name', package_name) return logger.log_progress( f"\nInstrumenting app: {os.path.basename(self.app_path)}") # copy sources and instrument application instrumented_app_path, package_name = self.prepare_app_for_instrumentation( ) # compile with emma data result_code = os.system( f"ant clean emma debug 2>&1 >{self.result_dir}/build.log") if result_code != 0: raise Exception("Unable run ant clean emma debug") # copy emma generated file result_code = os.system( f"cp bin/coverage.em {self.result_dir}/{logger.redirect_string()}") if result_code != 0: raise Exception("Unable to copy coverage.em file") os.chdir(settings.WORKING_DIR) features.provide('package_name', package_name) features.provide('instrumented_app_path', instrumented_app_path) # assume same compiled package name as the one declard in AndroidManifest.xml file features.provide('compiled_package_name', package_name)
def generate_test_coverage( self, device: Device, coverage_folder_local_path: str, accumulated_output: str, accumulated_errors: str, script_path: str, generation: int, individual_index: int, test_case_index: int, unique_crashes: Set[str], scripts_crash_status: Dict[str, bool] ) -> bool: compiled_package_name: str = RequiredFeature('compiled_package_name').request() result_dir: str = RequiredFeature('result_dir').request() # clear app's data and state output, errors, result_code = adb.shell_command(device, f"pm clear {compiled_package_name}") accumulated_output += output accumulated_errors += errors if result_code != 0: adb.log_evaluation_result(device, result_dir, script_path, False) if self.verbose_level > 0: logger.log_progress(f"\n{accumulated_output}\n{accumulated_errors}") raise Exception(f"Unable to clear package for script_path {script_path} in device: {device.name}") output, errors, result_code = adb.shell_command( device, f"am instrument {compiled_package_name}/{compiled_package_name}.EmmaInstrument.EmmaInstrumentation" ) accumulated_output += output accumulated_errors += errors if result_code != 0: adb.log_evaluation_result(device, result_dir, script_path, False) if self.verbose_level > 0: logger.log_progress(f"\n{accumulated_output}\n{accumulated_errors}") raise Exception(f"Unable to instrument for script_path {script_path} in device: {device.name}") script_name = script_path.split("/")[-1] test_runner = RequiredFeature('test_runner').request() test_runner.run(device, compiled_package_name, script_name) return self.dump_script_coverage( device, coverage_folder_local_path, accumulated_output, accumulated_errors, script_path, generation, individual_index, test_case_index, unique_crashes, scripts_crash_status)
def run(self) -> List[Individual]: self.package_name = RequiredFeature('compiled_package_name').request() success = self.initPopulation() if not success: logger.log_progress( "\nThere was an error initializing population for app.") raise Exception( "There was an error initializing population for app.") return self.evolve()
def show_best_historic_fitness(self) -> None: self.logbook = RequiredFeature('logbook').request() fitness_by_gen = self.logbook.select("fitness") # best independent (i.e., from different individuals) historic values for each objective max_coverage = 0.0 min_length = float(sys.maxsize) max_crashes = 0.0 # the fitness of the best multi-objective individual best_individual_fitness = (max_coverage, min_length, max_crashes) for gen, population in enumerate(fitness_by_gen): for fitness in population: individual_coverage = fitness['coverage'] individual_length = fitness['length'] individual_crashes = fitness['crashes'] # is this a better individual than the one found so far? at_least_as_good = individual_coverage >= max_coverage \ and individual_length <= min_length \ and individual_crashes >= max_crashes partially_better = individual_coverage > max_coverage \ or individual_length < min_length \ or individual_crashes > max_crashes if at_least_as_good and partially_better: best_individual_fitness = (individual_coverage, individual_length, individual_crashes) if individual_coverage > max_coverage: max_coverage = individual_coverage if individual_length < min_length: min_length = individual_length if individual_crashes > max_crashes: max_crashes = individual_crashes logger.log_progress( f"\n- Best multi-objective individual: {best_individual_fitness}") # CAUTION: the following best values are from different individuals logger.log_progress(f"\n- Best historic coverage: {str(max_coverage)}") logger.log_progress(f"\n- Best historic crashes: {str(max_crashes)}") if max_crashes > 0: logger.log_progress(f"\n- Best historic length: {str(min_length)}") else: logger.log_progress("\n- Best historic length: --")
def generate(self, device: Device, package_name: str, destination_file_name: str) -> TestCase: verbose_level = RequiredFeature('verbose_level').request() start_time = time.time() self.prepare_device_for_run(device) evolutiz_events = random.randint(settings.SEQUENCE_LENGTH_MIN, settings.SEQUENCE_LENGTH_MAX) string_seeding_flag = "" # String seeding is NOT supported yet # if self.use_motifgene: # string_seeding_flag = f"--string-seeding /mnt/sdcard/{package_name}_strings.xml" evolutiz_cmd = f"evolutiz -p {package_name} --ignore-crashes --ignore-security-exceptions --ignore-timeouts" \ f" --bugreport {string_seeding_flag} -o {self.evolutiz_script_path_in_devices}" \ f" --throttle {self.throttle} -v {str(evolutiz_events)}" output, errors, result_code = adb.shell_command( device, evolutiz_cmd, timeout=settings.TEST_CASE_EVAL_TIMEOUT) if verbose_level > 1: print(f"Test case generation finished with output:\n{output}") if "Exception" in errors: if "** Error: " in errors: device_stacktrace = errors.split("** Error: ")[1] raise Exception( f"An error occurred when generating test case: {device_stacktrace}" ) else: raise Exception( f"An error occurred when generating test case: {errors}") # need to manually kill evolutiz when timeout adb.pkill(device, "evolutiz") time.sleep(5) test_case: TestCase = self.retrieve_generated_test( device, destination_file_name) self.clean_device_after_run(device) if verbose_level > 0: logger.log_progress( f'\nEvolutiz test generation took: {time.time() - start_time:.2f} seconds ' f'for {evolutiz_events:d} events') return test_case
def log_exception(self, e: Exception, stack_trace: str, device: Optional[Device] = None) -> None: verbose_level = RequiredFeature('verbose_level').request() if verbose_level == 0: return template_base = "\nAn error occurred when calling func in MultipleQueueConsumerThread" if device is None: if verbose_level > 0: formatted_string = f"{template_base}: \n{stack_trace}" logger.log_progress(formatted_string) else: logger.log_progress(f"{template_base}.\n") else: if verbose_level > 0: formatted_string = f"{template_base} on device {device}: \n{traceback.format_exc()}" logger.log_progress(formatted_string) else: formatted_string = f"{template_base} on device {device}.\n" logger.log_progress(formatted_string)
def random(cls, device: Device, evolutiz_connector: EvolutizConnector) -> WidgetActionResult: package_name = RequiredFeature('compiled_package_name').request() current_package_name = adb.get_current_package_name(device) current_activity = adb.get_current_activity(device) if 'com.google.android' in current_activity or 'com.google.android' in current_package_name: # we are still in the HOME, and launch-app command failed raise Exception(f"An error ocurred launching app {package_name}") action_performed = None for i in range(0, 3): # indicate Evolutiz test runner to perform any executable action randomly. # the runner should return the action performed action_performed = evolutiz_connector.send_command( device, package_name, f"performview random-action {current_activity}") if action_performed.startswith("FAILURE"): logger.log_progress( f"An error occurred when performing random action onto activity {current_activity}." f"Retrying.") else: break if action_performed.startswith("FAILURE"): raise Exception( f"An error occurred when performing random action onto activity {current_activity}" ) elif action_performed.startswith("SUCCESS_APP_CRASH"): result = action_performed.split("SUCCESS_APP_CRASH:")[1] return WidgetActionResult(result) elif action_performed.startswith("SUCCESS_OUTBOUND"): result = action_performed.split("SUCCESS_OUTBOUND:")[1] return WidgetActionResult(result) elif action_performed.startswith("SUCCESS_NEW_STATE"): result = action_performed.split("SUCCESS_NEW_STATE:")[1] return WidgetActionResult(result) elif action_performed.startswith("SUCCESS"): result = action_performed.split("SUCCESS:")[1] return WidgetActionResult(result) else: raise Exception( f"An error occurred when performing random action onto activity {current_activity}" )
def boot_emulators(self, wait_to_be_ready: bool = False) -> None: while True: self.next_available_emulator_port = 5554 self.next_available_adb_server_port = 5038 self.devices = [] logger.log_progress( f"\nBooting devices: {str(0)}/{str(self.emulators_number)}") for i in range(0, self.emulators_number): logger.log_progress( f"\nBooting devices: {str(i + 1)}/{str(self.emulators_number)}" ) emulator = Emulator(self) emulator.boot() self.devices.append(emulator) if wait_to_be_ready: try: self.wait_devices_to_be_ready() except WaitDevicesTimeout as e: logger.log_progress(f"\n{str(e)}") logger.log_progress( "\nForce kill on all current emulator processes") run_cmd("pgrep emu | xargs kill -9", discard_output=True) time.sleep(5) continue break
def wait_for_battery_threshold(self, battery_threshold: int = 20) -> None: while True: all_devices_above_threshold = all([ level is None or level >= battery_threshold for level in (device.battery_level() for device in self.get_devices()) ]) if all_devices_above_threshold: break else: logger.log_progress( f"\nWaiting for some devices to reach {str(battery_threshold)}% battery level" ) time.sleep(60) # sleep 1 minute
def run(self) -> None: app_path = RequiredFeature('app_path').request() self.population_generator.generate(self.sampling_size, gen=0) # did we crash? devices = self.device_manager.get_devices() assert len(devices) == 1 device = devices[0] crashed = crash_handler.handle(device, '', 0, 0, 0, set()) if crashed: logger.log_progress(f"\nApp {app_path} CRASHED") else: logger.log_progress(f"\nApp {app_path} PASSED")
def evolve(self) -> List[Individual]: verbose_level: bool = RequiredFeature('verbose_level').request() self.parent = self.population[0] for gen in range(1, self.max_generations): if not self.budget_manager.is_budget_available(): print("Budget ran out, exiting evolve") break logger.log_progress( f"\n---> Starting generation {str(gen)}" f"at {str(self.budget_manager.get_time_budget_used())}") # Create and evaluate mutants mutants = self.generate_mutants(gen) success = self.parallel_evaluator.evaluate(mutants) if not success: print( "Budget ran out during parallel evaluation, exiting evolve" ) break # select best mutant and generate offspring by applying crossover with parent best_mutant, = self.toolbox.selectBest(mutants, 1) offspring = self.generate_offspring(gen, best_mutant) success = self.parallel_evaluator.evaluate(offspring) if not success: print( "Budget ran out during parallel evaluation, exiting evolve" ) break # select best offspring and set as new parent # the parent is included in the selection to avoid degrading the current solution offspring.append(self.parent) best_offspring, = self.toolbox.selectBest(offspring, 1) self.parent = best_offspring self.population = [self.parent] self.parallel_evaluator.test_suite_evaluator.update_logbook( gen, self.population) if verbose_level > 0: logger.log_progress( f"\nFinished generation {str(gen)} " f"at {str(self.budget_manager.get_time_budget_used())}") return self.population
def analyse(self) -> None: self.result_dir: str = RequiredFeature('result_dir').request() self.package_name: str = RequiredFeature( 'compiled_package_name').request() self.instrumented_app_path: str = RequiredFeature( 'instrumented_app_path').request() self.get_apk_path() print("### Working on apk:", self.package_name) # static analysis self.decoded_dir = f"{self.result_dir}/decoded-apk" if settings.ENABLE_STRING_SEEDING: logger.log_progress("\nRunning static analysis on apk") self.decode_apk()
def collectCoverage(device, package_name, result_dir, suffix=""): logger.log_progress("\nSending coverage broadcast in device: " + device + " at: " + datetime.today().strftime("%H:%M:%S")) os.system(adb.adb_cmd_prefix + " -s " + device + " shell am broadcast -a evolutiz.emma.COLLECT_COVERAGE" + logger.redirect_string()) logger.log_progress("\nPulling coverage from device: " + device + " at: " + datetime.today().strftime("%H:%M:%S")) coverageFilePath = "/data/data/" + package_name + "/files/coverage.ec" os.system(adb.adb_cmd_prefix + " -s " + device + " pull " + coverageFilePath + " " + result_dir + "/coverage.ec" + suffix + logger.redirect_string()) return True
def generate_test_coverage(self, device: Device, coverage_folder_local_path: str, accumulated_output: str, accumulated_errors: str, script_path: str, generation: int, individual_index: int, test_case_index: int, unique_crashes: Set[str], scripts_crash_status: Dict[str, bool]) -> bool: compiled_package_name: str = RequiredFeature( 'compiled_package_name').request() result_dir: str = RequiredFeature('result_dir').request() # clear app's data and state output, errors, result_code = adb.shell_command( device, f"pm clear {compiled_package_name}") accumulated_output += str(output) accumulated_errors += str(errors) if result_code != 0: adb.log_evaluation_result(device, result_dir, script_path, False) if self.verbose_level > 0: logger.log_progress( f"\n{accumulated_output}\n{accumulated_errors}") raise Exception( f"Unable to clear package for script_path {script_path} in device: {device.name}" ) # copy the covids file to the coverage_folder_local_path ella_coverage_ids_file_path = f"{self.get_current_apk_output_folder(device)}/covids" output, errors, result_code = run_cmd( f"cp {ella_coverage_ids_file_path} {coverage_folder_local_path}/") if result_code != 0: raise Exception( f"Unable to copy the coverage ids file for test script, path is: {ella_coverage_ids_file_path}" ) # Run test case using the requested test runner. # Internally, the test runner will take care of calling the methods AppInstrumentator#setup_for_test_run and # AppInstrumentator#teardown_after_test_run, which will start and stop the ELLA server correspondingly. script_name = script_path.split("/")[-1] test_runner = RequiredFeature('test_runner').request() test_runner.run(device, compiled_package_name, script_name) return self.dump_script_coverage(device, coverage_folder_local_path, accumulated_output, accumulated_errors, script_path, generation, individual_index, test_case_index, unique_crashes, scripts_crash_status)
def boot(self, port: Optional[int] = None, adb_port: Optional[int] = None) -> None: verbose_level = RequiredFeature('verbose_level').request() Device.boot(self) # ensure the emulator configuration is correct self.port = port if port is not None else self.device_manager.get_next_available_emulator_port( ) self.avd_name = self.avd_manager.get_avd_name_for_emulator_port( self.port) if not self.avd_manager.avd_name_exists(self.avd_name): raise Exception( f"AVD name {self.avd_name} doesn't exist. " f"Check that the provided AVD series ({self.avd_manager.avd_series}) is correct." ) # start custom abd server for this emulator self.adb_port = adb_port if adb_port is not None else self.device_manager.get_next_available_adb_server_port( ) adb.restart_server(self.adb_port) # start emulator self.name = f"emulator-{str(self.port)}" emulator_cmd = f"{self.get_adb_server_port_prefix()} QEMU_AUDIO_DRV=none $ANDROID_HOME/emulator/emulator" flags: str = f" -no-snapshot -wipe-data -no-boot-anim -writable-system -port {str(self.port)}" if verbose_level < 3: # -no-window flag can't be at the end flags = f" -no-window{flags}" logs = " >/dev/null 2>/dev/null" if verbose_level > 0: logs = f" >logs/{self.avd_name}.log 2>logs/{self.avd_name}.err" flags = f"{flags} -verbose -debug all" cmd: str = f'{emulator_cmd} -avd {self.avd_name}{flags}{logs}' if verbose_level > 1: logger.log_progress(f"\nFiring up emulator with command: {cmd}") sub.Popen(cmd, shell=True)
def evolve(self) -> List[Individual]: verbose_level: bool = RequiredFeature('verbose_level').request() for gen in range(1, self.max_generations): if not self.budget_manager.is_budget_available(): print("Budget ran out, exiting evolve") break logger.log_progress( f"\n---> Starting generation {str(gen)} " f"at {str(self.budget_manager.get_time_budget_used())}") # select parents parents: List[Individual] = self.toolbox.select(self.population, 2) # generate offspring offspring = self.crossover(parents, gen, 2) self.mutation(offspring) success = self.parallel_evaluator.evaluate(offspring) if not success: print( "Budget ran out during parallel evaluation, exiting evolve" ) break best_ind, = self.toolbox.selectBest(offspring + parents, 1) if best_ind in offspring: # only update population if the best individual is one of the offspring self.population = [ ind for ind in self.population if ind not in parents ] + offspring self.parallel_evaluator.test_suite_evaluator.update_logbook( gen, self.population) if verbose_level > 0: logger.log_progress( f"\nFinished generation {str(gen)} " f"at {str(self.budget_manager.get_time_budget_used())}") return self.population