def run(self) -> None:
self.apk_preparer.prepare()
emulators_number = RequiredFeature('emulators_number').request()
if (emulators_number < 1):
return
logger.log_progress("\nSetting up devices.")
minimum_api = RequiredFeature('minimum_api').request()
mapper = MapperOnDevices(DeviceSetupThread.setup, minimum_api=minimum_api)
try:
mapper.run()
except Exception as e:
logger.log_progress(f"An error happened setting up devices: {str(traceback.format_exc())}")
return
devices = self.device_manager.get_devices()
for device in devices:
if device.state == State.setting_up:
raise Exception(f"An error occurred setting up devices before starting Evolutiz run")
# run the strategy
population = self.strategy.run()
self.write_summary_files()
def get_suite_coverage(self, scripts: List[str], device: Device,
generation: int,
individual_index: int) -> CoverageResult:
self.verbose_level = RequiredFeature('verbose_level').request()
compiled_package_name: str = RequiredFeature(
'compiled_package_name').request()
unique_crashes: Set[str] = set()
scripts_crash_status: Dict[str, bool] = {}
accumulated_output = ""
accumulated_errors = ""
# stop target app in the device (just in case)
adb.shell_command(device, f"am force-stop {compiled_package_name}")
coverage_folder_local_path = self.prepare_coverage_folder(
generation, individual_index)
# upload the test scripts to device
adb.push_all(device, scripts, "/mnt/sdcard")
# run scripts
for test_case_index, script_path in enumerate(scripts):
self.generate_test_coverage(device, coverage_folder_local_path,
accumulated_output, accumulated_errors,
script_path, generation,
individual_index, test_case_index,
unique_crashes, scripts_crash_status)
# collect coverage data
coverage = self.get_coverage(device, coverage_folder_local_path,
accumulated_output, accumulated_errors)
return coverage, unique_crashes, scripts_crash_status
def instrument(self) -> None:
self.app_path: str = RequiredFeature('app_path').request()
self.result_dir: str = RequiredFeature('result_dir').request()
self.instrumented_subjects_path: str = RequiredFeature('instrumented_subjects_path').request()
self.emma_instrument_path: str = RequiredFeature('emma_instrument_path').request()
# first, check if we should assume apps are already instrumented
assume_subjects_instrumented = RequiredFeature('assume_subjects_instrumented').request()
if assume_subjects_instrumented:
features.provide('instrumented_app_path', self.app_path)
output, errors, result_code = run_cmd(f"aapt dump badging {self.app_path} | grep package:\\ name")
package_name = output.split("package: name=\'")[1].split("\'")[0]
features.provide('package_name', package_name)
self.prepare_files_for_coverage()
return
logger.log_progress(f"\nInstrumenting app: {os.path.basename(self.app_path)}")
# copy sources and instrument application
instrumented_app_path, package_name = self.prepare_app_for_instrumentation()
features.provide('package_name', package_name)
features.provide('instrumented_app_path', instrumented_app_path)
self.instrument_gradle_file(instrumented_app_path, package_name)
result_code = os.system(f"./gradlew assembleDebug 2>&1 >{self.result_dir}/build.log")
if result_code != 0:
raise Exception("Unable run assembleDebug")
os.chdir(settings.WORKING_DIR)
self.prepare_files_for_coverage()
def run(self) -> List[Individual]:
gen = 0
while self.budget_manager.is_budget_available():
logger.log_progress(
f"\n---> Starting generation {str(gen)} "
f"at {str(self.budget_manager.get_time_budget_used())}")
new_individuals = self.population_generator.generate(
self.sampling_size, gen=gen)
if new_individuals is None:
# Timeout occurred
break
success = self.parallel_evaluator.evaluate(new_individuals)
if not success:
# Timeout occurred
break
# update logbook
self.parallel_evaluator.test_suite_evaluator.update_logbook(
gen, new_individuals)
history = RequiredFeature('history').request()
history.update(self.population)
# select best individuals between current population and new one
self.population[:] = self.toolbox.selectBest(
self.population + new_individuals, self.population_size)
gen += 1
return self.population
def dump_logbook_to_file(self) -> None:
self.logbook = RequiredFeature('logbook').request()
self.result_dir = RequiredFeature('result_dir').request()
logbook_file = open(f"{self.result_dir}/logbook.pickle", 'wb')
pickle.dump(self.logbook, logbook_file)
logbook_file.close()
def dump_hall_of_fame_to_file(self) -> None:
self.result_dir = RequiredFeature('result_dir').request()
hall_of_fame = RequiredFeature('hall_of_fame').request()
hof_file = open(f"{self.result_dir}/hall_of_fame.pickle", 'wb')
pickle.dump(hall_of_fame, hof_file)
hof_file.close()
def dump_individual_to_files(self, individual: Individual) -> Tuple[List[str], Dict[str, int]]:
self.result_dir = RequiredFeature('result_dir').request()
main_activity = RequiredFeature('main_activity').request()
package_name = RequiredFeature('compiled_package_name').request()
script_path: List[str] = []
suite_lengths: Dict[str, int] = {}
for test_case_index, test_case in enumerate(individual):
length = len(test_case)
# generate script file list
filename = f"{self.result_dir}/intermediate/script." \
f"{str(individual.generation)}.{str(individual.index_in_generation)}.{str(test_case_index)}"
with open(filename, "w") as script_file:
script_file.write(settings.SCRIPT_HEADER)
script_file.write(f"LaunchActivity({package_name},{main_activity})\n")
script_file.write("\n".join(map(lambda t: str(t), test_case)))
script_file.write("\n")
script = os.path.abspath(filename)
suite_lengths[script] = length
script_path.append(script)
return script_path, suite_lengths
class ApkPreparer(object):
def __init__(self) -> None:
self.app_instrumentator = RequiredFeature(
'app_instrumentator').request()
self.apk_analyser = ApkAnalyser()
def prepare(self) -> None:
self.app_instrumentator.instrument()
self.apk_analyser.analyse()
def install_on_device(self, device: Device) -> None:
package_name: str = RequiredFeature('compiled_package_name').request()
apk_path: str = RequiredFeature('apk_path').request()
successful = False
for i in range(3):
try:
self.apk_analyser.upload_string_xml(device)
adb.shell_command(device, "rm /mnt/sdcard/bugreport.crash")
adb.uninstall(device, package_name)
adb.install(device, package_name, apk_path)
successful = self.app_instrumentator.instrument_device(device)
if successful:
break
except Exception as e:
logger.log_progress(
f"\nThere was an error installing apk on device: {str(e)}")
time.sleep(5)
if not successful:
raise Exception(f"Unable to setup device: {device.name}")
def set_empty_fitness(self, individual: IndividualMultiObjective) -> None:
individual.fitness.values = (0, sys.maxsize, 0)
individual.evaluation_finish_timestamp = time.time()
individual.evaluation_elapsed_time = 0
hall_of_fame = RequiredFeature('hall_of_fame').request()
hall_of_fame.update([individual])
def __init__(self) -> None:
self.device_manager = RequiredFeature('device_manager').request()
self.budget_manager = RequiredFeature('budget_manager').request()
self.population_generator = RequiredFeature(
'population_generator').request()
self.parallel_evaluator = ParallelEvaluator()
self.toolbox = RequiredFeature('toolbox').request()
self.result_dir = RequiredFeature('result_dir').request()
def __init__(self) -> None:
super(RandomSearch, self).__init__()
# use expected number of devices as sampling size to leverage parallelism when using more than one emulator.
self.device_manager = RequiredFeature('device_manager').request()
self.sampling_size = self.device_manager.get_total_number_of_devices_expected(
)
self.population_size = 1
self.population: List[Individual] = []
def instrument(self) -> None:
self.app_path: str = RequiredFeature('app_path').request()
self.result_dir: str = RequiredFeature('result_dir').request()
self.instrumented_subjects_path: str = RequiredFeature(
'instrumented_subjects_path').request()
self.emma_instrument_path: str = RequiredFeature(
'emma_instrument_path').request()
# first, check if we should assume apps are already instrumented
assume_subjects_instrumented = RequiredFeature(
'assume_subjects_instrumented').request()
if assume_subjects_instrumented:
features.provide('instrumented_app_path', self.app_path)
# copy emma generated file
result_code = os.system(
f"cp bin/coverage.em {self.result_dir}/{logger.redirect_string()}"
)
if result_code != 0:
raise Exception("Unable to copy coverage.em file")
output, errors, result_code = run_cmd(
f"aapt dump badging {self.app_path} | grep package:\\ name")
package_name = output.split("package: name=\'")[1].split("\'")[0]
features.provide('package_name', package_name)
features.provide('compiled_package_name', package_name)
return
logger.log_progress(
f"\nInstrumenting app: {os.path.basename(self.app_path)}")
# copy sources and instrument application
instrumented_app_path, package_name = self.prepare_app_for_instrumentation(
)
# compile with emma data
result_code = os.system(
f"ant clean emma debug 2>&1 >{self.result_dir}/build.log")
if result_code != 0:
raise Exception("Unable run ant clean emma debug")
# copy emma generated file
result_code = os.system(
f"cp bin/coverage.em {self.result_dir}/{logger.redirect_string()}")
if result_code != 0:
raise Exception("Unable to copy coverage.em file")
os.chdir(settings.WORKING_DIR)
features.provide('package_name', package_name)
features.provide('instrumented_app_path', instrumented_app_path)
# assume same compiled package name as the one declard in AndroidManifest.xml file
features.provide('compiled_package_name', package_name)
def write_summary_files(self) -> None:
if RequiredFeature('write_hall_of_fame').request():
self.test_suite_evaluator.dump_hall_of_fame_to_file()
if RequiredFeature('write_history').request():
history_file = open(f"{self.result_dir}/history.pickle", 'wb')
pickle.dump(self.history, history_file)
history_file.close()
if RequiredFeature('write_logbook').request():
self.test_suite_evaluator.dump_logbook_to_file()
def setup(device: 'Device') -> None:
device.state = State.setting_up
# give test runner opportunity to install on devices
test_runner = RequiredFeature('test_runner').request()
test_runner.test_runner_installer.install(device)
apk_preparer = RequiredFeature('apk_preparer').request()
apk_preparer.install_on_device(device)
device.needs_setup = False
device.state = State.ready_idle
def get_suite_coverage(
self,
scripts: List[str],
device: Device,
generation: int,
individual_index: int
) -> CoverageResult:
self.verbose_level = RequiredFeature('verbose_level').request()
compiled_package_name: str = RequiredFeature('compiled_package_name').request()
result_dir: str = RequiredFeature('result_dir').request()
unique_crashes: Set[str] = set()
scripts_crash_status: Dict[str, bool] = {}
accumulated_output = ""
accumulated_errors = ""
self.clean_coverage_files_in_device(device)
coverage_folder_local_path = self.prepare_coverage_folder(generation, individual_index)
# copy coverage.em file to local folder
coverage_em_local_path = f"{coverage_folder_local_path}/coverage.em"
os.system(f"cp {result_dir}/coverage.em {coverage_em_local_path}{logger.redirect_string()}")
adb.shell_command(device, f"am force-stop {compiled_package_name}")
adb.push_all(device, scripts, "/mnt/sdcard")
# run scripts
there_is_coverage = False
for test_case_index, script_path in enumerate(scripts):
there_is_coverage = self.generate_test_coverage(
device,
coverage_folder_local_path,
accumulated_output,
accumulated_errors,
script_path,
generation,
individual_index,
test_case_index,
unique_crashes,
scripts_crash_status)
# collect coverage data
coverage = 0
if there_is_coverage:
coverage = self.get_coverage(device,
coverage_folder_local_path,
accumulated_output,
accumulated_errors)
return coverage, unique_crashes, scripts_crash_status
def show_best_historic_fitness(self) -> None:
self.logbook = RequiredFeature('logbook').request()
fitness_by_gen = self.logbook.select("fitness")
# best independent (i.e., from different individuals) historic values for each objective
max_coverage = 0.0
min_length = float(sys.maxsize)
max_crashes = 0.0
# the fitness of the best multi-objective individual
best_individual_fitness = (max_coverage, min_length, max_crashes)
for gen, population in enumerate(fitness_by_gen):
for fitness in population:
individual_coverage = fitness['coverage']
individual_length = fitness['length']
individual_crashes = fitness['crashes']
# is this a better individual than the one found so far?
at_least_as_good = individual_coverage >= max_coverage \
and individual_length <= min_length \
and individual_crashes >= max_crashes
partially_better = individual_coverage > max_coverage \
or individual_length < min_length \
or individual_crashes > max_crashes
if at_least_as_good and partially_better:
best_individual_fitness = (individual_coverage,
individual_length,
individual_crashes)
if individual_coverage > max_coverage:
max_coverage = individual_coverage
if individual_length < min_length:
min_length = individual_length
if individual_crashes > max_crashes:
max_crashes = individual_crashes
logger.log_progress(
f"\n- Best multi-objective individual: {best_individual_fitness}")
# CAUTION: the following best values are from different individuals
logger.log_progress(f"\n- Best historic coverage: {str(max_coverage)}")
logger.log_progress(f"\n- Best historic crashes: {str(max_crashes)}")
if max_crashes > 0:
logger.log_progress(f"\n- Best historic length: {str(min_length)}")
else:
logger.log_progress("\n- Best historic length: --")
def run(strategy_name: str, app_paths: List[str]) -> None:
compress = RequiredFeature('compress').request()
continue_on_subject_failure = RequiredFeature(
'continue_on_subject_failure').request()
for i in range(0, len(app_paths)):
features.provide('app_path', app_paths[i])
success = run_one_app(strategy_name)
if not success and not continue_on_subject_failure:
break
if compress:
compress_results(strategy_name)
def __init__(self) -> None:
self.emulators_number: int = RequiredFeature(
'emulators_number').request()
self.real_devices_number: int = RequiredFeature(
'real_devices_number').request()
self.next_available_emulator_port: int = 5554
self.next_available_adb_server_port: int = 5038
# all devices that can be used
self.devices: List[Device] = []
# init available devices
self.refresh_reachable_devices()
def analyse(self) -> None:
self.result_dir: str = RequiredFeature('result_dir').request()
self.package_name: str = RequiredFeature(
'compiled_package_name').request()
self.instrumented_app_path: str = RequiredFeature(
'instrumented_app_path').request()
self.get_apk_path()
print("### Working on apk:", self.package_name)
# static analysis
self.decoded_dir = f"{self.result_dir}/decoded-apk"
if settings.ENABLE_STRING_SEEDING:
logger.log_progress("\nRunning static analysis on apk")
self.decode_apk()
def run(self, device: Device, package_name: str, script_name: str) -> None:
verbose_level = RequiredFeature('verbose_level').request()
start_time = time.time()
self.prepare_device_for_run(device)
string_seeding_flag = ""
if self.use_motifgene:
string_seeding_flag = f"--string-seeding /mnt/sdcard/{package_name}_strings.xml"
motifcore_cmd = f"motifcore -p {package_name} --ignore-crashes --ignore-security-exceptions --ignore-timeouts" \
f" --bugreport {string_seeding_flag} -f /mnt/sdcard/{script_name} 1"
output, errors, result_code = adb.shell_command(
device, motifcore_cmd, timeout=settings.TEST_CASE_EVAL_TIMEOUT)
if verbose_level > 1:
print(f"Test case running finished with output:\n{output}")
if "Exception" in errors:
device_stacktrace = errors.split("** Error: ")[1]
raise Exception(
f"An error occurred when running test case: {device_stacktrace}"
)
# need to manually kill motifcore when timeout
adb.pkill(device, "motifcore")
self.clean_device_after_run(device)
if verbose_level > 0:
logger.log_progress(
f'\nMotifcore test run took: {time.time() - start_time:.2f} seconds'
)
def mutation(self, individual: EvolutizTestSuite) -> Tuple[EvolutizTestSuite]:
"""Implements a mutation function for test suites.
It consists of randomly choosing a test case of the test suite and mutating it.
"""
device_manager = RequiredFeature('device_manager').request()
devices = device_manager.get_idle_devices()
device = random.choice(devices)
device.mark_work_start()
random_index = random.randint(0, len(individual)-1)
individual[random_index] = self.mutate_test_case(device, individual[random_index])
device.mark_work_stop()
return individual,
def log_evaluation_result(device: 'Device', result_dir: str, script: str,
success: bool) -> None:
verbose_level = RequiredFeature('verbose_level').request()
if verbose_level > 0:
device_adb_log_file = f"{result_dir}/{device.name}-evaluations.log"
os.system(
f"echo \"{str(success)} -> {script}\" >> {device_adb_log_file}")
def evolve(self) -> List[Individual]:
verbose_level = RequiredFeature('verbose_level').request()
for gen in range(1, self.max_generations):
if not self.budget_manager.is_budget_available():
print("Budget ran out, exiting evolve")
break
logger.log_progress(f"\n---> Starting generation {str(gen)} "
f"at {str(self.budget_manager.get_time_budget_used())}")
# create and evaluate offspring
offspring = self.generate_offspring(self.population, gen)
success = self.parallel_evaluator.evaluate(offspring)
if not success:
print("Budget ran out during parallel evaluation, exiting evolve")
break
self.population[:] = self.toolbox.selectBest(self.population + offspring, self.population_size)
self.parallel_evaluator.test_suite_evaluator.update_logbook(gen, self.population)
if verbose_level > 0:
logger.log_progress(f"\nFinished generation {str(gen)} "
f"at {str(self.budget_manager.get_time_budget_used())}")
return self.population
def dump_script_coverage(self, device: Device,
coverage_folder_local_path: str,
accumulated_output: str, accumulated_errors: str,
script_path: str, generation: int,
individual_index: int, test_case_index: int,
unique_crashes: Set[str],
scripts_crash_status: Dict[str, bool]) -> bool:
result_dir: str = RequiredFeature('result_dir').request()
if crash_handler.handle(device, script_path, generation,
individual_index, test_case_index,
unique_crashes):
scripts_crash_status[script_path] = True
return False
else:
# no crash, collect coverage
scripts_crash_status[script_path] = False
adb.log_evaluation_result(device, result_dir, script_path, True)
# save the coverage.dat file of this test script
output, errors, result_code = run_cmd(
f"mv {self.get_coverage_dat_path(device)} {coverage_folder_local_path}/"
)
if result_code != 0:
raise Exception(
f"Unable to move the coverage dat file for test script, path is: {self.get_coverage_dat_path(device)}"
)
return True
def execute(self, device: Device,
evolutiz_connector: EvolutizConnector) -> bool:
package_name = RequiredFeature('compiled_package_name').request()
result = evolutiz_connector.send_command(
device, package_name,
f"performview id {self.id()} {self.action_type()}")
return result.startswith('OK')
def generate(self, device: Device, package_name: str,
destination_file_name: str) -> TestCase:
assert device.api_level() >= self.minimum_api
verbose_level = RequiredFeature('verbose_level').request()
start_time = time.time()
self.prepare_device_for_run(device)
evolutiz_events = settings.SEQUENCE_LENGTH_MAX
test_case = []
launch_result = self.evolutiz_connector.send_command(
device, package_name, f"performview launch-app")
for i in range(0, evolutiz_events):
widget_action_result = WidgetAction.random(device,
self.evolutiz_connector)
test_event: TestEvent = widget_action_result
test_case.append(test_event)
if widget_action_result.is_outbound():
break
if verbose_level > 0:
logger.log_progress(
f'\nEvolutiz test generation took: {time.time() - start_time:.2f} '
f'seconds for {len(test_case):d} events')
self.clean_device_after_run(device)
return test_case
def run(self, device: Device, package_name: str, script_name: str) -> None:
assert device.api_level() >= self.minimum_api
verbose_level = RequiredFeature('verbose_level').request()
start_time = time.time()
self.prepare_device_for_run(device)
evolutiz_cmd = f"evolutiz -p {package_name} -v -v -v --throttle 200 --ignore-crashes " \
f"--ignore-security-exceptions --ignore-timeouts --bugreport -f /mnt/sdcard/{script_name} 1"
output, errors, result_code = adb.shell_command(
device, evolutiz_cmd, timeout=settings.TEST_CASE_EVAL_TIMEOUT)
if verbose_level > 1:
print(f"Test case running finished with output:\n{output}")
if "Exception" in errors:
device_stacktrace = errors.split("** Error: ")[1]
raise Exception(
f"An error occurred when running test case: {device_stacktrace}"
)
# need to manually kill evolutiz when timeout
adb.pkill(device, "evolutiz")
self.clean_device_after_run(device)
if verbose_level > 0:
logger.log_progress(
f'\nEvolutiz test run took: {time.time() - start_time:.2f} seconds'
)
def log_exception(self,
e: Exception,
stack_trace: str,
device: Optional[Device] = None) -> None:
verbose_level = RequiredFeature('verbose_level').request()
if verbose_level == 0:
return
template_base = "\nAn error occurred when calling func in MultipleQueueConsumerThread"
if device is None:
if verbose_level > 0:
formatted_string = f"{template_base}: \n{stack_trace}"
logger.log_progress(formatted_string)
else:
logger.log_progress(f"{template_base}.\n")
else:
if verbose_level > 0:
formatted_string = f"{template_base} on device {device}: \n{traceback.format_exc()}"
logger.log_progress(formatted_string)
else:
formatted_string = f"{template_base} on device {device}.\n"
logger.log_progress(formatted_string)
def update_logbook(self, gen: int,
population: List[IndividualSingleObjective]) -> None:
self.result_dir = RequiredFeature('result_dir').request()
self.logbook = RequiredFeature('logbook').request()
record = {}
fitness = []
evaluation = []
creation = []
for individual in population:
fitness.append({
'evaluation': 'single-objective',
'generation': individual.generation,
'index_in_generation': individual.index_in_generation,
'coverage': individual.fitness.values[0],
# The following objective values are taken directly from the Individual Python's object, since for a
# single-objective algorithm the fitness values only contain the coverage data.
'length': individual.length,
'crashes': individual.crashes,
})
evaluation.append({
'generation':
individual.generation,
'index_in_generation':
individual.index_in_generation,
'evaluation_finish_timestamp':
individual.evaluation_finish_timestamp,
'evaluation_elapsed_time':
individual.evaluation_elapsed_time,
})
creation.append({
'generation':
individual.generation,
'index_in_generation':
individual.index_in_generation,
'creation_finish_timestamp':
individual.creation_finish_timestamp,
'creation_elapsed_time':
individual.creation_elapsed_time,
})
record['fitness'] = numpy.array(fitness)
record['evaluation'] = numpy.array(evaluation)
record['creation'] = numpy.array(creation)
self.logbook.record(gen=gen, **record)
self.show_best_historic_fitness()
def instrument_device(self, device: Device) -> bool:
package_name = RequiredFeature('compiled_package_name').request()
instrumentation_cmd = f"am instrument {package_name}/{package_name}.EmmaInstrument.EmmaInstrumentation"
output, errors, result_code = adb.shell_command(
device, instrumentation_cmd)
if result_code != 0:
raise Exception(f"Unable to instrument {package_name}")
return True