def run(self, device: Device, package_name: str, script_name: str) -> None:
assert device.api_level() >= self.minimum_api
verbose_level = RequiredFeature('verbose_level').request()
start_time = time.time()
self.prepare_device_for_run(device)
evolutiz_cmd = f"evolutiz -p {package_name} -v -v -v --throttle 200 --ignore-crashes " \
f"--ignore-security-exceptions --ignore-timeouts --bugreport -f /mnt/sdcard/{script_name} 1"
output, errors, result_code = adb.shell_command(
device, evolutiz_cmd, timeout=settings.TEST_CASE_EVAL_TIMEOUT)
if verbose_level > 1:
print(f"Test case running finished with output:\n{output}")
if "Exception" in errors:
device_stacktrace = errors.split("** Error: ")[1]
raise Exception(
f"An error occurred when running test case: {device_stacktrace}"
)
# need to manually kill evolutiz when timeout
adb.pkill(device, "evolutiz")
self.clean_device_after_run(device)
if verbose_level > 0:
logger.log_progress(
f'\nEvolutiz test run took: {time.time() - start_time:.2f} seconds'
)
def evolve(self) -> List[Individual]:
verbose_level = RequiredFeature('verbose_level').request()
for gen in range(1, self.max_generations):
if not self.budget_manager.is_budget_available():
print("Budget ran out, exiting evolve")
break
logger.log_progress(f"\n---> Starting generation {str(gen)} "
f"at {str(self.budget_manager.get_time_budget_used())}")
# create and evaluate offspring
offspring = self.generate_offspring(self.population, gen)
success = self.parallel_evaluator.evaluate(offspring)
if not success:
print("Budget ran out during parallel evaluation, exiting evolve")
break
self.population[:] = self.toolbox.selectBest(self.population + offspring, self.population_size)
self.parallel_evaluator.test_suite_evaluator.update_logbook(gen, self.population)
if verbose_level > 0:
logger.log_progress(f"\nFinished generation {str(gen)} "
f"at {str(self.budget_manager.get_time_budget_used())}")
return self.population
def generate(self, device: Device, package_name: str,
destination_file_name: str) -> TestCase:
assert device.api_level() >= self.minimum_api
verbose_level = RequiredFeature('verbose_level').request()
start_time = time.time()
self.prepare_device_for_run(device)
evolutiz_events = settings.SEQUENCE_LENGTH_MAX
test_case = []
launch_result = self.evolutiz_connector.send_command(
device, package_name, f"performview launch-app")
for i in range(0, evolutiz_events):
widget_action_result = WidgetAction.random(device,
self.evolutiz_connector)
test_event: TestEvent = widget_action_result
test_case.append(test_event)
if widget_action_result.is_outbound():
break
if verbose_level > 0:
logger.log_progress(
f'\nEvolutiz test generation took: {time.time() - start_time:.2f} '
f'seconds for {len(test_case):d} events')
self.clean_device_after_run(device)
return test_case
def instrument(self) -> None:
self.app_path: str = RequiredFeature('app_path').request()
self.result_dir: str = RequiredFeature('result_dir').request()
self.instrumented_subjects_path: str = RequiredFeature('instrumented_subjects_path').request()
self.emma_instrument_path: str = RequiredFeature('emma_instrument_path').request()
# first, check if we should assume apps are already instrumented
assume_subjects_instrumented = RequiredFeature('assume_subjects_instrumented').request()
if assume_subjects_instrumented:
features.provide('instrumented_app_path', self.app_path)
output, errors, result_code = run_cmd(f"aapt dump badging {self.app_path} | grep package:\\ name")
package_name = output.split("package: name=\'")[1].split("\'")[0]
features.provide('package_name', package_name)
self.prepare_files_for_coverage()
return
logger.log_progress(f"\nInstrumenting app: {os.path.basename(self.app_path)}")
# copy sources and instrument application
instrumented_app_path, package_name = self.prepare_app_for_instrumentation()
features.provide('package_name', package_name)
features.provide('instrumented_app_path', instrumented_app_path)
self.instrument_gradle_file(instrumented_app_path, package_name)
result_code = os.system(f"./gradlew assembleDebug 2>&1 >{self.result_dir}/build.log")
if result_code != 0:
raise Exception("Unable run assembleDebug")
os.chdir(settings.WORKING_DIR)
self.prepare_files_for_coverage()
def run(self) -> None:
self.apk_preparer.prepare()
emulators_number = RequiredFeature('emulators_number').request()
if (emulators_number < 1):
return
logger.log_progress("\nSetting up devices.")
minimum_api = RequiredFeature('minimum_api').request()
mapper = MapperOnDevices(DeviceSetupThread.setup, minimum_api=minimum_api)
try:
mapper.run()
except Exception as e:
logger.log_progress(f"An error happened setting up devices: {str(traceback.format_exc())}")
return
devices = self.device_manager.get_devices()
for device in devices:
if device.state == State.setting_up:
raise Exception(f"An error occurred setting up devices before starting Evolutiz run")
# run the strategy
population = self.strategy.run()
self.write_summary_files()
def run(self, device: Device, package_name: str, script_name: str) -> None:
verbose_level = RequiredFeature('verbose_level').request()
start_time = time.time()
self.prepare_device_for_run(device)
string_seeding_flag = ""
if self.use_motifgene:
string_seeding_flag = f"--string-seeding /mnt/sdcard/{package_name}_strings.xml"
motifcore_cmd = f"motifcore -p {package_name} --ignore-crashes --ignore-security-exceptions --ignore-timeouts" \
f" --bugreport {string_seeding_flag} -f /mnt/sdcard/{script_name} 1"
output, errors, result_code = adb.shell_command(
device, motifcore_cmd, timeout=settings.TEST_CASE_EVAL_TIMEOUT)
if verbose_level > 1:
print(f"Test case running finished with output:\n{output}")
if "Exception" in errors:
device_stacktrace = errors.split("** Error: ")[1]
raise Exception(
f"An error occurred when running test case: {device_stacktrace}"
)
# need to manually kill motifcore when timeout
adb.pkill(device, "motifcore")
self.clean_device_after_run(device)
if verbose_level > 0:
logger.log_progress(
f'\nMotifcore test run took: {time.time() - start_time:.2f} seconds'
)
def run(self) -> List[Individual]:
gen = 0
while self.budget_manager.is_budget_available():
logger.log_progress(
f"\n---> Starting generation {str(gen)} "
f"at {str(self.budget_manager.get_time_budget_used())}")
new_individuals = self.population_generator.generate(
self.sampling_size, gen=gen)
if new_individuals is None:
# Timeout occurred
break
success = self.parallel_evaluator.evaluate(new_individuals)
if not success:
# Timeout occurred
break
# update logbook
self.parallel_evaluator.test_suite_evaluator.update_logbook(
gen, new_individuals)
history = RequiredFeature('history').request()
history.update(self.population)
# select best individuals between current population and new one
self.population[:] = self.toolbox.selectBest(
self.population + new_individuals, self.population_size)
gen += 1
return self.population
def instrument_apk(folder_name, app_path, result_dir):
logger.log_progress("\nInstrumenting app: " + folder_name)
result_code = os.system("mkdir -p " + result_dir)
if result_code != 0: raise Exception("Unable to create result dir")
result_code = os.system("ant clean emma debug 2>&1 >" + result_dir +
"/build.log")
if result_code != 0: raise Exception("Unable run ant clean emma debug")
result_code = os.system("cp bin/coverage.em " + result_dir + "/" +
logger.redirect_string())
if result_code != 0: raise Exception("Unable to copy coverage.em file")
p = sub.Popen("ls bin/*-debug.apk",
stdout=sub.PIPE,
stderr=sub.PIPE,
shell=True)
apk_path, errors = p.communicate()
apk_path = apk_path.rstrip('\n')
p = sub.Popen(
settings.AAPT + " d xmltree " + apk_path +
" AndroidManifest.xml | grep package | awk 'BEGIN {FS=\"\\\"\"}{print $2}'",
stdout=sub.PIPE,
stderr=sub.PIPE,
shell=True)
package_name, errors = p.communicate()
package_name = package_name.rstrip('\n')
return apk_path, package_name
def reboot_devices(self, wait_to_be_ready: bool = False) -> None:
logger.log_progress("\nRebooting devices.")
for device in self.get_devices():
device.reboot()
if wait_to_be_ready:
self.wait_devices_to_be_ready()
def check_virtualbox_is_not_running() -> None:
if is_command_available("vboxmanage"):
output, errors, result_code = run_cmd("vboxmanage list runningvms")
if output.strip() != "":
cause = "Android emulators can't be run while VirtualBox is running.\n" \
"Shutdown VirtualBox before running Evolutiz."
logger.log_progress(cause)
raise Exception(cause)
def reboot(self) -> None:
Device.reboot(self)
output, errors, result_code = adb.adb_command(
self, "reboot", timeout=settings.ADB_REGULAR_COMMAND_TIMEOUT)
if result_code != 0:
logger.log_progress(f"\nUnable to reboot device: {self.name}")
logger.log_progress("\nPlease, turn it off and on manually.")
raise Exception(f"Unable to reboot device: {self.name}")
def get_subject_paths(arguments: argparse.Namespace) -> List[str]:
features.provide('assume_subjects_instrumented',
arguments.assume_subjects_instrumented)
subject_path = arguments.subject_path
if subject_path is not None:
absolute_path = os.path.abspath(subject_path)
absolute_path = absolute_path.rstrip('/')
features.provide('subjects_path', [absolute_path])
return [absolute_path]
else:
subjects_path = arguments.subjects_path.rstrip('/') + '/'
features.provide('subjects_path', subjects_path)
app_paths = []
if arguments.assume_subjects_instrumented:
output, errors, result_code = run_cmd(
f"find -L {subjects_path} -name \"*.apk\"")
for line in output.strip().split('\n'):
app_paths.append(
line.rstrip('/')) # remove trailing forward slash
else:
output, errors, result_code = run_cmd(
f"ls -1 \"{subjects_path}\"*")
for relative_path in output.strip().split('\n'):
# convert relative path to absolute path
absolute_path = os.path.abspath(relative_path)
absolute_path = absolute_path.rstrip('/')
if os.path.isdir(absolute_path):
app_paths.append(absolute_path)
elif os.path.isfile(absolute_path):
if absolute_path.endswith(".apk"):
if "hydrate" not in relative_path: # hydrate app doesn't compile yet, so don't bother
app_paths.append(
absolute_path) # remove trailing forward slash
else:
logger.log_progress(
f"Ignoring non-APK file {absolute_path} in subjects path"
)
else:
# special file (e.g., socket)
logger.log_progress(
f"Ignoring special file {absolute_path} in subjects path"
)
continue
if arguments.randomize_subjects:
random.shuffle(app_paths)
if arguments.limit_subjects_number != -1:
app_paths = app_paths[0:arguments.limit_subjects_number]
# return list(filter(lambda p: 'com.zhiliaoapp.musically' in p, app_paths))
return app_paths
def evolve(self) -> List[Individual]:
verbose_level: bool = RequiredFeature('verbose_level').request()
for gen in range(1, self.max_generations):
if not self.budget_manager.is_budget_available():
print("Budget ran out, exiting evolve")
break
logger.log_progress(
f"\n---> Starting generation {str(gen)}"
f" at {str(self.budget_manager.get_time_budget_used())}")
# create new population, starting with elitism
new_population: List[Individual] = self.toolbox.selectBest(
self.population, self.elitism_size)
while len(new_population) < self.population_size:
# select parents
parents: List[Individual] = self.toolbox.select(
self.population, 2)
# generate offspring
needed_offspring = min(
self.population_size - len(new_population), 2)
offspring: List[Individual] = self.crossover(
parents,
gen,
needed_offspring,
base_index_in_generation=len(new_population))
self.mutation(offspring)
success = self.parallel_evaluator.evaluate(offspring)
if not success:
print(
"Budget ran out during parallel evaluation, exiting evolve"
)
return self.population
# extend new population with offspring or parents, depending which ones have the best individual
best_ind, = self.toolbox.selectBest(offspring + parents, 1)
if best_ind in offspring:
new_population.extend(offspring)
else:
new_population.extend(parents)
self.population = new_population.copy()
self.parallel_evaluator.test_suite_evaluator.update_logbook(
gen, self.population)
if verbose_level > 0:
logger.log_progress(
f"\nFinished generation {str(gen)} "
f"at {str(self.budget_manager.get_time_budget_used())}")
return self.population
def install_in_all_devices(self, minimum_api: Optional[int] = None) -> None:
logger.log_progress(f"\nPreparing {self.test_runner_name} test runner in devices.")
mapper = MapperOnDevices(self.install, minimum_api=minimum_api)
try:
mapper.run()
except Exception as e:
str = traceback.format_exc()
print(str)
def instrument(self) -> None:
self.app_path: str = RequiredFeature('app_path').request()
self.result_dir: str = RequiredFeature('result_dir').request()
self.instrumented_subjects_path: str = RequiredFeature(
'instrumented_subjects_path').request()
self.emma_instrument_path: str = RequiredFeature(
'emma_instrument_path').request()
# first, check if we should assume apps are already instrumented
assume_subjects_instrumented = RequiredFeature(
'assume_subjects_instrumented').request()
if assume_subjects_instrumented:
features.provide('instrumented_app_path', self.app_path)
# copy emma generated file
result_code = os.system(
f"cp bin/coverage.em {self.result_dir}/{logger.redirect_string()}"
)
if result_code != 0:
raise Exception("Unable to copy coverage.em file")
output, errors, result_code = run_cmd(
f"aapt dump badging {self.app_path} | grep package:\\ name")
package_name = output.split("package: name=\'")[1].split("\'")[0]
features.provide('package_name', package_name)
features.provide('compiled_package_name', package_name)
return
logger.log_progress(
f"\nInstrumenting app: {os.path.basename(self.app_path)}")
# copy sources and instrument application
instrumented_app_path, package_name = self.prepare_app_for_instrumentation(
)
# compile with emma data
result_code = os.system(
f"ant clean emma debug 2>&1 >{self.result_dir}/build.log")
if result_code != 0:
raise Exception("Unable run ant clean emma debug")
# copy emma generated file
result_code = os.system(
f"cp bin/coverage.em {self.result_dir}/{logger.redirect_string()}")
if result_code != 0:
raise Exception("Unable to copy coverage.em file")
os.chdir(settings.WORKING_DIR)
features.provide('package_name', package_name)
features.provide('instrumented_app_path', instrumented_app_path)
# assume same compiled package name as the one declard in AndroidManifest.xml file
features.provide('compiled_package_name', package_name)
def generate_test_coverage(
self,
device: Device,
coverage_folder_local_path: str,
accumulated_output: str,
accumulated_errors: str,
script_path: str,
generation: int,
individual_index: int,
test_case_index: int,
unique_crashes: Set[str],
scripts_crash_status: Dict[str, bool]
) -> bool:
compiled_package_name: str = RequiredFeature('compiled_package_name').request()
result_dir: str = RequiredFeature('result_dir').request()
# clear app's data and state
output, errors, result_code = adb.shell_command(device, f"pm clear {compiled_package_name}")
accumulated_output += output
accumulated_errors += errors
if result_code != 0:
adb.log_evaluation_result(device, result_dir, script_path, False)
if self.verbose_level > 0:
logger.log_progress(f"\n{accumulated_output}\n{accumulated_errors}")
raise Exception(f"Unable to clear package for script_path {script_path} in device: {device.name}")
output, errors, result_code = adb.shell_command(
device,
f"am instrument {compiled_package_name}/{compiled_package_name}.EmmaInstrument.EmmaInstrumentation"
)
accumulated_output += output
accumulated_errors += errors
if result_code != 0:
adb.log_evaluation_result(device, result_dir, script_path, False)
if self.verbose_level > 0:
logger.log_progress(f"\n{accumulated_output}\n{accumulated_errors}")
raise Exception(f"Unable to instrument for script_path {script_path} in device: {device.name}")
script_name = script_path.split("/")[-1]
test_runner = RequiredFeature('test_runner').request()
test_runner.run(device, compiled_package_name, script_name)
return self.dump_script_coverage(
device,
coverage_folder_local_path,
accumulated_output,
accumulated_errors,
script_path,
generation,
individual_index,
test_case_index,
unique_crashes,
scripts_crash_status)
def run(self) -> List[Individual]:
self.package_name = RequiredFeature('compiled_package_name').request()
success = self.initPopulation()
if not success:
logger.log_progress(
"\nThere was an error initializing population for app.")
raise Exception(
"There was an error initializing population for app.")
return self.evolve()
def show_best_historic_fitness(self) -> None:
self.logbook = RequiredFeature('logbook').request()
fitness_by_gen = self.logbook.select("fitness")
# best independent (i.e., from different individuals) historic values for each objective
max_coverage = 0.0
min_length = float(sys.maxsize)
max_crashes = 0.0
# the fitness of the best multi-objective individual
best_individual_fitness = (max_coverage, min_length, max_crashes)
for gen, population in enumerate(fitness_by_gen):
for fitness in population:
individual_coverage = fitness['coverage']
individual_length = fitness['length']
individual_crashes = fitness['crashes']
# is this a better individual than the one found so far?
at_least_as_good = individual_coverage >= max_coverage \
and individual_length <= min_length \
and individual_crashes >= max_crashes
partially_better = individual_coverage > max_coverage \
or individual_length < min_length \
or individual_crashes > max_crashes
if at_least_as_good and partially_better:
best_individual_fitness = (individual_coverage,
individual_length,
individual_crashes)
if individual_coverage > max_coverage:
max_coverage = individual_coverage
if individual_length < min_length:
min_length = individual_length
if individual_crashes > max_crashes:
max_crashes = individual_crashes
logger.log_progress(
f"\n- Best multi-objective individual: {best_individual_fitness}")
# CAUTION: the following best values are from different individuals
logger.log_progress(f"\n- Best historic coverage: {str(max_coverage)}")
logger.log_progress(f"\n- Best historic crashes: {str(max_crashes)}")
if max_crashes > 0:
logger.log_progress(f"\n- Best historic length: {str(min_length)}")
else:
logger.log_progress("\n- Best historic length: --")
def generate(self, device: Device, package_name: str,
destination_file_name: str) -> TestCase:
verbose_level = RequiredFeature('verbose_level').request()
start_time = time.time()
self.prepare_device_for_run(device)
evolutiz_events = random.randint(settings.SEQUENCE_LENGTH_MIN,
settings.SEQUENCE_LENGTH_MAX)
string_seeding_flag = ""
# String seeding is NOT supported yet
# if self.use_motifgene:
# string_seeding_flag = f"--string-seeding /mnt/sdcard/{package_name}_strings.xml"
evolutiz_cmd = f"evolutiz -p {package_name} --ignore-crashes --ignore-security-exceptions --ignore-timeouts" \
f" --bugreport {string_seeding_flag} -o {self.evolutiz_script_path_in_devices}" \
f" --throttle {self.throttle} -v {str(evolutiz_events)}"
output, errors, result_code = adb.shell_command(
device, evolutiz_cmd, timeout=settings.TEST_CASE_EVAL_TIMEOUT)
if verbose_level > 1:
print(f"Test case generation finished with output:\n{output}")
if "Exception" in errors:
if "** Error: " in errors:
device_stacktrace = errors.split("** Error: ")[1]
raise Exception(
f"An error occurred when generating test case: {device_stacktrace}"
)
else:
raise Exception(
f"An error occurred when generating test case: {errors}")
# need to manually kill evolutiz when timeout
adb.pkill(device, "evolutiz")
time.sleep(5)
test_case: TestCase = self.retrieve_generated_test(
device, destination_file_name)
self.clean_device_after_run(device)
if verbose_level > 0:
logger.log_progress(
f'\nEvolutiz test generation took: {time.time() - start_time:.2f} seconds '
f'for {evolutiz_events:d} events')
return test_case
def log_exception(self,
e: Exception,
stack_trace: str,
device: Optional[Device] = None) -> None:
verbose_level = RequiredFeature('verbose_level').request()
if verbose_level == 0:
return
template_base = "\nAn error occurred when calling func in MultipleQueueConsumerThread"
if device is None:
if verbose_level > 0:
formatted_string = f"{template_base}: \n{stack_trace}"
logger.log_progress(formatted_string)
else:
logger.log_progress(f"{template_base}.\n")
else:
if verbose_level > 0:
formatted_string = f"{template_base} on device {device}: \n{traceback.format_exc()}"
logger.log_progress(formatted_string)
else:
formatted_string = f"{template_base} on device {device}.\n"
logger.log_progress(formatted_string)
def random(cls, device: Device,
evolutiz_connector: EvolutizConnector) -> WidgetActionResult:
package_name = RequiredFeature('compiled_package_name').request()
current_package_name = adb.get_current_package_name(device)
current_activity = adb.get_current_activity(device)
if 'com.google.android' in current_activity or 'com.google.android' in current_package_name:
# we are still in the HOME, and launch-app command failed
raise Exception(f"An error ocurred launching app {package_name}")
action_performed = None
for i in range(0, 3):
# indicate Evolutiz test runner to perform any executable action randomly.
# the runner should return the action performed
action_performed = evolutiz_connector.send_command(
device, package_name,
f"performview random-action {current_activity}")
if action_performed.startswith("FAILURE"):
logger.log_progress(
f"An error occurred when performing random action onto activity {current_activity}."
f"Retrying.")
else:
break
if action_performed.startswith("FAILURE"):
raise Exception(
f"An error occurred when performing random action onto activity {current_activity}"
)
elif action_performed.startswith("SUCCESS_APP_CRASH"):
result = action_performed.split("SUCCESS_APP_CRASH:")[1]
return WidgetActionResult(result)
elif action_performed.startswith("SUCCESS_OUTBOUND"):
result = action_performed.split("SUCCESS_OUTBOUND:")[1]
return WidgetActionResult(result)
elif action_performed.startswith("SUCCESS_NEW_STATE"):
result = action_performed.split("SUCCESS_NEW_STATE:")[1]
return WidgetActionResult(result)
elif action_performed.startswith("SUCCESS"):
result = action_performed.split("SUCCESS:")[1]
return WidgetActionResult(result)
else:
raise Exception(
f"An error occurred when performing random action onto activity {current_activity}"
)
def boot_emulators(self, wait_to_be_ready: bool = False) -> None:
while True:
self.next_available_emulator_port = 5554
self.next_available_adb_server_port = 5038
self.devices = []
logger.log_progress(
f"\nBooting devices: {str(0)}/{str(self.emulators_number)}")
for i in range(0, self.emulators_number):
logger.log_progress(
f"\nBooting devices: {str(i + 1)}/{str(self.emulators_number)}"
)
emulator = Emulator(self)
emulator.boot()
self.devices.append(emulator)
if wait_to_be_ready:
try:
self.wait_devices_to_be_ready()
except WaitDevicesTimeout as e:
logger.log_progress(f"\n{str(e)}")
logger.log_progress(
"\nForce kill on all current emulator processes")
run_cmd("pgrep emu | xargs kill -9", discard_output=True)
time.sleep(5)
continue
break
def wait_for_battery_threshold(self, battery_threshold: int = 20) -> None:
while True:
all_devices_above_threshold = all([
level is None or level >= battery_threshold
for level in (device.battery_level()
for device in self.get_devices())
])
if all_devices_above_threshold:
break
else:
logger.log_progress(
f"\nWaiting for some devices to reach {str(battery_threshold)}% battery level"
)
time.sleep(60) # sleep 1 minute
def run(self) -> None:
app_path = RequiredFeature('app_path').request()
self.population_generator.generate(self.sampling_size, gen=0)
# did we crash?
devices = self.device_manager.get_devices()
assert len(devices) == 1
device = devices[0]
crashed = crash_handler.handle(device, '', 0, 0, 0, set())
if crashed:
logger.log_progress(f"\nApp {app_path} CRASHED")
else:
logger.log_progress(f"\nApp {app_path} PASSED")
def evolve(self) -> List[Individual]:
verbose_level: bool = RequiredFeature('verbose_level').request()
self.parent = self.population[0]
for gen in range(1, self.max_generations):
if not self.budget_manager.is_budget_available():
print("Budget ran out, exiting evolve")
break
logger.log_progress(
f"\n---> Starting generation {str(gen)}"
f"at {str(self.budget_manager.get_time_budget_used())}")
# Create and evaluate mutants
mutants = self.generate_mutants(gen)
success = self.parallel_evaluator.evaluate(mutants)
if not success:
print(
"Budget ran out during parallel evaluation, exiting evolve"
)
break
# select best mutant and generate offspring by applying crossover with parent
best_mutant, = self.toolbox.selectBest(mutants, 1)
offspring = self.generate_offspring(gen, best_mutant)
success = self.parallel_evaluator.evaluate(offspring)
if not success:
print(
"Budget ran out during parallel evaluation, exiting evolve"
)
break
# select best offspring and set as new parent
# the parent is included in the selection to avoid degrading the current solution
offspring.append(self.parent)
best_offspring, = self.toolbox.selectBest(offspring, 1)
self.parent = best_offspring
self.population = [self.parent]
self.parallel_evaluator.test_suite_evaluator.update_logbook(
gen, self.population)
if verbose_level > 0:
logger.log_progress(
f"\nFinished generation {str(gen)} "
f"at {str(self.budget_manager.get_time_budget_used())}")
return self.population
def analyse(self) -> None:
self.result_dir: str = RequiredFeature('result_dir').request()
self.package_name: str = RequiredFeature(
'compiled_package_name').request()
self.instrumented_app_path: str = RequiredFeature(
'instrumented_app_path').request()
self.get_apk_path()
print("### Working on apk:", self.package_name)
# static analysis
self.decoded_dir = f"{self.result_dir}/decoded-apk"
if settings.ENABLE_STRING_SEEDING:
logger.log_progress("\nRunning static analysis on apk")
self.decode_apk()
def collectCoverage(device, package_name, result_dir, suffix=""):
logger.log_progress("\nSending coverage broadcast in device: " + device +
" at: " + datetime.today().strftime("%H:%M:%S"))
os.system(adb.adb_cmd_prefix + " -s " + device +
" shell am broadcast -a evolutiz.emma.COLLECT_COVERAGE" +
logger.redirect_string())
logger.log_progress("\nPulling coverage from device: " + device + " at: " +
datetime.today().strftime("%H:%M:%S"))
coverageFilePath = "/data/data/" + package_name + "/files/coverage.ec"
os.system(adb.adb_cmd_prefix + " -s " + device + " pull " +
coverageFilePath + " " + result_dir + "/coverage.ec" + suffix +
logger.redirect_string())
return True
def generate_test_coverage(self, device: Device,
coverage_folder_local_path: str,
accumulated_output: str,
accumulated_errors: str, script_path: str,
generation: int, individual_index: int,
test_case_index: int, unique_crashes: Set[str],
scripts_crash_status: Dict[str, bool]) -> bool:
compiled_package_name: str = RequiredFeature(
'compiled_package_name').request()
result_dir: str = RequiredFeature('result_dir').request()
# clear app's data and state
output, errors, result_code = adb.shell_command(
device, f"pm clear {compiled_package_name}")
accumulated_output += str(output)
accumulated_errors += str(errors)
if result_code != 0:
adb.log_evaluation_result(device, result_dir, script_path, False)
if self.verbose_level > 0:
logger.log_progress(
f"\n{accumulated_output}\n{accumulated_errors}")
raise Exception(
f"Unable to clear package for script_path {script_path} in device: {device.name}"
)
# copy the covids file to the coverage_folder_local_path
ella_coverage_ids_file_path = f"{self.get_current_apk_output_folder(device)}/covids"
output, errors, result_code = run_cmd(
f"cp {ella_coverage_ids_file_path} {coverage_folder_local_path}/")
if result_code != 0:
raise Exception(
f"Unable to copy the coverage ids file for test script, path is: {ella_coverage_ids_file_path}"
)
# Run test case using the requested test runner.
# Internally, the test runner will take care of calling the methods AppInstrumentator#setup_for_test_run and
# AppInstrumentator#teardown_after_test_run, which will start and stop the ELLA server correspondingly.
script_name = script_path.split("/")[-1]
test_runner = RequiredFeature('test_runner').request()
test_runner.run(device, compiled_package_name, script_name)
return self.dump_script_coverage(device, coverage_folder_local_path,
accumulated_output,
accumulated_errors, script_path,
generation, individual_index,
test_case_index, unique_crashes,
scripts_crash_status)
def boot(self,
port: Optional[int] = None,
adb_port: Optional[int] = None) -> None:
verbose_level = RequiredFeature('verbose_level').request()
Device.boot(self)
# ensure the emulator configuration is correct
self.port = port if port is not None else self.device_manager.get_next_available_emulator_port(
)
self.avd_name = self.avd_manager.get_avd_name_for_emulator_port(
self.port)
if not self.avd_manager.avd_name_exists(self.avd_name):
raise Exception(
f"AVD name {self.avd_name} doesn't exist. "
f"Check that the provided AVD series ({self.avd_manager.avd_series}) is correct."
)
# start custom abd server for this emulator
self.adb_port = adb_port if adb_port is not None else self.device_manager.get_next_available_adb_server_port(
)
adb.restart_server(self.adb_port)
# start emulator
self.name = f"emulator-{str(self.port)}"
emulator_cmd = f"{self.get_adb_server_port_prefix()} QEMU_AUDIO_DRV=none $ANDROID_HOME/emulator/emulator"
flags: str = f" -no-snapshot -wipe-data -no-boot-anim -writable-system -port {str(self.port)}"
if verbose_level < 3:
# -no-window flag can't be at the end
flags = f" -no-window{flags}"
logs = " >/dev/null 2>/dev/null"
if verbose_level > 0:
logs = f" >logs/{self.avd_name}.log 2>logs/{self.avd_name}.err"
flags = f"{flags} -verbose -debug all"
cmd: str = f'{emulator_cmd} -avd {self.avd_name}{flags}{logs}'
if verbose_level > 1:
logger.log_progress(f"\nFiring up emulator with command: {cmd}")
sub.Popen(cmd, shell=True)
def evolve(self) -> List[Individual]:
verbose_level: bool = RequiredFeature('verbose_level').request()
for gen in range(1, self.max_generations):
if not self.budget_manager.is_budget_available():
print("Budget ran out, exiting evolve")
break
logger.log_progress(
f"\n---> Starting generation {str(gen)} "
f"at {str(self.budget_manager.get_time_budget_used())}")
# select parents
parents: List[Individual] = self.toolbox.select(self.population, 2)
# generate offspring
offspring = self.crossover(parents, gen, 2)
self.mutation(offspring)
success = self.parallel_evaluator.evaluate(offspring)
if not success:
print(
"Budget ran out during parallel evaluation, exiting evolve"
)
break
best_ind, = self.toolbox.selectBest(offspring + parents, 1)
if best_ind in offspring:
# only update population if the best individual is one of the offspring
self.population = [
ind for ind in self.population if ind not in parents
] + offspring
self.parallel_evaluator.test_suite_evaluator.update_logbook(
gen, self.population)
if verbose_level > 0:
logger.log_progress(
f"\nFinished generation {str(gen)} "
f"at {str(self.budget_manager.get_time_budget_used())}")
return self.population
