def get_sequence(device, apk_dir, package_name, index, unique_crashes):
std_out_file = apk_dir + "/intermediate/" + "output.stdout"
random.seed()
motifcore_events = random.randint(settings.SEQUENCE_LENGTH_MIN,
settings.SEQUENCE_LENGTH_MAX)
ret = []
# clear data
os.system(settings.ADB + " -s " + device + " shell pm clear " +
package_name)
# start motifcore
print "... Start generating a sequence"
# command = Command(settings.ADB + " -s " + device + " shell motifcore -p " + package_name + " -v --throttle " + str(
# settings.THROTTLE) + " " + str(motifcore_events))
# command.run(timeout=600)
cmd = settings.ADB + " -s " + device + " shell motifcore -p " + package_name + " --ignore-crashes --ignore-security-exceptions --ignore-timeouts --bugreport --string-seeding /mnt/sdcard/" + package_name + "_strings.xml -v " + str(
motifcore_events)
os.system(settings.TIMEOUT_CMD + " " + str(settings.EVAL_TIMEOUT) + " " +
cmd)
# need to kill motifcore when timeout
kill_motifcore_cmd = "shell ps | awk '/com\.android\.commands\.motifcore/ { system(\"" + settings.ADB + " -s " + device + " shell kill \" $2) }'"
os.system(settings.ADB + " -s " + device + " " + kill_motifcore_cmd)
print "... Finish generating a sequence"
# access the generated script, should ignore the first launch activity
script_name = settings.MOTIFCORE_SCRIPT_PATH.split("/")[-1]
ts = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S.%f")[:-3]
os.system(settings.ADB + " -s " + device + " pull " +
settings.MOTIFCORE_SCRIPT_PATH + " " + apk_dir +
"/intermediate/" + script_name + ".init." + ts + "." +
str(index))
script = open(apk_dir + "/intermediate/" + script_name + ".init." + ts +
"." + str(index))
is_content = False
is_skipped_first = False
for line in script:
line = line.strip()
if line.find("start data >>") != -1:
is_content = True
continue
if is_content and line != "":
if is_skipped_first == False:
is_skipped_first = True
continue
if is_skipped_first:
ret.append(line)
script.close()
# deal with crash
crash_handler.handle(
device, apk_dir, apk_dir + "/intermediate/" + script_name + ".init." +
ts + "." + str(index), "init", ts, index, unique_crashes)
return ret
def get_sequence(device, apk_dir, package_name, index, unique_crashes):
std_out_file = apk_dir + "/intermediate/" + "output.stdout"
random.seed()
motifcore_events = random.randint(settings.SEQUENCE_LENGTH_MIN, settings.SEQUENCE_LENGTH_MAX)
ret = []
# clear data
os.system("adb -s " + device + " shell pm clear " + package_name)
# start motifcore
print "... Start generating a sequence"
# command = Command("adb -s " + device + " shell motifcore -p " + package_name + " -v --throttle " + str(
# settings.THROTTLE) + " " + str(motifcore_events))
# command.run(timeout=600)
cmd = "adb -s " + device + " shell motifcore -p " + package_name + " --ignore-crashes --ignore-security-exceptions --ignore-timeouts --bugreport --string-seeding /mnt/sdcard/" + package_name + "_strings.xml -v " + str(
motifcore_events)
os.system(settings.TIMEOUT_CMD + " " + str(settings.EVAL_TIMEOUT) + " " + cmd)
# need to kill motifcore when timeout
kill_motifcore_cmd = "shell ps | awk '/com\.android\.commands\.motifcore/ { system(\"adb -s " + device + " shell kill \" $2) }'"
os.system("adb -s " + device + " " + kill_motifcore_cmd)
print "... Finish generating a sequence"
# access the generated script, should ignore the first launch activity
script_name = settings.MOTIFCORE_SCRIPT_PATH.split("/")[-1]
ts = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S.%f")[:-3]
os.system(
"adb -s " + device + " pull " + settings.MOTIFCORE_SCRIPT_PATH + " " + apk_dir + "/intermediate/" + script_name + ".init." + ts + "." + str(
index))
script = open(apk_dir + "/intermediate/" + script_name + ".init." + ts + "." + str(index))
is_content = False
is_skipped_first = False
for line in script:
line = line.strip()
if line.find("start data >>") != -1:
is_content = True
continue
if is_content and line != "":
if is_skipped_first == False:
is_skipped_first = True
continue
if is_skipped_first:
ret.append(line)
script.close()
# deal with crash
crash_handler.handle(device, apk_dir, apk_dir + "/intermediate/" + script_name + ".init." + ts + "." + str(index),
"init", ts, index, unique_crashes)
return ret
def get_suite_coverage(scripts, device, apk_dir, package_name, gen, pop):
unique_crashes = set()
# clean states
os.system(settings.ADB + " -s " + device + " shell am force-stop " +
package_name)
os.system(settings.ADB + " -s " + device + " shell pm clear " +
package_name)
# os.system("rm " + apk_dir + "/intermediate/activity.coverage.*")
# run scripts
for index, script in enumerate(scripts):
start_target = settings.ADB + " -s " + device + " shell motifcore -p " + package_name + " -c android.intent.category.LAUNCHER 1"
os.system(start_target)
os.system(settings.ADB + " -s " + device + " push " + script +
" /mnt/sdcard/")
script_name = script.split("/")[-1]
# command = Command(settings.ADB + " -s " + device + " shell motifcore -p " + package_name + " --bugreport --throttle " + str(
# settings.THROTTLE) + " -f /mnt/sdcard/" + script_name + " 1")
# command = Command(settings.ADB + " -s " + device + " shell motifcore -p " + package_name + " --bugreport " + "-f /mnt/sdcard/" + script_name + " 1")
# command.run(timeout=600)
cmd = settings.ADB + " -s " + device + " shell motifcore -p " + package_name + " --bugreport --string-seeding /mnt/sdcard/" + package_name + "_strings.xml" + " -f /mnt/sdcard/" + script_name + " 1"
os.system(settings.TIMEOUT_CMD + " " + str(settings.EVAL_TIMEOUT) +
" " + cmd)
# need to manually kill motifcore when timeout
kill_motifcore_cmd = "shell ps | awk '/com\.android\.commands\.motifcore/ { system(\"" + settings.ADB + " -s " + device + " shell kill \" $2) }'"
os.system(settings.ADB + " -s " + device + " " + kill_motifcore_cmd)
os.system(settings.ADB + " -s " + device +
" pull /sdcard/activity.coverage " + apk_dir +
"/coverages/activity.coverage." + str(gen) + "." + str(pop) +
"." + str(index))
crash_handler.handle(device, apk_dir, script, gen, pop, index,
unique_crashes)
# close app
os.system(settings.ADB + " -s " + device + " shell pm clear " +
package_name)
os.system(settings.ADB + " -s " + device + " shell am force-stop " +
package_name)
coverage = cal_coverage(apk_dir + "/coverages/", gen, pop)
# print "\n\n\n### get_suite_coverage: Coverage, num crashes =", coverage, ",", len(unique_crashes)
return coverage, len(unique_crashes)
def dump_script_coverage(self, device: Device,
coverage_folder_local_path: str,
accumulated_output: str, accumulated_errors: str,
script_path: str, generation: int,
individual_index: int, test_case_index: int,
unique_crashes: Set[str],
scripts_crash_status: Dict[str, bool]) -> bool:
result_dir: str = RequiredFeature('result_dir').request()
if crash_handler.handle(device, script_path, generation,
individual_index, test_case_index,
unique_crashes):
scripts_crash_status[script_path] = True
return False
else:
# no crash, collect coverage
scripts_crash_status[script_path] = False
adb.log_evaluation_result(device, result_dir, script_path, True)
# save the coverage.dat file of this test script
output, errors, result_code = run_cmd(
f"mv {self.get_coverage_dat_path(device)} {coverage_folder_local_path}/"
)
if result_code != 0:
raise Exception(
f"Unable to move the coverage dat file for test script, path is: {self.get_coverage_dat_path(device)}"
)
return True
def get_suite_coverage(scripts, device, apk_dir, package_name, gen, pop):
unique_crashes = set()
# clean states
os.system("adb -s " + device + " shell am force-stop " + package_name)
os.system("adb -s " + device + " shell pm clear " + package_name)
os.system("adb -s " + device + " shell rm /mnt/sdcard/coverage.ec")
os.chdir(apk_dir)
ts = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
coverage_folder = str(gen) + "." + str(pop) + "." + ts
os.system("mkdir coverages/" + coverage_folder)
os.system("cp bin/coverage.em coverages/" + coverage_folder)
os.chdir("coverages/" + coverage_folder)
# run scripts
for index, script in enumerate(scripts):
start_target = "adb -s " + device + " shell am instrument " + package_name + "/" + package_name + ".EmmaInstrument.EmmaInstrumentation"
os.system(start_target)
os.system("adb -s " + device + " push " + script + " /mnt/sdcard/.")
script_name = script.split("/")[-1]
# command = Command("adb -s " + device + " shell motifcore -p " + package_name + " --bugreport " + "-f /mnt/sdcard/" + script_name + " 1")
# command.run(timeout = 180)
cmd = "adb -s " + device + " shell motifcore -p " + package_name + " --ignore-crashes --ignore-security-exceptions --ignore-timeouts --bugreport --string-seeding /mnt/sdcard/" + package_name + "_strings.xml" + " -f /mnt/sdcard/" + script_name + " 1"
os.system(settings.TIMEOUT_CMD + " " + str(settings.EVAL_TIMEOUT) + " " + cmd)
# need to manually kill motifcore when timeout
kill_motifcore_cmd = "shell ps | awk '/com\.android\.commands\.motifcore/ { system(\"adb -s " + device + " shell kill \" $2) }'"
os.system("adb -s " + device + " " + kill_motifcore_cmd)
if crash_handler.handle(device, apk_dir, script, gen, pop, index, unique_crashes):
pass
else:
# no crash, can broadcast
os.system("adb -s " + device + " shell am broadcast -a edu.gatech.m3.emma.COLLECT_COVERAGE")
# close app
os.system("adb -s " + device + " shell pm clear " + package_name)
print "### Getting EMMA coverage.ec and report ..."
os.system("adb -s " + device + " shell pm clear " + package_name)
time.sleep(0.5)
os.system("adb -s " + device + " pull /mnt/sdcard/coverage.ec")
os.system(
"java -cp " + settings.ANDROID_HOME + "tools/lib/emma.jar emma report -r html -in coverage.em,coverage.ec")
html_file = apk_dir + "/coverages/" + coverage_folder + "/coverage/index.html"
try:
coverage_str = extract_coverage(html_file)
except:
return 0, len(unique_crashes)
if coverage_str.find("%") != -1:
return int(coverage_str.split("%")[0]), len(unique_crashes)
else:
return 0, len(unique_crashes)
def get_ella_suite_coverage(scripts, device, apk_dir, package_name, gen):
num_crashes = 0
std_out_file = apk_dir + "/intermediate/" + "output.stdout"
# clean states
os.system("adb -s " + device + " shell am force-stop " + package_name)
os.system("adb -s " + device + " shell pm clear " + package_name)
os.system("rm " + apk_dir + "/coverage.dat*")
# run scripts
for index, script in enumerate(scripts):
start_target = "adb -s " + device + " shell motifcore -p " + package_name + " -c android.intent.category.LAUNCHER 1"
os.system(start_target + " > " + std_out_file)
# p = subprocess.Popen(start_target, shell=True, stdout=subprocess.PIPE).communicate()[0]
os.system("adb -s " + device + " push " + script + " /mnt/sdcard/" +
" > " + std_out_file)
script_name = script.split("/")[-1]
# start motifcore replay
# add timeout
command = Command("adb -s " + device + " shell motifcore -p " +
package_name + " --bugreport --throttle " +
str(settings.THROTTLE) + " -f /mnt/sdcard/" +
script_name + " 1")
command.run(timeout=600)
# print "... Start evaluating a script"
# p = subprocess.Popen("adb -s " + settings.DEVICE + " shell motifcore -p " + package_name + " --throttle " + str(settings.THROTTLE) + " -f /mnt/sdcard/" + script_name + " 1", shell=True, stdout=subprocess.PIPE).communicate()[0]
# os.system("adb -s " + device + " shell motifcore -p " + package_name + " --bugreport --throttle " + str(settings.THROTTLE) + " -f /mnt/sdcard/" + script_name + " 1" + " > " + std_out_file)
# print "... Finished evaluating a script"
# TODO: handle the bugreport: save to db together with generation info and the script
if crash_handler.handle(device, apk_dir, script):
num_crashes += 1
# close app
os.system("adb -s " + device + " shell pm clear " + package_name +
" > " + std_out_file)
os.system("adb -s " + device + " shell am force-stop " + package_name +
" > " + std_out_file)
coverage_file = ''
covids_file = apk_dir + "/covids"
coverage_file_count = 0
for file_name in os.listdir(apk_dir):
if file_name.startswith("coverage.dat"):
coverage_file_count += 1
coverage_file = apk_dir + "/" + file_name
assert coverage_file_count == 1
coverage = cal_coverage(coverage_file, covids_file)
return coverage, num_crashes
def run(self) -> None:
app_path = RequiredFeature('app_path').request()
self.population_generator.generate(self.sampling_size, gen=0)
# did we crash?
devices = self.device_manager.get_devices()
assert len(devices) == 1
device = devices[0]
crashed = crash_handler.handle(device, '', 0, 0, 0, set())
if crashed:
logger.log_progress(f"\nApp {app_path} CRASHED")
else:
logger.log_progress(f"\nApp {app_path} PASSED")
def get_ella_suite_coverage(scripts, device, apk_dir, package_name, gen):
num_crashes = 0
std_out_file = apk_dir + "/intermediate/" + "output.stdout"
# clean states
os.system("adb -s " + device + " shell am force-stop " + package_name)
os.system("adb -s " + device + " shell pm clear " + package_name)
os.system("rm " + apk_dir + "/coverage.dat*")
# run scripts
for index, script in enumerate(scripts):
start_target = "adb -s " + device + " shell motifcore -p " + package_name + " -c android.intent.category.LAUNCHER 1"
os.system(start_target + " > " + std_out_file)
# p = subprocess.Popen(start_target, shell=True, stdout=subprocess.PIPE).communicate()[0]
os.system("adb -s " + device + " push " + script + " /mnt/sdcard/" + " > " + std_out_file)
script_name = script.split("/")[-1]
# start motifcore replay
# add timeout
command = Command("adb -s " + device + " shell motifcore -p " + package_name + " --bugreport --throttle " + str(
settings.THROTTLE) + " -f /mnt/sdcard/" + script_name + " 1")
command.run(timeout=600)
# print "... Start evaluating a script"
# p = subprocess.Popen("adb -s " + settings.DEVICE + " shell motifcore -p " + package_name + " --throttle " + str(settings.THROTTLE) + " -f /mnt/sdcard/" + script_name + " 1", shell=True, stdout=subprocess.PIPE).communicate()[0]
# os.system("adb -s " + device + " shell motifcore -p " + package_name + " --bugreport --throttle " + str(settings.THROTTLE) + " -f /mnt/sdcard/" + script_name + " 1" + " > " + std_out_file)
# print "... Finished evaluating a script"
# TODO: handle the bugreport: save to db together with generation info and the script
if crash_handler.handle(device, apk_dir, script):
num_crashes += 1
# close app
os.system("adb -s " + device + " shell pm clear " + package_name + " > " + std_out_file)
os.system("adb -s " + device + " shell am force-stop " + package_name + " > " + std_out_file)
coverage_file = ''
covids_file = apk_dir + "/covids"
coverage_file_count = 0
for file_name in os.listdir(apk_dir):
if file_name.startswith("coverage.dat"):
coverage_file_count += 1
coverage_file = apk_dir + "/" + file_name
assert coverage_file_count == 1
coverage = cal_coverage(coverage_file, covids_file)
return coverage, num_crashes
def dump_script_coverage(
self,
device: Device,
coverage_folder_local_path: str,
accumulated_output: str,
accumulated_errors: str,
script_path: str,
generation: int,
individual_index: int,
test_case_index: int,
unique_crashes: Set[str],
scripts_crash_status: Dict[str, bool]
) -> bool:
compiled_package_name: str = RequiredFeature('compiled_package_name').request()
package_name: str = RequiredFeature('package_name').request()
result_dir: str = RequiredFeature('result_dir').request()
application_files = f"/data/data/{compiled_package_name}/files"
coverage_ec_device_path = f"{application_files}/coverage.ec"
if crash_handler.handle(device, script_path, generation, individual_index, test_case_index, unique_crashes):
scripts_crash_status[script_path] = True
return False
else:
# no crash, can broadcast
scripts_crash_status[script_path] = False
# don't check result_code of the following command, since it can fail if this is the first time running emma
# for this test suite and there is no coverage.ec file in /mnt/sdcard folder.
adb.sudo_shell_command(
device,
f"cp -p {self.coverage_ec_device_backup_path} {coverage_ec_device_path}"
)
broadcast = f"am broadcast -a evolutiz.emma.COLLECT_COVERAGE " \
f"-n {compiled_package_name}/{package_name}.EmmaInstrument.CollectCoverageReceiver"
output, errors, result_code = adb.shell_command(device, broadcast, timeout=60)
accumulated_output += output
accumulated_errors += errors
if "Exception" in errors:
adb.log_evaluation_result(device, result_dir, script_path, False)
if self.verbose_level > 0:
logger.log_progress(f"\n{accumulated_output}\n{accumulated_errors}")
raise Exception(f"Unable to broadcast coverage gathering for "
f"script_path {script_path} in device: {device.name}")
if not adb.exists_file(device, coverage_ec_device_path):
adb.log_evaluation_result(device, result_dir, script_path, False)
if self.verbose_level > 0:
logger.log_progress(f"\n{accumulated_output}\n{accumulated_errors}")
raise Exception(f"Coverage broadcast was sent for script_path {script_path} "
f"in device: {device.name} but there is not file: {coverage_ec_device_path}")
adb.log_evaluation_result(device, result_dir, script_path, True)
# save coverage.ec file to /mnt/sdcard before clearing app (files are deleted)
cp_command = f"cp -p {coverage_ec_device_path} {self.coverage_ec_device_backup_path}"
output, errors, result_code = adb.sudo_shell_command(device, cp_command)
accumulated_output += output
accumulated_errors += errors
if result_code != 0:
adb.log_evaluation_result(device, result_dir, script_path, False)
if self.verbose_level > 0:
logger.log_progress(f"\n{accumulated_output}\n{accumulated_errors}")
raise Exception(f"Unable to retrieve coverage.ec file after coverage broadcast for "
f"script_path {script_path} in device: {device.name}")
return True