def get_sequence(device, apk_dir, package_name, index, unique_crashes): std_out_file = apk_dir + "/intermediate/" + "output.stdout" random.seed() motifcore_events = random.randint(settings.SEQUENCE_LENGTH_MIN, settings.SEQUENCE_LENGTH_MAX) ret = [] # clear data os.system(settings.ADB + " -s " + device + " shell pm clear " + package_name) # start motifcore print "... Start generating a sequence" # command = Command(settings.ADB + " -s " + device + " shell motifcore -p " + package_name + " -v --throttle " + str( # settings.THROTTLE) + " " + str(motifcore_events)) # command.run(timeout=600) cmd = settings.ADB + " -s " + device + " shell motifcore -p " + package_name + " --ignore-crashes --ignore-security-exceptions --ignore-timeouts --bugreport --string-seeding /mnt/sdcard/" + package_name + "_strings.xml -v " + str( motifcore_events) os.system(settings.TIMEOUT_CMD + " " + str(settings.EVAL_TIMEOUT) + " " + cmd) # need to kill motifcore when timeout kill_motifcore_cmd = "shell ps | awk '/com\.android\.commands\.motifcore/ { system(\"" + settings.ADB + " -s " + device + " shell kill \" $2) }'" os.system(settings.ADB + " -s " + device + " " + kill_motifcore_cmd) print "... Finish generating a sequence" # access the generated script, should ignore the first launch activity script_name = settings.MOTIFCORE_SCRIPT_PATH.split("/")[-1] ts = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S.%f")[:-3] os.system(settings.ADB + " -s " + device + " pull " + settings.MOTIFCORE_SCRIPT_PATH + " " + apk_dir + "/intermediate/" + script_name + ".init." + ts + "." + str(index)) script = open(apk_dir + "/intermediate/" + script_name + ".init." + ts + "." + str(index)) is_content = False is_skipped_first = False for line in script: line = line.strip() if line.find("start data >>") != -1: is_content = True continue if is_content and line != "": if is_skipped_first == False: is_skipped_first = True continue if is_skipped_first: ret.append(line) script.close() # deal with crash crash_handler.handle( device, apk_dir, apk_dir + "/intermediate/" + script_name + ".init." + ts + "." + str(index), "init", ts, index, unique_crashes) return ret
def get_sequence(device, apk_dir, package_name, index, unique_crashes): std_out_file = apk_dir + "/intermediate/" + "output.stdout" random.seed() motifcore_events = random.randint(settings.SEQUENCE_LENGTH_MIN, settings.SEQUENCE_LENGTH_MAX) ret = [] # clear data os.system("adb -s " + device + " shell pm clear " + package_name) # start motifcore print "... Start generating a sequence" # command = Command("adb -s " + device + " shell motifcore -p " + package_name + " -v --throttle " + str( # settings.THROTTLE) + " " + str(motifcore_events)) # command.run(timeout=600) cmd = "adb -s " + device + " shell motifcore -p " + package_name + " --ignore-crashes --ignore-security-exceptions --ignore-timeouts --bugreport --string-seeding /mnt/sdcard/" + package_name + "_strings.xml -v " + str( motifcore_events) os.system(settings.TIMEOUT_CMD + " " + str(settings.EVAL_TIMEOUT) + " " + cmd) # need to kill motifcore when timeout kill_motifcore_cmd = "shell ps | awk '/com\.android\.commands\.motifcore/ { system(\"adb -s " + device + " shell kill \" $2) }'" os.system("adb -s " + device + " " + kill_motifcore_cmd) print "... Finish generating a sequence" # access the generated script, should ignore the first launch activity script_name = settings.MOTIFCORE_SCRIPT_PATH.split("/")[-1] ts = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S.%f")[:-3] os.system( "adb -s " + device + " pull " + settings.MOTIFCORE_SCRIPT_PATH + " " + apk_dir + "/intermediate/" + script_name + ".init." + ts + "." + str( index)) script = open(apk_dir + "/intermediate/" + script_name + ".init." + ts + "." + str(index)) is_content = False is_skipped_first = False for line in script: line = line.strip() if line.find("start data >>") != -1: is_content = True continue if is_content and line != "": if is_skipped_first == False: is_skipped_first = True continue if is_skipped_first: ret.append(line) script.close() # deal with crash crash_handler.handle(device, apk_dir, apk_dir + "/intermediate/" + script_name + ".init." + ts + "." + str(index), "init", ts, index, unique_crashes) return ret
def get_suite_coverage(scripts, device, apk_dir, package_name, gen, pop): unique_crashes = set() # clean states os.system(settings.ADB + " -s " + device + " shell am force-stop " + package_name) os.system(settings.ADB + " -s " + device + " shell pm clear " + package_name) # os.system("rm " + apk_dir + "/intermediate/activity.coverage.*") # run scripts for index, script in enumerate(scripts): start_target = settings.ADB + " -s " + device + " shell motifcore -p " + package_name + " -c android.intent.category.LAUNCHER 1" os.system(start_target) os.system(settings.ADB + " -s " + device + " push " + script + " /mnt/sdcard/") script_name = script.split("/")[-1] # command = Command(settings.ADB + " -s " + device + " shell motifcore -p " + package_name + " --bugreport --throttle " + str( # settings.THROTTLE) + " -f /mnt/sdcard/" + script_name + " 1") # command = Command(settings.ADB + " -s " + device + " shell motifcore -p " + package_name + " --bugreport " + "-f /mnt/sdcard/" + script_name + " 1") # command.run(timeout=600) cmd = settings.ADB + " -s " + device + " shell motifcore -p " + package_name + " --bugreport --string-seeding /mnt/sdcard/" + package_name + "_strings.xml" + " -f /mnt/sdcard/" + script_name + " 1" os.system(settings.TIMEOUT_CMD + " " + str(settings.EVAL_TIMEOUT) + " " + cmd) # need to manually kill motifcore when timeout kill_motifcore_cmd = "shell ps | awk '/com\.android\.commands\.motifcore/ { system(\"" + settings.ADB + " -s " + device + " shell kill \" $2) }'" os.system(settings.ADB + " -s " + device + " " + kill_motifcore_cmd) os.system(settings.ADB + " -s " + device + " pull /sdcard/activity.coverage " + apk_dir + "/coverages/activity.coverage." + str(gen) + "." + str(pop) + "." + str(index)) crash_handler.handle(device, apk_dir, script, gen, pop, index, unique_crashes) # close app os.system(settings.ADB + " -s " + device + " shell pm clear " + package_name) os.system(settings.ADB + " -s " + device + " shell am force-stop " + package_name) coverage = cal_coverage(apk_dir + "/coverages/", gen, pop) # print "\n\n\n### get_suite_coverage: Coverage, num crashes =", coverage, ",", len(unique_crashes) return coverage, len(unique_crashes)
def dump_script_coverage(self, device: Device, coverage_folder_local_path: str, accumulated_output: str, accumulated_errors: str, script_path: str, generation: int, individual_index: int, test_case_index: int, unique_crashes: Set[str], scripts_crash_status: Dict[str, bool]) -> bool: result_dir: str = RequiredFeature('result_dir').request() if crash_handler.handle(device, script_path, generation, individual_index, test_case_index, unique_crashes): scripts_crash_status[script_path] = True return False else: # no crash, collect coverage scripts_crash_status[script_path] = False adb.log_evaluation_result(device, result_dir, script_path, True) # save the coverage.dat file of this test script output, errors, result_code = run_cmd( f"mv {self.get_coverage_dat_path(device)} {coverage_folder_local_path}/" ) if result_code != 0: raise Exception( f"Unable to move the coverage dat file for test script, path is: {self.get_coverage_dat_path(device)}" ) return True
def get_suite_coverage(scripts, device, apk_dir, package_name, gen, pop): unique_crashes = set() # clean states os.system("adb -s " + device + " shell am force-stop " + package_name) os.system("adb -s " + device + " shell pm clear " + package_name) os.system("adb -s " + device + " shell rm /mnt/sdcard/coverage.ec") os.chdir(apk_dir) ts = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S") coverage_folder = str(gen) + "." + str(pop) + "." + ts os.system("mkdir coverages/" + coverage_folder) os.system("cp bin/coverage.em coverages/" + coverage_folder) os.chdir("coverages/" + coverage_folder) # run scripts for index, script in enumerate(scripts): start_target = "adb -s " + device + " shell am instrument " + package_name + "/" + package_name + ".EmmaInstrument.EmmaInstrumentation" os.system(start_target) os.system("adb -s " + device + " push " + script + " /mnt/sdcard/.") script_name = script.split("/")[-1] # command = Command("adb -s " + device + " shell motifcore -p " + package_name + " --bugreport " + "-f /mnt/sdcard/" + script_name + " 1") # command.run(timeout = 180) cmd = "adb -s " + device + " shell motifcore -p " + package_name + " --ignore-crashes --ignore-security-exceptions --ignore-timeouts --bugreport --string-seeding /mnt/sdcard/" + package_name + "_strings.xml" + " -f /mnt/sdcard/" + script_name + " 1" os.system(settings.TIMEOUT_CMD + " " + str(settings.EVAL_TIMEOUT) + " " + cmd) # need to manually kill motifcore when timeout kill_motifcore_cmd = "shell ps | awk '/com\.android\.commands\.motifcore/ { system(\"adb -s " + device + " shell kill \" $2) }'" os.system("adb -s " + device + " " + kill_motifcore_cmd) if crash_handler.handle(device, apk_dir, script, gen, pop, index, unique_crashes): pass else: # no crash, can broadcast os.system("adb -s " + device + " shell am broadcast -a edu.gatech.m3.emma.COLLECT_COVERAGE") # close app os.system("adb -s " + device + " shell pm clear " + package_name) print "### Getting EMMA coverage.ec and report ..." os.system("adb -s " + device + " shell pm clear " + package_name) time.sleep(0.5) os.system("adb -s " + device + " pull /mnt/sdcard/coverage.ec") os.system( "java -cp " + settings.ANDROID_HOME + "tools/lib/emma.jar emma report -r html -in coverage.em,coverage.ec") html_file = apk_dir + "/coverages/" + coverage_folder + "/coverage/index.html" try: coverage_str = extract_coverage(html_file) except: return 0, len(unique_crashes) if coverage_str.find("%") != -1: return int(coverage_str.split("%")[0]), len(unique_crashes) else: return 0, len(unique_crashes)
def get_ella_suite_coverage(scripts, device, apk_dir, package_name, gen): num_crashes = 0 std_out_file = apk_dir + "/intermediate/" + "output.stdout" # clean states os.system("adb -s " + device + " shell am force-stop " + package_name) os.system("adb -s " + device + " shell pm clear " + package_name) os.system("rm " + apk_dir + "/coverage.dat*") # run scripts for index, script in enumerate(scripts): start_target = "adb -s " + device + " shell motifcore -p " + package_name + " -c android.intent.category.LAUNCHER 1" os.system(start_target + " > " + std_out_file) # p = subprocess.Popen(start_target, shell=True, stdout=subprocess.PIPE).communicate()[0] os.system("adb -s " + device + " push " + script + " /mnt/sdcard/" + " > " + std_out_file) script_name = script.split("/")[-1] # start motifcore replay # add timeout command = Command("adb -s " + device + " shell motifcore -p " + package_name + " --bugreport --throttle " + str(settings.THROTTLE) + " -f /mnt/sdcard/" + script_name + " 1") command.run(timeout=600) # print "... Start evaluating a script" # p = subprocess.Popen("adb -s " + settings.DEVICE + " shell motifcore -p " + package_name + " --throttle " + str(settings.THROTTLE) + " -f /mnt/sdcard/" + script_name + " 1", shell=True, stdout=subprocess.PIPE).communicate()[0] # os.system("adb -s " + device + " shell motifcore -p " + package_name + " --bugreport --throttle " + str(settings.THROTTLE) + " -f /mnt/sdcard/" + script_name + " 1" + " > " + std_out_file) # print "... Finished evaluating a script" # TODO: handle the bugreport: save to db together with generation info and the script if crash_handler.handle(device, apk_dir, script): num_crashes += 1 # close app os.system("adb -s " + device + " shell pm clear " + package_name + " > " + std_out_file) os.system("adb -s " + device + " shell am force-stop " + package_name + " > " + std_out_file) coverage_file = '' covids_file = apk_dir + "/covids" coverage_file_count = 0 for file_name in os.listdir(apk_dir): if file_name.startswith("coverage.dat"): coverage_file_count += 1 coverage_file = apk_dir + "/" + file_name assert coverage_file_count == 1 coverage = cal_coverage(coverage_file, covids_file) return coverage, num_crashes
def run(self) -> None: app_path = RequiredFeature('app_path').request() self.population_generator.generate(self.sampling_size, gen=0) # did we crash? devices = self.device_manager.get_devices() assert len(devices) == 1 device = devices[0] crashed = crash_handler.handle(device, '', 0, 0, 0, set()) if crashed: logger.log_progress(f"\nApp {app_path} CRASHED") else: logger.log_progress(f"\nApp {app_path} PASSED")
def get_ella_suite_coverage(scripts, device, apk_dir, package_name, gen): num_crashes = 0 std_out_file = apk_dir + "/intermediate/" + "output.stdout" # clean states os.system("adb -s " + device + " shell am force-stop " + package_name) os.system("adb -s " + device + " shell pm clear " + package_name) os.system("rm " + apk_dir + "/coverage.dat*") # run scripts for index, script in enumerate(scripts): start_target = "adb -s " + device + " shell motifcore -p " + package_name + " -c android.intent.category.LAUNCHER 1" os.system(start_target + " > " + std_out_file) # p = subprocess.Popen(start_target, shell=True, stdout=subprocess.PIPE).communicate()[0] os.system("adb -s " + device + " push " + script + " /mnt/sdcard/" + " > " + std_out_file) script_name = script.split("/")[-1] # start motifcore replay # add timeout command = Command("adb -s " + device + " shell motifcore -p " + package_name + " --bugreport --throttle " + str( settings.THROTTLE) + " -f /mnt/sdcard/" + script_name + " 1") command.run(timeout=600) # print "... Start evaluating a script" # p = subprocess.Popen("adb -s " + settings.DEVICE + " shell motifcore -p " + package_name + " --throttle " + str(settings.THROTTLE) + " -f /mnt/sdcard/" + script_name + " 1", shell=True, stdout=subprocess.PIPE).communicate()[0] # os.system("adb -s " + device + " shell motifcore -p " + package_name + " --bugreport --throttle " + str(settings.THROTTLE) + " -f /mnt/sdcard/" + script_name + " 1" + " > " + std_out_file) # print "... Finished evaluating a script" # TODO: handle the bugreport: save to db together with generation info and the script if crash_handler.handle(device, apk_dir, script): num_crashes += 1 # close app os.system("adb -s " + device + " shell pm clear " + package_name + " > " + std_out_file) os.system("adb -s " + device + " shell am force-stop " + package_name + " > " + std_out_file) coverage_file = '' covids_file = apk_dir + "/covids" coverage_file_count = 0 for file_name in os.listdir(apk_dir): if file_name.startswith("coverage.dat"): coverage_file_count += 1 coverage_file = apk_dir + "/" + file_name assert coverage_file_count == 1 coverage = cal_coverage(coverage_file, covids_file) return coverage, num_crashes
def dump_script_coverage( self, device: Device, coverage_folder_local_path: str, accumulated_output: str, accumulated_errors: str, script_path: str, generation: int, individual_index: int, test_case_index: int, unique_crashes: Set[str], scripts_crash_status: Dict[str, bool] ) -> bool: compiled_package_name: str = RequiredFeature('compiled_package_name').request() package_name: str = RequiredFeature('package_name').request() result_dir: str = RequiredFeature('result_dir').request() application_files = f"/data/data/{compiled_package_name}/files" coverage_ec_device_path = f"{application_files}/coverage.ec" if crash_handler.handle(device, script_path, generation, individual_index, test_case_index, unique_crashes): scripts_crash_status[script_path] = True return False else: # no crash, can broadcast scripts_crash_status[script_path] = False # don't check result_code of the following command, since it can fail if this is the first time running emma # for this test suite and there is no coverage.ec file in /mnt/sdcard folder. adb.sudo_shell_command( device, f"cp -p {self.coverage_ec_device_backup_path} {coverage_ec_device_path}" ) broadcast = f"am broadcast -a evolutiz.emma.COLLECT_COVERAGE " \ f"-n {compiled_package_name}/{package_name}.EmmaInstrument.CollectCoverageReceiver" output, errors, result_code = adb.shell_command(device, broadcast, timeout=60) accumulated_output += output accumulated_errors += errors if "Exception" in errors: adb.log_evaluation_result(device, result_dir, script_path, False) if self.verbose_level > 0: logger.log_progress(f"\n{accumulated_output}\n{accumulated_errors}") raise Exception(f"Unable to broadcast coverage gathering for " f"script_path {script_path} in device: {device.name}") if not adb.exists_file(device, coverage_ec_device_path): adb.log_evaluation_result(device, result_dir, script_path, False) if self.verbose_level > 0: logger.log_progress(f"\n{accumulated_output}\n{accumulated_errors}") raise Exception(f"Coverage broadcast was sent for script_path {script_path} " f"in device: {device.name} but there is not file: {coverage_ec_device_path}") adb.log_evaluation_result(device, result_dir, script_path, True) # save coverage.ec file to /mnt/sdcard before clearing app (files are deleted) cp_command = f"cp -p {coverage_ec_device_path} {self.coverage_ec_device_backup_path}" output, errors, result_code = adb.sudo_shell_command(device, cp_command) accumulated_output += output accumulated_errors += errors if result_code != 0: adb.log_evaluation_result(device, result_dir, script_path, False) if self.verbose_level > 0: logger.log_progress(f"\n{accumulated_output}\n{accumulated_errors}") raise Exception(f"Unable to retrieve coverage.ec file after coverage broadcast for " f"script_path {script_path} in device: {device.name}") return True