def install_ape_and_make_snapshot(avd_name, force_snapshot=False):
avd_list = get_avd_list()
avd = next(avd for avd in avd_list if avd.name == avd_name)
if avd.running:
if not force_snapshot and APE_READY_SS in list_snapshots(
serial=avd.serial):
load_snapshot(APE_READY_SS, serial=avd.serial)
return avd
serial = avd.serial
else:
serial = emulator_run_and_wait(avd_name, snapshot=APE_READY_SS)
if force_snapshot or APE_READY_SS not in list_snapshots(serial=serial):
print(
'No saved snapshot on the device, rebooting and making snapshot...'
)
kill_emulator(serial=serial)
time.sleep(3)
serial = emulator_run_and_wait(avd_name, wipe_data=True)
print('Setup emulator...')
emulator_setup(serial=serial)
run_adb_cmd('push ape.jar {}'.format(APE_ROOT), serial=serial)
save_snapshot(APE_READY_SS, serial=serial)
avd.setRunning(serial)
return avd
def ape_stdout_callback(line):
f.write(line + '\n')
if line.startswith("[APE_MT] mt data "):
directory = line[len("[APE_MT] mt data "):]
run_adb_cmd("pull {} {}".format(directory, mt_data_dir),
serial=serial)
run_adb_cmd("shell rm -rf {}".format(directory), serial=serial)
def fetch_result(output_dir, serial):
ret = run_adb_cmd('shell ls /sdcard/', serial=serial)
folders = []
for line in ret.split('\n'):
if line.startswith('sata-'):
folders.append('/sdcard/{}'.format(line.rstrip()))
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
for folder in folders:
run_adb_cmd('pull {} {}'.format(folder, output_dir), serial=serial)
def start_catcher(output_fname, serial = None):
run_adb_cmd("logcat -c", serial = serial)
with open(output_fname, 'wt') as f:
try:
run_adb_cmd("logcat art:I AndroidRuntime:E CrashAnrDetector:D ActivityManager:E SQLiteDatabase:E WindowManager:E ActivityThread:E Parcel:E *:F *:S",
serial = serial,
stdout_callback = lambda t:f.write("O: " + t + '\n'),
stderr_callback = lambda t:f.write("E: " + t + '\n'))
except RunCmdError as e:
pass
def mt_task(package_name, serial, logging_flag, mt_is_running):
def stdout_callback(line):
if line.startswith('Server with uid'):
mt_is_running.value += 1
print('Start mtserver...')
run_adb_cmd('shell /data/local/tmp/mtserver server {} {}' \
.format(package_name, logging_flag),
stdout_callback = stdout_callback,
stderr_callback = stdout_callback,
serial=serial)
kill_mtserver(serial)
def release_file(self, socketfd, fname):
# This part could be called with gentle termination of the app (not SIGKILL)
if socketfd not in self.connections:
raise WrongConnectionState
state, data = self.connections[socketfd]
if state != STATE_RUNNING:
raise WrongConnectionState
print("{} release file {}".format(TAG, fname))
run_adb_cmd("pull {} {}".format(fname, self.output_folder),
serial=self.serial)
run_adb_cmd("shell rm {}".format(fname),
serial=self.serial)
def ape_task(avd_name, serial, package_name, output_dir, running_minutes,
mt_is_running):
sleep_cnt = 0
while mt_is_running.value == 0 and sleep_cnt < 30:
time.sleep(1)
sleep_cnt += 1
# Something wrong on mtserver, wait 30 seconds
if sleep_cnt == 30:
kill_mtserver(serial=serial)
return
print('ape_task(): Emulator[{}, {}] Running APE with package {}'.format(
avd_name, serial, package_name))
args = '-p {} --running-minutes {} --mt --ape sata'.format(
package_name, running_minutes)
with open(os.path.join(output_dir, 'ape_stdout_stderr.txt'), 'wt') as f:
ret = run_adb_cmd(
'shell CLASSPATH={} {} {} {} {}'.format(
os.path.join(TMP_LOCATION,
'ape.jar'), '/system/bin/app_process',
TMP_LOCATION, 'com.android.commands.monkey.Monkey', args),
stdout_callback=lambda t: f.write(t + '\n'),
stderr_callback=lambda t: f.write(t + '\n'),
serial=serial,
)
fetch_result(output_dir, serial)
def run_mtserver(package_name, output_folder, serial=None):
connections = Connections(package_name, serial, output_folder)
kill_mtserver(serial)
try:
print('{} Start mtserver...'.format(TAG))
out = run_adb_cmd('shell /data/local/tmp/mtserver server {}' \
.format(package_name),
stdout_callback = connections.stdout_callback,
stderr_callback = connections.stderr_callback,
serial=serial)
except WrongConnectionState:
print('{} WrongConnectionState. Check follow log...'.format(TAG))
for line in connections.log:
print('{} {}'.format(TAG, line))
kill_mtserver(serial)
except KeyboardInterrupt:
connections.clean_up('KeyboardInterrupt')
raise
except Exception as e:
connections.clean_up('Exception {}'.format(e))
raise
connections.clean_up('Normal run_mtserver')
def libart_check(libart_path, serial):
# Check our target libart.so is installed
# Since checking is done by its size, it may give wrong result.
size1 = os.path.getsize(libart_path)
output = run_adb_cmd("shell ls -l /system/lib/libart.so", serial=serial)
size2 = int(output.split()[3])
return size1 == size2
def ape_task(avd_name, serial, package_name, output_dir, running_minutes,
mt_is_running, mtdtarget_fname, target_all_thread, methodGuide):
sleep_cnt = 0
while mt_is_running.value == 0 and sleep_cnt < 30:
time.sleep(1)
sleep_cnt += 1
# Something wrong on mtserver, wait 30 seconds
if sleep_cnt == 30:
kill_mtserver(serial=serial)
return
mt_data_dir = os.path.join(output_dir, 'mt_data')
if not os.path.isdir(mt_data_dir):
os.makedirs(mt_data_dir)
def ape_stdout_callback(line):
f.write(line + '\n')
if line.startswith("[APE_MT] mt data "):
directory = line[len("[APE_MT] mt data "):]
run_adb_cmd("pull {} {}".format(directory, mt_data_dir),
serial=serial)
run_adb_cmd("shell rm -rf {}".format(directory), serial=serial)
print('ape_task(): Emulator[{}, {}] Running APE with package {}'.format(
avd_name, serial, package_name))
args = '-p {} --running-minutes {} --mt --mtdtarget {} --ape {} {}--countlim 2'.format(
package_name, running_minutes, mtdtarget_fname,
"target" if methodGuide else "sata",
"--target-all-thread " if target_all_thread else "")
with open(os.path.join(output_dir, 'ape_stdout_stderr.txt'), 'wt') as f:
ret = run_adb_cmd(
'shell CLASSPATH={} {} {} {} {}'.format(
os.path.join(TMP_LOCATION,
'ape.jar'), '/system/bin/app_process',
TMP_LOCATION, 'com.android.commands.monkey.Monkey', args),
stdout_callback=ape_stdout_callback,
stderr_callback=lambda t: f.write(t + '\n'),
serial=serial,
)
# pull ape results
try:
run_adb_cmd('shell rmdir /data/ape/mt_data', serial=serial)
except RunCmdError as e:
print(e.message, file=sys.stderr)
ret = run_adb_cmd('pull /data/ape {}'.format(output_dir), serial=serial)
def close_connection(self, socketfd, prefix):
if socketfd not in self.connections:
raise WrongConnectionState
state, data = self.connections[socketfd]
if state == STATE_RUNNING:
pid, prefix_ = data
assert prefix == prefix_, (prefix, prefix_)
prefix_local = os.path.join(self.output_folder, os.path.split(prefix)[1])
# clean up given prefix
print("{} close connection on {}".format(TAG, prefix_local))
out = run_adb_cmd('shell ls {}*'.format(prefix),
serial=self.serial)
if "No such file or directory" not in out:
pulled_files = []
for line in out.split():
if line == '':
break
fname = line.rstrip()
print("{} - pull and remove {}".format(TAG, fname))
try:
run_adb_cmd("pull {} {}".format(fname, self.output_folder),
serial=self.serial)
pulled_files.append(
os.path.join(self.output_folder,
os.path.split(fname)[1]))
except RunCmdError as e:
# @TODO How could it be happen?
print("{} - failed to pull and remove {}".format(TAG, fname), file=sys.stderr)
for file in pulled_files:
os.remove(file)
prefix_local = ""
break
run_adb_cmd("shell rm {}".format(fname),
serial=self.serial)
del self.connections[socketfd]
return prefix_local
else:
raise WrongConnectionState
def run_ape(apk_path, avd_name, output_dir, running_minutes=1):
package_name = get_package_name(apk_path)
print('run_ape(): given apk_path {} avd_name {}'.format(
apk_path, avd_name))
avd = install_ape_and_make_snapshot(avd_name)
try:
install_package(apk_path, serial=avd.serial)
except RuntimeError as e:
print(e)
return
# run ape
print('run_ape(): Emulator[{}, {}] Running APE with apk={}'.format(
avd_name, avd.serial, apk_path))
args = '-p {} --running-minutes {} --ape sata'.format(
package_name, running_minutes)
ret = run_adb_cmd('shell CLASSPATH={} {} {} {} {}'.format(
os.path.join(APE_ROOT, 'ape.jar'), '/system/bin/app_process', APE_ROOT,
'com.android.commands.monkey.Monkey', args),
serial=avd.serial)
fetch_result(output_dir, avd.serial)
def mt_task(package_name, output_folder, serial, logging_flag, mt_is_running):
connections = ConnectionsWithValue(package_name, serial, output_folder,
mt_is_running)
try:
print('Start mtserver...')
out = run_adb_cmd('shell /data/local/tmp/mtserver server {} {}' \
.format(package_name, logging_flag),
stdout_callback = connections.stdout_callback,
stderr_callback = connections.stderr_callback,
serial=serial)
except WrongConnectionState:
print('CONNECTION: WrongConnectionState. Check follow log...')
for line in connections.log:
print(line)
kill_mtserver(serial)
except KeyboardInterrupt:
connections.clean_up("KeyboardInterrupt")
raise
except Exception as e:
connections.clean_up("Exception " + repr(e))
raise
connections.clean_up("Normal")
def install_art_ape_mt(avd_name,
libart_path,
ape_jar_path,
mtserver_path,
force_clear=False):
avd_list = get_avd_list()
avd = next(avd for avd in avd_list if avd.name == avd_name)
serial = None
if not force_clear: # use snapshot if it is possible
if avd.running and ART_APE_MT_READY_SS in list_snapshots(
serial=avd.serial):
load_snapshot(ART_APE_MT_READY_SS, serial=avd.serial)
time.sleep(3)
assert libart_check(libart_path, serial=avd.serial)
return avd
serial = emulator_run_and_wait(avd_name,
snapshot=ART_APE_MT_READY_SS,
writable_system=True,
ram_size_in_mb=4096,
partition_size_in_mb=8192)
if ART_APE_MT_READY_SS in list_snapshots(serial=serial):
# now running
avd.setRunning(serial)
return avd
print(
"No saved snapshot on the device, rebooting and making snapshot..."
)
kill_emulator(serial=serial)
time.sleep(3)
elif avd.running: # if force_clear, turn off running emulators
kill_emulator(serial=avd.serial)
time.sleep(3)
serial = emulator_run_and_wait(avd_name,
wipe_data=True,
writable_system=True,
ram_size_in_mb=4096,
partition_size_in_mb=8192)
print("Installing libart.so")
run_adb_cmd("remount", serial=serial)
run_adb_cmd("shell su root mount -o remount,rw /system", serial=serial)
# run_adb_cmd("shell su root mount -o remount,rw rootfs /", serial=serial)
# run_adb_cmd("shell su root chmod 777 /storage/sdcard", serial=serial)
run_adb_cmd("push {} /sdcard/libart.so".format(libart_path), serial=serial)
run_adb_cmd("shell su root mv /sdcard/libart.so /system/lib/libart.so",
serial=serial)
run_adb_cmd("shell su root chmod 644 /system/lib/libart.so", serial=serial)
run_adb_cmd("shell su root chown root:root /system/lib/libart.so",
serial=serial)
run_adb_cmd("shell su root reboot", serial=serial)
print("Wait for emulator")
emulator_wait_for_boot(avd_name, r_fd=None, serial=serial)
print("Setup emulator...")
emulator_setup(serial=serial)
print("Installing ape.jar")
run_adb_cmd("push {} {}".format(ape_jar_path,
os.path.join(TMP_LOCATION, "ape.jar")),
serial=serial)
print("Installing minitrace")
run_adb_cmd("push {} {}".format(mtserver_path, TMP_LOCATION),
serial=serial)
save_snapshot(ART_APE_MT_READY_SS, serial=serial)
assert libart_check(libart_path, serial=serial)
avd.setRunning(serial)
return avd
def run_ape_with_mt(apk_path,
avd_name,
libart_path,
ape_path,
mtserver_path,
output_dir,
running_minutes,
force_clear,
methods,
target_all_thread,
methodGuide=True):
package_name = get_package_name(apk_path)
print('run_ape_with_mt(): given apk_path {} avd_name {}'.format(
apk_path, avd_name))
assert os.path.split(libart_path)[1] == 'libart.so'
assert os.path.split(mtserver_path)[1] == 'mtserver'
assert os.path.split(ape_path)[1] == 'ape.jar'
avd = install_art_ape_mt(avd_name, libart_path, ape_path, mtserver_path,
force_clear)
try:
install_package(apk_path, serial=avd.serial)
except RuntimeError as e:
print(e)
return "install"
mtdtarget_fname = 'mtdtarget.txt'
mtdtarget_emulpath = '/data/local/tmp/mtdtarget.txt'
with open(mtdtarget_fname, 'wt') as f:
for clsname, mtdname, signature in methods:
f.write("%s\t%s\t%s\t1\n" %
(clsname, mtdname, signature)) # 1: method enter
run_adb_cmd("push {} {}".format(mtdtarget_fname, mtdtarget_emulpath),
serial=avd.serial)
kill_mtserver(serial=avd.serial)
mt_is_running = Value('i', 0)
mtserver_thread = threading.Thread(target=mt_task,
args=(package_name, avd.serial,
"00010180", mt_is_running))
apetask_thread = threading.Thread(target=ape_task,
args=(avd_name, avd.serial, package_name,
output_dir, running_minutes,
mt_is_running, mtdtarget_emulpath,
target_all_thread, methodGuide))
set_multiprocessing_mode()
generate_catcher_thread(os.path.join(output_dir, "logcat.txt"),
serial=avd.serial)
mtserver_thread.start()
apetask_thread.start()
apetask_thread.join()
kill_mtserver(serial=avd.serial)
mtserver_thread.join()
kill_generated_logcat_processes()
unset_multiprocessing_mode()
if mt_is_running.value == 0: # It failed to run ape/mt
print('run_ape_with_mt(): failed to run')
return "rerun"
# feedback
# 1. method is not found, then stop experiment this (apk, targeting methods)
# 2. method is not called
methods_registered_over_exp = [] # list of registered methods for each run
logs = []
try:
with open(os.path.join(output_dir, "logcat.txt"), 'rt') as f:
cur_methods_registered = None
lazy_methods = []
for line in f:
'''
MiniTrace: TargetMethod %s:%s[%s] lazy
MiniTrace: TargetMethod %s:%s[%s] registered
MiniTrace: TargetMethod %s:%s[%s] registered lazily
'''
if 'MiniTrace' not in line:
continue
line = line.rstrip()
line = line[line.index('MiniTrace'):]
if line.startswith("MiniTrace: connection success, received"):
lazy_methods = []
if cur_methods_registered is not None:
methods_registered_over_exp.append(
cur_methods_registered)
cur_methods_registered = []
logs.append(line)
continue
gp = re.match(
r'MiniTrace: TargetMethod (.*):(.*)\[(.*)\] ([a-z ]+)',
line)
if gp:
logs.append(line)
clsname, mtdname, signature, status = gp.groups()
if status == 'lazy':
assert (clsname, mtdname,
signature) in methods, (clsname, mtdname,
signature, methods)
lazy_methods.append((clsname, mtdname, signature))
elif status == 'registered':
assert (clsname, mtdname,
signature) in methods, (clsname, mtdname,
signature, methods)
cur_methods_registered.append(
methods.index((clsname, mtdname, signature)))
else:
assert status == 'registered lazily', status
assert clsname[0] == 'L' and clsname[-1] == ';', clsname
clsname = clsname[1:-1]
assert (clsname, mtdname,
signature) in lazy_methods, (clsname, mtdname,
signature,
lazy_methods)
cur_methods_registered.append(
methods.index((clsname, mtdname, signature)))
if cur_methods_registered is not None:
methods_registered_over_exp.append(cur_methods_registered)
except AssertionError as e:
print("run_ape_with_mt(): Feedback - failed to register methods")
print(methods_registered_over_exp)
print('\n'.join(logs))
print(e)
traceback.print_exc()
return "unregistered"
if all(executed == [] for executed in methods_registered_over_exp):
print("run_ape_with_mt(): Feedback - failed to register any methods")
return "unregistered"
print('run_ape_with_mt(): methods registered...')
print(methods_registered_over_exp)
if not methodGuide:
return "success"
# 2. method is not called
with open(os.path.join(output_dir, 'ape_stdout_stderr.txt'), 'rt') as logf:
for line in logf:
line = line.rstrip()
if "met target" in line:
return "success"
return "notsearched"
def kill_mtserver(serial = None):
pids = get_pids('/data/local/tmp/mtserver', serial=serial)
for pid in pids:
run_adb_cmd('shell kill {}'.format(pid), serial=serial)
