def run():
"""Run Android initialization."""
init_runner.run()
# Set cuttlefish device serial if needed.
if environment.is_android_cuttlefish():
android.adb.set_cuttlefish_device_serial()
# Check if we need to reflash device to latest build.
android.flash.flash_to_latest_build_if_needed()
# Reconnect to cuttlefish device if connection is ever lost.
if environment.is_android_cuttlefish():
android.adb.connect_to_cuttlefish_device()
# Reboot to bring device in a good state if not done recently.
if android.adb.time_since_last_reboot() > TIME_SINCE_REBOOT_MIN_THRESHOLD:
android.device.reboot()
# Make sure that device is in a good condition before we move forward.
android.adb.wait_until_fully_booted()
# Wait until battery charges to a minimum level and temperature threshold.
android.battery.wait_until_good_state()
# Initialize environment settings.
android.device.initialize_environment()
def main():
"""Run a cycle of heartbeat checks to ensure Android device is running."""
logs.configure('android_heartbeat')
dates.initialize_timezone_from_environment()
environment.set_bot_environment()
monitor.initialize()
if environment.is_android_cuttlefish():
android.adb.set_cuttlefish_device_serial()
device_serial = environment.get_value('ANDROID_SERIAL')
while True:
state = android.adb.get_device_state()
if state == android.adb.DEVICE_NOT_FOUND_STRING.format(
serial=device_serial):
android.adb.connect_to_cuttlefish_device()
state = android.adb.get_device_state()
logs.log('Android device %s state: %s' % (device_serial, state))
monitoring_metrics.ANDROID_UPTIME.increment_by(
int(state == 'device'), {
'serial': device_serial or '',
'platform': environment.get_platform_group() or '',
})
time.sleep(data_types.ANDROID_HEARTBEAT_WAIT_INTERVAL)
if data_handler.bot_run_timed_out():
break
def reset_usb():
"""Reset USB bus for a device serial."""
if environment.is_android_cuttlefish() or environment.is_android_emulator():
# Nothing to do here.
return True
# App Engine does not let us import this.
import fcntl
# We need to get latest device path since it could be changed in reboots or
# adb root restarts.
try:
device_path = get_device_path()
except IOError:
# We may reach this state if the device is no longer available.
device_path = None
if not device_path:
# Try pulling from cache (if available).
device_path = environment.get_value('DEVICE_PATH')
if not device_path:
logs.log_warn('No device path found, unable to reset usb.')
return False
try:
with open(device_path, 'w') as f:
fcntl.ioctl(f, USBDEVFS_RESET)
except:
logs.log_warn('Failed to reset usb.')
return False
# Wait for usb to recover.
wait_for_device(recover=False)
return True
def get_config():
"""Get arguments for a given fuzz target."""
device_serial = environment.get_value('ANDROID_SERIAL')
build_dir = environment.get_value('BUILD_DIR')
temp_dir = fuzzer_utils.get_temp_dir()
binary_path = os.path.join(build_dir, 'syzkaller')
json_config_path = os.path.join(temp_dir, 'config.json')
default_vmlinux_path = os.path.join('/tmp', device_serial, 'vmlinux')
vmlinux_path = environment.get_value('VMLINUX_PATH', default_vmlinux_path)
syzhub_address = environment.get_value('SYZHUB_ADDRESS')
syzhub_client = environment.get_value('SYZHUB_CLIENT')
syzhub_key = environment.get_value('SYZHUB_KEY')
on_cuttlefish = environment.is_android_cuttlefish()
config.generate(serial=device_serial,
work_dir_path=get_work_dir(),
binary_path=binary_path,
vmlinux_path=vmlinux_path,
config_path=json_config_path,
kcov=True,
reproduce=False,
syzhub_address=syzhub_address,
syzhub_client=syzhub_client,
syzhub_key=syzhub_key,
on_cuttlefish=on_cuttlefish)
return ['-config', json_config_path]
def wait_until_good_state():
"""Check battery and make sure it is charged beyond minimum level and
temperature thresholds."""
# Battery levels are not applicable on GCE.
if environment.is_android_cuttlefish() or settings.is_automotive():
return
# Make sure device is online.
adb.wait_for_device()
# Skip battery check if done recently.
last_battery_check_time = persistent_cache.get_value(
LAST_BATTERY_CHECK_TIME_KEY,
constructor=datetime.datetime.utcfromtimestamp)
if last_battery_check_time and not dates.time_has_expired(
last_battery_check_time, seconds=BATTERY_CHECK_INTERVAL):
return
# Initialize variables.
battery_level_threshold = environment.get_value('LOW_BATTERY_LEVEL_THRESHOLD',
LOW_BATTERY_LEVEL_THRESHOLD)
battery_temperature_threshold = environment.get_value(
'MAX_BATTERY_TEMPERATURE_THRESHOLD', MAX_BATTERY_TEMPERATURE_THRESHOLD)
device_restarted = False
while True:
battery_information = get_battery_level_and_temperature()
if battery_information is None:
logs.log_error('Failed to get battery information, skipping check.')
return
battery_level = battery_information['level']
battery_temperature = battery_information['temperature']
logs.log('Battery information: level (%d%%), temperature (%.1f celsius).' %
(battery_level, battery_temperature))
if (battery_level >= battery_level_threshold and
battery_temperature <= battery_temperature_threshold):
persistent_cache.set_value(LAST_BATTERY_CHECK_TIME_KEY, time.time())
return
logs.log('Battery in bad battery state, putting device in sleep mode.')
if not device_restarted:
adb.reboot()
device_restarted = True
# Change thresholds to expected levels (only if they were below minimum
# thresholds).
if battery_level < battery_level_threshold:
battery_level_threshold = EXPECTED_BATTERY_LEVEL
if battery_temperature > battery_temperature_threshold:
battery_temperature_threshold = EXPECTED_BATTERY_TEMPERATURE
# Stopping shell should help with shutting off a lot of services that would
# otherwise use up the battery. However, we need to turn it back on to get
# battery status information.
adb.stop_shell()
time.sleep(BATTERY_CHARGE_INTERVAL)
adb.start_shell()
def get_device_path():
"""Gets a device path to be cached and used by reset_usb."""
def _get_usb_devices():
"""Returns a list of device objects containing a serial and USB path."""
usb_list_cmd = 'lsusb -v'
output = execute_command(usb_list_cmd, timeout=RECOVERY_CMD_TIMEOUT)
if output is None:
logs.log_error('Failed to populate usb devices using lsusb, '
'host restart might be needed.')
bad_state_reached()
devices = []
path = None
for line in output.splitlines():
match = LSUSB_BUS_RE.match(line)
if match:
path = '/dev/bus/usb/%s/%s' % (match.group(1), match.group(2))
continue
match = LSUSB_SERIAL_RE.match(line)
if path and match and match.group(1):
serial = match.group(1)
devices.append(DEVICE(serial, path))
return devices
def _get_device_path_for_serial():
"""Return device path. Assumes a simple ANDROID_SERIAL."""
devices = _get_usb_devices()
for device in devices:
if device_serial == device.serial:
return device.path
return None
def _get_device_path_for_usb():
"""Returns a device path.
Assumes ANDROID_SERIAL in the form "usb:<identifier>"."""
# Android serial may reference a usb device rather than a serial number.
device_id = device_serial[len('usb:'):]
bus_number = int(
open('/sys/bus/usb/devices/%s/busnum' % device_id).read().strip())
device_number = int(
open('/sys/bus/usb/devices/%s/devnum' % device_id).read().strip())
return '/dev/bus/usb/%03d/%03d' % (bus_number, device_number)
if environment.is_android_cuttlefish():
return None
device_serial = environment.get_value('ANDROID_SERIAL')
if device_serial.startswith('usb:'):
return _get_device_path_for_usb()
return _get_device_path_for_serial()
def configure(force_enable=False):
"""Configure airplane mode and wifi on device."""
# The reproduce tool shouldn't inherit wifi settings from jobs.
if environment.get_value('REPRODUCE_TOOL'):
return
# Airplane mode should be disabled in all cases. This can get inadvertently
# turned on via gestures.
disable_airplane_mode()
# Need to disable wifi before changing configuration.
disable()
# Check if wifi needs to be enabled. If not, then no need to modify the
# supplicant file.
wifi_enabled = force_enable or environment.get_value('WIFI', True)
if not wifi_enabled:
# No more work to do, we already disabled it at start.
return
# Wait 2 seconds to allow the wifi to be enabled.
enable()
time.sleep(2)
# Install helper apk to configure wifi.
wifi_util_apk_path = os.path.join(
environment.get_platform_resources_directory(), 'wifi_util.apk')
if not app.is_installed(WIFI_UTIL_PACKAGE_NAME):
app.install(wifi_util_apk_path)
# Get ssid and password from admin configuration.
if environment.is_android_cuttlefish():
wifi_ssid = 'VirtWifi'
wifi_password = ''
else:
config = db_config.get()
if not config.wifi_ssid:
logs.log('No wifi ssid is set, skipping wifi config.')
return
wifi_ssid = config.wifi_ssid
wifi_password = config.wifi_password or ''
connect_wifi_command = (
'am instrument -e method connectToNetwork -e ssid {ssid} ')
if wifi_password:
connect_wifi_command += '-e psk {password} '
connect_wifi_command += '-w {call_path}'
output = adb.run_shell_command(
connect_wifi_command.format(ssid=quote(wifi_ssid),
password=quote(wifi_password),
call_path=WIFI_UTIL_CALL_PATH))
if 'result=true' not in output:
logs.log_warn('Failed to connect to wifi.', output=output)
def execute_command(cmd, timeout=None, log_error=True,
on_cuttlefish_host=False):
"""Spawns a subprocess to run the given shell command."""
if on_cuttlefish_host and environment.is_android_cuttlefish():
# Auto accept key fingerprint for ssh command.
cmd = ('ssh -o StrictHostKeyChecking=no '
f'{get_cuttlefish_ssh_target()} "{cmd}"')
so = []
# pylint: disable=consider-using-with
output_dest = tempfile.TemporaryFile()
# pylint: disable=subprocess-popen-preexec-fn,consider-using-with
pipe = subprocess.Popen(
cmd,
executable='/bin/bash',
stdout=output_dest,
stderr=subprocess.STDOUT,
shell=True,
preexec_fn=lambda: signal.signal(signal.SIGPIPE, signal.SIG_DFL),
bufsize=0)
def run():
"""Thread target function that waits for subprocess to complete."""
try:
pipe.communicate()
output_dest.seek(0)
output = output_dest.read()
output_dest.close()
if output:
so.append(output)
except OSError as _:
logs.log_warn('Failed to retrieve stdout from: %s' % cmd)
if pipe.returncode:
if log_error:
logs.log_warn(
'%s returned %d error code.' % (cmd, pipe.returncode),
output=output)
thread = threading.Thread(target=run)
thread.start()
thread.join(timeout)
if thread.is_alive():
logs.log_warn('Command %s timed out. Killing process.' % cmd)
try:
pipe.kill()
except OSError:
# Can't kill a dead process.
pass
return None
bytes_output = b''.join(so)
return bytes_output.strip().decode('utf-8', errors='ignore')
def run_fastboot_command(cmd, log_output=True, log_error=True, timeout=None):
"""Run a command in fastboot shell."""
if environment.is_android_cuttlefish():
# We can't run fastboot commands on Android cuttlefish instances.
return None
if isinstance(cmd, list):
cmd = ' '.join([str(i) for i in cmd])
if log_output:
logs.log('Running: fastboot %s' % cmd)
if not timeout:
timeout = ADB_TIMEOUT
output = execute_command(get_fastboot_command_line(cmd), timeout, log_error)
return output
def hard_reset():
"""Perform a hard reset of the device."""
if environment.is_android_cuttlefish() or environment.is_android_emulator():
# There is no recovery step at this point for a cuttlefish bot, so just exit
# and wait for reimage on next iteration.
bad_state_reached()
# For physical device.
# Try hard-reset via sysrq-trigger (requires root).
hard_reset_sysrq_cmd = get_adb_command_line(
'shell echo b \\> /proc/sysrq-trigger')
execute_command(hard_reset_sysrq_cmd, timeout=RECOVERY_CMD_TIMEOUT)
# Try soft-reset now (does not require root).
soft_reset_cmd = get_adb_command_line('reboot')
execute_command(soft_reset_cmd, timeout=RECOVERY_CMD_TIMEOUT)
def reset_device_connection():
"""Reset the connection to the physical device through USB. Returns whether
or not the reset succeeded."""
if environment.is_android_cuttlefish():
restart_cuttlefish_device()
else:
# Physical device. Try restarting usb.
reset_usb()
# Check device status.
state = get_device_state()
if state != 'device':
logs.log_warn('Device state is %s, unable to recover using usb reset/'
'cuttlefish reconnect.' % str(state))
return False
return True
def linkify_kernel_or_lkl_stacktrace_if_needed(crash_info):
"""Linkify Android Kernel or lkl stacktrace."""
kernel_prefix = ''
kernel_hash = ''
if (environment.is_android_kernel()
and not environment.is_android_cuttlefish() and
(crash_info.found_android_kernel_crash or crash_info.is_kasan)):
kernel_prefix, kernel_hash = \
android_kernel.get_kernel_prefix_and_full_hash()
elif (environment.is_lkl_job() and crash_info.is_lkl
and crash_info.lkl_kernel_build_id):
kernel_prefix, kernel_hash = \
lkl_kernel.get_kernel_prefix_and_full_hash(crash_info.lkl_kernel_build_id)
if kernel_prefix and kernel_hash:
_linkify_android_kernel_stacktrace(crash_info, kernel_prefix,
kernel_hash)
def factory_reset():
"""Reset device to factory state."""
if environment.is_android_cuttlefish() or environment.is_android_emulator():
# We cannot recover from this since there can be cases like userdata image
# corruption in /data/data. Till the bug is fixed, we just need to wait
# for reimage in next iteration.
bad_state_reached()
# A device can be stuck in a boot loop due to a bad clang library update.
# Reverting that can bring a device back to good state.
revert_asan_device_setup_if_needed()
run_as_root()
run_shell_command([
'am', 'broadcast', '-a', 'android.intent.action.MASTER_CLEAR', '-n',
'android/com.android.server.MasterClearReceiver'
])
# Wait until the reset is complete.
time.sleep(FACTORY_RESET_WAIT)
def flash_to_latest_build_if_needed():
"""Wipes user data, resetting the device to original factory state."""
if environment.get_value('LOCAL_DEVELOPMENT'):
# Don't reimage local development devices.
return
run_timeout = environment.get_value('RUN_TIMEOUT')
if run_timeout:
# If we have a run timeout, then we are already scheduled to bail out and
# will be probably get re-imaged. E.g. using frameworks like Tradefed.
return
# Check if a flash is needed based on last recorded flash time.
last_flash_time = persistent_cache.get_value(
constants.LAST_FLASH_TIME_KEY,
constructor=datetime.datetime.utcfromtimestamp)
needs_flash = last_flash_time is None or dates.time_has_expired(
last_flash_time, seconds=FLASH_INTERVAL)
if not needs_flash:
return
is_google_device = settings.is_google_device()
if is_google_device is None:
logs.log_error('Unable to query device. Reimaging failed.')
adb.bad_state_reached()
elif not is_google_device:
# We can't reimage these, skip.
logs.log('Non-Google device found, skipping reimage.')
return
# Check if both |BUILD_BRANCH| and |BUILD_TARGET| environment variables
# are set. If not, we don't have enough data for reimaging and hence
# we bail out.
branch = environment.get_value('BUILD_BRANCH')
target = environment.get_value('BUILD_TARGET')
if not target:
# We default to userdebug configuration.
build_params = settings.get_build_parameters()
if build_params:
target = build_params.get('target') + '-userdebug'
# Cache target in environment. This is also useful for cases when
# device is bricked and we don't have this information available.
environment.set_value('BUILD_TARGET', target)
if not branch or not target:
logs.log_warn(
'BUILD_BRANCH and BUILD_TARGET are not set, skipping reimage.')
return
image_directory = environment.get_value('IMAGES_DIR')
build_info = fetch_artifact.get_latest_artifact_info(branch, target)
if not build_info:
logs.log_error('Unable to fetch information on latest build artifact for '
'branch %s and target %s.' % (branch, target))
return
if environment.is_android_cuttlefish():
download_latest_build(build_info, FLASH_CUTTLEFISH_REGEXES, image_directory)
adb.recreate_cuttlefish_device()
adb.connect_to_cuttlefish_device()
else:
download_latest_build(build_info, FLASH_IMAGE_REGEXES, image_directory)
# We do one device flash at a time on one host, otherwise we run into
# failures and device being stuck in a bad state.
flash_lock_key_name = 'flash:%s' % socket.gethostname()
if not locks.acquire_lock(flash_lock_key_name, by_zone=True):
logs.log_error('Failed to acquire lock for reimaging, exiting.')
return
logs.log('Reimaging started.')
logs.log('Rebooting into bootloader mode.')
for _ in range(FLASH_RETRIES):
adb.run_as_root()
adb.run_command(['reboot-bootloader'])
time.sleep(FLASH_REBOOT_BOOTLOADER_WAIT)
adb.run_fastboot_command(['oem', 'off-mode-charge', '0'])
adb.run_fastboot_command(['-w', 'reboot-bootloader'])
for partition, partition_image_filename in FLASH_IMAGE_FILES:
partition_image_file_path = os.path.join(image_directory,
partition_image_filename)
adb.run_fastboot_command(
['flash', partition, partition_image_file_path])
if partition in ['bootloader', 'radio']:
adb.run_fastboot_command(['reboot-bootloader'])
# Disable ramdump to avoid capturing ramdumps during kernel crashes.
# This causes device lockup of several minutes during boot and we intend
# to analyze them ourselves.
adb.run_fastboot_command(['oem', 'ramdump', 'disable'])
adb.run_fastboot_command('reboot')
time.sleep(FLASH_REBOOT_WAIT)
if adb.get_device_state() == 'device':
break
logs.log_error('Reimaging failed, retrying.')
locks.release_lock(flash_lock_key_name, by_zone=True)
if adb.get_device_state() != 'device':
logs.log_error('Unable to find device. Reimaging failed.')
adb.bad_state_reached()
logs.log('Reimaging finished.')
# Reset all of our persistent keys after wipe.
persistent_cache.delete_value(constants.BUILD_PROP_MD5_KEY)
persistent_cache.delete_value(constants.LAST_TEST_ACCOUNT_CHECK_KEY)
persistent_cache.set_value(constants.LAST_FLASH_BUILD_KEY, build_info)
persistent_cache.set_value(constants.LAST_FLASH_TIME_KEY, time.time())