def run(): """Run Android initialization.""" init_runner.run() # Set cuttlefish device serial if needed. if environment.is_android_cuttlefish(): android.adb.set_cuttlefish_device_serial() # Check if we need to reflash device to latest build. android.flash.flash_to_latest_build_if_needed() # Reconnect to cuttlefish device if connection is ever lost. if environment.is_android_cuttlefish(): android.adb.connect_to_cuttlefish_device() # Reboot to bring device in a good state if not done recently. if android.adb.time_since_last_reboot() > TIME_SINCE_REBOOT_MIN_THRESHOLD: android.device.reboot() # Make sure that device is in a good condition before we move forward. android.adb.wait_until_fully_booted() # Wait until battery charges to a minimum level and temperature threshold. android.battery.wait_until_good_state() # Initialize environment settings. android.device.initialize_environment()
def main(): """Run a cycle of heartbeat checks to ensure Android device is running.""" logs.configure('android_heartbeat') dates.initialize_timezone_from_environment() environment.set_bot_environment() monitor.initialize() if environment.is_android_cuttlefish(): android.adb.set_cuttlefish_device_serial() device_serial = environment.get_value('ANDROID_SERIAL') while True: state = android.adb.get_device_state() if state == android.adb.DEVICE_NOT_FOUND_STRING.format( serial=device_serial): android.adb.connect_to_cuttlefish_device() state = android.adb.get_device_state() logs.log('Android device %s state: %s' % (device_serial, state)) monitoring_metrics.ANDROID_UPTIME.increment_by( int(state == 'device'), { 'serial': device_serial or '', 'platform': environment.get_platform_group() or '', }) time.sleep(data_types.ANDROID_HEARTBEAT_WAIT_INTERVAL) if data_handler.bot_run_timed_out(): break
def reset_usb(): """Reset USB bus for a device serial.""" if environment.is_android_cuttlefish() or environment.is_android_emulator(): # Nothing to do here. return True # App Engine does not let us import this. import fcntl # We need to get latest device path since it could be changed in reboots or # adb root restarts. try: device_path = get_device_path() except IOError: # We may reach this state if the device is no longer available. device_path = None if not device_path: # Try pulling from cache (if available). device_path = environment.get_value('DEVICE_PATH') if not device_path: logs.log_warn('No device path found, unable to reset usb.') return False try: with open(device_path, 'w') as f: fcntl.ioctl(f, USBDEVFS_RESET) except: logs.log_warn('Failed to reset usb.') return False # Wait for usb to recover. wait_for_device(recover=False) return True
def get_config(): """Get arguments for a given fuzz target.""" device_serial = environment.get_value('ANDROID_SERIAL') build_dir = environment.get_value('BUILD_DIR') temp_dir = fuzzer_utils.get_temp_dir() binary_path = os.path.join(build_dir, 'syzkaller') json_config_path = os.path.join(temp_dir, 'config.json') default_vmlinux_path = os.path.join('/tmp', device_serial, 'vmlinux') vmlinux_path = environment.get_value('VMLINUX_PATH', default_vmlinux_path) syzhub_address = environment.get_value('SYZHUB_ADDRESS') syzhub_client = environment.get_value('SYZHUB_CLIENT') syzhub_key = environment.get_value('SYZHUB_KEY') on_cuttlefish = environment.is_android_cuttlefish() config.generate(serial=device_serial, work_dir_path=get_work_dir(), binary_path=binary_path, vmlinux_path=vmlinux_path, config_path=json_config_path, kcov=True, reproduce=False, syzhub_address=syzhub_address, syzhub_client=syzhub_client, syzhub_key=syzhub_key, on_cuttlefish=on_cuttlefish) return ['-config', json_config_path]
def wait_until_good_state(): """Check battery and make sure it is charged beyond minimum level and temperature thresholds.""" # Battery levels are not applicable on GCE. if environment.is_android_cuttlefish() or settings.is_automotive(): return # Make sure device is online. adb.wait_for_device() # Skip battery check if done recently. last_battery_check_time = persistent_cache.get_value( LAST_BATTERY_CHECK_TIME_KEY, constructor=datetime.datetime.utcfromtimestamp) if last_battery_check_time and not dates.time_has_expired( last_battery_check_time, seconds=BATTERY_CHECK_INTERVAL): return # Initialize variables. battery_level_threshold = environment.get_value('LOW_BATTERY_LEVEL_THRESHOLD', LOW_BATTERY_LEVEL_THRESHOLD) battery_temperature_threshold = environment.get_value( 'MAX_BATTERY_TEMPERATURE_THRESHOLD', MAX_BATTERY_TEMPERATURE_THRESHOLD) device_restarted = False while True: battery_information = get_battery_level_and_temperature() if battery_information is None: logs.log_error('Failed to get battery information, skipping check.') return battery_level = battery_information['level'] battery_temperature = battery_information['temperature'] logs.log('Battery information: level (%d%%), temperature (%.1f celsius).' % (battery_level, battery_temperature)) if (battery_level >= battery_level_threshold and battery_temperature <= battery_temperature_threshold): persistent_cache.set_value(LAST_BATTERY_CHECK_TIME_KEY, time.time()) return logs.log('Battery in bad battery state, putting device in sleep mode.') if not device_restarted: adb.reboot() device_restarted = True # Change thresholds to expected levels (only if they were below minimum # thresholds). if battery_level < battery_level_threshold: battery_level_threshold = EXPECTED_BATTERY_LEVEL if battery_temperature > battery_temperature_threshold: battery_temperature_threshold = EXPECTED_BATTERY_TEMPERATURE # Stopping shell should help with shutting off a lot of services that would # otherwise use up the battery. However, we need to turn it back on to get # battery status information. adb.stop_shell() time.sleep(BATTERY_CHARGE_INTERVAL) adb.start_shell()
def get_device_path(): """Gets a device path to be cached and used by reset_usb.""" def _get_usb_devices(): """Returns a list of device objects containing a serial and USB path.""" usb_list_cmd = 'lsusb -v' output = execute_command(usb_list_cmd, timeout=RECOVERY_CMD_TIMEOUT) if output is None: logs.log_error('Failed to populate usb devices using lsusb, ' 'host restart might be needed.') bad_state_reached() devices = [] path = None for line in output.splitlines(): match = LSUSB_BUS_RE.match(line) if match: path = '/dev/bus/usb/%s/%s' % (match.group(1), match.group(2)) continue match = LSUSB_SERIAL_RE.match(line) if path and match and match.group(1): serial = match.group(1) devices.append(DEVICE(serial, path)) return devices def _get_device_path_for_serial(): """Return device path. Assumes a simple ANDROID_SERIAL.""" devices = _get_usb_devices() for device in devices: if device_serial == device.serial: return device.path return None def _get_device_path_for_usb(): """Returns a device path. Assumes ANDROID_SERIAL in the form "usb:<identifier>".""" # Android serial may reference a usb device rather than a serial number. device_id = device_serial[len('usb:'):] bus_number = int( open('/sys/bus/usb/devices/%s/busnum' % device_id).read().strip()) device_number = int( open('/sys/bus/usb/devices/%s/devnum' % device_id).read().strip()) return '/dev/bus/usb/%03d/%03d' % (bus_number, device_number) if environment.is_android_cuttlefish(): return None device_serial = environment.get_value('ANDROID_SERIAL') if device_serial.startswith('usb:'): return _get_device_path_for_usb() return _get_device_path_for_serial()
def configure(force_enable=False): """Configure airplane mode and wifi on device.""" # The reproduce tool shouldn't inherit wifi settings from jobs. if environment.get_value('REPRODUCE_TOOL'): return # Airplane mode should be disabled in all cases. This can get inadvertently # turned on via gestures. disable_airplane_mode() # Need to disable wifi before changing configuration. disable() # Check if wifi needs to be enabled. If not, then no need to modify the # supplicant file. wifi_enabled = force_enable or environment.get_value('WIFI', True) if not wifi_enabled: # No more work to do, we already disabled it at start. return # Wait 2 seconds to allow the wifi to be enabled. enable() time.sleep(2) # Install helper apk to configure wifi. wifi_util_apk_path = os.path.join( environment.get_platform_resources_directory(), 'wifi_util.apk') if not app.is_installed(WIFI_UTIL_PACKAGE_NAME): app.install(wifi_util_apk_path) # Get ssid and password from admin configuration. if environment.is_android_cuttlefish(): wifi_ssid = 'VirtWifi' wifi_password = '' else: config = db_config.get() if not config.wifi_ssid: logs.log('No wifi ssid is set, skipping wifi config.') return wifi_ssid = config.wifi_ssid wifi_password = config.wifi_password or '' connect_wifi_command = ( 'am instrument -e method connectToNetwork -e ssid {ssid} ') if wifi_password: connect_wifi_command += '-e psk {password} ' connect_wifi_command += '-w {call_path}' output = adb.run_shell_command( connect_wifi_command.format(ssid=quote(wifi_ssid), password=quote(wifi_password), call_path=WIFI_UTIL_CALL_PATH)) if 'result=true' not in output: logs.log_warn('Failed to connect to wifi.', output=output)
def execute_command(cmd, timeout=None, log_error=True, on_cuttlefish_host=False): """Spawns a subprocess to run the given shell command.""" if on_cuttlefish_host and environment.is_android_cuttlefish(): # Auto accept key fingerprint for ssh command. cmd = ('ssh -o StrictHostKeyChecking=no ' f'{get_cuttlefish_ssh_target()} "{cmd}"') so = [] # pylint: disable=consider-using-with output_dest = tempfile.TemporaryFile() # pylint: disable=subprocess-popen-preexec-fn,consider-using-with pipe = subprocess.Popen( cmd, executable='/bin/bash', stdout=output_dest, stderr=subprocess.STDOUT, shell=True, preexec_fn=lambda: signal.signal(signal.SIGPIPE, signal.SIG_DFL), bufsize=0) def run(): """Thread target function that waits for subprocess to complete.""" try: pipe.communicate() output_dest.seek(0) output = output_dest.read() output_dest.close() if output: so.append(output) except OSError as _: logs.log_warn('Failed to retrieve stdout from: %s' % cmd) if pipe.returncode: if log_error: logs.log_warn( '%s returned %d error code.' % (cmd, pipe.returncode), output=output) thread = threading.Thread(target=run) thread.start() thread.join(timeout) if thread.is_alive(): logs.log_warn('Command %s timed out. Killing process.' % cmd) try: pipe.kill() except OSError: # Can't kill a dead process. pass return None bytes_output = b''.join(so) return bytes_output.strip().decode('utf-8', errors='ignore')
def run_fastboot_command(cmd, log_output=True, log_error=True, timeout=None): """Run a command in fastboot shell.""" if environment.is_android_cuttlefish(): # We can't run fastboot commands on Android cuttlefish instances. return None if isinstance(cmd, list): cmd = ' '.join([str(i) for i in cmd]) if log_output: logs.log('Running: fastboot %s' % cmd) if not timeout: timeout = ADB_TIMEOUT output = execute_command(get_fastboot_command_line(cmd), timeout, log_error) return output
def hard_reset(): """Perform a hard reset of the device.""" if environment.is_android_cuttlefish() or environment.is_android_emulator(): # There is no recovery step at this point for a cuttlefish bot, so just exit # and wait for reimage on next iteration. bad_state_reached() # For physical device. # Try hard-reset via sysrq-trigger (requires root). hard_reset_sysrq_cmd = get_adb_command_line( 'shell echo b \\> /proc/sysrq-trigger') execute_command(hard_reset_sysrq_cmd, timeout=RECOVERY_CMD_TIMEOUT) # Try soft-reset now (does not require root). soft_reset_cmd = get_adb_command_line('reboot') execute_command(soft_reset_cmd, timeout=RECOVERY_CMD_TIMEOUT)
def reset_device_connection(): """Reset the connection to the physical device through USB. Returns whether or not the reset succeeded.""" if environment.is_android_cuttlefish(): restart_cuttlefish_device() else: # Physical device. Try restarting usb. reset_usb() # Check device status. state = get_device_state() if state != 'device': logs.log_warn('Device state is %s, unable to recover using usb reset/' 'cuttlefish reconnect.' % str(state)) return False return True
def linkify_kernel_or_lkl_stacktrace_if_needed(crash_info): """Linkify Android Kernel or lkl stacktrace.""" kernel_prefix = '' kernel_hash = '' if (environment.is_android_kernel() and not environment.is_android_cuttlefish() and (crash_info.found_android_kernel_crash or crash_info.is_kasan)): kernel_prefix, kernel_hash = \ android_kernel.get_kernel_prefix_and_full_hash() elif (environment.is_lkl_job() and crash_info.is_lkl and crash_info.lkl_kernel_build_id): kernel_prefix, kernel_hash = \ lkl_kernel.get_kernel_prefix_and_full_hash(crash_info.lkl_kernel_build_id) if kernel_prefix and kernel_hash: _linkify_android_kernel_stacktrace(crash_info, kernel_prefix, kernel_hash)
def factory_reset(): """Reset device to factory state.""" if environment.is_android_cuttlefish() or environment.is_android_emulator(): # We cannot recover from this since there can be cases like userdata image # corruption in /data/data. Till the bug is fixed, we just need to wait # for reimage in next iteration. bad_state_reached() # A device can be stuck in a boot loop due to a bad clang library update. # Reverting that can bring a device back to good state. revert_asan_device_setup_if_needed() run_as_root() run_shell_command([ 'am', 'broadcast', '-a', 'android.intent.action.MASTER_CLEAR', '-n', 'android/com.android.server.MasterClearReceiver' ]) # Wait until the reset is complete. time.sleep(FACTORY_RESET_WAIT)
def flash_to_latest_build_if_needed(): """Wipes user data, resetting the device to original factory state.""" if environment.get_value('LOCAL_DEVELOPMENT'): # Don't reimage local development devices. return run_timeout = environment.get_value('RUN_TIMEOUT') if run_timeout: # If we have a run timeout, then we are already scheduled to bail out and # will be probably get re-imaged. E.g. using frameworks like Tradefed. return # Check if a flash is needed based on last recorded flash time. last_flash_time = persistent_cache.get_value( constants.LAST_FLASH_TIME_KEY, constructor=datetime.datetime.utcfromtimestamp) needs_flash = last_flash_time is None or dates.time_has_expired( last_flash_time, seconds=FLASH_INTERVAL) if not needs_flash: return is_google_device = settings.is_google_device() if is_google_device is None: logs.log_error('Unable to query device. Reimaging failed.') adb.bad_state_reached() elif not is_google_device: # We can't reimage these, skip. logs.log('Non-Google device found, skipping reimage.') return # Check if both |BUILD_BRANCH| and |BUILD_TARGET| environment variables # are set. If not, we don't have enough data for reimaging and hence # we bail out. branch = environment.get_value('BUILD_BRANCH') target = environment.get_value('BUILD_TARGET') if not target: # We default to userdebug configuration. build_params = settings.get_build_parameters() if build_params: target = build_params.get('target') + '-userdebug' # Cache target in environment. This is also useful for cases when # device is bricked and we don't have this information available. environment.set_value('BUILD_TARGET', target) if not branch or not target: logs.log_warn( 'BUILD_BRANCH and BUILD_TARGET are not set, skipping reimage.') return image_directory = environment.get_value('IMAGES_DIR') build_info = fetch_artifact.get_latest_artifact_info(branch, target) if not build_info: logs.log_error('Unable to fetch information on latest build artifact for ' 'branch %s and target %s.' % (branch, target)) return if environment.is_android_cuttlefish(): download_latest_build(build_info, FLASH_CUTTLEFISH_REGEXES, image_directory) adb.recreate_cuttlefish_device() adb.connect_to_cuttlefish_device() else: download_latest_build(build_info, FLASH_IMAGE_REGEXES, image_directory) # We do one device flash at a time on one host, otherwise we run into # failures and device being stuck in a bad state. flash_lock_key_name = 'flash:%s' % socket.gethostname() if not locks.acquire_lock(flash_lock_key_name, by_zone=True): logs.log_error('Failed to acquire lock for reimaging, exiting.') return logs.log('Reimaging started.') logs.log('Rebooting into bootloader mode.') for _ in range(FLASH_RETRIES): adb.run_as_root() adb.run_command(['reboot-bootloader']) time.sleep(FLASH_REBOOT_BOOTLOADER_WAIT) adb.run_fastboot_command(['oem', 'off-mode-charge', '0']) adb.run_fastboot_command(['-w', 'reboot-bootloader']) for partition, partition_image_filename in FLASH_IMAGE_FILES: partition_image_file_path = os.path.join(image_directory, partition_image_filename) adb.run_fastboot_command( ['flash', partition, partition_image_file_path]) if partition in ['bootloader', 'radio']: adb.run_fastboot_command(['reboot-bootloader']) # Disable ramdump to avoid capturing ramdumps during kernel crashes. # This causes device lockup of several minutes during boot and we intend # to analyze them ourselves. adb.run_fastboot_command(['oem', 'ramdump', 'disable']) adb.run_fastboot_command('reboot') time.sleep(FLASH_REBOOT_WAIT) if adb.get_device_state() == 'device': break logs.log_error('Reimaging failed, retrying.') locks.release_lock(flash_lock_key_name, by_zone=True) if adb.get_device_state() != 'device': logs.log_error('Unable to find device. Reimaging failed.') adb.bad_state_reached() logs.log('Reimaging finished.') # Reset all of our persistent keys after wipe. persistent_cache.delete_value(constants.BUILD_PROP_MD5_KEY) persistent_cache.delete_value(constants.LAST_TEST_ACCOUNT_CHECK_KEY) persistent_cache.set_value(constants.LAST_FLASH_BUILD_KEY, build_info) persistent_cache.set_value(constants.LAST_FLASH_TIME_KEY, time.time())