def setup_host_and_device_forwarder_if_needed(): """Sets up http(s) forwarding between device and host.""" # Get list of ports to map. http_port_1 = environment.get_value('HTTP_PORT_1', 8000) http_port_2 = environment.get_value('HTTP_PORT_2', 8080) ports = [http_port_1, http_port_2] # Reverse map socket connections from device to host machine. for port in ports: port_string = 'tcp:%d' % port adb.run_adb_command(['reverse', port_string, port_string])
def setup_memory_monitor_script_if_needed(): """Run check_process_mem.sh to monitor the memory usage""" # The script should only start if this is a low end device. device_codename = environment.get_value('DEVICE_CODENAME', get_codename()) if device_codename not in MEMORY_CONSTRAINED_DEVICES: return adb.run_as_root() if get_pid_for_script(MEMORY_MONITOR_SCRIPT): # The script is already running, no work to do. return android_directory = environment.get_platform_resources_directory() script_host_path = os.path.join(android_directory, MEMORY_MONITOR_SCRIPT) script_device_path = os.path.join(adb.DEVICE_TMP_DIR, MEMORY_MONITOR_SCRIPT) # Push memory monitor script onto device and make it executable (if needed). if not adb.file_exists(script_device_path): adb.run_adb_command(['push', script_host_path, adb.DEVICE_TMP_DIR]) adb.run_adb_shell_command(['chmod', '0755', script_device_path]) # Run the memory monitor script. adb.run_adb_shell_command('sh %s 2>/dev/null 1>/dev/null &' % script_device_path) # Wait one second to allow the script to run. time.sleep(1) # Change the priority of the process so that it will not be easily killed # by lowmemorykiller. pid = get_pid_for_script(MEMORY_MONITOR_SCRIPT) if not pid: logs.log_error('Memory monitor script failed to run.') return adb.run_adb_shell_command('echo -1000 \\> /proc/%s/oom_score_adj' % pid) adb.run_adb_shell_command('echo 0 \\> /proc/%s/oom_score' % pid) adb.run_adb_shell_command('echo -17 \\> /proc/%s/oom_adj' % pid)
def get_codename(): """Return the device codename.""" device_serial = environment.get_value('ANDROID_SERIAL') devices_output = adb.run_adb_command(['devices', '-l']) for line in devices_output.splitlines(): values = line.strip().split() serial = values[0] if serial != device_serial: continue for value in values: if not value.startswith('device:'): continue device_codename = value.split(':')[-1] if device_codename: return device_codename # Unable to get code name. return ''
def get_codename(): """Return the device codename.""" serial = environment.get_value('ANDROID_SERIAL') devices_output = adb.run_adb_command(['devices', '-l']) serial_pattern = r'(^|\s){serial}\s'.format(serial=re.escape(serial)) serial_regex = re.compile(serial_pattern) for line in devices_output.splitlines(): values = line.strip().split() if not serial_regex.search(line): continue for value in values: if not value.startswith('device:'): continue device_codename = value.split(':')[-1] if device_codename: return device_codename # Unable to get code name. return ''
def filter_binary_path(binary_path): """Filters binary path to provide a local copy.""" platform = environment.platform() if platform == 'ANDROID': # Make sure we have a valid device codename. device_codename = environment.get_value('DEVICE_CODENAME') if not device_codename: return '' # Skip symbolization when running it on bad entries like [stack:XYZ]. if not binary_path.startswith('/') or '(deleted)' in binary_path: return '' # Initialize some helper variables. app_path = environment.get_value('APP_PATH') app_directory = os.path.dirname(app_path) binary_filename = os.path.basename(binary_path) nfs_directory = environment.get_value('NFS_ROOT') symbols_directory = environment.get_value('SYMBOLS_DIR') # Try to find the library in the build directory first. local_binary_path = utils.find_binary_path(app_directory, binary_path) if local_binary_path: return local_binary_path # Try finding the symbols on NFS server. if nfs_directory: nfs_symbols_directory = os.path.join(nfs_directory, 'symbols', device_codename) if os.path.exists(nfs_symbols_directory): local_binary_path = utils.find_binary_path( nfs_symbols_directory, binary_path) if local_binary_path: return local_binary_path # We didn't find the library locally in the build directory. # Try finding the library in the local system library cache. download_system_symbols_if_needed(symbols_directory) local_binary_path = utils.find_binary_path(symbols_directory, binary_path) if local_binary_path: return local_binary_path # Try pulling in the binary directly from the device into the # system library cache directory. local_binary_path = os.path.join(symbols_directory, binary_filename) adb.run_adb_command('pull %s %s' % (binary_path, local_binary_path)) if os.path.exists(local_binary_path): return local_binary_path # Unable to find library. logs.log_error('Unable to find library %s for symbolization.' % binary_path) return '' if platform == 'CHROMEOS': # FIXME: Add code to pull binaries from ChromeOS device. return binary_path if environment.is_chromeos_system_job(): # This conditional is True for ChromeOS system fuzzers that are running on # Linux. Ensure that the binary is always looked for in the chroot and not # in system directories. build_dir = environment.get_value('BUILD_DIR') if not binary_path.startswith(build_dir): # Fixup path so |binary_path| points to a binary in the chroot (probably # a system library). return os.path.join(build_dir, binary_path[1:]) # For Linux and Mac, the binary exists locally. No work to do, # just return the same binary path. return binary_path
def get_crash_info(output): """Parse crash output to get (local) minidump path and any other information useful for crash uploading, and store in a CrashReportInfo object.""" crash_stacks_directory = environment.get_value('CRASH_STACKTRACES_DIR') platform = environment.platform() output_lines = output.splitlines() num_lines = len(output_lines) for i, line in enumerate(output_lines): if platform == 'ANDROID': # If we are on Android, the dump extraction is more complicated. # The location placed in the crash-stacktrace is of the dump itself but # in fact only the MIME of the dump exists, and will have a different # extension. We need to pull the MIME and process it. match = re.match(CRASH_DUMP_PATH_MARKER, line) if not match: continue minidump_mime_filename_base = None for j in range(i + 1, num_lines): line = output_lines[j] match = re.match(r'(.*)\.dmp', line) if match: minidump_mime_filename_base = os.path.basename( match.group(1).strip()) break if not minidump_mime_filename_base: logs.log_error( 'Minidump marker was found, but no path in stacktrace.') return None # Look for MIME. If none found, bail. # We might not have copied over the crash dumps yet (copying is buffered), # so we want to search both the original directory and the one to which # the minidumps should later be copied. device_directories_to_search = [ adb.DEVICE_CRASH_DUMPS_DIR, os.path.dirname(line.strip()) ] device_minidump_search_paths = [] device_minidump_mime_path = None for device_directory in device_directories_to_search: device_minidump_mime_potential_paths = adb.run_adb_shell_command( ['ls', '"%s"' % device_directory], root=True).splitlines() device_minidump_search_paths += device_minidump_mime_potential_paths for potential_path in device_minidump_mime_potential_paths: # Check that we actually found a file, and the right one (not logcat). if 'No such file or directory' in potential_path: continue if minidump_mime_filename_base not in potential_path: continue if '.up' in potential_path or '.dmp' in potential_path: device_minidump_mime_path = os.path.join( device_directory, potential_path) break # Break if we found a path. if device_minidump_mime_path is not None: break # If we still didn't find a minidump path, bail. if device_minidump_mime_path is None: logs.log_error('Could not get MIME path from ls:\n%s' % str(device_minidump_search_paths)) return None # Pull out MIME and parse to minidump file and MIME parameters. minidump_mime_filename = '%s.mime' % minidump_mime_filename_base local_minidump_mime_path = os.path.join(crash_stacks_directory, minidump_mime_filename) adb.run_adb_command([ 'pull', '"%s"' % device_minidump_mime_path, local_minidump_mime_path ]) if not os.path.exists(local_minidump_mime_path): logs.log_error( 'Could not pull MIME from %s to %s.' % (device_minidump_mime_path, local_minidump_mime_path)) return None crash_info = parse_mime_to_crash_report_info( local_minidump_mime_path) if crash_info is None: return None crash_info.unsymbolized_stacktrace = output return crash_info else: # Other platforms are not currently supported. logs.log_error( 'Unable to fetch crash information for this platform.') return None # Could not find dump location, bail out. This could also happen when we don't # have a minidump location in stack at all, e.g. when testcase does not crash # during minimization. return None
def configure_build_properties_if_needed(): """Edits /system/build.prop for better boot speed and power use.""" # Check md5 checksum of build.prop to see if already updated, # in which case exit. If build.prop does not exist, something # is very wrong with the device, so bail. old_md5 = persistent_cache.get_value(BUILD_PROP_MD5_KEY) current_md5 = adb.get_file_checksum(BUILD_PROP_PATH) if current_md5 is None: logs.log_error('Unable to find %s on device.' % BUILD_PROP_PATH) return if old_md5 == current_md5: return # Pull to tmp file. bot_tmp_directory = environment.get_value('BOT_TMPDIR') old_build_prop_path = os.path.join(bot_tmp_directory, 'old.prop') adb.run_adb_command(['pull', BUILD_PROP_PATH, old_build_prop_path]) if not os.path.exists(old_build_prop_path): logs.log_error('Unable to fetch %s from device.' % BUILD_PROP_PATH) return # Write new build.prop. new_build_prop_path = os.path.join(bot_tmp_directory, 'new.prop') old_build_prop_file_content = open(old_build_prop_path, 'r') new_build_prop_file_content = open(new_build_prop_path, 'w') new_content_notification = '### CHANGED OR ADDED PROPERTIES ###' for line in old_build_prop_file_content: property_name = line.split('=')[0].strip() if property_name in BUILD_PROPERTIES: continue if new_content_notification in line: continue new_build_prop_file_content.write(line) new_build_prop_file_content.write(new_content_notification + '\n') for flag, value in BUILD_PROPERTIES.iteritems(): new_build_prop_file_content.write('%s=%s\n' % (flag, value)) old_build_prop_file_content.close() new_build_prop_file_content.close() # Keep verified boot disabled for M and higher releases. This makes it easy # to modify system's app_process to load asan libraries. build_version = get_build_version() if is_build_at_least(build_version, 'M'): adb.run_as_root() adb.run_adb_command('disable-verity') reboot() # Make /system writable. adb.run_as_root() adb.remount() # Remove seccomp policies (on N and higher) as ASan requires extra syscalls. if is_build_at_least(build_version, 'N'): policy_files = adb.run_adb_shell_command( ['find', '/system/etc/seccomp_policy/', '-type', 'f']) for policy_file in policy_files.splitlines(): adb.run_adb_shell_command(['rm', policy_file.strip()]) # Remove Google Plus app from non-Google devices. Makes it easy to install # older Gallery app on these devices. Otherwise, we run into duplicate # permission errors. if not google_device(): adb.run_adb_shell_command(['rm', '/system/app/PlusOne.apk']) adb.run_adb_shell_command(['rm', '/system/app/PlusOne/PlusOne.apk']) # Push new build.prop and backup to device. logs.log('Pushing new build properties file on device.') adb.run_adb_command( ['push', '-p', old_build_prop_path, BUILD_PROP_BACKUP_PATH]) adb.run_adb_command(['push', '-p', new_build_prop_path, BUILD_PROP_PATH]) adb.run_adb_shell_command(['chmod', '644', BUILD_PROP_PATH]) # Set persistent cache key containing and md5sum. current_md5 = adb.get_file_checksum(BUILD_PROP_PATH) persistent_cache.set_value(BUILD_PROP_MD5_KEY, current_md5)
def flash_to_latest_build_if_needed(): """Wipes user data, resetting the device to original factory state.""" if environment.get_value('LOCAL_DEVELOPMENT'): # Don't reimage local development devices. return run_timeout = environment.get_value('RUN_TIMEOUT') if run_timeout: # If we have a run timeout, then we are already scheduled to bail out and # will be probably get re-imaged. E.g. using frameworks like Tradefed. return # Check if a flash is needed based on last recorded flash time. last_flash_time = persistent_cache.get_value( LAST_FLASH_TIME_KEY, constructor=datetime.datetime.utcfromtimestamp) needs_flash = last_flash_time is None or dates.time_has_expired( last_flash_time, seconds=adb.FLASH_INTERVAL) if not needs_flash: return build_info = {} if adb.is_gce(): adb.recreate_gce_device() else: # Physical device. is_google_device = google_device() if is_google_device is None: logs.log_error('Unable to query device. Reimaging failed.') adb.bad_state_reached() elif not is_google_device: # We can't reimage these, skip. logs.log('Non-Google device found, skipping reimage.') return else: # For Google devices. # Check if both |BUILD_BRANCH| and |BUILD_TARGET| environment variables # are set. If not, we don't have enough data for reimaging and hence # we bail out. branch = environment.get_value('BUILD_BRANCH') target = environment.get_value('BUILD_TARGET') if not target: # We default to userdebug configuration. build_params = get_build_parameters() if build_params: target = build_params.get('target') + '-userdebug' # Cache target in environment. This is also useful for cases when # device is bricked and we don't have this information available. environment.set_value('BUILD_TARGET', target) if not branch or not target: logs.log_warn( 'BUILD_BRANCH and BUILD_TARGET are not set, skipping reimage.' ) return # Download the latest build artifact for this branch and target. build_info = fetch_artifact.get_latest_artifact_info( branch, target) if not build_info: logs.log_error( 'Unable to fetch information on latest build artifact for ' 'branch %s and target %s.' % (branch, target)) return # Check if our local build matches the latest build. If not, we will # download it. build_id = build_info['bid'] target = build_info['target'] image_directory = environment.get_value('IMAGES_DIR') last_build_info = persistent_cache.get_value(LAST_FLASH_BUILD_KEY) if not last_build_info or last_build_info['bid'] != build_id: # Clean up the images directory first. shell.remove_directory(image_directory, recreate=True) # We have a new build, download the build artifacts for it. for image_regex in FLASH_IMAGE_REGEXES: image_file_path = fetch_artifact.get( build_id, target, image_regex, image_directory) if not image_file_path: logs.log_error( 'Failed to download image artifact %s for ' 'branch %s and target %s.' % (image_file_path, branch, target)) return if image_file_path.endswith('.zip'): archive.unpack(image_file_path, image_directory) # We do one device flash at a time on one host, otherwise we run into # failures and device being stuck in a bad state. flash_lock_key_name = 'flash:%s' % socket.gethostname() if not locks.acquire_lock(flash_lock_key_name, by_zone=True): logs.log_error( 'Failed to acquire lock for reimaging, exiting.') return logs.log('Reimaging started.') logs.log('Rebooting into bootloader mode.') for _ in xrange(FLASH_RETRIES): adb.run_as_root() adb.run_adb_command(['reboot-bootloader']) time.sleep(FLASH_REBOOT_BOOTLOADER_WAIT) adb.run_fastboot_command(['oem', 'off-mode-charge', '0']) adb.run_fastboot_command(['-w', 'reboot-bootloader']) for partition, partition_image_filename in FLASH_IMAGE_FILES: partition_image_file_path = os.path.join( image_directory, partition_image_filename) adb.run_fastboot_command( ['flash', partition, partition_image_file_path]) if partition in ['bootloader', 'radio']: adb.run_fastboot_command(['reboot-bootloader']) adb.run_fastboot_command('reboot') time.sleep(FLASH_REBOOT_WAIT) if adb.get_device_state() == 'device': break logs.log_error('Reimaging failed, retrying.') locks.release_lock(flash_lock_key_name, by_zone=True) if adb.get_device_state() != 'device': logs.log_error('Unable to find device. Reimaging failed.') adb.bad_state_reached() logs.log('Reimaging finished.') # Reset all of our persistent keys after wipe. persistent_cache.delete_value(BUILD_PROP_MD5_KEY) persistent_cache.delete_value(LAST_TEST_ACCOUNT_CHECK_KEY) persistent_cache.set_value(LAST_FLASH_BUILD_KEY, build_info) persistent_cache.set_value(LAST_FLASH_TIME_KEY, time.time())