def download_artifact_if_needed(build_id, artifact_directory, artifact_archive_path, targets_with_type_and_san, artifact_file_name, output_filename_override, build_params, build_params_check_path): """Downloads artifact to actifacts_archive_path if needed""" # Check if we already have the symbols in cache. cached_build_params = utils.read_data_from_file(build_params_check_path, eval_data=True) if cached_build_params and cached_build_params == build_params: # No work to do, same system symbols already in cache. return # Delete existing symbols directory first. shell.remove_directory(artifact_directory, recreate=True) # Fetch symbol file from cloud storage cache (if available). found_in_cache = storage.get_file_from_cache_if_exists( artifact_archive_path, update_modification_time_on_access=False) if not found_in_cache: for target_with_type_and_san in targets_with_type_and_san: # Fetch the artifact now. fetch_artifact.get(build_id, target_with_type_and_san, artifact_file_name, artifact_directory, output_filename_override) if os.path.exists(artifact_archive_path): break
def download_system_symbols_if_needed(symbols_directory): """Download system libraries from |SYMBOLS_URL| and cache locally.""" # For local testing, we do not have access to the cloud storage bucket with # the symbols. In this case, just bail out. if environment.get_value('LOCAL_DEVELOPMENT'): return # When running reproduce tool locally, we do not have access to the cloud # storage bucket with the symbols. In this case, just bail out. if environment.get_value('REPRODUCE_TOOL'): return # We have archived symbols for google builds only. if not settings.is_google_device(): return # Get the build fingerprint parameters. build_params = settings.get_build_parameters() if not build_params: logs.log_error('Unable to determine build parameters.') return build_id = build_params.get('build_id') target = build_params.get('target') type = build_params.get('type') if not build_id or not target or not type: logs.log_error('Null build parameters found, exiting.') return # Check if we already have the symbols in cache. build_params_check_path = os.path.join(symbols_directory, '.cached_build_params') cached_build_params = utils.read_data_from_file( build_params_check_path, eval_data=True) if cached_build_params and cmp(cached_build_params, build_params) == 0: # No work to do, same system symbols already in cache. return symbols_archive_filename = '%s-symbols-%s.zip' % (target, build_id) symbols_archive_path = os.path.join(symbols_directory, symbols_archive_filename) # Delete existing symbols directory first. shell.remove_directory(symbols_directory, recreate=True) # Fetch symbol file from cloud storage cache (if available). found_in_cache = storage.get_file_from_cache_if_exists( symbols_archive_path, update_modification_time_on_access=False) if not found_in_cache: # Include type and sanitizer information in the target. target_with_type_and_san = '%s-%s' % (target, type) tool_suffix = environment.get_value('SANITIZER_TOOL_NAME') if tool_suffix and not tool_suffix in target_with_type_and_san: target_with_type_and_san += '_%s' % tool_suffix # Fetch the artifact now. fetch_artifact.get(build_id, target_with_type_and_san, symbols_archive_filename, symbols_directory) if not os.path.exists(symbols_archive_path): logs.log_error( 'Unable to locate symbols archive %s.' % symbols_archive_path) return # Store the artifact for later use or for use by other bots. storage.store_file_in_cache(symbols_archive_path) archive.unpack(symbols_archive_path, symbols_directory, trusted=True) shell.remove_file(symbols_archive_path) utils.write_data_to_file(build_params, build_params_check_path)
def flash_to_latest_build_if_needed(): """Wipes user data, resetting the device to original factory state.""" if environment.get_value('LOCAL_DEVELOPMENT'): # Don't reimage local development devices. return run_timeout = environment.get_value('RUN_TIMEOUT') if run_timeout: # If we have a run timeout, then we are already scheduled to bail out and # will be probably get re-imaged. E.g. using frameworks like Tradefed. return # Check if a flash is needed based on last recorded flash time. last_flash_time = persistent_cache.get_value( LAST_FLASH_TIME_KEY, constructor=datetime.datetime.utcfromtimestamp) needs_flash = last_flash_time is None or dates.time_has_expired( last_flash_time, seconds=adb.FLASH_INTERVAL) if not needs_flash: return build_info = {} if adb.is_gce(): adb.recreate_gce_device() else: # Physical device. is_google_device = google_device() if is_google_device is None: logs.log_error('Unable to query device. Reimaging failed.') adb.bad_state_reached() elif not is_google_device: # We can't reimage these, skip. logs.log('Non-Google device found, skipping reimage.') return else: # For Google devices. # Check if both |BUILD_BRANCH| and |BUILD_TARGET| environment variables # are set. If not, we don't have enough data for reimaging and hence # we bail out. branch = environment.get_value('BUILD_BRANCH') target = environment.get_value('BUILD_TARGET') if not target: # We default to userdebug configuration. build_params = get_build_parameters() if build_params: target = build_params.get('target') + '-userdebug' # Cache target in environment. This is also useful for cases when # device is bricked and we don't have this information available. environment.set_value('BUILD_TARGET', target) if not branch or not target: logs.log_warn( 'BUILD_BRANCH and BUILD_TARGET are not set, skipping reimage.' ) return # Download the latest build artifact for this branch and target. build_info = fetch_artifact.get_latest_artifact_info( branch, target) if not build_info: logs.log_error( 'Unable to fetch information on latest build artifact for ' 'branch %s and target %s.' % (branch, target)) return # Check if our local build matches the latest build. If not, we will # download it. build_id = build_info['bid'] target = build_info['target'] image_directory = environment.get_value('IMAGES_DIR') last_build_info = persistent_cache.get_value(LAST_FLASH_BUILD_KEY) if not last_build_info or last_build_info['bid'] != build_id: # Clean up the images directory first. shell.remove_directory(image_directory, recreate=True) # We have a new build, download the build artifacts for it. for image_regex in FLASH_IMAGE_REGEXES: image_file_path = fetch_artifact.get( build_id, target, image_regex, image_directory) if not image_file_path: logs.log_error( 'Failed to download image artifact %s for ' 'branch %s and target %s.' % (image_file_path, branch, target)) return if image_file_path.endswith('.zip'): archive.unpack(image_file_path, image_directory) # We do one device flash at a time on one host, otherwise we run into # failures and device being stuck in a bad state. flash_lock_key_name = 'flash:%s' % socket.gethostname() if not locks.acquire_lock(flash_lock_key_name, by_zone=True): logs.log_error( 'Failed to acquire lock for reimaging, exiting.') return logs.log('Reimaging started.') logs.log('Rebooting into bootloader mode.') for _ in xrange(FLASH_RETRIES): adb.run_as_root() adb.run_adb_command(['reboot-bootloader']) time.sleep(FLASH_REBOOT_BOOTLOADER_WAIT) adb.run_fastboot_command(['oem', 'off-mode-charge', '0']) adb.run_fastboot_command(['-w', 'reboot-bootloader']) for partition, partition_image_filename in FLASH_IMAGE_FILES: partition_image_file_path = os.path.join( image_directory, partition_image_filename) adb.run_fastboot_command( ['flash', partition, partition_image_file_path]) if partition in ['bootloader', 'radio']: adb.run_fastboot_command(['reboot-bootloader']) adb.run_fastboot_command('reboot') time.sleep(FLASH_REBOOT_WAIT) if adb.get_device_state() == 'device': break logs.log_error('Reimaging failed, retrying.') locks.release_lock(flash_lock_key_name, by_zone=True) if adb.get_device_state() != 'device': logs.log_error('Unable to find device. Reimaging failed.') adb.bad_state_reached() logs.log('Reimaging finished.') # Reset all of our persistent keys after wipe. persistent_cache.delete_value(BUILD_PROP_MD5_KEY) persistent_cache.delete_value(LAST_TEST_ACCOUNT_CHECK_KEY) persistent_cache.set_value(LAST_FLASH_BUILD_KEY, build_info) persistent_cache.set_value(LAST_FLASH_TIME_KEY, time.time())
def download_system_symbols_if_needed(symbols_directory, is_kernel=False): """Download system libraries from |SYMBOLS_URL| and cache locally.""" # For local testing, we do not have access to the cloud storage bucket with # the symbols. In this case, just bail out. if environment.get_value('LOCAL_DEVELOPMENT'): return # When running reproduce tool locally, we do not have access to the cloud # storage bucket with the symbols. In this case, just bail out. if environment.get_value('REPRODUCE_TOOL'): return # We have archived symbols for google builds only. if not settings.is_google_device(): return # For Android kernel we want to get the repro.prop # Note: kasan and non-kasan kernel should have the same repo.prop for a given # build_id. if is_kernel: _, build_id = kernel_utils.get_kernel_hash_and_build_id() target = kernel_utils.get_kernel_name() if not build_id or not target: logs.log_error('Could not get kernel parameters, exiting.') return artifact_file_name = 'repo.prop' symbols_archive_filename = get_symbols_archive_filename(build_id, target) output_filename_override = symbols_archive_filename # We create our own build_params for cache build_params = {'build_id': build_id, 'target': target, 'type': 'kernel'} else: # Get the build fingerprint parameters. build_params = settings.get_build_parameters() if not build_params: logs.log_error('Unable to determine build parameters.') return build_id = build_params.get('build_id') target = build_params.get('target') build_type = build_params.get('type') if not build_id or not target or not build_type: logs.log_error('Null build parameters found, exiting.') return symbols_archive_filename = '%s-symbols-%s.zip' % (target, build_id) artifact_file_name = symbols_archive_filename output_filename_override = None # Check if we already have the symbols in cache. build_params_check_path = os.path.join(symbols_directory, '.cached_build_params') cached_build_params = utils.read_data_from_file( build_params_check_path, eval_data=True) if cached_build_params and cached_build_params == build_params: # No work to do, same system symbols already in cache. return symbols_archive_path = os.path.join(symbols_directory, symbols_archive_filename) # Delete existing symbols directory first. shell.remove_directory(symbols_directory, recreate=True) # Fetch symbol file from cloud storage cache (if available). found_in_cache = storage.get_file_from_cache_if_exists( symbols_archive_path, update_modification_time_on_access=False) if not found_in_cache: tool_suffix = environment.get_value('SANITIZER_TOOL_NAME') if is_kernel: # Some kernels are just 'kernel', some are kernel_target if tool_suffix: targets_with_type_and_san = [ 'kernel_%s' % tool_suffix, 'kernel_%s_%s' % (tool_suffix, target) ] else: targets_with_type_and_san = ['kernel', 'kernel_%s' % target] else: # Include type and sanitizer information in the target. target_with_type_and_san = '%s-%s' % (target, build_type) if tool_suffix and not tool_suffix in target_with_type_and_san: target_with_type_and_san += '_%s' % tool_suffix targets_with_type_and_san = [target_with_type_and_san] for target_with_type_and_san in targets_with_type_and_san: # Fetch the artifact now. fetch_artifact.get(build_id, target_with_type_and_san, artifact_file_name, symbols_directory, output_filename_override) if os.path.exists(symbols_archive_path): break if not os.path.exists(symbols_archive_path): logs.log_error( 'Unable to locate symbols archive %s.' % symbols_archive_path) return # Store the artifact for later use or for use by other bots. storage.store_file_in_cache(symbols_archive_path) # repo.prop is not a zip archive. if not is_kernel: archive.unpack(symbols_archive_path, symbols_directory, trusted=True) shell.remove_file(symbols_archive_path) utils.write_data_to_file(build_params, build_params_check_path)