예제 #1
0
def _fetch_lock_for_data_bundle_update(data_bundle):
    """Fetch exclusive lock for a data bundle update."""
    # Data bundle on local disk can be modified without race conditions.
    if data_bundle.is_local:
        return True

    data_bundle_lock_name = _get_data_bundle_update_lock_name(data_bundle.name)
    return locks.acquire_lock(
        data_bundle_lock_name,
        max_hold_seconds=_DATA_BUNDLE_LOCK_INTERVAL_IN_SECONDS,
        by_zone=True)
예제 #2
0
def flash_to_latest_build_if_needed():
  """Wipes user data, resetting the device to original factory state."""
  if environment.get_value('LOCAL_DEVELOPMENT'):
    # Don't reimage local development devices.
    return

  run_timeout = environment.get_value('RUN_TIMEOUT')
  if run_timeout:
    # If we have a run timeout, then we are already scheduled to bail out and
    # will be probably get re-imaged. E.g. using frameworks like Tradefed.
    return

  # Check if a flash is needed based on last recorded flash time.
  last_flash_time = persistent_cache.get_value(
      constants.LAST_FLASH_TIME_KEY,
      constructor=datetime.datetime.utcfromtimestamp)
  needs_flash = last_flash_time is None or dates.time_has_expired(
      last_flash_time, seconds=FLASH_INTERVAL)
  if not needs_flash:
    return

  is_google_device = settings.is_google_device()
  if is_google_device is None:
    logs.log_error('Unable to query device. Reimaging failed.')
    adb.bad_state_reached()

  elif not is_google_device:
    # We can't reimage these, skip.
    logs.log('Non-Google device found, skipping reimage.')
    return

  # Check if both |BUILD_BRANCH| and |BUILD_TARGET| environment variables
  # are set. If not, we don't have enough data for reimaging and hence
  # we bail out.
  branch = environment.get_value('BUILD_BRANCH')
  target = environment.get_value('BUILD_TARGET')
  if not target:
    # We default to userdebug configuration.
    build_params = settings.get_build_parameters()
    if build_params:
      target = build_params.get('target') + '-userdebug'

      # Cache target in environment. This is also useful for cases when
      # device is bricked and we don't have this information available.
      environment.set_value('BUILD_TARGET', target)

  if not branch or not target:
    logs.log_warn(
        'BUILD_BRANCH and BUILD_TARGET are not set, skipping reimage.')
    return

  image_directory = environment.get_value('IMAGES_DIR')
  build_info = fetch_artifact.get_latest_artifact_info(branch, target)
  if not build_info:
    logs.log_error('Unable to fetch information on latest build artifact for '
                   'branch %s and target %s.' % (branch, target))
    return

  if environment.is_android_cuttlefish():
    download_latest_build(build_info, FLASH_CUTTLEFISH_REGEXES, image_directory)
    adb.recreate_cuttlefish_device()
    adb.connect_to_cuttlefish_device()
  else:
    download_latest_build(build_info, FLASH_IMAGE_REGEXES, image_directory)
    # We do one device flash at a time on one host, otherwise we run into
    # failures and device being stuck in a bad state.
    flash_lock_key_name = 'flash:%s' % socket.gethostname()
    if not locks.acquire_lock(flash_lock_key_name, by_zone=True):
      logs.log_error('Failed to acquire lock for reimaging, exiting.')
      return

    logs.log('Reimaging started.')
    logs.log('Rebooting into bootloader mode.')
    for _ in range(FLASH_RETRIES):
      adb.run_as_root()
      adb.run_command(['reboot-bootloader'])
      time.sleep(FLASH_REBOOT_BOOTLOADER_WAIT)
      adb.run_fastboot_command(['oem', 'off-mode-charge', '0'])
      adb.run_fastboot_command(['-w', 'reboot-bootloader'])

      for partition, partition_image_filename in FLASH_IMAGE_FILES:
        partition_image_file_path = os.path.join(image_directory,
                                                 partition_image_filename)
        adb.run_fastboot_command(
            ['flash', partition, partition_image_file_path])
        if partition in ['bootloader', 'radio']:
          adb.run_fastboot_command(['reboot-bootloader'])

      # Disable ramdump to avoid capturing ramdumps during kernel crashes.
      # This causes device lockup of several minutes during boot and we intend
      # to analyze them ourselves.
      adb.run_fastboot_command(['oem', 'ramdump', 'disable'])

      adb.run_fastboot_command('reboot')
      time.sleep(FLASH_REBOOT_WAIT)

      if adb.get_device_state() == 'device':
        break
      logs.log_error('Reimaging failed, retrying.')

    locks.release_lock(flash_lock_key_name, by_zone=True)

  if adb.get_device_state() != 'device':
    logs.log_error('Unable to find device. Reimaging failed.')
    adb.bad_state_reached()

  logs.log('Reimaging finished.')

  # Reset all of our persistent keys after wipe.
  persistent_cache.delete_value(constants.BUILD_PROP_MD5_KEY)
  persistent_cache.delete_value(constants.LAST_TEST_ACCOUNT_CHECK_KEY)
  persistent_cache.set_value(constants.LAST_FLASH_BUILD_KEY, build_info)
  persistent_cache.set_value(constants.LAST_FLASH_TIME_KEY, time.time())
예제 #3
0
def store_file_in_cache(file_path,
                        cached_files_per_directory_limit=True,
                        force_update=False):
  """Get file from nfs cache if available."""
  if not os.path.exists(file_path):
    logs.log_error(
        'Local file %s does not exist, nothing to store in cache.' % file_path)
    return

  if os.path.getsize(file_path) > CACHE_SIZE_LIMIT:
    logs.log('File %s is too large to store in cache, skipping.' % file_path)
    return

  nfs_root = environment.get_value('NFS_ROOT')
  if not nfs_root:
    # No NFS, nothing to store in cache.
    return

  # If NFS server is not available due to heavy load, skip storage operation
  # altogether as we would fail to store file.
  if not os.path.exists(os.path.join(nfs_root, '.')):  # Use . to iterate mount.
    logs.log_warn('Cache %s not available.' % nfs_root)
    return

  cache_file_path = get_cache_file_path(file_path)
  cache_directory = os.path.dirname(cache_file_path)
  filename = os.path.basename(file_path)

  if not os.path.exists(cache_directory):
    if not shell.create_directory(cache_directory, create_intermediates=True):
      logs.log_error('Failed to create cache directory %s.' % cache_directory)
      return

  # Check if the file already exists in cache.
  if file_exists_in_cache(cache_file_path):
    if not force_update:
      return

    # If we are forcing update, we need to remove current cached file and its
    # metadata.
    remove_cache_file_and_metadata(cache_file_path)

  # Delete old cached files beyond our maximum storage limit.
  if cached_files_per_directory_limit:
    # Get a list of cached files.
    cached_files_list = []
    for cached_filename in os.listdir(cache_directory):
      if cached_filename.endswith(CACHE_METADATA_FILE_EXTENSION):
        continue
      cached_file_path = os.path.join(cache_directory, cached_filename)
      cached_files_list.append(cached_file_path)

    mtime = lambda f: os.stat(f).st_mtime
    last_used_cached_files_list = list(
        sorted(cached_files_list, key=mtime, reverse=True))
    for cached_file_path in (
        last_used_cached_files_list[MAX_CACHED_FILES_PER_DIRECTORY - 1:]):
      remove_cache_file_and_metadata(cached_file_path)

  # Start storing the actual file in cache now.
  logs.log('Started storing file %s into cache.' % filename)

  # Fetch lock to store this file. Try only once since if any other bot has
  # started to store it, we don't need to do it ourselves. Just bail out.
  lock_name = 'store:cache_file:%s' % utils.string_hash(cache_file_path)
  if not locks.acquire_lock(
      lock_name, max_hold_seconds=CACHE_LOCK_TIMEOUT, retries=1, by_zone=True):
    logs.log_warn(
        'Unable to fetch lock to update cache file %s, skipping.' % filename)
    return

  # Check if another bot already updated it.
  if file_exists_in_cache(cache_file_path):
    locks.release_lock(lock_name, by_zone=True)
    return

  shell.copy_file(file_path, cache_file_path)
  write_cache_file_metadata(cache_file_path, file_path)
  time.sleep(CACHE_COPY_WAIT_TIME)
  error_occurred = not file_exists_in_cache(cache_file_path)
  locks.release_lock(lock_name, by_zone=True)

  if error_occurred:
    logs.log_error('Failed to store file %s into cache.' % filename)
  else:
    logs.log('Completed storing file %s into cache.' % filename)
예제 #4
0
def flash_to_latest_build_if_needed():
    """Wipes user data, resetting the device to original factory state."""
    if environment.get_value('LOCAL_DEVELOPMENT'):
        # Don't reimage local development devices.
        return

    run_timeout = environment.get_value('RUN_TIMEOUT')
    if run_timeout:
        # If we have a run timeout, then we are already scheduled to bail out and
        # will be probably get re-imaged. E.g. using frameworks like Tradefed.
        return

    # Check if a flash is needed based on last recorded flash time.
    last_flash_time = persistent_cache.get_value(
        LAST_FLASH_TIME_KEY, constructor=datetime.datetime.utcfromtimestamp)
    needs_flash = last_flash_time is None or dates.time_has_expired(
        last_flash_time, seconds=adb.FLASH_INTERVAL)
    if not needs_flash:
        return

    build_info = {}
    if adb.is_gce():
        adb.recreate_gce_device()
    else:
        # Physical device.
        is_google_device = google_device()
        if is_google_device is None:
            logs.log_error('Unable to query device. Reimaging failed.')
            adb.bad_state_reached()

        elif not is_google_device:
            # We can't reimage these, skip.
            logs.log('Non-Google device found, skipping reimage.')
            return

        else:
            # For Google devices.
            # Check if both |BUILD_BRANCH| and |BUILD_TARGET| environment variables
            # are set. If not, we don't have enough data for reimaging and hence
            # we bail out.
            branch = environment.get_value('BUILD_BRANCH')
            target = environment.get_value('BUILD_TARGET')
            if not target:
                # We default to userdebug configuration.
                build_params = get_build_parameters()
                if build_params:
                    target = build_params.get('target') + '-userdebug'

                    # Cache target in environment. This is also useful for cases when
                    # device is bricked and we don't have this information available.
                    environment.set_value('BUILD_TARGET', target)

            if not branch or not target:
                logs.log_warn(
                    'BUILD_BRANCH and BUILD_TARGET are not set, skipping reimage.'
                )
                return

            # Download the latest build artifact for this branch and target.
            build_info = fetch_artifact.get_latest_artifact_info(
                branch, target)
            if not build_info:
                logs.log_error(
                    'Unable to fetch information on latest build artifact for '
                    'branch %s and target %s.' % (branch, target))
                return

            # Check if our local build matches the latest build. If not, we will
            # download it.
            build_id = build_info['bid']
            target = build_info['target']
            image_directory = environment.get_value('IMAGES_DIR')
            last_build_info = persistent_cache.get_value(LAST_FLASH_BUILD_KEY)
            if not last_build_info or last_build_info['bid'] != build_id:
                # Clean up the images directory first.
                shell.remove_directory(image_directory, recreate=True)

                # We have a new build, download the build artifacts for it.
                for image_regex in FLASH_IMAGE_REGEXES:
                    image_file_path = fetch_artifact.get(
                        build_id, target, image_regex, image_directory)
                    if not image_file_path:
                        logs.log_error(
                            'Failed to download image artifact %s for '
                            'branch %s and target %s.' %
                            (image_file_path, branch, target))
                        return
                    if image_file_path.endswith('.zip'):
                        archive.unpack(image_file_path, image_directory)

            # We do one device flash at a time on one host, otherwise we run into
            # failures and device being stuck in a bad state.
            flash_lock_key_name = 'flash:%s' % socket.gethostname()
            if not locks.acquire_lock(flash_lock_key_name, by_zone=True):
                logs.log_error(
                    'Failed to acquire lock for reimaging, exiting.')
                return

            logs.log('Reimaging started.')
            logs.log('Rebooting into bootloader mode.')
            for _ in xrange(FLASH_RETRIES):
                adb.run_as_root()
                adb.run_adb_command(['reboot-bootloader'])
                time.sleep(FLASH_REBOOT_BOOTLOADER_WAIT)
                adb.run_fastboot_command(['oem', 'off-mode-charge', '0'])
                adb.run_fastboot_command(['-w', 'reboot-bootloader'])

                for partition, partition_image_filename in FLASH_IMAGE_FILES:
                    partition_image_file_path = os.path.join(
                        image_directory, partition_image_filename)
                    adb.run_fastboot_command(
                        ['flash', partition, partition_image_file_path])
                    if partition in ['bootloader', 'radio']:
                        adb.run_fastboot_command(['reboot-bootloader'])
                adb.run_fastboot_command('reboot')
                time.sleep(FLASH_REBOOT_WAIT)

                if adb.get_device_state() == 'device':
                    break
                logs.log_error('Reimaging failed, retrying.')

            locks.release_lock(flash_lock_key_name, by_zone=True)

    if adb.get_device_state() != 'device':
        logs.log_error('Unable to find device. Reimaging failed.')
        adb.bad_state_reached()

    logs.log('Reimaging finished.')

    # Reset all of our persistent keys after wipe.
    persistent_cache.delete_value(BUILD_PROP_MD5_KEY)
    persistent_cache.delete_value(LAST_TEST_ACCOUNT_CHECK_KEY)
    persistent_cache.set_value(LAST_FLASH_BUILD_KEY, build_info)
    persistent_cache.set_value(LAST_FLASH_TIME_KEY, time.time())