示例#1
0
 def test_clear_all(self):
     """Ensure clear all works."""
     persistent_cache.set_value('key', 'a')
     persistent_cache.set_value('key2', 'b', persist_across_reboots=True)
     persistent_cache.clear_values(clear_all=True)
     self.assertEqual(persistent_cache.get_value('key'), None)
     self.assertEqual(persistent_cache.get_value('key2'), None)
示例#2
0
def wait_until_good_state():
  """Check battery and make sure it is charged beyond minimum level and
  temperature thresholds."""
  # Battery levels are not applicable on GCE.
  if environment.is_android_cuttlefish() or settings.is_automotive():
    return

  # Make sure device is online.
  adb.wait_for_device()

  # Skip battery check if done recently.
  last_battery_check_time = persistent_cache.get_value(
      LAST_BATTERY_CHECK_TIME_KEY,
      constructor=datetime.datetime.utcfromtimestamp)
  if last_battery_check_time and not dates.time_has_expired(
      last_battery_check_time, seconds=BATTERY_CHECK_INTERVAL):
    return

  # Initialize variables.
  battery_level_threshold = environment.get_value('LOW_BATTERY_LEVEL_THRESHOLD',
                                                  LOW_BATTERY_LEVEL_THRESHOLD)
  battery_temperature_threshold = environment.get_value(
      'MAX_BATTERY_TEMPERATURE_THRESHOLD', MAX_BATTERY_TEMPERATURE_THRESHOLD)
  device_restarted = False

  while True:
    battery_information = get_battery_level_and_temperature()
    if battery_information is None:
      logs.log_error('Failed to get battery information, skipping check.')
      return

    battery_level = battery_information['level']
    battery_temperature = battery_information['temperature']
    logs.log('Battery information: level (%d%%), temperature (%.1f celsius).' %
             (battery_level, battery_temperature))
    if (battery_level >= battery_level_threshold and
        battery_temperature <= battery_temperature_threshold):
      persistent_cache.set_value(LAST_BATTERY_CHECK_TIME_KEY, time.time())
      return

    logs.log('Battery in bad battery state, putting device in sleep mode.')

    if not device_restarted:
      adb.reboot()
      device_restarted = True

    # Change thresholds to expected levels (only if they were below minimum
    # thresholds).
    if battery_level < battery_level_threshold:
      battery_level_threshold = EXPECTED_BATTERY_LEVEL
    if battery_temperature > battery_temperature_threshold:
      battery_temperature_threshold = EXPECTED_BATTERY_TEMPERATURE

    # Stopping shell should help with shutting off a lot of services that would
    # otherwise use up the battery. However, we need to turn it back on to get
    # battery status information.
    adb.stop_shell()
    time.sleep(BATTERY_CHARGE_INTERVAL)
    adb.start_shell()
示例#3
0
def track_task_start(task, task_duration):
    """Cache task information."""
    persistent_cache.set_value(TASK_PAYLOAD_KEY, task.payload())
    persistent_cache.set_value(TASK_END_TIME_KEY, time.time() + task_duration)

    # Don't wait on |run_heartbeat|, update task information as soon as it starts.
    from clusterfuzz._internal.datastore import data_handler
    data_handler.update_heartbeat(force_update=True)
示例#4
0
 def test_get_invalid(self):
     """Ensure it returns default_value when constructor fails."""
     time_now = datetime.datetime.utcnow()
     persistent_cache.set_value('key', 'random')
     self.assertEqual(persistent_cache.get_value('key'), 'random')
     self.assertEqual(
         persistent_cache.get_value(
             'key',
             default_value=time_now,
             constructor=datetime.datetime.utcfromtimestamp), time_now)
示例#5
0
 def test_set_get_datetime(self):
     """Ensure it works with datetime value."""
     epoch = datetime.datetime.utcfromtimestamp(0)
     end_time = datetime.datetime.utcfromtimestamp(10)
     diff_time = end_time - epoch
     persistent_cache.set_value('key', diff_time.total_seconds())
     self.assertEqual(
         persistent_cache.get_value(
             'key', constructor=datetime.datetime.utcfromtimestamp),
         end_time)
示例#6
0
def update_tests_if_needed():
    """Updates layout tests every day."""
    data_directory = environment.get_value('FUZZ_DATA')
    error_occured = False
    expected_task_duration = 60 * 60  # 1 hour.
    retry_limit = environment.get_value('FAIL_RETRIES')
    temp_archive = os.path.join(data_directory, 'temp.zip')
    tests_url = environment.get_value('WEB_TESTS_URL')

    # Check if we have a valid tests url.
    if not tests_url:
        return

    # Layout test updates are usually disabled to speedup local testing.
    if environment.get_value('LOCAL_DEVELOPMENT'):
        return

    # |UPDATE_WEB_TESTS| env variable can be used to control our update behavior.
    if not environment.get_value('UPDATE_WEB_TESTS'):
        return

    last_modified_time = persistent_cache.get_value(
        TESTS_LAST_UPDATE_KEY, constructor=datetime.datetime.utcfromtimestamp)
    if (last_modified_time is not None
            and not dates.time_has_expired(last_modified_time,
                                           days=TESTS_UPDATE_INTERVAL_DAYS)):
        return

    logs.log('Updating layout tests.')
    tasks.track_task_start(tasks.Task('update_tests', '', ''),
                           expected_task_duration)

    # Download and unpack the tests archive.
    for _ in range(retry_limit):
        try:
            shell.remove_directory(data_directory, recreate=True)
            storage.copy_file_from(tests_url, temp_archive)
            archive.unpack(temp_archive, data_directory, trusted=True)
            shell.remove_file(temp_archive)
            error_occured = False
            break
        except:
            logs.log_error(
                'Could not retrieve and unpack layout tests archive. Retrying.'
            )
            error_occured = True

    if not error_occured:
        persistent_cache.set_value(TESTS_LAST_UPDATE_KEY,
                                   time.time(),
                                   persist_across_reboots=True)

    tasks.track_task_end()
示例#7
0
    def put(self, key, value):
        """Put (key, value) into cache."""
        # Lock to avoid race condition in pop.
        self.lock.acquire()

        if len(self.keys) >= self.capacity:
            key_to_remove = self.keys.pop(0)
            persistent_cache.delete_value(key_to_remove)

        persistent_cache.set_value(key, value)
        self.keys.append(key)

        self.lock.release()
示例#8
0
def add_test_accounts_if_needed():
    """Add test account to work with GmsCore, etc."""
    last_test_account_check_time = persistent_cache.get_value(
        constants.LAST_TEST_ACCOUNT_CHECK_KEY,
        constructor=datetime.datetime.utcfromtimestamp)
    needs_test_account_update = (last_test_account_check_time is None
                                 or dates.time_has_expired(
                                     last_test_account_check_time,
                                     seconds=ADD_TEST_ACCOUNT_CHECK_INTERVAL))
    if not needs_test_account_update:
        return

    config = db_config.get()
    if not config:
        return

    test_account_email = config.test_account_email
    test_account_password = config.test_account_password
    if not test_account_email or not test_account_password:
        return

    adb.run_as_root()
    wifi.configure(force_enable=True)

    if not app.is_installed(ADD_TEST_ACCOUNT_PKG_NAME):
        logs.log('Installing helper apk for adding test account.')
        android_directory = environment.get_platform_resources_directory()
        add_test_account_apk_path = os.path.join(android_directory,
                                                 ADD_TEST_ACCOUNT_APK_NAME)
        app.install(add_test_account_apk_path)

    logs.log('Trying to add test account.')
    output = adb.run_shell_command(
        'am instrument -e account %s -e password %s -w %s' %
        (test_account_email, test_account_password,
         ADD_TEST_ACCOUNT_CALL_PATH),
        timeout=ADD_TEST_ACCOUNT_TIMEOUT)
    if not output or test_account_email not in output:
        logs.log('Failed to add test account, probably due to wifi issues.')
        return

    logs.log('Test account added successfully.')
    persistent_cache.set_value(constants.LAST_TEST_ACCOUNT_CHECK_KEY,
                               time.time())
示例#9
0
def remove_running_handle(handle):
  """Remove a handle from the tracked set."""
  new_handle_list = list(set(get_running_handles()) - set([handle]))
  persistent_cache.set_value(
      HANDLE_CACHE_KEY, new_handle_list, persist_across_reboots=True)
示例#10
0
def add_running_handle(handle):
  """Record a handle as potentially needing to be cleaned up on restart."""
  new_handle_list = list(set(get_running_handles()) | set([handle]))
  persistent_cache.set_value(
      HANDLE_CACHE_KEY, new_handle_list, persist_across_reboots=True)
示例#11
0
def configure_system_build_properties():
    """Modifies system build properties in /system/build.prop for better boot
  speed and power use."""
    adb.run_as_root()

    # Check md5 checksum of build.prop to see if already updated,
    # in which case exit. If build.prop does not exist, something
    # is very wrong with the device, so bail.
    old_md5 = persistent_cache.get_value(constants.BUILD_PROP_MD5_KEY)
    current_md5 = adb.get_file_checksum(BUILD_PROP_PATH)
    if current_md5 is None:
        logs.log_error('Unable to find %s on device.' % BUILD_PROP_PATH)
        return
    if old_md5 == current_md5:
        return

    # Pull to tmp file.
    bot_tmp_directory = environment.get_value('BOT_TMPDIR')
    old_build_prop_path = os.path.join(bot_tmp_directory, 'old.prop')
    adb.run_command(['pull', BUILD_PROP_PATH, old_build_prop_path])
    if not os.path.exists(old_build_prop_path):
        logs.log_error('Unable to fetch %s from device.' % BUILD_PROP_PATH)
        return

    # Write new build.prop.
    new_build_prop_path = os.path.join(bot_tmp_directory, 'new.prop')
    old_build_prop_file_content = open(old_build_prop_path, 'r')
    new_build_prop_file_content = open(new_build_prop_path, 'w')
    new_content_notification = '### CHANGED OR ADDED PROPERTIES ###'
    for line in old_build_prop_file_content:
        property_name = line.split('=')[0].strip()
        if property_name in BUILD_PROPERTIES:
            continue
        if new_content_notification in line:
            continue
        new_build_prop_file_content.write(line)

    new_build_prop_file_content.write(new_content_notification + '\n')
    for flag, value in six.iteritems(BUILD_PROPERTIES):
        new_build_prop_file_content.write('%s=%s\n' % (flag, value))
    old_build_prop_file_content.close()
    new_build_prop_file_content.close()

    # Keep verified boot disabled for M and higher releases. This makes it easy
    # to modify system's app_process to load asan libraries.
    build_version = settings.get_build_version()
    if is_build_at_least(build_version, 'M'):
        adb.run_as_root()
        adb.run_command('disable-verity')
        reboot()

    # Make /system writable.
    adb.run_as_root()
    adb.remount()

    # Remove seccomp policies (on N and higher) as ASan requires extra syscalls.
    if is_build_at_least(build_version, 'N'):
        policy_files = adb.run_shell_command(
            ['find', '/system/etc/seccomp_policy/', '-type', 'f'])
        for policy_file in policy_files.splitlines():
            adb.run_shell_command(['rm', policy_file.strip()])

    # Push new build.prop and backup to device.
    logs.log('Pushing new build properties file on device.')
    adb.run_command(
        ['push', '-p', old_build_prop_path, BUILD_PROP_BACKUP_PATH])
    adb.run_command(['push', '-p', new_build_prop_path, BUILD_PROP_PATH])
    adb.run_shell_command(['chmod', '644', BUILD_PROP_PATH])

    # Set persistent cache key containing and md5sum.
    current_md5 = adb.get_file_checksum(BUILD_PROP_PATH)
    persistent_cache.set_value(constants.BUILD_PROP_MD5_KEY, current_md5)
示例#12
0
def recreate_instance_with_disks(instance_name,
                                 project,
                                 zone,
                                 additional_metadata=None,
                                 wait_for_completion=False):
  """Recreate an instance and its disk."""
  # Get existing instance information.
  # First, try to get instance info from cache.
  # TODO(ochang): Make this more general in case anything else needs to use
  # this method (e.g. appengine).
  instance_info = persistent_cache.get_value(GCE_INSTANCE_INFO_KEY)
  if instance_info is None:
    instance_info = _get_instance_info(instance_name, project, zone)

  # Bail out if we don't have a valid instance information.
  if (not instance_info or 'disks' not in instance_info or
      not instance_info['disks']):
    logs.log_error(
        'Failed to get disk info from existing instance, bailing on instance '
        'recreation.')
    return False

  # Add any additional metadata required for instance booting.
  if additional_metadata:
    for key, value in six.iteritems(additional_metadata):
      items = instance_info.setdefault('metadata', {}).setdefault('items', [])
      _add_metadata_key_value(items, key, value)

  # Cache the latest instance information.
  persistent_cache.set_value(
      GCE_INSTANCE_INFO_KEY, instance_info, persist_across_reboots=True)

  # Delete the instance.
  if not _do_instance_operation(
      'delete', instance_name, project, zone, wait_for_completion=True):
    logs.log_error('Failed to delete instance.')
    return False

  # Get existing disks information, and recreate.
  api = _get_api()
  disks = instance_info['disks']
  for disk in disks:
    disk_source = disk['source']
    disk_name = disk_source.split('/')[-1]

    disk_info_func = api.disks().get(disk=disk_name, project=project, zone=zone)
    disk_info = _execute_api_call_with_retries(disk_info_func)
    if 'sourceImage' not in disk_info or 'sizeGb' not in disk_info:
      logs.log_error(
          'Failed to get source image and size from existing disk, bailing on '
          'instance recreation.')
      return False

    size_gb = disk_info['sizeGb']
    source_image = disk_info['sourceImage']

    # Recreate the disk.
    if not delete_disk(disk_name, project, zone, wait_for_completion=True):
      logs.log_error('Failed to delete disk.')
      return False

    if not create_disk(
        disk_name,
        source_image,
        size_gb,
        project,
        zone,
        wait_for_completion=True):
      logs.log_error('Failed to recreate disk.')
      return False

  # Recreate the instance with the exact same configurations, but not
  # necessarily the same IPs.
  try:
    del instance_info['networkInterfaces'][0]['accessConfigs'][0]['natIP']
  except:
    # This is not a failure. When a bot is stopped, it has no ip/interface.
    pass
  try:
    del instance_info['networkInterfaces'][0]['networkIP']
  except:
    # This is not a failure. When a bot is stopped, it has no ip/interface.
    pass

  operation = api.instances().insert(
      body=instance_info, project=project, zone=zone)

  return _do_operation_with_retries(
      operation, project, zone, wait_for_completion=wait_for_completion)
示例#13
0
 def test_delete(self):
     """Ensure it deletes key."""
     persistent_cache.set_value('key', 'value')
     persistent_cache.delete_value('key')
     self.assertEqual(persistent_cache.get_value('key'), None)
示例#14
0
 def test_set_get_string(self):
     """Ensure it works with string value."""
     persistent_cache.set_value('key', 'value')
     self.assertEqual(persistent_cache.get_value('key'), 'value')
示例#15
0
def flash_to_latest_build_if_needed():
  """Wipes user data, resetting the device to original factory state."""
  if environment.get_value('LOCAL_DEVELOPMENT'):
    # Don't reimage local development devices.
    return

  run_timeout = environment.get_value('RUN_TIMEOUT')
  if run_timeout:
    # If we have a run timeout, then we are already scheduled to bail out and
    # will be probably get re-imaged. E.g. using frameworks like Tradefed.
    return

  # Check if a flash is needed based on last recorded flash time.
  last_flash_time = persistent_cache.get_value(
      constants.LAST_FLASH_TIME_KEY,
      constructor=datetime.datetime.utcfromtimestamp)
  needs_flash = last_flash_time is None or dates.time_has_expired(
      last_flash_time, seconds=FLASH_INTERVAL)
  if not needs_flash:
    return

  is_google_device = settings.is_google_device()
  if is_google_device is None:
    logs.log_error('Unable to query device. Reimaging failed.')
    adb.bad_state_reached()

  elif not is_google_device:
    # We can't reimage these, skip.
    logs.log('Non-Google device found, skipping reimage.')
    return

  # Check if both |BUILD_BRANCH| and |BUILD_TARGET| environment variables
  # are set. If not, we don't have enough data for reimaging and hence
  # we bail out.
  branch = environment.get_value('BUILD_BRANCH')
  target = environment.get_value('BUILD_TARGET')
  if not target:
    # We default to userdebug configuration.
    build_params = settings.get_build_parameters()
    if build_params:
      target = build_params.get('target') + '-userdebug'

      # Cache target in environment. This is also useful for cases when
      # device is bricked and we don't have this information available.
      environment.set_value('BUILD_TARGET', target)

  if not branch or not target:
    logs.log_warn(
        'BUILD_BRANCH and BUILD_TARGET are not set, skipping reimage.')
    return

  image_directory = environment.get_value('IMAGES_DIR')
  build_info = fetch_artifact.get_latest_artifact_info(branch, target)
  if not build_info:
    logs.log_error('Unable to fetch information on latest build artifact for '
                   'branch %s and target %s.' % (branch, target))
    return

  if environment.is_android_cuttlefish():
    download_latest_build(build_info, FLASH_CUTTLEFISH_REGEXES, image_directory)
    adb.recreate_cuttlefish_device()
    adb.connect_to_cuttlefish_device()
  else:
    download_latest_build(build_info, FLASH_IMAGE_REGEXES, image_directory)
    # We do one device flash at a time on one host, otherwise we run into
    # failures and device being stuck in a bad state.
    flash_lock_key_name = 'flash:%s' % socket.gethostname()
    if not locks.acquire_lock(flash_lock_key_name, by_zone=True):
      logs.log_error('Failed to acquire lock for reimaging, exiting.')
      return

    logs.log('Reimaging started.')
    logs.log('Rebooting into bootloader mode.')
    for _ in range(FLASH_RETRIES):
      adb.run_as_root()
      adb.run_command(['reboot-bootloader'])
      time.sleep(FLASH_REBOOT_BOOTLOADER_WAIT)
      adb.run_fastboot_command(['oem', 'off-mode-charge', '0'])
      adb.run_fastboot_command(['-w', 'reboot-bootloader'])

      for partition, partition_image_filename in FLASH_IMAGE_FILES:
        partition_image_file_path = os.path.join(image_directory,
                                                 partition_image_filename)
        adb.run_fastboot_command(
            ['flash', partition, partition_image_file_path])
        if partition in ['bootloader', 'radio']:
          adb.run_fastboot_command(['reboot-bootloader'])

      # Disable ramdump to avoid capturing ramdumps during kernel crashes.
      # This causes device lockup of several minutes during boot and we intend
      # to analyze them ourselves.
      adb.run_fastboot_command(['oem', 'ramdump', 'disable'])

      adb.run_fastboot_command('reboot')
      time.sleep(FLASH_REBOOT_WAIT)

      if adb.get_device_state() == 'device':
        break
      logs.log_error('Reimaging failed, retrying.')

    locks.release_lock(flash_lock_key_name, by_zone=True)

  if adb.get_device_state() != 'device':
    logs.log_error('Unable to find device. Reimaging failed.')
    adb.bad_state_reached()

  logs.log('Reimaging finished.')

  # Reset all of our persistent keys after wipe.
  persistent_cache.delete_value(constants.BUILD_PROP_MD5_KEY)
  persistent_cache.delete_value(constants.LAST_TEST_ACCOUNT_CHECK_KEY)
  persistent_cache.set_value(constants.LAST_FLASH_BUILD_KEY, build_info)
  persistent_cache.set_value(constants.LAST_FLASH_TIME_KEY, time.time())