def testRunWithBackoff(self):
    # Returning None forces exponential backoff to occur, and within 5 seconds
    # the callable should be called ~3 times.
    results = utils.run_with_backoff(func=lambda: None, runtime=5)
    # Weakly verify it fits in some arbitrary range:
    self.assertGreater(len(results), 1)
    self.assertLess(len(results), 6)

    # Test stop_on_success.
    results = utils.run_with_backoff(func=lambda: True, stop_on_success=True)
    self.assertEqual([True], results)
Beispiel #2
0
    def testRunWithBackoff(self):
        # Returning None forces exponential backoff to occur, and within 5 seconds
        # the callable should be called ~3 times.
        results = utils.run_with_backoff(func=lambda: None, runtime=5)
        # Weakly verify it fits in some arbitrary range:
        self.assertGreater(len(results), 1)
        self.assertLess(len(results), 6)

        # Test stop_on_success.
        results = utils.run_with_backoff(func=lambda: True,
                                         stop_on_success=True)
        self.assertEqual([True], results)
Beispiel #3
0
  def process_windows_with_backoff(self, runtime=DEFAULT_CRON_RUNTIME_SECONDS):
    """Long-running function to process multiple windows.

    Args:
      runtime: How long to process data for.
    Returns:
      A list of results from process_next_window().
    """
    results = utils.run_with_backoff(
        func=self.process_next_window,
        runtime=runtime,
        max_backoff=TASKQUEUE_LEASE_SECONDS)
    return results
Beispiel #4
0
    def process_windows_with_backoff(self,
                                     runtime=DEFAULT_CRON_RUNTIME_SECONDS):
        """Long-running function to process multiple windows.

    Args:
      runtime: How long to process data for.
    Returns:
      A list of results from process_next_window().
    """
        results = utils.run_with_backoff(func=self.process_next_window,
                                         runtime=runtime,
                                         max_backoff=TASKQUEUE_LEASE_SECONDS)
        return results
Beispiel #5
0
def process_data_with_backoff(
    timeout_seconds=DEFAULT_PROCESSING_TIMEOUT_SECONDS,
    max_tasks=DEFAULT_MAX_TASKS):
  """Like process_data, but with exponential backoff."""
  results = utils.run_with_backoff(
      func=process_data,
      runtime=timeout_seconds,
      max_tasks=max_tasks,
      allow_transient_errors=True)
  for result in results:
    if result is not False and 'error' in result:
      logging.error('Microversion failed (will retry): %s - %s',
                    results['path'], results['error'])
  return results