def testRunWithBackoff(self):
# Returning None forces exponential backoff to occur, and within 5 seconds
# the callable should be called ~3 times.
results = utils.run_with_backoff(func=lambda: None, runtime=5)
# Weakly verify it fits in some arbitrary range:
self.assertGreater(len(results), 1)
self.assertLess(len(results), 6)
# Test stop_on_success.
results = utils.run_with_backoff(func=lambda: True, stop_on_success=True)
self.assertEqual([True], results)
def testRunWithBackoff(self):
# Returning None forces exponential backoff to occur, and within 5 seconds
# the callable should be called ~3 times.
results = utils.run_with_backoff(func=lambda: None, runtime=5)
# Weakly verify it fits in some arbitrary range:
self.assertGreater(len(results), 1)
self.assertLess(len(results), 6)
# Test stop_on_success.
results = utils.run_with_backoff(func=lambda: True,
stop_on_success=True)
self.assertEqual([True], results)
def process_windows_with_backoff(self, runtime=DEFAULT_CRON_RUNTIME_SECONDS):
"""Long-running function to process multiple windows.
Args:
runtime: How long to process data for.
Returns:
A list of results from process_next_window().
"""
results = utils.run_with_backoff(
func=self.process_next_window,
runtime=runtime,
max_backoff=TASKQUEUE_LEASE_SECONDS)
return results
def process_windows_with_backoff(self,
runtime=DEFAULT_CRON_RUNTIME_SECONDS):
"""Long-running function to process multiple windows.
Args:
runtime: How long to process data for.
Returns:
A list of results from process_next_window().
"""
results = utils.run_with_backoff(func=self.process_next_window,
runtime=runtime,
max_backoff=TASKQUEUE_LEASE_SECONDS)
return results
def process_data_with_backoff(
timeout_seconds=DEFAULT_PROCESSING_TIMEOUT_SECONDS,
max_tasks=DEFAULT_MAX_TASKS):
"""Like process_data, but with exponential backoff."""
results = utils.run_with_backoff(
func=process_data,
runtime=timeout_seconds,
max_tasks=max_tasks,
allow_transient_errors=True)
for result in results:
if result is not False and 'error' in result:
logging.error('Microversion failed (will retry): %s - %s',
results['path'], results['error'])
return results