def testRunWithBackoff(self): # Returning None forces exponential backoff to occur, and within 5 seconds # the callable should be called ~3 times. results = utils.run_with_backoff(func=lambda: None, runtime=5) # Weakly verify it fits in some arbitrary range: self.assertGreater(len(results), 1) self.assertLess(len(results), 6) # Test stop_on_success. results = utils.run_with_backoff(func=lambda: True, stop_on_success=True) self.assertEqual([True], results)
def process_windows_with_backoff(self, runtime=DEFAULT_CRON_RUNTIME_SECONDS): """Long-running function to process multiple windows. Args: runtime: How long to process data for. Returns: A list of results from process_next_window(). """ results = utils.run_with_backoff( func=self.process_next_window, runtime=runtime, max_backoff=TASKQUEUE_LEASE_SECONDS) return results
def process_windows_with_backoff(self, runtime=DEFAULT_CRON_RUNTIME_SECONDS): """Long-running function to process multiple windows. Args: runtime: How long to process data for. Returns: A list of results from process_next_window(). """ results = utils.run_with_backoff(func=self.process_next_window, runtime=runtime, max_backoff=TASKQUEUE_LEASE_SECONDS) return results
def process_data_with_backoff( timeout_seconds=DEFAULT_PROCESSING_TIMEOUT_SECONDS, max_tasks=DEFAULT_MAX_TASKS): """Like process_data, but with exponential backoff.""" results = utils.run_with_backoff( func=process_data, runtime=timeout_seconds, max_tasks=max_tasks, allow_transient_errors=True) for result in results: if result is not False and 'error' in result: logging.error('Microversion failed (will retry): %s - %s', results['path'], results['error']) return results