def is_throttled(e):
"""
Function that tries to determine if an exception was caused by throttling
TODO - this implementation is incomplete
"""
if hasattr(e, 'message') and 'Google Cloud' in e.message:
return False
else:
return aws_is_throttled(e)
async def run_concurrently(function, backoff_seconds=15):
try:
async with asyncio.get_event_loop().throttler:
return await run_function_concurrently(function)
except Exception as e:
# Determine whether the exception is due to API throttling
throttled = aws_is_throttled(e) # FIXME - this only works for AWS
if throttled:
print_info('Hitting API rate limiting, will retry in {}s'.format(
backoff_seconds))
await asyncio.sleep(backoff_seconds)
return await run_concurrently(function, backoff_seconds + 15)
else:
raise