async def _reconnect_async(self): # pylint: disable=too-many-statements
# pylint: disable=protected-access
alt_creds = {
"username": self.client._auth_config.get("iot_username"),
"password":self.client._auth_config.get("iot_password")}
await self._handler.close_async()
source = Source(self.source)
if self.offset is not None:
source.set_filter(self.offset.selector())
self._handler = ReceiveClientAsync(
source,
auth=self.client.get_auth(**alt_creds),
debug=self.client.debug,
prefetch=self.prefetch,
link_properties=self.properties,
timeout=self.timeout,
error_policy=self.retry_policy,
keep_alive_interval=self.keep_alive,
client_name=self.name,
properties=self.client.create_properties(),
loop=self.loop)
try:
await self._handler.open_async()
while not await self._handler.client_ready_async():
await asyncio.sleep(0.05)
return True
except errors.TokenExpired as shutdown:
log.info("AsyncReceiver disconnected due to token expiry. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
await self.close_async(exception=error)
raise error
except (errors.LinkDetach, errors.ConnectionClose) as shutdown:
if shutdown.action.retry and self.auto_reconnect:
log.info("AsyncReceiver detached. Attempting reconnect.")
return False
log.info("AsyncReceiver detached. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
await self.close_async(exception=error)
raise error
except errors.MessageHandlerError as shutdown:
if self.auto_reconnect:
log.info("AsyncReceiver detached. Attempting reconnect.")
return False
log.info("AsyncReceiver detached. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
await self.close_async(exception=error)
raise error
except errors.AMQPConnectionError as shutdown:
if str(shutdown).startswith("Unable to open authentication session") and self.auto_reconnect:
log.info("AsyncReceiver couldn't authenticate. Attempting reconnect.")
return False
log.info("AsyncReceiver connection error (%r). Shutting down.", shutdown)
error = EventHubError(str(shutdown))
await self.close_async(exception=error)
raise error
except Exception as e:
log.info("Unexpected error occurred (%r). Shutting down.", e)
error = EventHubError("Receiver reconnect failed: {}".format(e))
await self.close_async(exception=error)
raise error
async def close_async(self, exception=None):
"""
Close down the handler. If the handler has already closed,
this will be a no op. An optional exception can be passed in to
indicate that the handler was shutdown due to error.
:param exception: An optional exception if the handler is closing
due to an error.
:type exception: Exception
Example:
.. literalinclude:: ../examples/async_examples/test_examples_eventhub_async.py
:start-after: [START eventhub_client_async_receiver_close]
:end-before: [END eventhub_client_async_receiver_close]
:language: python
:dedent: 4
:caption: Close down the handler.
"""
self.running = False
if self.error:
return
if isinstance(exception, errors.LinkRedirect):
self.redirected = exception
elif isinstance(exception, EventHubError):
self.error = exception
elif isinstance(exception, (errors.LinkDetach, errors.ConnectionClose)):
self.error = EventHubError(str(exception), exception)
elif exception:
self.error = EventHubError(str(exception))
else:
self.error = EventHubError("This receive handler is now closed.")
await self._handler.close_async()
async def wait_async(self):
"""
Wait until all transferred events have been sent.
"""
if self.error:
raise self.error
if not self.running:
raise ValueError("Unable to send until client has been started.")
try:
await self._handler.wait_async()
except (errors.TokenExpired, errors.AuthenticationException):
log.info("AsyncSender disconnected due to token error. Attempting reconnect.")
await self.reconnect_async()
except (errors.LinkDetach, errors.ConnectionClose) as shutdown:
if shutdown.action.retry and self.auto_reconnect:
log.info("AsyncSender detached. Attempting reconnect.")
await self.reconnect_async()
else:
log.info("AsyncSender detached. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
await self.close_async(exception=error)
raise error
except errors.MessageHandlerError as shutdown:
if self.auto_reconnect:
log.info("AsyncSender detached. Attempting reconnect.")
await self.reconnect_async()
else:
log.info("AsyncSender detached. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
await self.close_async(exception=error)
raise error
except Exception as e:
log.info("Unexpected error occurred (%r).", e)
raise EventHubError("Send failed: {}".format(e))
async def receive(self, max_batch_size=None, timeout=None):
"""
Receive events asynchronously from the EventHub.
:param max_batch_size: Receive a batch of events. Batch size will
be up to the maximum specified, but will return as soon as service
returns no new events. If combined with a timeout and no events are
retrieve before the time, the result will be empty. If no batch
size is supplied, the prefetch size will be the maximum.
:type max_batch_size: int
:rtype: list[~azure.eventhub.common.EventData]
"""
if self.error:
raise self.error
try:
timeout_ms = 1000 * timeout if timeout else 0
message_batch = await self._handler.receive_message_batch_async(
max_batch_size=max_batch_size, timeout=timeout_ms)
data_batch = []
for message in message_batch:
event_data = EventData(message=message)
self.offset = event_data.offset
data_batch.append(event_data)
return data_batch
except errors.LinkDetach as detach:
error = EventHubError(str(detach))
await self.close_async(exception=error)
raise error
except Exception as e:
error = EventHubError("Receive failed: {}".format(e))
await self.close_async(exception=error)
raise error
async def send(self, event_data):
"""
Sends an event data and asynchronously waits until
acknowledgement is received or operation times out.
:param event_data: The event to be sent.
:type event_data: ~azure.eventhub.common.EventData
:raises: ~azure.eventhub.common.EventHubError if the message fails to
send.
"""
if self.error:
raise self.error
if event_data.partition_key and self.partition:
raise ValueError(
"EventData partition key cannot be used with a partition sender."
)
event_data.message.on_send_complete = self._on_outcome
try:
await self._handler.send_message_async(event_data.message)
if self._outcome != constants.MessageSendResult.Ok:
raise Sender._error(self._outcome, self._condition)
except errors.LinkDetach as detach:
error = EventHubError(str(detach))
await self.close_async(exception=error)
raise error
except Exception as e:
error = EventHubError("Send failed: {}".format(e))
await self.close_async(exception=error)
raise error
else:
return self._outcome
async def wait_async(self):
"""
Wait until all transferred events have been sent.
"""
if self.error:
raise self.error
try:
await self._handler.wait_async()
except (errors.LinkDetach, errors.ConnectionClose) as shutdown:
if shutdown.action.retry and self.auto_reconnect:
log.info("AsyncSender detached. Attempting reconnect.")
await self.reconnect_async()
else:
log.info("AsyncSender detached. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
await self.close_async(exception=error)
raise error
except errors.MessageHandlerError as shutdown:
if self.auto_reconnect:
log.info("AsyncSender detached. Attempting reconnect.")
await self.reconnect_async()
else:
log.info("AsyncSender detached. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
await self.close_async(exception=error)
raise error
except Exception as e:
log.info("Unexpected error occurred (%r).", e)
raise EventHubError("Send failed: {}".format(e))
async def close_async(self, exception=None):
"""
Close down the handler. If the handler has already closed,
this will be a no op. An optional exception can be passed in to
indicate that the handler was shutdown due to error.
:param exception: An optional exception if the handler is closing
due to an error.
:type exception: Exception
"""
self.running = False
if self.error:
return
if isinstance(exception, errors.LinkRedirect):
self.redirected = exception
elif isinstance(exception, EventHubError):
self.error = exception
elif isinstance(exception,
(errors.LinkDetach, errors.ConnectionClose)):
self.error = EventHubError(str(exception), exception)
elif exception:
self.error = EventHubError(str(exception))
else:
self.error = EventHubError("This receive handler is now closed.")
await self._handler.close_async()
async def receive(self, max_batch_size=None, callback=None, timeout=None):
"""
Receive events asynchronously from the EventHub.
:param max_batch_size: Receive a batch of events. Batch size will
be up to the maximum specified, but will return as soon as service
returns no new events. If combined with a timeout and no events are
retrieve before the time, the result will be empty. If no batch
size is supplied, the prefetch size will be the maximum.
:type max_batch_size: int
:param callback: A callback to be run for each received event. This must
be a function that accepts a single argument - the event data. This callback
will be run before the message is returned in the result generator.
:type callback: func[~azure.eventhub.EventData]
:returns: list[~azure.eventhub.EventData]
"""
try:
self._callback = callback
timeout_ms = 1000 * timeout if timeout else 0
batch = await self._handler.receive_message_batch_async(
max_batch_size=max_batch_size,
on_message_received=self.on_message,
timeout=timeout_ms)
return batch
except errors.AMQPConnectionError as e:
message = "Failed to open receiver: {}".format(e)
message += "\nPlease check that the partition key is valid "
if self.epoch:
message += "and that a higher epoch receiver is " \
"not already running for this partition."
else:
message += "and whether an epoch receiver is " \
"already running for this partition."
raise EventHubError(message)
except Exception as e:
raise EventHubError("Receive failed: {}".format(e))
async def send(self, event_data):
"""
Sends an event data and asynchronously waits until
acknowledgement is received or operation times out.
:param event_data: The event to be sent.
:type event_data: ~azure.eventhub.common.EventData
:raises: ~azure.eventhub.common.EventHubError if the message fails to
send.
Example:
.. literalinclude:: ../examples/async_examples/test_examples_eventhub_async.py
:start-after: [START eventhub_client_async_send]
:end-before: [END eventhub_client_async_send]
:language: python
:dedent: 4
:caption: Sends an event data and asynchronously waits
until acknowledgement is received or operation times out.
"""
if self.error:
raise self.error
if not self.running:
raise ValueError("Unable to send until client has been started.")
if event_data.partition_key and self.partition:
raise ValueError("EventData partition key cannot be used with a partition sender.")
event_data.message.on_send_complete = self._on_outcome
try:
await self._handler.send_message_async(event_data.message)
if self._outcome != constants.MessageSendResult.Ok:
raise Sender._error(self._outcome, self._condition)
except (errors.TokenExpired, errors.AuthenticationException):
log.info("AsyncSender disconnected due to token error. Attempting reconnect.")
await self.reconnect_async()
except (errors.LinkDetach, errors.ConnectionClose) as shutdown:
if shutdown.action.retry and self.auto_reconnect:
log.info("AsyncSender detached. Attempting reconnect.")
await self.reconnect_async()
else:
log.info("AsyncSender detached. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
await self.close_async(exception=error)
raise error
except errors.MessageHandlerError as shutdown:
if self.auto_reconnect:
log.info("AsyncSender detached. Attempting reconnect.")
await self.reconnect_async()
else:
log.info("AsyncSender detached. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
await self.close_async(exception=error)
raise error
except Exception as e:
log.info("Unexpected error occurred (%r). Shutting down.", e)
error = EventHubError("Send failed: {}".format(e))
await self.close_async(exception=error)
raise error
else:
return self._outcome
async def receive(self, max_batch_size=None, timeout=None):
"""
Receive events asynchronously from the EventHub.
:param max_batch_size: Receive a batch of events. Batch size will
be up to the maximum specified, but will return as soon as service
returns no new events. If combined with a timeout and no events are
retrieve before the time, the result will be empty. If no batch
size is supplied, the prefetch size will be the maximum.
:type max_batch_size: int
:rtype: list[~azure.eventhub.common.EventData]
"""
if self.error:
raise self.error
if not self.running:
raise ValueError(
"Unable to receive until client has been started.")
data_batch = []
try:
timeout_ms = 1000 * timeout if timeout else 0
message_batch = await self._handler.receive_message_batch_async(
max_batch_size=max_batch_size, timeout=timeout_ms)
for message in message_batch:
event_data = EventData(message=message)
self.offset = event_data.offset
data_batch.append(event_data)
return data_batch
except (errors.TokenExpired, errors.AuthenticationException):
log.info(
"AsyncReceiver disconnected due to token error. Attempting reconnect."
)
await self.reconnect_async()
return data_batch
except (errors.LinkDetach, errors.ConnectionClose) as shutdown:
if shutdown.action.retry and self.auto_reconnect:
log.info("AsyncReceiver detached. Attempting reconnect.")
await self.reconnect_async()
return data_batch
log.info("AsyncReceiver detached. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
await self.close_async(exception=error)
raise error
except errors.MessageHandlerError as shutdown:
if self.auto_reconnect:
log.info("AsyncReceiver detached. Attempting reconnect.")
await self.reconnect_async()
return data_batch
log.info("AsyncReceiver detached. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
await self.close_async(exception=error)
raise error
except Exception as e:
log.info("Unexpected error occurred (%r). Shutting down.", e)
error = EventHubError("Receive failed: {}".format(e))
await self.close_async(exception=error)
raise error
async def _reconnect_async(self):
await self._handler.close_async()
unsent_events = self._handler.pending_messages
self._handler = SendClientAsync(
self.target,
auth=self.client.get_auth(),
debug=self.client.debug,
msg_timeout=self.timeout,
error_policy=self.retry_policy,
keep_alive_interval=self.keep_alive,
client_name=self.name,
properties=self.client.create_properties(),
loop=self.loop)
try:
await self._handler.open_async()
self._handler.queue_message(*unsent_events)
await self._handler.wait_async()
return True
except errors.TokenExpired as shutdown:
log.info(
"AsyncSender disconnected due to token expiry. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
await self.close_async(exception=error)
raise error
except (errors.LinkDetach, errors.ConnectionClose) as shutdown:
if shutdown.action.retry and self.auto_reconnect:
log.info("AsyncSender detached. Attempting reconnect.")
return False
log.info("AsyncSender reconnect failed. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
await self.close_async(exception=error)
raise error
except errors.MessageHandlerError as shutdown:
if self.auto_reconnect:
log.info("AsyncSender detached. Attempting reconnect.")
return False
log.info("AsyncSender reconnect failed. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
await self.close_async(exception=error)
raise error
except errors.AMQPConnectionError as shutdown:
if str(shutdown).startswith("Unable to open authentication session"
) and self.auto_reconnect:
log.info(
"AsyncSender couldn't authenticate. Attempting reconnect.")
return False
log.info("AsyncSender connection error (%r). Shutting down.",
shutdown)
error = EventHubError(str(shutdown))
await self.close_async(exception=error)
raise error
except Exception as e:
log.info("Unexpected error occurred (%r). Shutting down.", e)
error = EventHubError("Sender reconnect failed: {}".format(e))
await self.close_async(exception=error)
raise error
async def _handle_redirect(self, redirects):
if len(redirects) != len(self.clients):
not_redirected = [c for c in self.clients if not c.redirected]
_, timeout = await asyncio.wait([self._wait_for_client(c) for c in not_redirected], timeout=5)
if timeout:
raise EventHubError("Some clients are attempting to redirect the connection.")
redirects = [c.redirected for c in self.clients if c.redirected]
if not all(r.hostname == redirects[0].hostname for r in redirects):
raise EventHubError("Multiple clients attempting to redirect to different hosts.")
self._process_redirect_uri(redirects[0])
await asyncio.gather(*[c.open_async() for c in self.clients])
async def _handle_redirect(self, redirects):
if len(redirects) != len(self.clients):
not_redirected = [c for c in self.clients if not c.redirected]
_, timeout = await asyncio.wait([self._wait_for_client(c) for c in not_redirected], timeout=5)
if timeout:
raise EventHubError("Some clients are attempting to redirect the connection.")
redirects = [c.redirected for c in self.clients if c.redirected]
if not all(r.hostname == redirects[0].hostname for r in redirects):
raise EventHubError("Multiple clients attempting to redirect to different hosts.")
self.auth = self._create_auth(redirects[0].address.decode('utf-8'), **self._auth_config)
await self.connection.redirect_async(redirects[0], self.auth)
await asyncio.gather(*[c.open_async(self.connection) for c in self.clients])
async def reconnect_async(self):
"""If the Receiver was disconnected from the service with
a retryable error - attempt to reconnect."""
# pylint: disable=protected-access
alt_creds = {
"username": self.client._auth_config.get("iot_username"),
"password": self.client._auth_config.get("iot_password")
}
await self._handler.close_async()
source = Source(self.source)
if self.offset is not None:
source.set_filter(self.offset.selector())
self._handler = ReceiveClientAsync(
source,
auth=self.client.get_auth(**alt_creds),
debug=self.client.debug,
prefetch=self.prefetch,
link_properties=self.properties,
timeout=self.timeout,
error_policy=self.retry_policy,
keep_alive_interval=self.keep_alive,
client_name=self.name,
properties=self.client.create_properties(),
loop=self.loop)
try:
await self._handler.open_async()
while not await self.has_started():
await self._handler._connection.work_async()
except (errors.LinkDetach, errors.ConnectionClose) as shutdown:
if shutdown.action.retry and self.auto_reconnect:
log.info("AsyncReceiver detached. Attempting reconnect.")
await self.reconnect_async()
else:
log.info("AsyncReceiver detached. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
await self.close_async(exception=error)
raise error
except errors.MessageHandlerError as shutdown:
if self.auto_reconnect:
log.info("AsyncReceiver detached. Attempting reconnect.")
await self.reconnect_async()
else:
log.info("AsyncReceiver detached. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
await self.close_async(exception=error)
raise error
except Exception as e:
log.info("Unexpected error occurred (%r). Shutting down.", e)
error = EventHubError("Receiver reconnect failed: {}".format(e))
await self.close_async(exception=error)
raise error
async def run_async(self):
"""
Run the EventHubClient asynchronously.
Opens the connection and starts running all AsyncSender/AsyncReceiver clients.
Returns a list of the start up results. For a succcesful client start the
result will be `None`, otherwise the exception raised.
If all clients failed to start, then run will fail, shut down the connection
and raise an exception.
If at least one client starts up successfully the run command will succeed.
:rtype: list[~azure.eventhub.common.EventHubError]
"""
log.info("%r: Starting %r clients", self.container_id, len(self.clients))
tasks = [self._start_client_async(c) for c in self.clients]
try:
await asyncio.gather(*tasks)
redirects = [c.redirected for c in self.clients if c.redirected]
failed = [c.error for c in self.clients if c.error]
if failed and len(failed) == len(self.clients):
log.warning("%r: All clients failed to start.", self.container_id)
raise failed[0]
elif failed:
log.warning("%r: %r clients failed to start.", self.container_id, len(failed))
elif redirects:
await self._handle_redirect(redirects)
except EventHubError:
await self.stop_async()
raise
except Exception as exp:
await self.stop_async()
raise EventHubError(str(exp))
return failed
async def send(self, event_data):
"""
Sends an event data and asynchronously waits until
acknowledgement is received or operation times out.
:param event_data: The event to be sent.
:type event_data: ~azure.eventhub.common.EventData
:raises: ~azure.eventhub.common.EventHubError if the message fails to
send.
"""
if self.error:
raise self.error
if event_data.partition_key and self.partition:
raise ValueError(
"EventData partition key cannot be used with a partition sender."
)
event_data.message.on_send_complete = self._on_outcome
try:
await self._handler.send_message_async(event_data.message)
if self._outcome != constants.MessageSendResult.Ok:
raise Sender._error(self._outcome, self._condition)
except (errors.LinkDetach, errors.ConnectionClose) as shutdown:
if shutdown.action.retry and self.auto_reconnect:
log.info("AsyncSender detached. Attempting reconnect.")
await self.reconnect_async()
else:
log.info("AsyncSender detached. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
await self.close_async(exception=error)
raise error
except errors.MessageHandlerError as shutdown:
if self.auto_reconnect:
log.info("AsyncSender detached. Attempting reconnect.")
await self.reconnect_async()
else:
log.info("AsyncSender detached. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
await self.close_async(exception=error)
raise error
except Exception as e:
log.info("Unexpected error occurred (%r). Shutting down.", e)
error = EventHubError("Send failed: {}".format(e))
await self.close_async(exception=error)
raise error
else:
return self._outcome
def flush(self):
try:
start = time.time()
self.sender.send(EventData(self.buffer))
self.flush_cb(time.time() - start, self.item_count)
except Exception as e:
raise EventHubError("lost the following {} records\n".format(self.item_count))
self.buffer = ""
self.item_count = 0
async def wait_async(self):
"""
Wait until all transferred events have been sent.
"""
if self.error:
raise self.error
try:
await self._handler.wait_async()
except Exception as e:
raise EventHubError("Send failed: {}".format(e))
async def reconnect_async(self):
"""If the Receiver was disconnected from the service with
a retryable error - attempt to reconnect."""
await self._handler.close_async()
unsent_events = self._handler.pending_messages
self._handler = SendClientAsync(
self.target,
auth=self.client.get_auth(),
debug=self.client.debug,
msg_timeout=self.timeout,
error_policy=self.retry_policy,
keep_alive_interval=self.keep_alive,
client_name=self.name,
properties=self.client.create_properties(),
loop=self.loop)
try:
await self._handler.open_async()
self._handler.queue_message(*unsent_events)
await self._handler.wait_async()
except (errors.LinkDetach, errors.ConnectionClose) as shutdown:
if shutdown.action.retry and self.auto_reconnect:
log.info("AsyncSender detached. Attempting reconnect.")
await self.reconnect_async()
else:
log.info("AsyncSender reconnect failed. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
await self.close_async(exception=error)
raise error
except errors.MessageHandlerError as shutdown:
if self.auto_reconnect:
log.info("AsyncSender detached. Attempting reconnect.")
await self.reconnect_async()
else:
log.info("AsyncSender reconnect failed. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
await self.close_async(exception=error)
raise error
except Exception as e:
log.info("Unexpected error occurred (%r). Shutting down.", e)
error = EventHubError("Sender reconnect failed: {}".format(e))
await self.close_async(exception=error)
raise error
def push(self, item):
_item = "{}{}".format(self.serializer(item), self.item_seperator)
if len(_item.encode("utf-8")) > self.max_size:
raise EventHubError(
"Item {} is to big ({}) where as limit is {}. Ignoring.".format(
item["id"], len(_item.encode("utf-8")), self.max_size
)
)
if len((self.buffer + _item).encode("utf-8")) > self.max_size:
self.flush()
self.buffer += _item
self.item_count += 1
async def wait_async(self):
"""
Wait until all transferred events have been sent.
"""
if self.error:
raise self.error
try:
await self._handler.wait_async()
except (errors.LinkDetach, errors.ConnectionClose) as shutdown:
if shutdown.action.retry and self.auto_reconnect:
await self.reconnect_async()
else:
error = EventHubError(str(shutdown), shutdown)
await self.close_async(exception=error)
raise error
except errors.MessageHandlerError as shutdown:
if self.auto_reconnect:
await self.reconnect_async()
else:
error = EventHubError(str(shutdown), shutdown)
await self.close_async(exception=error)
raise error
except Exception as e:
raise EventHubError("Send failed: {}".format(e))
async def has_started(self):
"""
Whether the handler has completed all start up processes such as
establishing the connection, session, link and authentication, and
is not ready to process messages.
:rtype: bool
"""
# pylint: disable=protected-access
timeout = False
auth_in_progress = False
if self._handler._connection.cbs:
timeout, auth_in_progress = await self._handler._auth.handle_token_async()
if timeout:
raise EventHubError("Authorization timeout.")
elif auth_in_progress:
return False
elif not await self._handler._client_ready_async():
return False
else:
return True
async def receive(self):
raise EventHubError("Mock EventHubConsumer EventHubError")
def run():
info("STARTED github to eventhub")
loadConfig()
headers = {"Authorization": "token {}".format(token)}
# monitor = Monitor(debug)
senders = []
for eh_client in eventhubs:
try:
senders.append(
BufferedEventHubSender(
eh_client.add_sender(),
flush_cb=lambda took, count: debug(
"EVENTHUB REQUEST | took: {} sec, sent {} records to {}.".format(took, count, eh_client.eh_name)
),
)
)
failed = eh_client.run()
if failed:
raise EventHubError("Couldn't connect to EH {}".format(eh_client.eh_name))
except EventHubError as e:
error("EventHubError", e.message)
seconds_per_request = round(
1.0 / (5000 / 60 / 60), 2
) # requests / minutes / seconds = requests per sec, ^-1=secs per request
cache = SlidingCache()
events = 0
loop = True
while loop:
loop_start_time = time.time()
# monitor.report()
try:
resp = requests.get(ENDPOINT, headers=headers)
resp.raise_for_status()
# monitor.requests_issued += 1
data = sorted(resp.json(), key=lambda x: x["id"])
payload = ""
# debug("GITHUB REQUEST | took {} sec, got {} events.".format(resp.elapsed.total_seconds(), len(data)))
for d in data:
if d["id"] not in cache:
for buffered_sender in senders:
try:
buffered_sender.push(d)
except EventHubError as e:
error("EventHubError", e.message)
# monitor.events_sent += 1
cache.add(d.get("id"))
cycle_took = time.time() - loop_start_time
delay = seconds_per_request - cycle_took
# debug("CYCLE DONE | took {}, waiting for {}".format(cycle_took, max(delay, 0)))
if delay > 0:
time.sleep(delay)
except requests.HTTPError as e:
if resp.status_code in [429, 403]:
time_to_wait = int(
float(resp.headers.get("X-RateLimit-Reset", 60)) - datetime.datetime.utcnow().timestamp()
)
# info("waiting for {}".format(time_to_wait))
if time_to_wait > 0:
time.sleep(time_to_wait)
error("HTTP EXCEPTION", repr(e))
except EventHubError as e:
error("Failed to send events to eventhub, skipping", repr(e))
except Exception as e:
error("UNEXPECTED ERROR", repr(e))
traceback.print_exc()
def receive(self):
time.sleep(0.5)
raise EventHubError("Mock EventHubConsumer EventHubError")
