def start_receiving(self, on_message_received_callback): # pragma: no cover self.loop = asyncio.get_event_loop() try: host = EventProcessorHost(EventProcessor, self.eph_client, self.storage_manager, ep_params=[on_message_received_callback], eph_options=self.eh_options, loop=self.loop) self.tasks = asyncio.gather( host.open_async(), self.wait_and_close(host, self.timeout)) self.loop.run_until_complete(self.tasks) except KeyboardInterrupt: self.logger.info( "Handling keyboard interrupt or SIGINT gracefully.") # Canceling pending tasks and stopping the loop for task in asyncio.Task.all_tasks(): task.cancel() self.loop.run_forever() self.tasks.exception() raise finally: if self.loop.is_running(): self.loop.stop()
def get(): """ Main method of this module """ try: loop = asyncio.get_event_loop() # Storage Account Credentials storage_account_name = os.environ.get('AZURE_STORAGE_ACCOUNT') storage_key = os.environ.get('AZURE_STORAGE_ACCESS_KEY') storage_container_lease = os.environ.get('EVENT_HUB_STORAGE_CONTAINER') # namespace = os.environ.get('EVENT_HUB_NAMESPACE') # eventhub = os.environ.get('EVENT_HUB_NAME') # user = os.environ.get('EVENT_HUB_SAS_POLICY') # key = os.environ.get('EVENT_HUB_SAS_KEY') # consumer_group = os.environ.get('EVENT_HUB_CONSUMER_GROUP') namespace = '' # os.environ.get('address') eventhub = os.environ.get('address') user = os.environ.get('user') key = os.environ.get('key') consumer_group = os.environ.get('consumergroup') # Eventhub config and storage manager eh_config = EventHubConfig(namespace, eventhub, user, key, consumer_group=consumer_group) pprint(eh_config) eh_options = EPHOptions() eh_options.release_pump_on_timeout = True # eh_options. eh_options.debug_trace = False storage_manager = AzureStorageCheckpointLeaseManager(\ storage_account_name, storage_key, storage_container_lease) # Event loop and host host = EventProcessorHost(EventProcessor, eh_config, storage_manager, ep_params=[], eph_options=eh_options, loop=loop) tasks = asyncio.gather(host.open_async(), wait_and_close(host)) loop.run_until_complete(tasks) except KeyboardInterrupt: # Canceling pending tasks and stopping the loop for task in asyncio.Task.all_tasks(): task.cancel() loop.run_forever() tasks.exception() finally: loop.stop()
def run(self): try: print('Loading EventHub Config...') ehConfig = self.loadEventHubConfig() print('Loading Storage Manager...') storageManager = self.loadStorageManager() print('Clear Storage Old Data...') self.clearStorageOldData() print('Loading Event Host Options...') ehOptions = self.loadEventHostOptions() except Exception as ex: print('Exception on loading config. Error:' + str(ex)) return try: # Event loop and host print('Start Event Processor Host Loop...') loop = asyncio.get_event_loop() host = EventProcessorHost( EventProcessor, ehConfig, storageManager, ep_params=[self.webAppURL,self.rtMessageRoomId], eph_options=ehOptions, loop=loop) # Below line was must for China Azure storageManager.storage_client = BlockBlobService(account_name=self.storageAccountName, account_key=self.storageAccountKey, endpoint_suffix=self.storageEndpointSuffix) tasks = asyncio.gather( host.open_async(), noneStop(host)) loop.run_until_complete(tasks) except Exception as ex: # Canceling pending tasks and stopping the loop print('Exception, leave loop. Error:' + str(ex)) for task in asyncio.Task.all_tasks(): task.cancel() loop.run_forever() tasks.exception() finally: loop.stop()
def start_receiving(self, on_message_received_callback): # pragma: no cover loop = asyncio.get_event_loop() try: host = EventProcessorHost( EventProcessor, self.eph_client, self.storage_manager, ep_params=[on_message_received_callback], eph_options=self.eh_options, loop=loop) tasks = asyncio.gather(host.open_async(), self.wait_and_close(host, self.timeout)) loop.run_until_complete(tasks) finally: loop.stop()
def eph(): try: storage_clm = AzureStorageCheckpointLeaseManager( os.environ['AZURE_STORAGE_ACCOUNT'], os.environ['AZURE_STORAGE_ACCESS_KEY'], "lease") NAMESPACE = os.environ.get('EVENT_HUB_NAMESPACE') EVENTHUB = os.environ.get('EVENT_HUB_NAME') USER = os.environ.get('EVENT_HUB_SAS_POLICY') KEY = os.environ.get('EVENT_HUB_SAS_KEY') eh_config = EventHubConfig(NAMESPACE, EVENTHUB, USER, KEY, consumer_group="$default") host = EventProcessorHost(MockEventProcessor, eh_config, storage_clm) except KeyError: pytest.skip("Live EventHub configuration not found.") return host
def consume(self): try: loop = asyncio.get_event_loop() # Eventhub config and storage manager eh_config = EventHubConfig(self.namespace, self.evenhub, self.user, self.key, consumer_group=self.consumer_group) eh_options = EPHOptions() eh_options.release_pump_on_timeout = False eh_options.debug_trace = False if self._is_storage_checkpoint_enabled(): storage_manager = AzureStorageCheckpointLeaseManager( self.storage_account, self.storage_key, self.storage_container) else: storage_manager = DummyStorageCheckpointLeaseManager() # Event loop and host host = EventProcessorHost(EventProcessor, eh_config, storage_manager, ep_params=[self.on_receive_callback], eph_options=eh_options, loop=loop) tasks = asyncio.gather(host.open_async(), wait_and_close(host)) loop.run_until_complete(tasks) except KeyboardInterrupt: # Canceling pending tasks and stopping the loop for task in asyncio.Task.all_tasks(): task.cancel() loop.run_forever() tasks.exception() finally: loop.stop()
def start_receiving(self, on_message_received_callback): """ Receive messages from an EventHubStreamingClient. :param on_message_received_callback: Callback function. """ loop = asyncio.get_event_loop() try: host = EventProcessorHost(EventProcessor, self.eph_client, self.storage_manager, ep_params=[on_message_received_callback], eph_options=self.eh_options, loop=loop) tasks = asyncio.gather(host.open_async(), self.wait_and_close(host, self.timeout)) loop.run_until_complete(tasks) finally: loop.stop()
def run(self): try: print('Loading EventHub Config...') ehConfig = self.loadEventHubConfig() print('Loading Storage Manager...') storageManager = self.loadStorageManager() print('Clear Storage Old Data...') self.clearStorageOldData() print('Loading Event Host Options...') ehOptions = self.loadEventHostOptions() except Exception as ex: print('Exception on loading config. Error:' + str(ex)) return try: # Event loop and host print('Start Event Processor Host Loop...') loop = asyncio.get_event_loop() host = EventProcessorHost(EventProcessor, ehConfig, storageManager, ep_params=["param1", "param2"], eph_options=ehOptions, loop=loop) tasks = asyncio.gather(host.open_async(), noneStop(host)) loop.run_until_complete(tasks) except Exception as ex: # Canceling pending tasks and stopping the loop print('Exception, leave loop. Error:' + str(ex)) for task in asyncio.Task.all_tasks(): task.cancel() loop.run_forever() tasks.exception() finally: loop.stop()
USER = os.environ.get('EVENT_HUB_SAS_POLICY') KEY = os.environ.get('EVENT_HUB_SAS_KEY') # Eventhub config and storage manager eh_config = EventHubConfig(NAMESPACE, EVENTHUB, USER, KEY, consumer_group="$default") eh_options = EPHOptions() eh_options.release_pump_on_timeout = True eh_options.debug_trace = False storage_manager = AzureStorageCheckpointLeaseManager( STORAGE_ACCOUNT_NAME, STORAGE_KEY, LEASE_CONTAINER_NAME) # Event loop and host host = EventProcessorHost( EventProcessor, eh_config, storage_manager, ep_params=["param1","param2"], eph_options=eh_options, loop=loop) tasks = asyncio.gather( host.open_async(), wait_and_close(host)) loop.run_until_complete(tasks) except KeyboardInterrupt: # Canceling pending tasks and stopping the loop for task in asyncio.Task.all_tasks(): task.cancel() loop.run_forever() tasks.exception()
# Set algunas opciones ehOptions.release_pump_on_timeout = True ehOptions.debug_trace = False """ Configuración del Storage Párametros: storage_account_name = Nombre del storage storage_account_key = Llave del storage lease_container_name = Nombre del contenedor """ stgManager = AzureStorageCheckpointLeaseManager(stgName, stgKey, blobName) # Host del Event Hub Processor ehHost = EventProcessorHost(EventProcessor, ehConfig, stgManager, ep_params=["param1", "param2"], eph_options=ehOptions, loop=ephLoop) # Prepara los procedimientos a ejecutar en loop ephTasks = asyncio.gather(ehHost.open_async(), wait_and_close(ehHost)) # Corre el loop ephLoop.run_until_complete(ephTasks) # En caso de ocurrri excepciones de teclado except KeyboardInterrupt: # Cancela las tareas y el loop for task in asyncio.Task.all_tasks(): task.cancel() ephLoop.run_forever()
def test_long_running_eph(): parser = argparse.ArgumentParser() parser.add_argument("--duration", help="Duration in seconds of the test", type=int, default=30) parser.add_argument("--storage-account", help="Storage account name", default=os.environ.get('AZURE_STORAGE_ACCOUNT')) parser.add_argument("--storage-key", help="Storage account access key", default=os.environ.get('AZURE_STORAGE_ACCESS_KEY')) parser.add_argument("--container", help="Lease container name", default="leases") parser.add_argument("--eventhub", help="Name of EventHub", default=os.environ.get('EVENT_HUB_NAME')) parser.add_argument("--namespace", help="Namespace of EventHub", default=os.environ.get('EVENT_HUB_NAMESPACE')) parser.add_argument("--suffix", help="Namespace of EventHub", default="servicebus.windows.net") parser.add_argument( "--sas-policy", help="Name of the shared access policy to authenticate with", default=os.environ.get('EVENT_HUB_SAS_POLICY')) parser.add_argument("--sas-key", help="Shared access key", default=os.environ.get('EVENT_HUB_SAS_KEY')) loop = asyncio.get_event_loop() args, _ = parser.parse_known_args() if not args.namespace or not args.eventhub: try: import pytest pytest.skip("Must specify '--namespace' and '--eventhub'") except ImportError: raise ValueError("Must specify '--namespace' and '--eventhub'") # Eventhub config and storage manager eh_config = EventHubConfig(args.namespace, args.eventhub, args.sas_policy, args.sas_key, consumer_group="$default", namespace_suffix=args.suffix) eh_options = EPHOptions() eh_options.release_pump_on_timeout = True eh_options.debug_trace = False eh_options.receive_timeout = 120 storage_manager = AzureStorageCheckpointLeaseManager( storage_account_name=args.storage_account, storage_account_key=args.storage_key, lease_renew_interval=30, lease_container_name=args.container, lease_duration=60) # Event loop and host host = EventProcessorHost(EventProcessor, eh_config, storage_manager, ep_params=["param1", "param2"], eph_options=eh_options, loop=loop) tasks = asyncio.gather(host.open_async(), wait_and_close(host, args.duration), return_exceptions=True) results = loop.run_until_complete(tasks) assert not any(results)
async def test_long_running_context_eph(live_eventhub): parser = argparse.ArgumentParser() parser.add_argument("--duration", help="Duration in seconds of the test", type=int, default=30) parser.add_argument("--storage-account", help="Storage account name", default=os.environ.get('AZURE_STORAGE_ACCOUNT')) parser.add_argument("--storage-key", help="Storage account access key", default=os.environ.get('AZURE_STORAGE_ACCESS_KEY')) parser.add_argument("--container", help="Lease container name", default="contextleases") parser.add_argument("--eventhub", help="Name of EventHub", default=live_eventhub['event_hub']) parser.add_argument("--namespace", help="Namespace of EventHub", default=live_eventhub['namespace']) parser.add_argument("--suffix", help="Namespace of EventHub", default="servicebus.windows.net") parser.add_argument( "--sas-policy", help="Name of the shared access policy to authenticate with", default=live_eventhub['key_name']) parser.add_argument("--sas-key", help="Shared access key", default=live_eventhub['access_key']) loop = asyncio.get_event_loop() args, _ = parser.parse_known_args() if not args.namespace or not args.eventhub: try: import pytest pytest.skip("Must specify '--namespace' and '--eventhub'") except ImportError: raise ValueError("Must specify '--namespace' and '--eventhub'") # Queue up some events in the Eventhub conn_str = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}".format( live_eventhub['hostname'], live_eventhub['key_name'], live_eventhub['access_key'], live_eventhub['event_hub']) send_client = EventHubClient.from_connection_string(conn_str) pumps = [] for pid in ["0", "1"]: sender = send_client.create_producer(partition_id=pid, send_timeout=0) pumps.append(pump(pid, sender, 15)) results = await asyncio.gather(*pumps, return_exceptions=True) assert not any(results) # Eventhub config and storage manager eh_config = EventHubConfig(args.namespace, args.eventhub, args.sas_policy, args.sas_key, consumer_group="$default", namespace_suffix=args.suffix) eh_options = EPHOptions() eh_options.release_pump_on_timeout = True eh_options.debug_trace = False eh_options.receive_timeout = 120 storage_manager = AzureStorageCheckpointLeaseManager( storage_account_name=args.storage_account, storage_account_key=args.storage_key, lease_renew_interval=30, lease_container_name=args.container, lease_duration=60) # Event loop and host host = EventProcessorHost(EventProcessor, eh_config, storage_manager, ep_params=["param1", "param2"], eph_options=eh_options, loop=loop) tasks = asyncio.gather(host.open_async(), wait_and_close(host, args.duration), return_exceptions=True) results = await tasks assert not any(results)
Párametros: lease_container_name = Nombre del contenedor connection_string = Link de conexión al storage account """ STG_Manager = AzureStorageCheckpointLeaseManager( lease_container_name=Credentials.STG_BlobName, connection_string=Credentials.STG_ConnectionString) # while True: # Regresa un loop asincrono EPH_Loop = asyncio.get_event_loop() # Host del Event Hub Processor EPH_Host = EventProcessorHost(EventProcessor, EPH_Config, STG_Manager, ep_params=["param1", "param2"], eph_options=EPH_Options, loop=EPH_Loop) # Prepara los procedimientos a ejecutar en loop EPH_Tasks = asyncio.gather(EPH_Host.open_async(), wait_and_close(EPH_Host)) # Corre el loop EPH_Loop.run_until_complete(EPH_Tasks) # En caso de ocurrri excepciones de teclado except KeyboardInterrupt: # Cancela las tareas y el loop for task in asyncio.Task.all_tasks(): task.cancel() EPH_Loop.run_forever()