def storage_clm(eph):
try:
container = str(uuid.uuid4())
storage_clm = AzureStorageCheckpointLeaseManager(
os.environ['AZURE_STORAGE_ACCOUNT'],
os.environ['AZURE_STORAGE_ACCESS_KEY'], container)
except KeyError:
pytest.skip("Live Storage configuration not found.")
try:
storage_clm.initialize(eph)
storage_clm.storage_client.create_container(container)
yield storage_clm
finally:
storage_clm.storage_client.delete_container(container)
def storage_clm(eph):
try:
container = str(uuid.uuid4())
storage_clm = AzureStorageCheckpointLeaseManager(
os.environ['AZURE_STORAGE_ACCOUNT'],
os.environ['AZURE_STORAGE_ACCESS_KEY'],
container)
except KeyError:
pytest.skip("Live Storage configuration not found.")
try:
storage_clm.initialize(eph)
storage_clm.storage_client.create_container(container)
yield storage_clm
finally:
storage_clm.storage_client.delete_container(container)
def get():
"""
Main method of this module
"""
try:
loop = asyncio.get_event_loop()
# Storage Account Credentials
storage_account_name = os.environ.get('AZURE_STORAGE_ACCOUNT')
storage_key = os.environ.get('AZURE_STORAGE_ACCESS_KEY')
storage_container_lease = os.environ.get('EVENT_HUB_STORAGE_CONTAINER')
# namespace = os.environ.get('EVENT_HUB_NAMESPACE')
# eventhub = os.environ.get('EVENT_HUB_NAME')
# user = os.environ.get('EVENT_HUB_SAS_POLICY')
# key = os.environ.get('EVENT_HUB_SAS_KEY')
# consumer_group = os.environ.get('EVENT_HUB_CONSUMER_GROUP')
namespace = '' # os.environ.get('address')
eventhub = os.environ.get('address')
user = os.environ.get('user')
key = os.environ.get('key')
consumer_group = os.environ.get('consumergroup')
# Eventhub config and storage manager
eh_config = EventHubConfig(namespace,
eventhub,
user,
key,
consumer_group=consumer_group)
pprint(eh_config)
eh_options = EPHOptions()
eh_options.release_pump_on_timeout = True
# eh_options.
eh_options.debug_trace = False
storage_manager = AzureStorageCheckpointLeaseManager(\
storage_account_name, storage_key, storage_container_lease)
# Event loop and host
host = EventProcessorHost(EventProcessor,
eh_config,
storage_manager,
ep_params=[],
eph_options=eh_options,
loop=loop)
tasks = asyncio.gather(host.open_async(), wait_and_close(host))
loop.run_until_complete(tasks)
except KeyboardInterrupt:
# Canceling pending tasks and stopping the loop
for task in asyncio.Task.all_tasks():
task.cancel()
loop.run_forever()
tasks.exception()
finally:
loop.stop()
def storage_clm():
try:
storage_clm = AzureStorageCheckpointLeaseManager(
os.environ['AZURE_STORAGE_ACCOUNT'],
os.environ['AZURE_STORAGE_ACCESS_KEY'], "lease")
return storage_clm
except KeyError:
pytest.skip("Live Storage configuration not found.")
def __init__(self, config):
"""
Class to create an EventHubStreamingClient instance.
:param config: Dictionary file with all the relevant parameters.
"""
super().__init__()
self.message_callback = None
self.config = config
self.storage_account_name = self.config.get("AZURE_STORAGE_ACCOUNT")
self.storage_key = self.config.get("AZURE_STORAGE_ACCESS_KEY")
self.lease_container_name = self.config.get("LEASE_CONTAINER_NAME")
self.namespace = self.config.get("EVENT_HUB_NAMESPACE")
self.eventhub = self.config.get("EVENT_HUB_NAME")
self.consumer_group = self.config.get("EVENT_HUB_CONSUMER_GROUP")
if self.consumer_group is None:
self.consumer_group = '$default'
self.user = self.config.get("EVENT_HUB_SAS_POLICY")
self.key = self.config.get("EVENT_HUB_SAS_KEY")
if self.config.get("TIMEOUT"):
try:
self.timeout = int(self.config.get("TIMEOUT"))
except ValueError:
self.timeout = None
else:
self.timeout = None
# Create EPH Client
if self.storage_account_name is not None and self.storage_key is not None:
self.eph_client = EventHubConfig(
sb_name=self.namespace,
eh_name=self.eventhub,
policy=self.user,
sas_key=self.key,
consumer_group=self.consumer_group)
self.eh_options = EPHOptions()
self.eh_options.release_pump_on_timeout = True
self.eh_options.auto_reconnect_on_error = False
self.eh_options.debug_trace = False
self.storage_manager = AzureStorageCheckpointLeaseManager(
self.storage_account_name, self.storage_key,
self.lease_container_name)
# Create Send client
else:
address = "amqps://" + self.namespace + \
".servicebus.windows.net/" + self.eventhub
try:
self.send_client = EventHubClient(address,
debug=False,
username=self.user,
password=self.key)
self.sender = self.send_client.add_sender()
self.send_client.run()
except Exception as e:
logger.error('Failed to init EH send client: ' + str(e))
raise
def loadStorageManager(self):
try:
storageManager = AzureStorageCheckpointLeaseManager(
storage_account_name=self.storageAccountName,
storage_account_key=self.storageAccountKey,
lease_container_name=self.storageContainer)
except:
raise
return storageManager
def eph():
try:
storage_clm = AzureStorageCheckpointLeaseManager(
os.environ['AZURE_STORAGE_ACCOUNT'],
os.environ['AZURE_STORAGE_ACCESS_KEY'], "lease")
NAMESPACE = os.environ.get('EVENT_HUB_NAMESPACE')
EVENTHUB = os.environ.get('EVENT_HUB_NAME')
USER = os.environ.get('EVENT_HUB_SAS_POLICY')
KEY = os.environ.get('EVENT_HUB_SAS_KEY')
eh_config = EventHubConfig(NAMESPACE,
EVENTHUB,
USER,
KEY,
consumer_group="$default")
host = EventProcessorHost(MockEventProcessor, eh_config, storage_clm)
except KeyError:
pytest.skip("Live EventHub configuration not found.")
return host
def consume(self):
try:
loop = asyncio.get_event_loop()
# Eventhub config and storage manager
eh_config = EventHubConfig(self.namespace,
self.evenhub,
self.user,
self.key,
consumer_group=self.consumer_group)
eh_options = EPHOptions()
eh_options.release_pump_on_timeout = False
eh_options.debug_trace = False
if self._is_storage_checkpoint_enabled():
storage_manager = AzureStorageCheckpointLeaseManager(
self.storage_account, self.storage_key,
self.storage_container)
else:
storage_manager = DummyStorageCheckpointLeaseManager()
# Event loop and host
host = EventProcessorHost(EventProcessor,
eh_config,
storage_manager,
ep_params=[self.on_receive_callback],
eph_options=eh_options,
loop=loop)
tasks = asyncio.gather(host.open_async(), wait_and_close(host))
loop.run_until_complete(tasks)
except KeyboardInterrupt:
# Canceling pending tasks and stopping the loop
for task in asyncio.Task.all_tasks():
task.cancel()
loop.run_forever()
tasks.exception()
finally:
loop.stop()
def __init__(self, config): # pragma: no cover
"""
Azure EventHub streaming client implementation.
Configuration keys:
AZURE_STORAGE_ACCESS_KEY
AZURE_STORAGE_ACCOUNT
EVENT_HUB_CONSUMER_GROUP
EVENT_HUB_NAME
EVENT_HUB_NAMESPACE
EVENT_HUB_SAS_KEY
EVENT_HUB_SAS_POLICY
LEASE_CONTAINER_NAME
TIMEOUT
"""
storage_account_name = config.get("AZURE_STORAGE_ACCOUNT")
storage_key = config.get("AZURE_STORAGE_ACCESS_KEY")
lease_container_name = config.get("LEASE_CONTAINER_NAME")
namespace = config.get("EVENT_HUB_NAMESPACE")
eventhub = config.get("EVENT_HUB_NAME")
consumer_group = config.get("EVENT_HUB_CONSUMER_GROUP", '$Default')
user = config.get("EVENT_HUB_SAS_POLICY")
key = config.get("EVENT_HUB_SAS_KEY")
try:
self.timeout = int(config['TIMEOUT'])
except (KeyError, ValueError):
self.timeout = None
self.logger = Logger()
self.loop = None
# Create EPH Client
if storage_account_name is not None and storage_key is not None:
self.eph_client = EventHubConfig(sb_name=namespace,
eh_name=eventhub,
policy=user,
sas_key=key,
consumer_group=consumer_group)
self.eh_options = EPHOptions()
self.eh_options.release_pump_on_timeout = True
self.eh_options.auto_reconnect_on_error = False
self.eh_options.debug_trace = False
self.storage_manager = AzureStorageCheckpointLeaseManager(
storage_account_name, storage_key, lease_container_name)
self.tasks = None
signal.signal(signal.SIGTERM, self.exit_gracefully)
# Create Send client
else:
address = "amqps://" + namespace + \
".servicebus.windows.net/" + eventhub
try:
self.send_client = EventHubClient(address,
debug=False,
username=user,
password=key)
self.sender = self.send_client.add_sender()
self.send_client.run()
except Exception as ex:
self.logger.error('Failed to init EH send client: %s', ex)
raise
# Storage Account Credentials
STORAGE_ACCOUNT_NAME = os.environ.get('AZURE_STORAGE_ACCOUNT')
STORAGE_KEY = os.environ.get('AZURE_STORAGE_ACCESS_KEY')
LEASE_CONTAINER_NAME = "leases"
NAMESPACE = os.environ.get('EVENT_HUB_NAMESPACE')
EVENTHUB = os.environ.get('EVENT_HUB_NAME')
USER = os.environ.get('EVENT_HUB_SAS_POLICY')
KEY = os.environ.get('EVENT_HUB_SAS_KEY')
# Eventhub config and storage manager
eh_config = EventHubConfig(NAMESPACE, EVENTHUB, USER, KEY, consumer_group="$default")
eh_options = EPHOptions()
eh_options.release_pump_on_timeout = True
eh_options.debug_trace = False
storage_manager = AzureStorageCheckpointLeaseManager(
STORAGE_ACCOUNT_NAME, STORAGE_KEY, LEASE_CONTAINER_NAME)
# Event loop and host
host = EventProcessorHost(
EventProcessor,
eh_config,
storage_manager,
ep_params=["param1","param2"],
eph_options=eh_options,
loop=loop)
tasks = asyncio.gather(
host.open_async(),
wait_and_close(host))
loop.run_until_complete(tasks)
"""
ehConfig = EventHubConfig(ehNamespace, ehName, SASUser, SASKey)
# Opciones por default
ehOptions = EPHOptions()
# Set algunas opciones
ehOptions.release_pump_on_timeout = True
ehOptions.debug_trace = False
"""
Configuración del Storage
Párametros:
storage_account_name = Nombre del storage
storage_account_key = Llave del storage
lease_container_name = Nombre del contenedor
"""
stgManager = AzureStorageCheckpointLeaseManager(stgName, stgKey, blobName)
# Host del Event Hub Processor
ehHost = EventProcessorHost(EventProcessor,
ehConfig,
stgManager,
ep_params=["param1", "param2"],
eph_options=ehOptions,
loop=ephLoop)
# Prepara los procedimientos a ejecutar en loop
ephTasks = asyncio.gather(ehHost.open_async(), wait_and_close(ehHost))
# Corre el loop
ephLoop.run_until_complete(ephTasks)
def test_long_running_eph():
parser = argparse.ArgumentParser()
parser.add_argument("--duration",
help="Duration in seconds of the test",
type=int,
default=30)
parser.add_argument("--storage-account",
help="Storage account name",
default=os.environ.get('AZURE_STORAGE_ACCOUNT'))
parser.add_argument("--storage-key",
help="Storage account access key",
default=os.environ.get('AZURE_STORAGE_ACCESS_KEY'))
parser.add_argument("--container",
help="Lease container name",
default="leases")
parser.add_argument("--eventhub",
help="Name of EventHub",
default=os.environ.get('EVENT_HUB_NAME'))
parser.add_argument("--namespace",
help="Namespace of EventHub",
default=os.environ.get('EVENT_HUB_NAMESPACE'))
parser.add_argument("--suffix",
help="Namespace of EventHub",
default="servicebus.windows.net")
parser.add_argument(
"--sas-policy",
help="Name of the shared access policy to authenticate with",
default=os.environ.get('EVENT_HUB_SAS_POLICY'))
parser.add_argument("--sas-key",
help="Shared access key",
default=os.environ.get('EVENT_HUB_SAS_KEY'))
loop = asyncio.get_event_loop()
args, _ = parser.parse_known_args()
if not args.namespace or not args.eventhub:
try:
import pytest
pytest.skip("Must specify '--namespace' and '--eventhub'")
except ImportError:
raise ValueError("Must specify '--namespace' and '--eventhub'")
# Eventhub config and storage manager
eh_config = EventHubConfig(args.namespace,
args.eventhub,
args.sas_policy,
args.sas_key,
consumer_group="$default",
namespace_suffix=args.suffix)
eh_options = EPHOptions()
eh_options.release_pump_on_timeout = True
eh_options.debug_trace = False
eh_options.receive_timeout = 120
storage_manager = AzureStorageCheckpointLeaseManager(
storage_account_name=args.storage_account,
storage_account_key=args.storage_key,
lease_renew_interval=30,
lease_container_name=args.container,
lease_duration=60)
# Event loop and host
host = EventProcessorHost(EventProcessor,
eh_config,
storage_manager,
ep_params=["param1", "param2"],
eph_options=eh_options,
loop=loop)
tasks = asyncio.gather(host.open_async(),
wait_and_close(host, args.duration),
return_exceptions=True)
results = loop.run_until_complete(tasks)
assert not any(results)
async def test_long_running_context_eph(live_eventhub):
parser = argparse.ArgumentParser()
parser.add_argument("--duration",
help="Duration in seconds of the test",
type=int,
default=30)
parser.add_argument("--storage-account",
help="Storage account name",
default=os.environ.get('AZURE_STORAGE_ACCOUNT'))
parser.add_argument("--storage-key",
help="Storage account access key",
default=os.environ.get('AZURE_STORAGE_ACCESS_KEY'))
parser.add_argument("--container",
help="Lease container name",
default="contextleases")
parser.add_argument("--eventhub",
help="Name of EventHub",
default=live_eventhub['event_hub'])
parser.add_argument("--namespace",
help="Namespace of EventHub",
default=live_eventhub['namespace'])
parser.add_argument("--suffix",
help="Namespace of EventHub",
default="servicebus.windows.net")
parser.add_argument(
"--sas-policy",
help="Name of the shared access policy to authenticate with",
default=live_eventhub['key_name'])
parser.add_argument("--sas-key",
help="Shared access key",
default=live_eventhub['access_key'])
loop = asyncio.get_event_loop()
args, _ = parser.parse_known_args()
if not args.namespace or not args.eventhub:
try:
import pytest
pytest.skip("Must specify '--namespace' and '--eventhub'")
except ImportError:
raise ValueError("Must specify '--namespace' and '--eventhub'")
# Queue up some events in the Eventhub
conn_str = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}".format(
live_eventhub['hostname'], live_eventhub['key_name'],
live_eventhub['access_key'], live_eventhub['event_hub'])
send_client = EventHubClient.from_connection_string(conn_str)
pumps = []
for pid in ["0", "1"]:
sender = send_client.create_producer(partition_id=pid, send_timeout=0)
pumps.append(pump(pid, sender, 15))
results = await asyncio.gather(*pumps, return_exceptions=True)
assert not any(results)
# Eventhub config and storage manager
eh_config = EventHubConfig(args.namespace,
args.eventhub,
args.sas_policy,
args.sas_key,
consumer_group="$default",
namespace_suffix=args.suffix)
eh_options = EPHOptions()
eh_options.release_pump_on_timeout = True
eh_options.debug_trace = False
eh_options.receive_timeout = 120
storage_manager = AzureStorageCheckpointLeaseManager(
storage_account_name=args.storage_account,
storage_account_key=args.storage_key,
lease_renew_interval=30,
lease_container_name=args.container,
lease_duration=60)
# Event loop and host
host = EventProcessorHost(EventProcessor,
eh_config,
storage_manager,
ep_params=["param1", "param2"],
eph_options=eh_options,
loop=loop)
tasks = asyncio.gather(host.open_async(),
wait_and_close(host, args.duration),
return_exceptions=True)
results = await tasks
assert not any(results)
EPH_Config = EventHubConfig(Credentials.EH_Namespace, Credentials.EH_Name,
Credentials.EH_SASUser, Credentials.EH_SASKey)
# Opciones por default
EPH_Options = EPHOptions()
# Set algunas opciones
EPH_Options.release_pump_on_timeout = True
EPH_Options.debug_trace = False
"""
Configuración del Storage
Párametros:
lease_container_name = Nombre del contenedor
connection_string = Link de conexión al storage account
"""
STG_Manager = AzureStorageCheckpointLeaseManager(
lease_container_name=Credentials.STG_BlobName,
connection_string=Credentials.STG_ConnectionString)
# while True:
# Regresa un loop asincrono
EPH_Loop = asyncio.get_event_loop()
# Host del Event Hub Processor
EPH_Host = EventProcessorHost(EventProcessor,
EPH_Config,
STG_Manager,
ep_params=["param1", "param2"],
eph_options=EPH_Options,
loop=EPH_Loop)
# Prepara los procedimientos a ejecutar en loop