def __init__(self, storage_account_name, storage_account_key, lease_container_name,
storage_blob_prefix=None, lease_renew_interval=10, lease_duration=30):
AbstractCheckpointManager.__init__(self)
AbstractLeaseManager.__init__(self, lease_renew_interval, lease_duration)
self.storage_account_name = storage_account_name
self.storage_account_key = storage_account_key
self.lease_container_name = lease_container_name
self.storage_blob_prefix = storage_blob_prefix
self.storage_client = None
self.consumer_group_directory = None
self.host = None
self.storage_max_execution_time = 120
self.request_session = requests.Session()
self.request_session.mount('https://', requests.adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100))
self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=32)
# Validate storage inputs
if not self.storage_account_name or not self.storage_account_key:
raise ValueError("Need a valid storage account name and key")
if not re.compile(r"^[a-z0-9](([a-z0-9\-[^\-])){1,61}[a-z0-9]$").match(self.lease_container_name):
raise ValueError("Azure Storage lease container name is invalid.\
Please check naming conventions at\
https:# msdn.microsoft.com/en-us/library/azure/dd135715.aspx")
if self.storage_blob_prefix:
self.storage_blob_prefix.replace(" ", "") # Convert all-whitespace to empty string.
else:
self.storage_blob_prefix = "" # Convert null prefix to empty string.
def __init__(self, storage_account_name=None, storage_account_key=None, lease_container_name="eph-leases",
storage_blob_prefix=None, lease_renew_interval=10, lease_duration=30,
sas_token=None, endpoint_suffix="core.windows.net", connection_string=None):
AbstractCheckpointManager.__init__(self)
AbstractLeaseManager.__init__(self, lease_renew_interval, lease_duration)
self.storage_account_name = storage_account_name
self.storage_account_key = storage_account_key
self.storage_sas_token = sas_token
self.endpoint_suffix = endpoint_suffix
self.connection_string = connection_string
self.lease_container_name = lease_container_name
self.storage_blob_prefix = storage_blob_prefix
self.storage_client = None
self.consumer_group_directory = None
self.host = None
self.storage_max_execution_time = 120
self.request_session = requests.Session()
self.request_session.mount('https://', requests.adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100))
self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=32)
# Validate storage inputs
if not self.storage_account_name and not self.connection_string:
raise ValueError("Need a valid storage account name or connection string.")
if not re.compile(r"^[a-z0-9](([a-z0-9\-[^\-])){1,61}[a-z0-9]$").match(self.lease_container_name):
raise ValueError("Azure Storage lease container name is invalid.\
Please check naming conventions at\
https:# msdn.microsoft.com/en-us/library/azure/dd135715.aspx")
if self.storage_blob_prefix:
self.storage_blob_prefix.replace(" ", "") # Convert all-whitespace to empty string.
else:
self.storage_blob_prefix = "" # Convert null prefix to empty string.
def __init__(self):
AbstractCheckpointManager.__init__(self)
AbstractLeaseManager.__init__(self,
lease_renew_interval=10,
lease_duration=30)
self.checkpoints = defaultdict(Checkpoint)
self.leases = defaultdict(DummyLease)