def invoke(self, name, payload, invocation_type='RequestResponse'):
"""
Invokes a given lambda
:param name: name of the lambda function
:param payload: lambda event
:param invocation_type: default requestresponse
:return: response
"""
assert util.is_not_empty(name), \
"Lambda Name cannot be empty."
assert util.is_not_empty(payload), \
"Lambda Payload cannot be empty."
l_client = self.get_or_create_client()
response = l_client.invoke(FunctionName=name,
InvocationType=invocation_type,
Payload=payload)
if invocation_type == 'Event':
return
payload_str = response['Payload'].read().decode('utf-8')
payload = literal_eval(payload_str)
return payload
def get_aws_client(service_name,
region_name=None,
aws_access_key_id=None,
aws_secret_access_key=None,
aws_session_token=None,
endpoint_url=None,
config=None):
"""
Utility method to get aws (boto3) client instance.
:param config:
:param aws_session_token:
:param endpoint_url:
:param aws_secret_access_key:
:param aws_access_key_id:
:param region_name:
:param service_name:
:param region:
:return: client
"""
assert util.is_not_empty(region_name), \
"AWS Region cannot be empty."
assert util.is_not_empty(service_name), \
"client_type cannot be empty."
from boto3 import client as boto3_client
return boto3_client(service_name,
region_name=region_name,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
endpoint_url=endpoint_url,
config=config)
def __init__(self, stream_name=None, stream_client=None):
assert is_not_empty(stream_name), \
"Firehose delivery stream name cannot be empty."
assert is_not_empty(stream_client), \
"Stream Client not provided."
self.stream_name = stream_name
self.stream_client = stream_client
def __handle_err(self, err_code, err_msg, **kwargs):
if not ENABLE_ALERTS_HANDLING:
return
assert is_not_empty(err_code) and is_not_empty(err_msg), \
"'err_code', 'err_msg' can not be empty."
alert_payload = self.create_alert_payload(err_code, err_msg, **kwargs)
alert_payload_str = dumps(alert_payload) + '\n'
self.stream_client.write_record(stream_name=self.stream_name,
record=alert_payload_str.encode())
def write_record(self, stream_name=None, partition_key=None, record=None):
"""
:param record: list of kinesis Records, batch to be published to kinesis
:param stream_name: kinesis stream name to publish to.
:return:
"""
assert util.is_not_empty(stream_name), \
"Kinesis Stream Name cannot be empty."
assert record is not None, \
"Record is empty, No records to persist."
if not partition_key:
partition_key = str(hash(record))
def put_record(stream, rec):
k_client = self.get_or_create_client()
return k_client.put_record(Data=rec,
StreamName=stream,
PartitionKey=partition_key)
try:
return put_record(stream_name, record)
except Exception:
self.client = None
return put_record(stream_name, record)
def check_lambda_exist(self, lambda_name):
assert util.is_not_empty(lambda_name), \
"lamdba name cannot be empty."
l_client = self.get_or_create_client()
exist = False
try:
resp = l_client.get_function(FunctionName=lambda_name)
return resp is not None
except Exception as e:
self.logger.exception(str(e))
return exist
def write_record(self, stream_name=None, record=None):
"""
:param stream_name: firehose stream name to publish to.
:param record: , data in bytes to be published to firehose
:return: firehose response
"""
assert util.is_not_empty(stream_name), \
"Kinesis Firehose StreamName cannot be empty."
assert record is not None, \
"Record cannot be empty."
f_client = self.get_or_create_client()
return f_client.put_record(DeliveryStreamName=stream_name,
Record={'Data': record})
def write_batch(self, stream_name=None, batch=None):
"""
:param stream_name: firehose stream name to publish to.
:param data_bytes: , data in bytes to be published to firehose
:return: firehose response
"""
assert util.is_not_empty(stream_name), \
"Kinesis Firehose StreamName cannot be empty."
assert batch and len(batch) != 0, \
"Batch cannot be empty."
f_client = self.get_or_create_client()
record_batch = [{'Data': data} for data in batch]
return f_client.put_record_batch(DeliveryStreamName=stream_name,
Records=record_batch)
def is_stream_exist(self, stream_name):
"""
:param stream_name:
:return:
"""
assert util.is_not_empty(stream_name), \
"kinesis stream name cannot be empty."
k_client = self.get_or_create_client()
exist = False
try:
resp = k_client.describe_stream(StreamName=stream_name, Limit=1)
return resp is not None
except Exception as e:
self.logger.exception(str(e))
return exist
def write_batch(self, stream_name=None, batch=None):
"""
:param batch: list of kinesis Records, batch to be published to kinesis
:param stream_name: kinesis stream name to publish to.
:return:
"""
assert util.is_not_empty(stream_name), \
"Kinesis Stream Name cannot be empty."
assert batch and len(batch) != 0, \
"Record list is empty, No records to persist."
def put_records(stream, records):
k_client = self.get_or_create_client()
return k_client.put_records(Records=records, StreamName=stream)
try:
return put_records(stream_name, batch)
except Exception as e:
self.logger.warning(str(e))
self.client = None
return put_records(stream_name, batch)
def test_is_not_empty(self):
self.assertFalse(util.is_not_empty(None))
self.assertFalse(util.is_not_empty(''))
self.assertFalse(util.is_not_empty(' '))
self.assertFalse(util.is_not_empty({}))
self.assertFalse(util.is_not_empty([]))
self.assertTrue(util.is_not_empty(False))
self.assertTrue(util.is_not_empty(' A '))
self.assertTrue(util.is_not_empty([1]))
self.assertTrue(util.is_not_empty({"key": None}))
self.assertTrue(util.is_not_empty(1))
self.assertTrue(util.is_not_empty(1.000))
self.assertTrue(util.is_not_empty(0))
self.assertTrue(util.is_not_empty(0.0))
def is_present(self) -> bool:
return is_not_empty(self._value)