def tag_file_as_processed(bucket_name: str, key_prefix: str):
"""
This method appends processed status tags. This utility method can be called once the files are processed by
the respective processors
"""
s3 = get_service_client("s3")
# get existing tags
tags = s3.get_object_tagging(Bucket=bucket_name, Key=key_prefix)["TagSet"]
logger.debug(f"Old tags retrieved are: {tags}")
# merge old tags and new tags
tags.extend([
{
"Key": "processed_on",
"Value": datetime.now(timezone.utc).strftime(TIMESTAMP_FORMAT)
},
{
"Key": "processing_status",
"Value": "COMPLETE"
},
])
s3.put_object_tagging(
Bucket=bucket_name,
Key=key_prefix,
Tagging={"TagSet": tags},
)
def publish_config(config_event: ConfigEvent, event_bus=None):
if not event_bus:
event_bus = service_helper.get_service_client("events")
for url in config_event.url_list:
event = {
"platform": config_event.platform,
"account": config_event.account,
"query": config_event.query,
"url": url,
}
if config_event.topic:
event["topic"] = config_event.topic
service_response = event_bus.put_events(
Entries=[
{
"EventBusName": os.environ["EVENT_BUS_NAME"],
"Source": os.environ["INGESTION_NAMESPACE"],
"Detail": json.dumps(event),
"DetailType": "config",
}
]
)
if service_response["FailedEntryCount"]:
logger.error(f"Failed to publish following event: {event}")
logger.info(f"Published event {event} on event bus with response {service_response}")
def update_query_timestamp(video_id):
ddb = get_service_client("dynamodb")
table_name = os.environ["TARGET_DDB_TABLE"]
current_time = datetime.now(timezone.utc)
# defaulting to 7 days if ingestion window is not provided
expiry_window = str(
int((current_time + timedelta(days=int(
os.environ.get("VIDEO_SEARCH_INGESTION_WINDOW", 7)))).timestamp() *
1000))
ddb_response = ddb.put_item(
TableName=table_name,
Item={
"VIDEO_ID": {
"S": video_id
},
"LAST_QUERIED_TIMESTAMP": {
"S": current_time.isoformat()
},
"EXP_DATE": {
"N": expiry_window
},
},
)
logger.debug(
f"Response from ddb transaction write: {json.dumps(ddb_response)}")
def get_event_bus_stubber():
global event_bus_stubber
if not event_bus_stubber:
event_bus_client = get_service_client("events")
event_bus_stubber = Stubber(event_bus_client)
return event_bus_stubber
def get_event_bus_stubber():
global event_bus_stubber
from shared_util.service_helper import get_service_client
if not event_bus_stubber:
event_bus_client = get_service_client("events")
event_bus_stubber = Stubber(event_bus_client)
return event_bus_stubber
def buffer_data_into_stream(data, partition_key=None):
"""
This method buffers data in to a Kinesis Data Stream. The lambda function calling it, should have an environment
variable 'STREAM_NAME' whose value should be the name of the stream.
"""
kds_client = get_service_client("kinesis")
stream_name = os.environ["STREAM_NAME"]
if not partition_key:
partition_key = str(uuid.uuid4())
response = kds_client.put_record(StreamName=stream_name,
Data=json.dumps(data),
PartitionKey=partition_key)
logger.debug(f"Response from buffering the stream is {response}")
return response
def process_response(youtube_response, video_search_params):
event_bus = get_service_client("events")
count = 1
comments = []
logger.debug(f"video search parameters {json.dumps(video_search_params)}")
search_query = f'{video_search_params.get("q", None)}#{video_search_params.get("channelId", None)}'
for index, item in enumerate(youtube_response["items"]):
logger.debug(f"Item is {item}")
comments.append({
"EventBusName":
os.environ["EVENT_BUS_NAME"],
"Source":
os.environ["VIDEO_NAMESPACE"],
"Detail":
json.dumps({
"VideoId": item["id"]["videoId"],
"SearchQuery": search_query,
"Title": item["snippet"]["title"]
}),
"DetailType":
"Video",
})
logger.debug(f"Count is {count}")
# optimize the loop to perform put_events with every 10 items
if count == 10 or len(youtube_response["items"]) - index == 1:
service_response = event_bus.put_events(Entries=comments)
logger.debug(
f"Put events response is {json.dumps(service_response)}")
failed_entry_count = service_response.get("FailedEntryCount", None)
if failed_entry_count > 0:
logger.error(
f"Error in put events {json.dumps(service_response['Entries'])}"
)
count = 1
comments = []
else:
count = count + 1 # optimize the loop to perform put_events with every 10 items
def send_event(payload, detail_type, source):
"""
Method to publish json data as events to custom bus. The method expects the event bus
name to be set as a lambda environment variable
"""
if os.environ.get("INTEGRATION_BUS_NAME", None):
events = get_service_client("events")
response = events.put_events(
Entries=[{
"EventBusName": os.environ["INTEGRATION_BUS_NAME"],
"Detail": payload,
"DetailType": detail_type,
"Source": source,
}])
if response["FailedEntryCount"] and response["FailedEntryCount"] > 0:
err_msg = f"Following record failed publishing {payload}"
logger.error(err_msg)
raise EventPublishException(err_msg)
return response
else:
err_msg = "Event bus name not set in environment variable configuration for this lambda function"
logger.error(err_msg)
raise IncorrectEnvSetup(err_msg)
def get_api_key():
api_key = get_service_client("ssm").get_parameter(
Name=os.environ["SSM_API_KEY"],
WithDecryption=True)["Parameter"]["Value"]
return api_key
def test_service_client(self):
service_client = service_helper.get_service_client("s3")
self.assertIsNotNone(service_client)
self.assertTrue("https://s3." in service_client.meta.endpoint_url)
def stream_tear_down(stream_name):
kds_client = service_helper.get_service_client("kinesis")
kds_client.delete_stream(StreamName=stream_name)
def stream_setup(stream_name):
kds_client = service_helper.get_service_client("kinesis")
kds_client.create_stream(StreamName=stream_name, ShardCount=1)
return kds_client
def get_service_client(service_name):
"""Get the global service boto3 client"""
return service_helper.get_service_client(service_name)