def medialive_channel_mediapackage_channel_ddb_items():
"""
Identify and format MediaLive to MediaPackage channel connections for cache storage.
"""
items = []
try:
# get medialive channels
medialive_ch_cached = cache.cached_by_service("medialive-channel")
# get mediapackage channels
mediapackage_ch_cached = cache.cached_by_service("mediapackage-channel")
# compare each medialive output url to a mediapackage ingest url
for ml_channel in medialive_ch_cached:
ml_channel_data = json.loads(ml_channel["data"])
for destination in ml_channel_data["Destinations"]:
for setting in destination["Settings"]:
ml_url = setting["Url"]
# convert a mediapackage v1 ingest url to a v2 url before
# checking
parsed = urlparse(ml_url)
if parsed.path.startswith("/in/v1/"):
pieces = parsed.path.split("/")
if len(pieces) == 5:
ml_url = "{scheme}://{netloc}/in/v2/{uid}/{uid}/channel".format(scheme=parsed.scheme, netloc=parsed.netloc, uid=pieces[3])
for mp_channel in mediapackage_ch_cached:
mp_channel_data = json.loads(mp_channel["data"])
for ingest_endpoint in mp_channel_data["HlsIngest"]["IngestEndpoints"]:
if ml_url == ingest_endpoint["Url"]:
# create a 'connection' out of matches
config = {"from": ml_channel_data["Arn"], "to": mp_channel_data["Arn"], "pipeline": destination["Settings"].index(setting)}
print(config)
items.append(connection_to_ddb_item(ml_channel_data["Arn"], mp_channel_data["Arn"], "medialive-channel-mediapackage-channel", config))
except ClientError as error:
print(error)
return items
def test_cached_by_service(self, patched_env, patched_resource,
patched_client):
"""
Test the cached_by_service function
"""
from chalicelib import cache
cache.cached_by_service(SERVICE)
def delete_disconnected():
"""
This function will clean nodes without connections from the content database table.
"""
node_type = "S3"
nodes = []
connections = []
for node_type in NODE_TYPES:
nodes = nodes + cached_by_service(node_type)
for conn_type in CONNECTION_TYPES:
connections = connections + cached_by_service(conn_type)
remove_nodes = []
# scan for connections with 'to' or 'from' set with
for node in nodes:
found = False
for conn in connections:
if node["arn"] == conn["from"] or node["arn"] == conn["to"]:
found = True
if not found:
remove_nodes.append(node)
resource = boto3.resource("dynamodb")
table = resource.Table(os.environ["CONTENT_TABLE_NAME"])
for node in remove_nodes:
print(node)
table.delete_item(Key={"arn": node["arn"]})
def mediapackage_endpoint_mediatailor_configuration_ddb_items():
"""
Identify and format MediaPackage endpoints to a MediaTailor configuration for cache storage.
"""
items = []
connection_type = "mediapackage-origin-endpoint-mediatailor-configuration"
try:
mediapackage_ep_cached = cache.cached_by_service(
"mediapackage-origin-endpoint")
mediatailor_configs_cached = cache.cached_by_service(
"mediatailor-configuration")
# get the URL from data and compare to the VideoContentSourceUrl of MediaTailor
for mp_endpoint in mediapackage_ep_cached:
mp_endpoint_data = json.loads(mp_endpoint["data"])
mp_endpoint_channel_id = mp_endpoint_data["Url"]
for mt_config in mediatailor_configs_cached:
mt_config_data = json.loads(mt_config["data"])
mt_config_video_source = mt_config_data[
"VideoContentSourceUrl"]
if mt_config_video_source in mp_endpoint_channel_id:
config = {
"from": mp_endpoint_data["Arn"],
"to": mt_config_data["PlaybackConfigurationArn"],
"scheme": urlparse(mt_config_video_source).scheme
}
print(config)
items.append(
connection_to_ddb_item(
mp_endpoint_data["Arn"],
mt_config_data["PlaybackConfigurationArn"],
connection_type, config))
except ClientError as error:
print(error)
return items
def medialive_channel_mediastore_container_ddb_items():
"""
Identify and format MediaLive channel to MediaStore container connections for cache storage.
"""
items = []
try:
# get medialive channels
medialive_ch_cached = cache.cached_by_service("medialive-channel")
# get mediastore containers
mediastore_con_cached = cache.cached_by_service("mediastore-container")
# compare each medialive output url to a mediastore container endpoint
# url
for ml_channel in medialive_ch_cached:
ml_channel_data = json.loads(ml_channel["data"])
for destination in ml_channel_data["Destinations"]:
for setting in destination["Settings"]:
ml_url = setting["Url"]
parsed_destination = urlparse(ml_url)
if "mediastore" in parsed_destination.netloc:
for ms_container in mediastore_con_cached:
container_data = json.loads(ms_container["data"])
parsed_endpoint = urlparse(container_data["Endpoint"])
if parsed_destination.netloc == parsed_endpoint.netloc:
# create a 'connection' out of matches
config = {"from": ml_channel_data["Arn"], "to": container_data["ARN"], "scheme": parsed_destination.scheme}
print(config)
items.append(connection_to_ddb_item(ml_channel_data["Arn"], container_data["ARN"], "medialive-channel-mediastore-container", config))
except ClientError as error:
print(error)
return items
def link_device_medialive_input_ddb_items():
"""
Identify and format Elemental Link device to MediaLive input for cache storage.
"""
items = []
try:
# get medialive inputs
medialive_input_cached = cache.cached_by_service("medialive-input")
# get link devices
link_devices_cached = cache.cached_by_service("link-device")
# find matching ids in the attached inputs to attached channels
for ml_input in medialive_input_cached:
ml_input_data = json.loads(ml_input["data"])
for input_device in ml_input_data["InputDevices"]:
for link_device in link_devices_cached:
link_device_data = json.loads(link_device["data"])
if input_device["Id"] == link_device_data["Id"]:
config = {
"from":
link_device["arn"],
"to":
ml_input["arn"],
"scheme":
"ARQ",
"info":
"https://en.wikipedia.org/wiki/Automatic_repeat_request"
}
print(config)
items.append(
connection_to_ddb_item(
link_device["arn"], ml_input["arn"],
"link-device-medialive-input", config))
except ClientError as error:
print(error)
return items
def mediapackage_endpoint_cloudfront_distribution_by_origin_url_ddb_items():
"""
Identify and format MediaPackage origin endpoints to CloudFront Distributions by tags for cache storage.
"""
min_ratio = 80
items = []
try:
# get CloudFront distros
cloudfront_distros_cached = cache.cached_by_service("cloudfront-distribution")
# get MediaPackage origin endpoints
mediapackage_ep_cached = cache.cached_by_service("mediapackage-origin-endpoint")
# iterate over all distributions
for distro in cloudfront_distros_cached:
distro_data = json.loads(distro["data"])
for item in distro_data["Origins"]["Items"]:
origin_partial_url = "{}/{}".format(item["DomainName"], item["OriginPath"])
for mp_endpoint in mediapackage_ep_cached:
mp_endpoint_data = json.loads(mp_endpoint["data"])
ratio = fuzz.ratio(origin_partial_url, mp_endpoint_data["Url"])
# print("{} {} :: {}".format(ratio, origin_partial_url, mp_endpoint_data["Url"]))
if ratio >= min_ratio:
config = {"from": mp_endpoint["arn"], "to": distro["arn"], "scheme": urlparse(mp_endpoint_data["Url"]).scheme, "connected_by": "url", "match": "{}%".format(ratio)}
print(config)
items.append(connection_to_ddb_item(mp_endpoint["arn"], distro["arn"], "mediapackage-origin-endpoint-cloudfront-distribution", config))
except ClientError as error:
print(error)
return items
def mediapackage_endpoint_cloudfront_distribution_by_origin_url_ddb_items():
"""
Identify and format MediaPackage origin endpoints to CloudFront Distributions by URL for cache storage.
"""
min_ratio = 80
items = []
try:
# get CloudFront distros
cloudfront_distros_cached = cache.cached_by_service("cloudfront-distribution")
# get MediaPackage origin endpoints
mediapackage_ep_cached = cache.cached_by_service("mediapackage-origin-endpoint")
# iterate over all distributions
for distro in cloudfront_distros_cached:
distro_data = json.loads(distro["data"])
for item in distro_data["Origins"]["Items"]:
origin_partial_url = "{}/{}".format(item["DomainName"], item["OriginPath"])
for mp_endpoint in mediapackage_ep_cached:
mp_endpoint_data = json.loads(mp_endpoint["data"])
ratio = fuzz.ratio(origin_partial_url, mp_endpoint_data["Url"])
# print("{} {} :: {}".format(ratio, origin_partial_url, mp_endpoint_data["Url"]))
if ratio >= min_ratio:
config = {"from": mp_endpoint["arn"], "to": distro["arn"], "scheme": urlparse(mp_endpoint_data["Url"]).scheme, "connected_by": "url", "match": "{}%".format(ratio)}
print(config)
items.append(connection_to_ddb_item(mp_endpoint["arn"], distro["arn"], "mediapackage-origin-endpoint-cloudfront-distribution", config))
except ClientError as error:
print(error)
return items
def mediapackage_channel_mediapackage_endpoint_ddb_items():
"""
Identify and format MediaPackage channel to MediaPackage endpoint connections for cache storage.
"""
items = []
package_key = re.compile("^(.+)Package$")
try:
# get mediapackage channels
mediapackage_ch_cached = cache.cached_by_service("mediapackage-channel")
# get mediapackage endpoints
mediapackage_ep_cached = cache.cached_by_service("mediapackage-origin-endpoint")
# find matching ids in the attached inputs to attached channels
for mp_channel in mediapackage_ch_cached:
mp_channel_data = json.loads(mp_channel["data"])
mp_channel_id = mp_channel_data["Id"]
for mp_endpoint in mediapackage_ep_cached:
mp_endpoint_data = json.loads(mp_endpoint["data"])
mp_endpoint_channel_id = mp_endpoint_data["ChannelId"]
if mp_channel_id == mp_endpoint_channel_id:
package_type = ""
for key in mp_endpoint_data.keys():
matcher = package_key.match(key)
if matcher:
package_type = matcher.group(1).upper()
config = {"from": mp_channel_data["Arn"], "to": mp_endpoint_data["Arn"], "package": package_type}
print(config)
items.append(connection_to_ddb_item(mp_channel_data["Arn"], mp_endpoint_data["Arn"], "mediapackage-channel-mediapackage-origin-endpoint", config))
except ClientError as error:
print(error)
return items
def multiplex_mediaconnect_flow_ddb_items():
"""
Identify and format Multiplex to MediaConnect flow connections for cache storage.
"""
source_arn_expr = parse('$..Source.EntitlementArn')
destination_arn_expr = parse('$..Destinations[*].MediaConnectSettings.EntitlementArn')
items = []
try:
# get multiplexes
multiplex_cached = cache.cached_by_service("medialive-multiplex")
# get mediaconnect flows
mediaconnect_flows_cached = cache.cached_by_service("mediaconnect-flow")
for multiplex in multiplex_cached:
multiplex_data = json.loads(multiplex["data"])
# retrieve the multiplex's exported entitlements
entitlement_arns = [match.value for match in destination_arn_expr.find(multiplex_data)]
# print(entitlement_arns)
# search each flow for the same entitlement arns as sources
for flow in mediaconnect_flows_cached:
flow_data = json.loads(flow["data"])
source_arns = [match.value for match in source_arn_expr.find(flow_data)]
# print(source_arns)
for arn in source_arns:
if arn in entitlement_arns:
# create a 'connection' out of matches
config = {"from": multiplex_data["Arn"], "to": flow_data["FlowArn"], "entitlement": arn}
print(config)
items.append(connection_to_ddb_item(multiplex_data["Arn"], flow_data["FlowArn"], "multiplex-mediaconnect-flow", config))
except ClientError as error:
print(error)
return items
def medialive_channel_mediastore_container_ddb_items():
"""
Identify and format MediaLive channel to MediaStore container connections for cache storage.
"""
items = []
try:
# get medialive channels
medialive_ch_cached = cache.cached_by_service("medialive-channel")
# get mediastore containers
mediastore_con_cached = cache.cached_by_service("mediastore-container")
# compare each medialive output url to a mediastore container endpoint
# url
for ml_channel in medialive_ch_cached:
ml_channel_data = json.loads(ml_channel["data"])
for destination in ml_channel_data["Destinations"]:
for setting in destination["Settings"]:
ml_url = setting["Url"]
parsed_destination = urlparse(ml_url)
if "mediastore" in parsed_destination.netloc:
for ms_container in mediastore_con_cached:
container_data = json.loads(ms_container["data"])
parsed_endpoint = urlparse(container_data["Endpoint"])
if parsed_destination.netloc == parsed_endpoint.netloc:
# create a 'connection' out of matches
config = {"from": ml_channel_data["Arn"], "to": container_data["ARN"], "scheme": parsed_destination.scheme}
print(config)
items.append(connection_to_ddb_item(ml_channel_data["Arn"], container_data["ARN"], "medialive-channel-mediastore-container", config))
except ClientError as error:
print(error)
return items
def delete_disconnected():
"""
This function will clean nodes without connections from the content database table.
"""
node_type = "S3"
nodes = []
connections = []
for node_type in NODE_TYPES:
nodes = nodes + cached_by_service(node_type)
for conn_type in CONNECTION_TYPES:
connections = connections + cached_by_service(conn_type)
remove_nodes = []
# scan for connections with 'to' or 'from' set with
for node in nodes:
found = False
for conn in connections:
if node["arn"] == conn["from"] or node["arn"] == conn["to"]:
found = True
if not found:
remove_nodes.append(node)
resource = boto3.resource("dynamodb")
table = resource.Table(os.environ["CONTENT_TABLE_NAME"])
for node in remove_nodes:
print(node)
table.delete_item(Key={"arn": node["arn"]})
def mediastore_container_mediatailor_configuration_ddb_items():
"""
Identify and format MediaStore containers to a MediaTailor configuration for cache storage.
"""
items = []
try:
# get mediatailor configs
mediatailor_configs_cached = cache.cached_by_service("mediatailor-configuration")
# get mediastore containers
mediastore_con_cached = cache.cached_by_service("mediastore-container")
# iterate over mediatailor configs
for mt_config in mediatailor_configs_cached:
mt_config_data = json.loads(mt_config["data"])
mt_config_video_source = mt_config_data["VideoContentSourceUrl"]
parsed_source = urlparse(mt_config_video_source)
if "mediastore" in parsed_source.netloc:
for ms_container in mediastore_con_cached:
container_data = json.loads(ms_container["data"])
parsed_endpoint = urlparse(container_data["Endpoint"])
if parsed_source.netloc == parsed_endpoint.netloc:
# create a 'connection' out of matches
config = {"from": container_data["ARN"], "to": mt_config_data["PlaybackConfigurationArn"], "scheme": parsed_source.scheme}
print(config)
items.append(connection_to_ddb_item(container_data["ARN"], mt_config_data["PlaybackConfigurationArn"], "mediastore-container-mediatailor-configuration", config))
except ClientError as error:
print(error)
return items
def mediapackage_channel_mediapackage_endpoint_ddb_items():
"""
Identify and format MediaPackage channel to MediaPackage endpoint connections for cache storage.
"""
items = []
package_key = re.compile("^(.+)Package$")
try:
# get mediapackage channels
mediapackage_ch_cached = cache.cached_by_service("mediapackage-channel")
# get mediapackage endpoints
mediapackage_ep_cached = cache.cached_by_service("mediapackage-origin-endpoint")
# find matching ids in the attached inputs to attached channels
for mp_channel in mediapackage_ch_cached:
mp_channel_data = json.loads(mp_channel["data"])
mp_channel_id = mp_channel_data["Id"]
for mp_endpoint in mediapackage_ep_cached:
mp_endpoint_data = json.loads(mp_endpoint["data"])
mp_endpoint_channel_id = mp_endpoint_data["ChannelId"]
if mp_channel_id == mp_endpoint_channel_id:
package_type = ""
for key in mp_endpoint_data.keys():
matcher = package_key.match(key)
if matcher:
package_type = matcher.group(1).upper()
config = {"from": mp_channel_data["Arn"], "to": mp_endpoint_data["Arn"], "package": package_type}
print(config)
items.append(connection_to_ddb_item(mp_channel_data["Arn"], mp_endpoint_data["Arn"], "mediapackage-channel-mediapackage-origin-endpoint", config))
except ClientError as error:
print(error)
return items
def mediastore_container_medialive_input_ddb_items():
"""
Identify and format MediaStore container to MediaLive input connections for cache storage.
"""
items = []
try:
# get medialive inputs
medialive_in_cached = cache.cached_by_service("medialive-input")
# get mediastore containers
mediastore_con_cached = cache.cached_by_service("mediastore-container")
# check the inputs that pull from mediastore containers
for ml_input in medialive_in_cached:
ml_input_data = json.loads(ml_input["data"])
for source in ml_input_data["Sources"]:
ml_url = source["Url"]
parsed_source = urlparse(ml_url)
if "mediastore" in parsed_source.netloc:
for ms_container in mediastore_con_cached:
container_data = json.loads(ms_container["data"])
parsed_endpoint = urlparse(container_data["Endpoint"])
if parsed_source.netloc == parsed_endpoint.netloc:
# create a 'connection' out of matches
config = {"from": container_data["ARN"], "to": ml_input_data["Arn"], "scheme": parsed_source.scheme}
print(config)
items.append(connection_to_ddb_item(container_data["ARN"], ml_input_data["Arn"], "mediastore-container-medialive-input", config))
except ClientError as error:
print(error)
return items
def mediapackage_endpoint_speke_keyserver_ddb_items():
"""
Identify and format MediaPackage origin endpoints to SPEKE keyservers for cache storage.
"""
items = []
# create an expression to find speke server urls
jsonpath_expr = parse('$..SpekeKeyProvider.Url')
try:
# get SPEKE keyservers
speke_keyservers_cached = cache.cached_by_service("speke-keyserver")
# get MediaPackage origin endpoints
mediapackage_ep_cached = cache.cached_by_service("mediapackage-origin-endpoint")
# iterate over all distributions
for keyserver in speke_keyservers_cached:
keyserver_data = json.loads(keyserver["data"])
keyserver_endpoint = keyserver_data["endpoint"]
for mp_endpoint in mediapackage_ep_cached:
mp_endpoint_data = json.loads(mp_endpoint["data"])
for server_url in [match.value for match in jsonpath_expr.find(mp_endpoint_data)]:
if server_url == keyserver_endpoint:
config = {"from": mp_endpoint["arn"], "to": keyserver["arn"], "scheme": keyserver_data["scheme"]}
print(config)
items.append(connection_to_ddb_item(mp_endpoint["arn"], keyserver["arn"], "mediapackage-origin-endpoint-speke-keyserver", config))
except ClientError as error:
print(error)
return items
def mediastore_container_medialive_input_ddb_items():
"""
Identify and format MediaStore container to MediaLive input connections for cache storage.
"""
items = []
try:
# get medialive inputs
medialive_in_cached = cache.cached_by_service("medialive-input")
# get mediastore containers
mediastore_con_cached = cache.cached_by_service("mediastore-container")
# check the inputs that pull from mediastore containers
for ml_input in medialive_in_cached:
ml_input_data = json.loads(ml_input["data"])
for source in ml_input_data["Sources"]:
ml_url = source["Url"]
parsed_source = urlparse(ml_url)
if "mediastore" in parsed_source.netloc:
for ms_container in mediastore_con_cached:
container_data = json.loads(ms_container["data"])
parsed_endpoint = urlparse(container_data["Endpoint"])
if parsed_source.netloc == parsed_endpoint.netloc:
# create a 'connection' out of matches
config = {"from": container_data["ARN"], "to": ml_input_data["Arn"], "scheme": parsed_source.scheme}
print(config)
items.append(connection_to_ddb_item(container_data["ARN"], ml_input_data["Arn"], "mediastore-container-medialive-input", config))
except ClientError as error:
print(error)
return items
def medialive_input_medialive_channel_ddb_items():
"""
Identify and format MediaLive input to MediaLive channel connections for cache storage.
"""
items = []
try:
# get medialive channels
medialive_ch_cached = cache.cached_by_service("medialive-channel")
# get medialive inputs
medialive_in_cached = cache.cached_by_service("medialive-input")
# find matching ids in the attached inputs to attached channels
for ml_channel in medialive_ch_cached:
ml_channel_data = json.loads(ml_channel["data"])
ml_channel_id = ml_channel_data["Id"]
for ml_input in medialive_in_cached:
ml_input_data = json.loads(ml_input["data"])
for attached_id in ml_input_data["AttachedChannels"]:
if ml_channel_id == attached_id:
config = {
"from": ml_input_data["Arn"],
"to": ml_channel_data["Arn"],
"type": ml_input_data["Type"]
}
print(config)
items.append(
connection_to_ddb_item(
ml_input_data["Arn"], ml_channel_data["Arn"],
"medialive-input-medialive-channel", config))
except ClientError as error:
print(error)
return items
def mediapackage_endpoint_speke_keyserver_ddb_items():
"""
Identify and format MediaPackage origin endpoints to SPEKE keyservers for cache storage.
"""
items = []
# create an expression to find speke server urls
jsonpath_expr = parse('$..SpekeKeyProvider.Url')
try:
# get SPEKE keyservers
speke_keyservers_cached = cache.cached_by_service("speke-keyserver")
# get MediaPackage origin endpoints
mediapackage_ep_cached = cache.cached_by_service("mediapackage-origin-endpoint")
# iterate over all distributions
for keyserver in speke_keyservers_cached:
keyserver_data = json.loads(keyserver["data"])
keyserver_endpoint = keyserver_data["endpoint"]
for mp_endpoint in mediapackage_ep_cached:
mp_endpoint_data = json.loads(mp_endpoint["data"])
for server_url in [match.value for match in jsonpath_expr.find(mp_endpoint_data)]:
if server_url == keyserver_endpoint:
config = {"from": mp_endpoint["arn"], "to": keyserver["arn"], "scheme": keyserver_data["scheme"]}
print(config)
items.append(connection_to_ddb_item(mp_endpoint["arn"], keyserver["arn"], "mediapackage-origin-endpoint-speke-keyserver", config))
except ClientError as error:
print(error)
return items
def cloudfront_distribution_medialive_input_ddb_items():
"""
Identify and format CloudFront Distribution to MediaLive Input connections for cache storage.
"""
items = []
cloudfront_url = re.compile(r"http.?\:\/\/(\S+\.cloudfront\.net)\/.*")
try:
# get CloudFront distros
cloudfront_distros_cached = cache.cached_by_service("cloudfront-distribution")
# get MediaLive inputs
medialive_in_cached = cache.cached_by_service("medialive-input")
# iterate over all inputs
for ml_input in medialive_in_cached:
ml_input_data = json.loads(ml_input["data"])
for source in ml_input_data["Sources"]:
domain_name = None
scheme = None
# is this a cloudfront url?
match = cloudfront_url.match(source["Url"])
if match:
# yes
domain_name = match.group(1)
scheme = urlparse(source["Url"]).scheme
# find the distribution
for distro in cloudfront_distros_cached:
distro_data = json.loads(distro["data"])
if domain_name == distro_data["DomainName"]:
config = {"from": distro["arn"], "to": ml_input["arn"], "scheme": scheme}
print(config)
items.append(connection_to_ddb_item(distro["arn"], ml_input["arn"], "cloudfront-distribution-medialive-input", config))
except ClientError as error:
print(error)
return items
def s3_bucket_cloudfront_distribution_ddb_items():
"""
Identify and format S3 Bucket to CloudFront Distribution connections for cache storage.
"""
items = []
s3_origin = re.compile(r"(\S+)\.s3([^\.])*\.amazonaws\.com")
try:
# get S3 buckets
s3_buckets_cached = cache.cached_by_service("s3")
# get CloudFront distributions
cloudfront_dist_cached = cache.cached_by_service(
"cloudfront-distribution")
for s3_bucket in s3_buckets_cached:
s3_bucket_data = json.loads(s3_bucket["data"])
for cloudfront_distro in cloudfront_dist_cached:
cloudfront_distro_data = json.loads(cloudfront_distro["data"])
for origin_item in cloudfront_distro_data["Origins"]["Items"]:
matcher = s3_origin.match(origin_item["DomainName"])
if matcher:
bucket_name = matcher.group(1)
if bucket_name == s3_bucket_data["Name"]:
config = {
"from": s3_bucket["arn"],
"to": cloudfront_distro["arn"],
"label": "S3"
}
print(config)
items.append(
connection_to_ddb_item(
s3_bucket["arn"], cloudfront_distro["arn"],
"s3-bucket-cloudfront-distribution",
config))
except ClientError as error:
print(error)
return items
def s3_bucket_cloudfront_distribution_ddb_items():
"""
Identify and format S3 Bucket to CloudFront Distribution connections for cache storage.
"""
items = []
s3_origin = re.compile(r"(\S+)\.s3\.amazonaws\.com")
try:
# get S3 buckets
s3_buckets_cached = cache.cached_by_service("s3")
# get CloudFront distributions
cloudfront_dist_cached = cache.cached_by_service("cloudfront-distribution")
for s3_bucket in s3_buckets_cached:
s3_bucket_data = json.loads(s3_bucket["data"])
for cloudfront_distro in cloudfront_dist_cached:
cloudfront_distro_data = json.loads(cloudfront_distro["data"])
for origin_item in cloudfront_distro_data["Origins"]["Items"]:
matcher = s3_origin.match(origin_item["DomainName"])
if matcher:
bucket_name = matcher.group(1)
if bucket_name == s3_bucket_data["Name"]:
config = {"from": s3_bucket["arn"], "to": cloudfront_distro["arn"], "label": "S3"}
print(config)
items.append(connection_to_ddb_item(s3_bucket["arn"], cloudfront_distro["arn"], "s3-bucket-cloudfront-distribution", config))
except ClientError as error:
print(error)
return items
def cloudfront_distribution_medialive_input_ddb_items():
"""
Identify and format CloudFront Distribution to MediaLive Input connections for cache storage.
"""
items = []
cloudfront_url = re.compile(r"http.?\:\/\/(\S+\.cloudfront\.net)\/.*")
try:
# get CloudFront distros
cloudfront_distros_cached = cache.cached_by_service("cloudfront-distribution")
# get MediaLive inputs
medialive_in_cached = cache.cached_by_service("medialive-input")
# iterate over all inputs
for ml_input in medialive_in_cached:
ml_input_data = json.loads(ml_input["data"])
for source in ml_input_data["Sources"]:
domain_name = None
scheme = None
# is this a cloudfront url?
match = cloudfront_url.match(source["Url"])
if match:
# yes
domain_name = match.group(1)
scheme = urlparse(source["Url"]).scheme
# find the distribution
for distro in cloudfront_distros_cached:
distro_data = json.loads(distro["data"])
if domain_name == distro_data["DomainName"]:
config = {"from": distro["arn"], "to": ml_input["arn"], "scheme": scheme}
print(config)
items.append(connection_to_ddb_item(distro["arn"], ml_input["arn"], "cloudfront-distribution-medialive-input", config))
except ClientError as error:
print(error)
return items
def medialive_channel_mediapackage_channel_ddb_items():
"""
Identify and format MediaLive to MediaPackage channel connections for cache storage.
"""
items = []
try:
# get medialive channels
medialive_ch_cached = cache.cached_by_service("medialive-channel")
# get mediapackage channels
mediapackage_ch_cached = cache.cached_by_service("mediapackage-channel")
# compare each medialive output url to a mediapackage ingest url
for ml_channel in medialive_ch_cached:
ml_channel_data = json.loads(ml_channel["data"])
for destination in ml_channel_data["Destinations"]:
for setting in destination["Settings"]:
ml_url = setting["Url"]
# convert a mediapackage v1 ingest url to a v2 url before
# checking
parsed = urlparse(ml_url)
if parsed.path.startswith("/in/v1/"):
pieces = parsed.path.split("/")
if len(pieces) == 5:
ml_url = "{scheme}://{netloc}/in/v2/{uid}/{uid}/channel".format(scheme=parsed.scheme, netloc=parsed.netloc, uid=pieces[3])
for mp_channel in mediapackage_ch_cached:
mp_channel_data = json.loads(mp_channel["data"])
for ingest_endpoint in mp_channel_data["HlsIngest"]["IngestEndpoints"]:
if ml_url == ingest_endpoint["Url"]:
# create a 'connection' out of matches
config = {"from": ml_channel_data["Arn"], "to": mp_channel_data["Arn"], "pipeline": destination["Settings"].index(setting)}
print(config)
items.append(connection_to_ddb_item(ml_channel_data["Arn"], mp_channel_data["Arn"], "medialive-channel-mediapackage-channel", config))
except ClientError as error:
print(error)
return items
def medialive_channel_multiplex_ddb_items():
"""
Identify and format MediaLive channel to EML Multiplex connections for cache storage.
"""
items = []
ml_service_name = "medialive-channel-multiplex"
try:
# get medialive channels
medialive_ch_cached = cache.cached_by_service("medialive-channel")
# get multiplexes
medialive_mp_cached = cache.cached_by_service("medialive-multiplex")
for ml_channel in medialive_ch_cached:
ml_channel_data = json.loads(ml_channel["data"])
for destination in ml_channel_data["Destinations"]:
if "MultiplexSettings" in destination:
multiplex_id = destination["MultiplexSettings"]["MultiplexId"]
program_name = destination["MultiplexSettings"]["ProgramName"]
for ml_multiplex in medialive_mp_cached:
ml_multiplex_data = json.loads(ml_multiplex["data"])
if multiplex_id == ml_multiplex_data["Id"]:
pipelines_count = fetch_running_pipelines_count(ml_channel_data)
for pl in range(pipelines_count):
# create a 'connection' out of matches
config = {"from": ml_channel_data["Arn"], "to": ml_multiplex_data["Arn"], "program": program_name, "pipeline": pl}
print(config)
items.append(connection_to_ddb_item_pl(ml_channel_data["Arn"], ml_multiplex_data["Arn"], ml_service_name, config))
except ClientError as error:
print(error)
return items
def mediapackage_endpoint_cloudfront_distribution_by_tag_ddb_items():
"""
Identify and format MediaPackage origin endpoints to CloudFront Distributions by tags for cache storage.
"""
items = []
try:
# get CloudFront distros
cloudfront_distros_cached = cache.cached_by_service(
"cloudfront-distribution")
# get MediaPackage channels
mediapackage_ch_cached = cache.cached_by_service(
"mediapackage-channel")
# get MediaPackage origin endpoints
mediapackage_ep_cached = cache.cached_by_service(
"mediapackage-origin-endpoint")
# iterate over all distributions
for distro in cloudfront_distros_cached:
distro_data = json.loads(distro["data"])
for key, value in distro_data["Tags"].items():
if (key in [
"MP-Endpoint-ARN", "mediapackage:cloudfront_assoc"
]) and ":channels/" in value:
channel_arn = value
channel_id = None
# find the channel
for channel in mediapackage_ch_cached:
if channel_arn == channel["arn"]:
channel_data = json.loads(channel["data"])
channel_id = channel_data["Id"]
break
if channel_id:
# add a connection to each endpoint
for endpoint in mediapackage_ep_cached:
endpoint_data = json.loads(endpoint["data"])
if endpoint_data["ChannelId"] == channel_id:
config = {
"from": endpoint["arn"],
"to": distro["arn"],
"scheme":
urlparse(endpoint_data["Url"]).scheme,
"connected_by": "tag",
"tag": key
}
print(config)
items.append(
connection_to_ddb_item(
endpoint["arn"], distro["arn"],
"mediapackage-origin-endpoint-cloudfront-distribution",
config))
except ClientError as error:
print(error)
return items
def medialive_channel_mediapackage_channel_ddb_items():
"""
Identify and format MediaLive to MediaPackage channel connections for cache storage.
"""
items = []
ml_service_name = "medialive-channel-mediapackage-channel"
try:
# get medialive channels
medialive_ch_cached = cache.cached_by_service("medialive-channel")
# get mediapackage channels
mediapackage_ch_cached = cache.cached_by_service("mediapackage-channel")
# compare each medialive output url to a mediapackage ingest url
for ml_channel in medialive_ch_cached:
ml_channel_data = json.loads(ml_channel["data"])
for destination in ml_channel_data["Destinations"]:
# if setting is empty, we have to connect medialive with mediapackage via channel ID
if destination["MediaPackageSettings"]:
for mp_setting in destination["MediaPackageSettings"]:
for mp_channel in mediapackage_ch_cached:
mp_channel_data = json.loads(mp_channel["data"])
if mp_channel_data['Id'] == mp_setting['ChannelId']:
pipelines_count = fetch_running_pipelines_count(ml_channel_data)
for pl in range(pipelines_count):
# create a 'connection' out of matches
config = {"from": ml_channel_data["Arn"], "to": mp_channel_data["Arn"], "pipeline": pl}
print(config)
items.append(connection_to_ddb_item_pl(ml_channel_data["Arn"], mp_channel_data["Arn"], ml_service_name, config))
# otherwise we check via URL endpoints
else:
for setting in destination["Settings"]:
ml_url = setting["Url"]
ml_url_v2 = None
# convert a mediapackage v1 ingest url to a v2 url before
# checking
parsed = urlparse(ml_url)
if parsed.path.startswith("/in/v1/"):
pieces = parsed.path.split("/")
if len(pieces) == 5:
ml_url_v2 = "{scheme}://{netloc}/in/v2/{uid}/{uid}/channel".format(scheme=parsed.scheme, netloc=parsed.netloc, uid=pieces[3])
for mp_channel in mediapackage_ch_cached:
mp_channel_data = json.loads(mp_channel["data"])
for ingest_endpoint in mp_channel_data["HlsIngest"]["IngestEndpoints"]:
if ml_url == ingest_endpoint["Url"] or ml_url_v2 == ingest_endpoint["Url"]:
# create a 'connection' out of matches
config = {"from": ml_channel_data["Arn"], "to": mp_channel_data["Arn"], "pipeline": destination["Settings"].index(setting)}
print(config)
items.append(connection_to_ddb_item_pl(ml_channel_data["Arn"], mp_channel_data["Arn"], ml_service_name, config))
except ClientError as error:
print(error)
return items
def s3_bucket_mediatailor_configuration_ddb_items():
"""
Identify and format S3 buckets to a MediaTailor configuration for cache storage.
"""
items = []
s3_url_expressions = [
re.compile(r"http.?\:\/\/(\S+)\.s3\-website.+"),
re.compile(r"http.?\:\/\/s3\-\S+\.amazonaws\.com\/([^\/]+)\/.+"),
re.compile(r"http.?\:\/\/(\S+)\.s3\.amazonaws\.com\/.+"),
re.compile(r"http.?\:\/\/(\S+)\.s3\-(\S+)\.amazonaws\.com")
]
try:
# get S3 buckets
s3_buckets_cached = cache.cached_by_service("s3")
# get MediaTailor configurations
mediatailor_configs_cached = cache.cached_by_service(
"mediatailor-configuration")
# iterate over configs
for mt_config in mediatailor_configs_cached:
bucket_name = None
scheme = None
mt_config_data = json.loads(mt_config["data"])
mt_config_video_source = mt_config_data["VideoContentSourceUrl"]
# is this a bucket url?
for expr in s3_url_expressions:
match = expr.match(mt_config_video_source)
if match:
# yes
bucket_name = match.group(1)
scheme = urlparse(mt_config_video_source).scheme
break
if bucket_name:
# find the bucket
for s3_bucket in s3_buckets_cached:
s3_bucket_data = json.loads(s3_bucket["data"])
if bucket_name == s3_bucket_data["Name"]:
config = {
"from": s3_bucket["arn"],
"to": mt_config_data["PlaybackConfigurationArn"],
"scheme": scheme
}
print(config)
items.append(
connection_to_ddb_item(
s3_bucket["arn"],
mt_config_data["PlaybackConfigurationArn"],
"s3-bucket-mediatailor-configuration", config))
except ClientError as error:
print(error)
return items
def s3_bucket_medialive_input_ddb_items():
"""
Identify and format S3 Bucket to MediaLive Input connections for cache storage.
"""
items = []
s3_url_expressions = [
re.compile(r"http.?\:\/\/(\S+)\.s3\-website.+"),
re.compile(r"http.?\:\/\/s3\-\S+\.amazonaws\.com\/([^\/]+)\/.+"),
re.compile(r"http.?\:\/\/(\S+)\.s3\.amazonaws\.com\/.+"),
re.compile(r"http.?\:\/\/(\S+)\.s3\-(\S+)\.amazonaws\.com"),
re.compile(r"s3\:\/\/([^\/]+)")
]
try:
# get S3 buckets
s3_buckets_cached = cache.cached_by_service("s3")
# get MediaLive inputs
medialive_in_cached = cache.cached_by_service("medialive-input")
# iterate over all inputs
for ml_input in medialive_in_cached:
ml_input_data = json.loads(ml_input["data"])
for source in ml_input_data["Sources"]:
bucket_name = None
scheme = None
# is this a bucket url?
for expr in s3_url_expressions:
match = expr.match(source["Url"])
if match:
# yes
bucket_name = match.group(1)
scheme = urlparse(source["Url"]).scheme
break
if bucket_name:
# find the bucket
for s3_bucket in s3_buckets_cached:
s3_bucket_data = json.loads(s3_bucket["data"])
if bucket_name == s3_bucket_data["Name"]:
config = {
"from": s3_bucket["arn"],
"to": ml_input["arn"],
"scheme": scheme
}
print(config)
items.append(
connection_to_ddb_item(
s3_bucket["arn"], ml_input["arn"],
"s3-bucket-medialive-input", config))
except ClientError as error:
print(error)
return items
def mediaconnect_flow_medialive_input_ddb_items():
"""
Identify and format MediaConnect Flow to MediaLive Input connections for cache storage.
"""
items = []
connection_type = "mediaconnect-flow-medialive-input"
try:
# get MediaConnect flows
mediaconnect_flows_cached = cache.cached_by_service("mediaconnect-flow")
# process each flow
for flow in mediaconnect_flows_cached:
flow_data = json.loads(flow["data"])
# for each flow, process each outputs
for flow_output in flow_data["Outputs"]:
match_found = False
# check for MediaLiveInputArn first
try:
if flow_output["MediaLiveInputArn"]:
config = {"from": flow_data["FlowArn"], "to": flow_output["MediaLiveInputArn"], "scheme": "MEDIACONNECT"}
print(config)
items.append(connection_to_ddb_item(flow_data["FlowArn"], flow_output["MediaLiveInputArn"], connection_type, config))
# if that didn't work, then check for IPs (Destination)
except KeyError as error:
# for each output, look for the matching MediaLive input
medialive_in_cached = cache.cached_by_service("medialive-input")
# iterate over all medialive inputs
for ml_input in medialive_in_cached:
if not match_found:
ml_input_data = json.loads(ml_input["data"])
# there are 2 ip addresses in ml_input
for destination in ml_input_data["Destinations"]:
# match the flow output ip address to a mediaLive input ip address
try:
if destination["Ip"] == flow_output["Destination"]:
config = {"from": flow["arn"], "to": ml_input["arn"], "scheme": ml_input_data["Type"]}
print(config)
items.append(connection_to_ddb_item(flow["arn"], ml_input["arn"], connection_type, config))
match_found = True
break
except Exception as error:
print(error)
else:
break
except Exception as error:
print(error)
except ClientError as error:
print(error)
return items
def generate_metrics(stackname):
"""
This function is responsible for generating a resource count
of each service type in inventory. It puts the resulting
data to metrics with dimensions in CloudWatch.
"""
client = boto3.client('cloudwatch', config=MSAM_BOTO3_CONFIG)
for resource_type in MONITORED_SERVICES:
resources = cache.cached_by_service(resource_type)
client.put_metric_data(Namespace=METRICS_NAMESPACE,
MetricData=[
{
'MetricName':
METRICS_NAME,
'Dimensions': [{
'Name': 'Stack Name',
'Value': stackname
}, {
'Name': 'Resource Type',
'Value': resource_type
}],
'Value':
len(resources),
'Unit':
'Count'
},
])
def mediaconnect_flow_mediaconnect_flow_ddb_items():
"""
Identify and format MediaConnect Flow to another MediaConnect Flow for cache storage.
"""
items = []
connection_type = "mediaconnect-flow-mediaconnect-flow"
try:
# get MediaConnect flows
mediaconnect_flows_cached = cache.cached_by_service("mediaconnect-flow")
for outer_flow in mediaconnect_flows_cached:
outer_flow_data = json.loads(outer_flow["data"])
# process each flow for entitlement
try:
if outer_flow_data["Source"]["EntitlementArn"]:
config = {"from": outer_flow_data["Source"]["EntitlementArn"], "to": outer_flow_data["FlowArn"], "scheme": "ENTITLEMENT"}
items.append(connection_to_ddb_item(outer_flow_data["Source"]["EntitlementArn"], outer_flow_data["FlowArn"], connection_type, config))
except Exception as error:
print(error)
# also, process each flow against each of the same set of flows for regular IP push (standard)
outer_flow_egress_ip = outer_flow_data["EgressIp"]
# check this egress ip against all the output IPs of each of the flows
for inner_flow in mediaconnect_flows_cached:
inner_flow_data = json.loads(inner_flow["data"])
for flow_output in inner_flow_data["Outputs"]:
try:
if flow_output["Destination"] == outer_flow_egress_ip:
config = {"from": inner_flow_data["FlowArn"], "to": outer_flow_data["FlowArn"], "scheme": flow_output["Transport"]["Protocol"].upper()}
items.append(connection_to_ddb_item(inner_flow_data["FlowArn"], outer_flow_data["FlowArn"], connection_type, config))
except Exception as error:
print(error)
except ClientError as error:
print(error)
return items
def mediastore_container_cloudfront_distribution_ddb_items():
"""
Identify and format MediaStore Container to CloudFront Distribution connections for cache storage.
"""
items = []
try:
# get CloudFront distros
cloudfront_distros_cached = cache.cached_by_service(
"cloudfront-distribution")
# get cached MediaStore containers
mediastore_con_cached = cache.cached_by_service("mediastore-container")
# iterate over all distributions
for distro in cloudfront_distros_cached:
distro_data = json.loads(distro["data"])
for origin in distro_data["Origins"]["Items"]:
origin_domain_name = origin["DomainName"]
if "mediastore" in origin_domain_name:
for ms_container in mediastore_con_cached:
ms_container_data = json.loads(ms_container["data"])
if origin_domain_name in ms_container_data["Endpoint"]:
config = {
"from":
ms_container["arn"],
"to":
distro["arn"],
"scheme":
urlparse(ms_container_data["Endpoint"]).scheme
}
print(config)
items.append(
connection_to_ddb_item(
ms_container["arn"], distro["arn"],
"mediastore-container-cloudfront-distribution",
config))
except ClientError as error:
print(error)
return items
def medialive_channel_s3_bucket_ddb_items():
"""
Identify and format MediaLive channel to S3 bucket connections for cache storage.
"""
items = []
try:
# get medialive channels
medialive_ch_cached = cache.cached_by_service("medialive-channel")
# get s3 buckets
s3_buckets_cached = cache.cached_by_service("s3")
# compare each medialive output url to an s3 bucket location
# protocols allowed for writing to s3 buckets are s3 and s3ssl
for ml_channel in medialive_ch_cached:
ml_channel_data = json.loads(ml_channel["data"])
for destination in ml_channel_data["Destinations"]:
for setting in destination["Settings"]:
ml_url = setting["Url"]
parsed_destination = urlparse(ml_url)
if parsed_destination.scheme == 's3' or parsed_destination.scheme == 's3ssl':
for s3_bucket in s3_buckets_cached:
s3_bucket_data = json.loads(s3_bucket["data"])
if parsed_destination.netloc == s3_bucket_data[
'Name']:
# create a 'connection' out of matches
config = {
"from": ml_channel["arn"],
"to": s3_bucket["arn"],
"scheme": parsed_destination.scheme
}
print(config)
items.append(
connection_to_ddb_item(
ml_channel["arn"], s3_bucket["arn"],
"medialive-channel-s3-bucket", config))
except ClientError as error:
print(error)
return items
def medialive_input_medialive_channel_ddb_items():
"""
Identify and format MediaLive input to MediaLive channel connections for cache storage.
"""
items = []
try:
# get medialive channels
medialive_ch_cached = cache.cached_by_service("medialive-channel")
# get medialive inputs
medialive_in_cached = cache.cached_by_service("medialive-input")
# find matching ids in the attached inputs to attached channels
for ml_channel in medialive_ch_cached:
ml_channel_data = json.loads(ml_channel["data"])
ml_channel_id = ml_channel_data["Id"]
for ml_input in medialive_in_cached:
ml_input_data = json.loads(ml_input["data"])
for attached_id in ml_input_data["AttachedChannels"]:
if ml_channel_id == attached_id:
config = {"from": ml_input_data["Arn"], "to": ml_channel_data["Arn"], "type": ml_input_data["Type"]}
print(config)
items.append(connection_to_ddb_item(ml_input_data["Arn"], ml_channel_data["Arn"], "medialive-input-medialive-channel", config))
except ClientError as error:
print(error)
return items
def s3_bucket_medialive_input_ddb_items():
"""
Identify and format S3 Bucket to MediaLive Input connections for cache storage.
"""
items = []
s3_url_expressions = [re.compile(r"http.?\:\/\/(\S+)\.s3\-website.+"), re.compile(r"http.?\:\/\/s3\-\S+\.amazonaws\.com\/([^\/]+)\/.+"), re.compile(r"http.?\:\/\/(\S+)\.s3\.amazonaws\.com\/.+")]
try:
# get S3 buckets
s3_buckets_cached = cache.cached_by_service("s3")
# get MediaLive inputs
medialive_in_cached = cache.cached_by_service("medialive-input")
# iterate over all inputs
for ml_input in medialive_in_cached:
ml_input_data = json.loads(ml_input["data"])
for source in ml_input_data["Sources"]:
bucket_name = None
scheme = None
# is this a bucket url?
for expr in s3_url_expressions:
match = expr.match(source["Url"])
if match:
# yes
bucket_name = match.group(1)
scheme = urlparse(source["Url"]).scheme
break
if bucket_name:
# find the bucket
for s3_bucket in s3_buckets_cached:
s3_bucket_data = json.loads(s3_bucket["data"])
if bucket_name == s3_bucket_data["Name"]:
config = {"from": s3_bucket["arn"], "to": ml_input["arn"], "scheme": scheme}
print(config)
items.append(connection_to_ddb_item(s3_bucket["arn"], ml_input["arn"], "s3-bucket-medialive-input", config))
except ClientError as error:
print(error)
return items
def mediapackage_endpoint_cloudfront_distribution_by_tag_ddb_items():
"""
Identify and format MediaPackage origin endpoints to CloudFront Distributions by tags for cache storage.
"""
items = []
try:
# get CloudFront distros
cloudfront_distros_cached = cache.cached_by_service("cloudfront-distribution")
# get MediaPackage channels
mediapackage_ch_cached = cache.cached_by_service("mediapackage-channel")
# get MediaPackage origin endpoints
mediapackage_ep_cached = cache.cached_by_service("mediapackage-origin-endpoint")
# iterate over all distributions
for distro in cloudfront_distros_cached:
distro_data = json.loads(distro["data"])
for tag in distro_data["Tags"]["Items"]:
if (tag["Key"] == "MP-Endpoint-ARN" or tag["Key"] == "mediapackage:cloudfront_assoc") and ":channels/" in tag["Value"]:
channel_arn = tag["Value"]
channel_id = None
# find the channel
for channel in mediapackage_ch_cached:
if channel_arn == channel["arn"]:
channel_data = json.loads(channel["data"])
channel_id = channel_data["Id"]
break
if channel_id:
# add a connection to each endpoint
for endpoint in mediapackage_ep_cached:
endpoint_data = json.loads(endpoint["data"])
if endpoint_data["ChannelId"] == channel_id:
config = {"from": endpoint["arn"], "to": distro["arn"], "scheme": urlparse(endpoint_data["Url"]).scheme, "connected_by": "tag", "tag": tag["Key"]}
print(config)
items.append(connection_to_ddb_item(endpoint["arn"], distro["arn"], "mediapackage-origin-endpoint-cloudfront-distribution", config))
except ClientError as error:
print(error)
return items
def cached_by_service_region(service, region):
"""
API entry point to retrieve items from the cache under the service.
"""
return cache.cached_by_service(service)
