def update_nodes():
"""
Entry point for the CloudWatch scheduled task to discover and cache services.
"""
try:
never_regions_key = "never-cache-regions"
never_regions = msam_settings.get_setting(never_regions_key)
if never_regions is None:
never_regions = []
settings_key = "cache-next-region"
# make a region name list
region_name_list = []
for region in regions():
region_name = region["RegionName"]
# exclude regions listed in never-cache setting
if region_name not in never_regions:
region_name_list.append(region_name)
else:
print("{} in {} setting".format(region_name,
never_regions_key))
# sort it
region_name_list.sort()
# get the next region to process
next_region = msam_settings.get_setting(settings_key)
# start at the beginning if no previous setting
if next_region is None:
next_region = region_name_list[0]
# otherwise it's saved for us
region_name = next_region
# store the region for the next schedule
try:
# process global after the end of the region list
if region_name_list.index(next_region) + 1 >= len(
region_name_list):
next_region = "global"
else:
next_region = region_name_list[
region_name_list.index(next_region) + 1]
except (IndexError, ValueError):
# start over if we don't recognize the region, ex. global
next_region = region_name_list[0]
# store it
msam_settings.put_setting(settings_key, next_region)
# update the region
print("updating region {}".format(region_name))
if region_name == "global":
content.put_ddb_items(node_cache.s3_bucket_ddb_items())
content.put_ddb_items(
node_cache.cloudfront_distribution_ddb_items())
else:
node_cache.update_regional_ddb_items(region_name)
except ClientError as error:
print(error)
return True
def update_nodes():
"""
Entry point for the CloudWatch scheduled task to discover and cache services.
"""
try:
never_regions_key = "never-cache-regions"
never_regions = msam_settings.get_setting(never_regions_key)
if never_regions is None:
never_regions = []
settings_key = "cache-next-region"
# make a region name list
region_name_list = []
for region in regions():
region_name = region["RegionName"]
# exclude regions listed in never-cache setting
if region_name not in never_regions:
region_name_list.append(region_name)
else:
print("{} in {} setting".format(region_name, never_regions_key))
# sort it
region_name_list.sort()
# get the next region to process
next_region = msam_settings.get_setting(settings_key)
# start at the beginning if no previous setting
if next_region is None:
next_region = region_name_list[0]
# otherwise it's saved for us
region_name = next_region
# store the region for the next schedule
try:
# process global after the end of the region list
if region_name_list.index(next_region) + 1 >= len(region_name_list):
next_region = "global"
else:
next_region = region_name_list[region_name_list.index(next_region) + 1]
except (IndexError, ValueError):
# start over if we don't recognize the region, ex. global
next_region = region_name_list[0]
# store it
msam_settings.put_setting(settings_key, next_region)
# update the region
print("updating region {}".format(region_name))
if region_name == "global":
content.put_ddb_items(node_cache.s3_bucket_ddb_items())
content.put_ddb_items(node_cache.cloudfront_distribution_ddb_items())
else:
node_cache.update_regional_ddb_items(region_name)
except ClientError as error:
print(error)
return True
def set_channel_nodes(name, node_ids):
"""
API entry point to set the nodes for a given channel name.
"""
try:
name = unquote(name)
table = DYNAMO_RESOURCE.Table(CHANNELS_TABLE_NAME)
# print(request.json_body)
# node_ids = request.json_body
# write the channel nodes to the database
for node_id in node_ids:
item = {"channel": name, "id": node_id}
table.put_item(Item=item)
# update the list of channels in settings
name_list = msam_settings.get_setting("channels")
if not name_list:
name_list = []
if name not in name_list:
name_list.append(name)
msam_settings.put_setting("channels", name_list)
result = {"message": "saved"}
print(result)
except ClientError as error:
# send the exception back in the object
print(error)
result = {"exception": str(error)}
return result
def set_channel_nodes(request, name):
"""
API entry point to set the nodes for a given channel name.
"""
try:
name = unquote(name)
table = DYNAMO_RESOURCE.Table(CHANNELS_TABLE_NAME)
print(request.json_body)
node_ids = request.json_body
# write the channel nodes to the database
for node_id in node_ids:
item = {"channel": name, "id": node_id}
table.put_item(Item=item)
# update the list of channels in settings
name_list = msam_settings.get_setting("channels")
if not name_list:
name_list = []
if name not in name_list:
name_list.append(name)
msam_settings.put_setting("channels", name_list)
result = {"message": "saved"}
print(result)
except ClientError as error:
# send the exception back in the object
print(error)
result = {"exception": str(error)}
return result
def delete_channel_nodes(request, name):
"""
API entry point to delete a channel.
"""
try:
name = unquote(name)
table_name = CHANNELS_TABLE_NAME
table = DYNAMO_RESOURCE.Table(table_name)
print(request.method)
try:
# get the settings object
response = table.query(KeyConditionExpression=Key('channel').eq(name))
print(response)
# return the response or an empty object
if "Items" in response:
for item in response["Items"]:
table.delete_item(Key={"channel": item["channel"], "id": item["id"]})
name_list = msam_settings.get_setting("channels")
if not name_list:
name_list = []
if name in name_list:
name_list.remove(name)
msam_settings.put_setting("channels", name_list)
print("channel items deleted, channel list updated")
except ClientError:
print("not found")
response = {"message": "done"}
except ClientError as outer_error:
# send the exception back in the object
print(outer_error)
response = {"exception": str(outer_error)}
return response
def delete_channel_nodes(request, name):
"""
API entry point to delete a channel.
"""
try:
name = unquote(name)
table_name = CHANNELS_TABLE_NAME
table = DYNAMO_RESOURCE.Table(table_name)
print(request.method)
try:
# get the settings object
response = table.query(KeyConditionExpression=Key('channel').eq(name))
print(response)
# return the response or an empty object
if "Items" in response:
for item in response["Items"]:
table.delete_item(Key={"channel": item["channel"], "id": item["id"]})
name_list = msam_settings.get_setting("channels")
if not name_list:
name_list = []
if name in name_list:
name_list.remove(name)
msam_settings.put_setting("channels", name_list)
print("channel items deleted, channel list updated")
except ClientError:
print("not found")
response = {"message": "done"}
except ClientError as outer_error:
# send the exception back in the object
print(outer_error)
response = {"exception": str(outer_error)}
return response
def get_channel_list():
"""
Return all the current channel names.
"""
channels = msam_settings.get_setting("channels")
if not channels:
channels = []
return channels
def get_channel_list():
"""
Return all the current channel names.
"""
channels = msam_settings.get_setting("channels")
if not channels:
channels = []
return channels
def update_nodes_generic(update_global_func, update_regional_func,
settings_key):
"""
Entry point for the CloudWatch scheduled task to discover and cache services.
"""
try:
inventory_regions_key = "inventory-regions"
inventory_regions = msam_settings.get_setting(inventory_regions_key)
if inventory_regions is None:
inventory_regions = []
inventory_regions.sort()
# get the next region to process
next_region = msam_settings.get_setting(settings_key)
# start at the beginning if no previous setting
if next_region is None and len(inventory_regions):
next_region = inventory_regions[0]
region_name = next_region
# proceed only if we have a region
if not region_name is None:
# store the region for the next invocation
try:
# use two copies in case we roll off the end
expanded = inventory_regions + inventory_regions
position = expanded.index(region_name)
# process global after the end of the region list
if position >= 0:
next_region = expanded[position + 1]
else:
next_region = expanded[0]
except (IndexError, ValueError):
# start over if we don't recognize the region, ex. global
next_region = expanded[0]
# store it
msam_settings.put_setting(settings_key, next_region)
# update the region
print(f"updating nodes for region {region_name}")
if region_name == "global":
update_global_func()
else:
update_regional_func(region_name)
except ClientError as error:
print(error)
return region_name
def update_diagrams():
"""
scan for data with tags with MSAM-Diagram name and include in those named diagrams
"""
try:
ddb_table_name = CONTENT_TABLE_NAME
ddb_resource = boto3.resource('dynamodb')
ddb_table = ddb_resource.Table(ddb_table_name)
# expensive textual scan
response = ddb_table.scan(FilterExpression="contains(#data, :tagname)", ExpressionAttributeNames={"#data": "data"}, ExpressionAttributeValues={":tagname": "MSAM-Diagram"})
items = response["Items"]
# check for paging
while "LastEvaluatedKey" in response:
# scan again with start key
response = ddb_table.scan(
FilterExpression="contains(#data, :tagname)",
ExpressionAttributeNames={"#data": "data"},
ExpressionAttributeValues={":tagname": "MSAM-Diagram"},
ExclusiveStartKey=response['LastEvaluatedKey'])
items = items + response["Items"]
# filter down the results
for record in items:
cloud_resource = json.loads(record["data"])
if "Tags" in cloud_resource:
if "MSAM-Diagram" in cloud_resource["Tags"]:
arn = record["arn"]
diagram_name = cloud_resource["Tags"]["MSAM-Diagram"]
print("arn {} needed on diagram {}".format(arn, diagram_name))
diagrams = settings.get_setting("diagrams")
if not diagrams:
diagrams = []
found_diagram = False
view_id = None
for diagram in diagrams:
if diagram["name"] == diagram_name:
view_id = diagram["view_id"]
found_diagram = True
print("found diagram id {}".format(view_id))
if not found_diagram:
view_id = stringcase.snakecase(diagram_name)
print("new diagram id {}".format(view_id))
diagrams.append({"name": diagram_name, "view_id": view_id})
settings.put_setting("diagrams", diagrams)
print("created diagram id {}".format(view_id))
# check if this node is already on the diagram layout
if not layout.has_node(view_id, arn):
print("adding node {} to diagram id {}".format(arn, view_id))
# add the node arn to the layout
layout_items = [{"view": view_id, "id": arn, "x": 0, "y": 0}]
layout.set_node_layout(layout_items)
else:
print("node {} already on diagram id {}".format(arn, view_id))
except ClientError as error:
print(error)
def report_metrics(stackname, hours):
"""
This function is responsible for reporting anonymous resource counts.
"""
cloudwatch = boto3.resource('cloudwatch', config=MSAM_BOTO3_CONFIG)
uuid = msam_settings.get_setting('uuid')
# verify the uuid format from settings
if UUID_RE.match(uuid) is None:
print("uuid in settings does not match required format")
else:
# get the metric
metric = cloudwatch.Metric(METRICS_NAMESPACE, METRICS_NAME)
# assemble the payload structure
_, solution_id, _ = SOLUTION_ID.split("/")
data = {
"Solution": solution_id,
"Version": VERSION,
"UUID": uuid,
"TimeStamp": str(datetime.fromtimestamp(int(time.time()))),
"Data": {}
}
# get the max count for each resource type and add to payload
for resource_type in MONITORED_SERVICES:
response = metric.get_statistics(Dimensions=[{
'Name': 'Stack Name',
'Value': stackname
}, {
'Name': 'Resource Type',
'Value': resource_type
}],
StartTime=datetime.now() -
timedelta(hours=hours),
EndTime=datetime.now(),
Period=(hours * 3600),
Statistics=['Maximum'])
datapoints = response.get('Datapoints', [])
if datapoints:
data["Data"][resource_type] = int(datapoints[0]["Maximum"])
print(json.dumps(data, default=str, indent=4))
if data["Data"]:
# send it
response = requests.post(METRICS_ENDPOINT, json=data)
print(f"POST status code = {response.status_code}")
else:
print("skipping POST because of empty data")
def delete_channel_nodes(name):
"""
API entry point to delete a channel.
"""
try:
name = unquote(name)
table_name = CHANNELS_TABLE_NAME
table = DYNAMO_RESOURCE.Table(table_name)
# update the settings object with the name
name_list = msam_settings.get_setting("channels")
if not name_list:
name_list = []
if name in name_list:
name_list.remove(name)
msam_settings.put_setting("channels", name_list)
# remove the members
try:
response = table.query(
ProjectionExpression="channel,id",
KeyConditionExpression=Key('channel').eq(name))
items = response.get("Items", [])
while "LastEvaluatedKey" in response:
response = table.query(
ProjectionExpression="channel,id",
KeyConditionExpression=Key('channel').eq(name),
ExclusiveStartKey=response["LastEvaluatedKey"])
items = items + response.get("Items", [])
# return the response or an empty object
for item in items:
table.delete_item(Key={
"channel": item["channel"],
"id": item["id"]
})
print("channel items deleted, channel list updated")
except ClientError:
print("not found")
response = {"message": "done"}
except ClientError as outer_error:
# send the exception back in the object
print(outer_error)
response = {"exception": str(outer_error)}
return response
