def update_nodes(): """ Entry point for the CloudWatch scheduled task to discover and cache services. """ try: never_regions_key = "never-cache-regions" never_regions = msam_settings.get_setting(never_regions_key) if never_regions is None: never_regions = [] settings_key = "cache-next-region" # make a region name list region_name_list = [] for region in regions(): region_name = region["RegionName"] # exclude regions listed in never-cache setting if region_name not in never_regions: region_name_list.append(region_name) else: print("{} in {} setting".format(region_name, never_regions_key)) # sort it region_name_list.sort() # get the next region to process next_region = msam_settings.get_setting(settings_key) # start at the beginning if no previous setting if next_region is None: next_region = region_name_list[0] # otherwise it's saved for us region_name = next_region # store the region for the next schedule try: # process global after the end of the region list if region_name_list.index(next_region) + 1 >= len( region_name_list): next_region = "global" else: next_region = region_name_list[ region_name_list.index(next_region) + 1] except (IndexError, ValueError): # start over if we don't recognize the region, ex. global next_region = region_name_list[0] # store it msam_settings.put_setting(settings_key, next_region) # update the region print("updating region {}".format(region_name)) if region_name == "global": content.put_ddb_items(node_cache.s3_bucket_ddb_items()) content.put_ddb_items( node_cache.cloudfront_distribution_ddb_items()) else: node_cache.update_regional_ddb_items(region_name) except ClientError as error: print(error) return True
def update_nodes(): """ Entry point for the CloudWatch scheduled task to discover and cache services. """ try: never_regions_key = "never-cache-regions" never_regions = msam_settings.get_setting(never_regions_key) if never_regions is None: never_regions = [] settings_key = "cache-next-region" # make a region name list region_name_list = [] for region in regions(): region_name = region["RegionName"] # exclude regions listed in never-cache setting if region_name not in never_regions: region_name_list.append(region_name) else: print("{} in {} setting".format(region_name, never_regions_key)) # sort it region_name_list.sort() # get the next region to process next_region = msam_settings.get_setting(settings_key) # start at the beginning if no previous setting if next_region is None: next_region = region_name_list[0] # otherwise it's saved for us region_name = next_region # store the region for the next schedule try: # process global after the end of the region list if region_name_list.index(next_region) + 1 >= len(region_name_list): next_region = "global" else: next_region = region_name_list[region_name_list.index(next_region) + 1] except (IndexError, ValueError): # start over if we don't recognize the region, ex. global next_region = region_name_list[0] # store it msam_settings.put_setting(settings_key, next_region) # update the region print("updating region {}".format(region_name)) if region_name == "global": content.put_ddb_items(node_cache.s3_bucket_ddb_items()) content.put_ddb_items(node_cache.cloudfront_distribution_ddb_items()) else: node_cache.update_regional_ddb_items(region_name) except ClientError as error: print(error) return True
def set_channel_nodes(name, node_ids): """ API entry point to set the nodes for a given channel name. """ try: name = unquote(name) table = DYNAMO_RESOURCE.Table(CHANNELS_TABLE_NAME) # print(request.json_body) # node_ids = request.json_body # write the channel nodes to the database for node_id in node_ids: item = {"channel": name, "id": node_id} table.put_item(Item=item) # update the list of channels in settings name_list = msam_settings.get_setting("channels") if not name_list: name_list = [] if name not in name_list: name_list.append(name) msam_settings.put_setting("channels", name_list) result = {"message": "saved"} print(result) except ClientError as error: # send the exception back in the object print(error) result = {"exception": str(error)} return result
def set_channel_nodes(request, name): """ API entry point to set the nodes for a given channel name. """ try: name = unquote(name) table = DYNAMO_RESOURCE.Table(CHANNELS_TABLE_NAME) print(request.json_body) node_ids = request.json_body # write the channel nodes to the database for node_id in node_ids: item = {"channel": name, "id": node_id} table.put_item(Item=item) # update the list of channels in settings name_list = msam_settings.get_setting("channels") if not name_list: name_list = [] if name not in name_list: name_list.append(name) msam_settings.put_setting("channels", name_list) result = {"message": "saved"} print(result) except ClientError as error: # send the exception back in the object print(error) result = {"exception": str(error)} return result
def delete_channel_nodes(request, name): """ API entry point to delete a channel. """ try: name = unquote(name) table_name = CHANNELS_TABLE_NAME table = DYNAMO_RESOURCE.Table(table_name) print(request.method) try: # get the settings object response = table.query(KeyConditionExpression=Key('channel').eq(name)) print(response) # return the response or an empty object if "Items" in response: for item in response["Items"]: table.delete_item(Key={"channel": item["channel"], "id": item["id"]}) name_list = msam_settings.get_setting("channels") if not name_list: name_list = [] if name in name_list: name_list.remove(name) msam_settings.put_setting("channels", name_list) print("channel items deleted, channel list updated") except ClientError: print("not found") response = {"message": "done"} except ClientError as outer_error: # send the exception back in the object print(outer_error) response = {"exception": str(outer_error)} return response
def delete_channel_nodes(request, name): """ API entry point to delete a channel. """ try: name = unquote(name) table_name = CHANNELS_TABLE_NAME table = DYNAMO_RESOURCE.Table(table_name) print(request.method) try: # get the settings object response = table.query(KeyConditionExpression=Key('channel').eq(name)) print(response) # return the response or an empty object if "Items" in response: for item in response["Items"]: table.delete_item(Key={"channel": item["channel"], "id": item["id"]}) name_list = msam_settings.get_setting("channels") if not name_list: name_list = [] if name in name_list: name_list.remove(name) msam_settings.put_setting("channels", name_list) print("channel items deleted, channel list updated") except ClientError: print("not found") response = {"message": "done"} except ClientError as outer_error: # send the exception back in the object print(outer_error) response = {"exception": str(outer_error)} return response
def get_channel_list(): """ Return all the current channel names. """ channels = msam_settings.get_setting("channels") if not channels: channels = [] return channels
def get_channel_list(): """ Return all the current channel names. """ channels = msam_settings.get_setting("channels") if not channels: channels = [] return channels
def update_nodes_generic(update_global_func, update_regional_func, settings_key): """ Entry point for the CloudWatch scheduled task to discover and cache services. """ try: inventory_regions_key = "inventory-regions" inventory_regions = msam_settings.get_setting(inventory_regions_key) if inventory_regions is None: inventory_regions = [] inventory_regions.sort() # get the next region to process next_region = msam_settings.get_setting(settings_key) # start at the beginning if no previous setting if next_region is None and len(inventory_regions): next_region = inventory_regions[0] region_name = next_region # proceed only if we have a region if not region_name is None: # store the region for the next invocation try: # use two copies in case we roll off the end expanded = inventory_regions + inventory_regions position = expanded.index(region_name) # process global after the end of the region list if position >= 0: next_region = expanded[position + 1] else: next_region = expanded[0] except (IndexError, ValueError): # start over if we don't recognize the region, ex. global next_region = expanded[0] # store it msam_settings.put_setting(settings_key, next_region) # update the region print(f"updating nodes for region {region_name}") if region_name == "global": update_global_func() else: update_regional_func(region_name) except ClientError as error: print(error) return region_name
def update_diagrams(): """ scan for data with tags with MSAM-Diagram name and include in those named diagrams """ try: ddb_table_name = CONTENT_TABLE_NAME ddb_resource = boto3.resource('dynamodb') ddb_table = ddb_resource.Table(ddb_table_name) # expensive textual scan response = ddb_table.scan(FilterExpression="contains(#data, :tagname)", ExpressionAttributeNames={"#data": "data"}, ExpressionAttributeValues={":tagname": "MSAM-Diagram"}) items = response["Items"] # check for paging while "LastEvaluatedKey" in response: # scan again with start key response = ddb_table.scan( FilterExpression="contains(#data, :tagname)", ExpressionAttributeNames={"#data": "data"}, ExpressionAttributeValues={":tagname": "MSAM-Diagram"}, ExclusiveStartKey=response['LastEvaluatedKey']) items = items + response["Items"] # filter down the results for record in items: cloud_resource = json.loads(record["data"]) if "Tags" in cloud_resource: if "MSAM-Diagram" in cloud_resource["Tags"]: arn = record["arn"] diagram_name = cloud_resource["Tags"]["MSAM-Diagram"] print("arn {} needed on diagram {}".format(arn, diagram_name)) diagrams = settings.get_setting("diagrams") if not diagrams: diagrams = [] found_diagram = False view_id = None for diagram in diagrams: if diagram["name"] == diagram_name: view_id = diagram["view_id"] found_diagram = True print("found diagram id {}".format(view_id)) if not found_diagram: view_id = stringcase.snakecase(diagram_name) print("new diagram id {}".format(view_id)) diagrams.append({"name": diagram_name, "view_id": view_id}) settings.put_setting("diagrams", diagrams) print("created diagram id {}".format(view_id)) # check if this node is already on the diagram layout if not layout.has_node(view_id, arn): print("adding node {} to diagram id {}".format(arn, view_id)) # add the node arn to the layout layout_items = [{"view": view_id, "id": arn, "x": 0, "y": 0}] layout.set_node_layout(layout_items) else: print("node {} already on diagram id {}".format(arn, view_id)) except ClientError as error: print(error)
def report_metrics(stackname, hours): """ This function is responsible for reporting anonymous resource counts. """ cloudwatch = boto3.resource('cloudwatch', config=MSAM_BOTO3_CONFIG) uuid = msam_settings.get_setting('uuid') # verify the uuid format from settings if UUID_RE.match(uuid) is None: print("uuid in settings does not match required format") else: # get the metric metric = cloudwatch.Metric(METRICS_NAMESPACE, METRICS_NAME) # assemble the payload structure _, solution_id, _ = SOLUTION_ID.split("/") data = { "Solution": solution_id, "Version": VERSION, "UUID": uuid, "TimeStamp": str(datetime.fromtimestamp(int(time.time()))), "Data": {} } # get the max count for each resource type and add to payload for resource_type in MONITORED_SERVICES: response = metric.get_statistics(Dimensions=[{ 'Name': 'Stack Name', 'Value': stackname }, { 'Name': 'Resource Type', 'Value': resource_type }], StartTime=datetime.now() - timedelta(hours=hours), EndTime=datetime.now(), Period=(hours * 3600), Statistics=['Maximum']) datapoints = response.get('Datapoints', []) if datapoints: data["Data"][resource_type] = int(datapoints[0]["Maximum"]) print(json.dumps(data, default=str, indent=4)) if data["Data"]: # send it response = requests.post(METRICS_ENDPOINT, json=data) print(f"POST status code = {response.status_code}") else: print("skipping POST because of empty data")
def delete_channel_nodes(name): """ API entry point to delete a channel. """ try: name = unquote(name) table_name = CHANNELS_TABLE_NAME table = DYNAMO_RESOURCE.Table(table_name) # update the settings object with the name name_list = msam_settings.get_setting("channels") if not name_list: name_list = [] if name in name_list: name_list.remove(name) msam_settings.put_setting("channels", name_list) # remove the members try: response = table.query( ProjectionExpression="channel,id", KeyConditionExpression=Key('channel').eq(name)) items = response.get("Items", []) while "LastEvaluatedKey" in response: response = table.query( ProjectionExpression="channel,id", KeyConditionExpression=Key('channel').eq(name), ExclusiveStartKey=response["LastEvaluatedKey"]) items = items + response.get("Items", []) # return the response or an empty object for item in items: table.delete_item(Key={ "channel": item["channel"], "id": item["id"] }) print("channel items deleted, channel list updated") except ClientError: print("not found") response = {"message": "done"} except ClientError as outer_error: # send the exception back in the object print(outer_error) response = {"exception": str(outer_error)} return response