def create_rms_service_dict( data: Iterable[Dict[str, str]], values: Iterable[str]) -> Dict[str, Dict[str, Iterable[str]]]: logger = get_logger() return_dict = {} logger.debug( f"Creating services dictionary from \n{data}\nwith values {values}") for item in data: if "service" in item: service = item["service"] else: logger.info(f"Object {item} has no associated service\n") continue if not rip.get_internal_services( [service]): # check if the service is internal continue for value in values: if service not in return_dict: return_dict[service] = {value: [item[value]]} else: if value in return_dict[service]: return_dict[service][value].append(item[value]) else: return_dict[service][value] = [item[value]] logger.info(f"Created dictionary:\n{return_dict}") return return_dict
def get_all_edges(): logger = get_logger() logger.info(f"Fetching all of the RMS edges...") rmsv2_auth = get_aws_auth(AWS_HOST_RMSV2, AWS_REGION_EAST) fetched_data = get_rms_data(AWS_HOST_RMSV2, rmsv2_auth, RmsEndpoint.EDGES.value) logger.info(f"Edges retrieved:\n{fetched_data}") return fetched_data
def get_services_with_plan(): logger = get_logger() recon_auth = get_aws_auth(AWS_HOST_RECON, AWS_REGION_EAST) logger.info(f"Fetching all services with a plan categorization...") data = get_recon_data(AWS_HOST_RECON, recon_auth, "services") logger.info(f"Data retrieved:\n{data}") return data
def get_recon_data(aws_host: str, auth: AWSRequestsAuth, endpoint: str, **kwargs): logger = get_logger() query_params = urllib.parse.urlencode(kwargs) url = urllib.parse.urlunparse( ("https", aws_host, f"api/{endpoint}", None, query_params, None)) logger.info(f'Making recon request with:\n{url}') json_object = get_recon_response(auth, url) logger.info(f'Response received:\n{json_object}') return json_object
def get_recon_response(recon_auth: AWSRequestsAuth, url: str): logger = get_logger() json_object = {} try: response = retry_get_requests(url, auth=recon_auth) response.raise_for_status() json_object = response.json() logger.info(f"Recon response received:\n{json_object}") except requests.exceptions.HTTPError as e: logger.error(e) return json_object
def update_service_plan(service: str, plan: ServicePlan): logger = get_logger() plan_str = "" if plan == ServicePlan.UNCATEGORIZED else plan.value recon_auth = get_aws_auth(AWS_HOST_RECON_BETA, AWS_REGION_EAST) patch_params = {"op": "replace", "path": "/plan", "value": plan_str} url = urllib.parse.urlunparse( ("https", AWS_HOST_RECON_BETA, f"api/services/{service}", None, None, None)) logger.info( f"Sending patch request to update the plan of {service} with new value \"{plan_str}\"..." ) with requests.patch(url, data=patch_params, auth=recon_auth) as response: logger.info(f"Response received:\n{response}")
def get_aws_auth(aws_host, aws_region): logger = get_logger() session = boto3.Session() credentials = session.get_credentials() current_credentials = credentials.get_frozen_credentials() return AWSRequestsAuth( aws_access_key=current_credentials.access_key, aws_secret_access_key=current_credentials.secret_key, aws_token=current_credentials.token, aws_host=aws_host, aws_region=aws_region, aws_service="execute-api")
def get_rms_response(rmsauth: AWSRequestsAuth, url: str, next_token=None): url = url_token_checker(url, next_token) logger = get_logger() response = retry_get_requests(url, auth=rmsauth) json_object = None try: response.raise_for_status() json_object = response.json() next_token = json_object.get("nextToken") logger.info(f"RMS response received:\n{json_object}") except requests.exceptions.HTTPError as e: logger.error(response.raise_for_status()) next_token = None return json_object, next_token
def get_cpt_data(endpoints: Iterable[str]) -> Iterable[Dict[str, str]]: logger = get_logger() rmsv2_auth = get_aws_auth(AWS_HOST_RMSV2, AWS_REGION_EAST) fetched_data = [] for endpoint in endpoints: kwargs = { "label": "COMMERCIAL_PARTITION_TEMPLATE" } if endpoint == RmsEndpoint.EDGES.value else { "dimension": "COMMERCIAL_PARTITION_TEMPLATE" } fetched_data.append( get_rms_data(AWS_HOST_RMSV2, rmsv2_auth, endpoint, **kwargs)) logger.info(f"Data retrieved:\n{fetched_data}") return fetched_data
def get_internal_services(service_identifiers: Iterable[str]=[]): logger = get_logger() helper = RIPHelperLocal(metapackage="RIPDataAllSQLite-1.0") services = [] # Check in case future use involves grabbing all internal services if service_identifiers: for identifier in service_identifiers: try: if helper.service(identifier).visibility == Visibility.INTERNAL: services.append(identifier) except ServiceNotFoundError: logger.warning(f"The service identifier, \"{identifier}\", could not be found. {ServiceNotFoundError}") return services
def find_plan(service: str): logger = get_logger() recon_auth = get_aws_auth(AWS_HOST_RECON, AWS_REGION_EAST) logger.info(f"Fetching the plan categorization for {service}...") data = get_recon_data(AWS_HOST_RECON, recon_auth, f"services/{service}/plans") logger.info(f"Data retrieved:\n{data}") plan = data.get("service", {}).get("plan") if plan == ServicePlan.LAUNCH_BLOCKING.value: return ServicePlan.LAUNCH_BLOCKING elif plan == ServicePlan.MANDATORY.value: return ServicePlan.MANDATORY else: return ServicePlan.NON_GLOBAL
def get_rms_data(aws_host: str, auth: AWSRequestsAuth, endpoint: str, **kwargs): data = [] logger = get_logger() query_params = urllib.parse.urlencode(kwargs) url = urllib.parse.urlunparse( ("https", aws_host, f"prod/{endpoint}", None, query_params, None)) json_object, next_token = get_rms_response(auth, url) data.extend(json_object[endpoint]) while next_token: logger.info( f"Sending request to fetch data from RMSV2 commercial partition template with token {next_token}..." ) json_object, next_token = get_rms_response(auth, url, next_token) data.extend(json_object[endpoint]) return data
def get_milestones_tasks_successors(arn: str, option: RmsEndpoint): logger = get_logger() rmsv2_auth = get_aws_auth(AWS_HOST_RMSV2, AWS_REGION_EAST) fetched_data = None if option == RmsEndpoint.MILESTONES or option == RmsEndpoint.TASKS: endpoints = [option.value] elif option == RmsEndpoint.MT: endpoints = [RmsEndpoint.MILESTONES.value, RmsEndpoint.TASKS.value] else: raise ValueError(f"{option} is not a supported option") fetched_data = [ get_rms_data(AWS_HOST_RMSV2, rmsv2_auth, endpoint, predecessor=arn) for endpoint in endpoints ] logger.info( f"Grabbed the {option.value} succesors for ARN: {arn}:\n{fetched_data}" ) return fetched_data
from typing import Iterator, List from regions_recon_python_common.buildables_dao_models.service_metadata import SERVICE_METADATA_LATEST_ARTIFACT, \ ServiceMetadataLatest from regions_recon_python_common.buildables_dao_models.service_plan import PLAN_BY_SERVICE_ARTIFACT, \ PlanByService from regions_recon_python_common.query_utils.region_metadata_query_utils import get_region_metadata from regions_recon_python_common.utils.log import get_logger from regions_recon_python_common.utils.rms_managed_regions import get_regions_within_one_day_post_launch, \ get_regions_within_ninety_business_days_post_launch logger = get_logger() LAUNCH_BLOCKING_SERVICE_PLAN = "Globally Expanding - Launch Blocking" MANDATORY_SERVICE_PLAN = "Globally Expanding - Mandatory" def get_launch_blocking_services() -> Iterator[ServiceMetadataLatest]: return ServiceMetadataLatest.artifact_plan_index.query( SERVICE_METADATA_LATEST_ARTIFACT, range_key_condition=ServiceMetadataLatest.plan == LAUNCH_BLOCKING_SERVICE_PLAN ) def get_mandatory_services() -> Iterator[ServiceMetadataLatest]: return ServiceMetadataLatest.artifact_plan_index.query( SERVICE_METADATA_LATEST_ARTIFACT, range_key_condition=ServiceMetadataLatest.plan == MANDATORY_SERVICE_PLAN )
def test_format_csvrow(): logger = get_logger() fake_services_seen_categories = { "ecytu": ServicePlan.MANDATORY, "stree": ServicePlan.MANDATORY } fake_rms_jsons = [{ "arn": "arn:aws:rmsv2:::milestones/001", "dimension": "COMMERCIAL_PARTITION_TEMPLATE", "service": "stree3", "name": "Milestone 1" }, { "arn": "arn:aws:rmsv2:::milestones/002", "dimension": "COMMERCIAL_PARTITION_TEMPLATE", "service": "ecytu", "name": "Milestone 2" }, { "arn": "arn:aws:rmsv2:::milestones/003", "dimension": "COMMERCIAL_PARTITION_TEMPLATE", "service": "llamada", "name": "Milestone 3" }, { "arn": "arn:aws:rmsv2:::milestones/004", "dimension": "COMMERCIAL_PARTITION_TEMPLATE", "service": "cloudobserve", "name": "Milestone 4" }, { "arn": "arn:aws:rmsv2:::milestones/005", "dimension": "COMMERCIAL_PARTITION_TEMPLATE", "service": "blueshift", "name": "Milestone 5" }, { "arn": "arn:aws:rmsv2:::milestones/006", "dimension": "COMMERCIAL_PARTITION_TEMPLATE", "service": "menervesah", "name": "Milestone 6" }] fake_service_to_arn = { "stree": { "arn": [ "arn:aws:rmsv2:::milestones/007", "arn:aws:rmsv2:::milestones/008" ] } } fake_successors = [ RmsMilestoneOrTaskData( rms_data=fake_rms_jsons[0], service_plan=ServicePlan.MANDATORY, on_path_predecessor=fake_service_to_arn["stree"]["arn"][0]), RmsMilestoneOrTaskData( rms_data=fake_rms_jsons[1], service_plan=ServicePlan.NON_GLOBAL, on_path_predecessor=fake_service_to_arn["stree"]["arn"][1]), RmsMilestoneOrTaskData(rms_data=fake_rms_jsons[2], service_plan=ServicePlan.MANDATORY, on_path_predecessor=fake_rms_jsons[0]["arn"]), RmsMilestoneOrTaskData(rms_data=fake_rms_jsons[3], service_plan=ServicePlan.MANDATORY, on_path_predecessor=fake_rms_jsons[1]["arn"]), RmsMilestoneOrTaskData(rms_data=fake_rms_jsons[4], service_plan=ServicePlan.NON_GLOBAL, on_path_predecessor=fake_rms_jsons[1]["arn"]), RmsMilestoneOrTaskData(rms_data=fake_rms_jsons[5], service_plan=ServicePlan.MANDATORY, on_path_predecessor=fake_rms_jsons[1]["arn"]) ] expected_basic_output = [ "stree", ServicePlan.MANDATORY.value, "stree3, ecytu, llamada, cloudobserve, blueshift, menervesah" ] expected_detailed_output = [ "stree", ServicePlan.MANDATORY.value, f"{fake_service_to_arn['stree']['arn'][0]}\n{fake_service_to_arn['stree']['arn'][1]}", f"({fake_rms_jsons[0]['name']}, arn:aws:rmsv2:::milestones/001, stree3, {ServicePlan.MANDATORY.value}, {fake_service_to_arn['stree']['arn'][0]})\n\n" + f"({fake_rms_jsons[1]['name']}, arn:aws:rmsv2:::milestones/002, ecytu, {ServicePlan.NON_GLOBAL.value}, {fake_service_to_arn['stree']['arn'][1]})\n\n" + f"({fake_rms_jsons[2]['name']}, arn:aws:rmsv2:::milestones/003, llamada, {ServicePlan.MANDATORY.value}, {fake_rms_jsons[0]['arn']})\n\n" + f"({fake_rms_jsons[3]['name']}, arn:aws:rmsv2:::milestones/004, cloudobserve, {ServicePlan.MANDATORY.value}, {fake_rms_jsons[1]['arn']})\n\n" + f"({fake_rms_jsons[4]['name']}, arn:aws:rmsv2:::milestones/005, blueshift, {ServicePlan.NON_GLOBAL.value}, {fake_rms_jsons[1]['arn']})\n\n" + f"({fake_rms_jsons[5]['name']}, arn:aws:rmsv2:::milestones/006, menervesah, {ServicePlan.MANDATORY.value}, {fake_rms_jsons[1]['arn']})" ] basic_output, detailed_output = format_csvrow( "stree", fake_successors, fake_services_seen_categories, fake_service_to_arn) assert expected_basic_output == basic_output assert expected_detailed_output == detailed_output
from dateutil import parser import pytz from regions_recon_python_common.utils.cloudwatch_metrics_utils import submit_cloudwatch_metrics, increment_metric from regions_recon_python_common.message_multiplexer_helper import MessageMultiplexerHelper, get_mm_endpoint from regions_recon_python_common.utils.constants import BUILDABLES_TABLE_NAME from regions_recon_python_common.utils.log import get_logger from regions_recon_python_common.utils.misc import get_rip_name_from_instance_field from regions_recon_lambda.utils.dynamo_query import DynamoQuery from regions_recon_lambda.slips_email.slips_email_dao import SlipsEmailDAO MESSAGE_GROUP_NAME = "region-slips" CONDENSED_DATE_FORMAT = "%Y-%m-%d" SLIP_DELTA = 30 NORMALIZED_DATE_FORMAT = "%Y-%m-%d %H:%M %Z" logger = get_logger(logging.INFO) mmlogger = logging.getLogger('messagemultiplexer') mmlogger.setLevel(logging.WARN) def send_slips_mail(event, context): try: if 'debug' in event: logger.setLevel(logging.DEBUG) if 'debug' in event or 'logmm' in event: mmlogger.setLevel(logging.INFO) dryrun = 'dryrun' in event metrics = {} metrics["message_send_success"] = 0 metrics["message_send_failure"] = 0