def add_spec_missing_services_patch(region, services): """ Go through spec and determine patching """ LOGGER.info('Create 07_ssm_service_addition patch for region %s', region) spec_string = get_url_content(SPEC_REGIONS.get(region)) spec_string_standard = get_url_content(SPEC_REGIONS.get('us-east-1')) spec = json.loads(spec_string) spec_standard = json.loads(spec_string_standard) patches = [] for spec_type in ['ResourceTypes']: for service in services: found = False for resource in sorted(spec.get(spec_type).keys()): for spec_name in service_map.get(service): if resource.startswith(spec_name): found = True if found is False: for standard_spec_type in ['ResourceTypes', 'PropertyTypes']: for resource in sorted( spec_standard.get(standard_spec_type).keys()): for spec_name in service_map.get(service): if resource.startswith(spec_name): if spec_standard.get(standard_spec_type).get( resource): element = { 'op': 'add', 'path': '/%s/%s' % (standard_spec_type, resource), 'value': spec_standard.get( standard_spec_type).get(resource) } patches.append(element) elif standard_spec_type == 'ResourceTypes': print('patch for %s not found' % service) if patches: filename = 'src/cfnlint/data/ExtendedSpecs/%s/07_ssm_service_addition.json' % region with open(filename, 'w+') as f: json.dump(patches, f, indent=2, sort_keys=True, separators=(',', ': '))
def search_and_replace_botocore_types(obj): if isinstance(obj, dict): new_obj = {} for key, value in obj.items(): if key == 'botocore': service_and_type = value.split('/') service = '/'.join(service_and_type[:-1]) botocore_type = service_and_type[-1] if service not in botocore_cache: botocore_cache[service] = json.loads( get_url_content( 'https://raw.githubusercontent.com/boto/botocore/master/botocore/data/' + service + '/service-2.json')) new_obj['AllowedValues'] = sorted( botocore_cache[service]['shapes'][botocore_type] ['enum']) else: new_obj[key] = search_and_replace_botocore_types(value) return new_obj if isinstance(obj, list): new_list = [] for item in obj: new_list.append(search_and_replace_botocore_types(item)) return new_list return obj
def update_resource_spec(region, url): """ Update a single resource spec """ filename = os.path.join(os.path.dirname(cfnlint.__file__), 'data/CloudSpecs/%s.json' % region) multiprocessing_logger = multiprocessing.log_to_stderr() multiprocessing_logger.debug('Downloading template %s into %s', url, filename) # Check to see if we already have the latest version, and if so stop if not url_has_newer_version(url): return spec_content = get_url_content(url, caching=True) multiprocessing_logger.debug( 'A more recent version of %s was found, and will be downloaded to %s', url, filename) spec = json.loads(spec_content) # Patch the files spec = patch_spec(spec, 'all') spec = patch_spec(spec, region) with open(filename, 'w') as f: json.dump(spec, f, indent=2, sort_keys=True, separators=(',', ': '))
def update_iam_policies(): """update iam policies file""" url = 'https://awspolicygen.s3.amazonaws.com/js/policies.js' filename = os.path.join( os.path.dirname(cfnlint.data.AdditionalSpecs.__file__), 'Policies.json') LOGGER.debug('Downloading policies %s into %s', url, filename) content = get_url_content(url) content = content.split('app.PolicyEditorConfig=')[1] content = json.loads(content) actions = { 'Manage Amazon API Gateway': ['HEAD', 'OPTIONS'], 'Amazon API Gateway Management': ['HEAD', 'OPTIONS'], 'Amazon API Gateway Management V2': ['HEAD', 'OPTIONS'], 'Amazon Kinesis Video Streams': ['StartStreamEncryption'], } for k, v in actions.items(): if content.get('serviceMap').get(k): content['serviceMap'][k]['Actions'].extend(v) else: LOGGER.debug('"%s" was not found in the policies file', k) with open(filename, 'w') as f: json.dump(content, f, indent=2, sort_keys=True, separators=(',', ': '))
def update_resource_spec(region, url): """ Update a single resource spec """ filename = os.path.join(os.path.dirname(cfnlint.__file__), 'data/CloudSpecs/%s.json' % region) multiprocessing_logger = multiprocessing.log_to_stderr() multiprocessing_logger.debug('Downloading template %s into %s', url, filename) # Check to see if we already have the latest version, and if so stop if not url_has_newer_version(url): return spec_content = get_url_content(url, caching=True) multiprocessing_logger.debug( 'A more recent version of %s was found, and will be downloaded to %s', url, filename) spec = json.loads(spec_content) # Patch the files spec = patch_spec(spec, 'all') spec = patch_spec(spec, region) botocore_cache = {} def search_and_replace_botocore_types(obj): if isinstance(obj, dict): new_obj = {} for key, value in obj.items(): if key == 'botocore': service_and_type = value.split('/') service = '/'.join(service_and_type[:-1]) botocore_type = service_and_type[-1] if service not in botocore_cache: botocore_cache[service] = json.loads( get_url_content( 'https://raw.githubusercontent.com/boto/botocore/master/botocore/data/' + service + '/service-2.json')) new_obj['AllowedValues'] = sorted( botocore_cache[service]['shapes'][botocore_type] ['enum']) else: new_obj[key] = search_and_replace_botocore_types(value) return new_obj if isinstance(obj, list): new_list = [] for item in obj: new_list.append(search_and_replace_botocore_types(item)) return new_list return obj spec = search_and_replace_botocore_types(spec) with open(filename, 'w') as f: json.dump(spec, f, indent=2, sort_keys=True, separators=(',', ': '))
def update_resource_specs(): """ Update Resource Specs """ for region, url in SPEC_REGIONS.items(): filename = pkg_resources.resource_filename( __name__, '/data/CloudSpecs/%s.json' % region, ) LOGGER.debug('Downloading template %s into %s', url, filename) spec = json.loads(get_url_content(url)) # Patch the files spec = patch_spec(spec, 'all') spec = patch_spec(spec, region) with open(filename, 'w') as f: json.dump(spec, f, indent=2, sort_keys=True, separators=(',', ': '))
def add_spec_patch(region, services): """ Go through spec and determine patching """ LOGGER.info('Create 06_ssm_service_removal patch for region %s', region) spec = json.loads(get_url_content(SPEC_REGIONS.get(region))) patches = [] for spec_type in ['ResourceTypes', 'PropertyTypes']: for resource in sorted(spec.get(spec_type).keys()): for service in services: for spec_name in service_map.get(service): if resource.startswith(spec_name): element = { 'op': 'remove', 'path': '/%s/%s' % (spec_type, resource) } patches.append(element) filename = 'src/cfnlint/data/ExtendedSpecs/%s/06_ssm_service_removal.json' % region with open(filename, 'w+') as f: json.dump(patches, f, indent=2, sort_keys=True, separators=(',', ': '))
def update_iam_policies(): """update iam policies file""" url = 'https://awspolicygen.s3.amazonaws.com/js/policies.js' filename = os.path.join( os.path.dirname(cfnlint.data.AdditionalSpecs.__file__), 'Policies.json') LOGGER.debug('Downloading policies %s into %s', url, filename) content = get_url_content(url) content = content.split('app.PolicyEditorConfig=')[1] content = json.loads(content) content['serviceMap']['Manage Amazon API Gateway']['Actions'].extend( ['HEAD', 'OPTIONS']) content['serviceMap']['Amazon Kinesis Video Streams']['Actions'].append( 'StartStreamEncryption') with open(filename, 'w') as f: json.dump(content, f, indent=2, sort_keys=True, separators=(',', ': '))
def update_resource_spec(region, url, schema_cache): """ Update a single resource spec """ filename = os.path.join(os.path.dirname(cfnlint.__file__), 'data/CloudSpecs/%s.json' % region) multiprocessing_logger = multiprocessing.log_to_stderr() multiprocessing_logger.debug('Downloading template %s into %s', url, filename) # Check to see if we already have the latest version, and if so stop if not url_has_newer_version(url): return spec_content = get_url_content(url, caching=True) multiprocessing_logger.debug( 'A more recent version of %s was found, and will be downloaded to %s', url, filename) spec = json.loads(spec_content) # Patch the files spec = patch_spec(spec, 'all') spec = patch_spec(spec, region) # do each patch individually so we can ignore errors for patch in schema_cache: try: # since there could be patched in values to ValueTypes # Ref/GetAtt as an example. So we want to add new # ValueTypes that don't exist for i_patch in patch: path_details = i_patch.get('path').split('/') if path_details[1] == 'ValueTypes': if not spec.get('ValueTypes').get(path_details[2]): spec['ValueTypes'][path_details[2]] = {} # do the patch jsonpatch.JsonPatch(patch).apply(spec, in_place=True) except jsonpatch.JsonPatchConflict: for i_patch in patch: path_details = i_patch.get('path').split('/') if path_details[1] == 'ValueTypes': if not spec.get('ValueTypes').get(path_details[2]): try: del spec['ValueTypes'][path_details[2]] except: # pylint: disable=bare-except pass LOGGER.debug('Patch (%s) not applied in region %s', patch, region) except jsonpatch.JsonPointerException: for i_patch in patch: path_details = i_patch.get('path').split('/') if path_details[1] == 'ValueTypes': if not spec.get('ValueTypes').get(path_details[2]): try: del spec['ValueTypes'][path_details[2]] except: # pylint: disable=bare-except pass # Debug as the parent element isn't supported in the region LOGGER.debug('Parent element not found for patch (%s) in region %s', patch, region) botocore_cache = {} def search_and_replace_botocore_types(obj): if isinstance(obj, dict): new_obj = {} for key, value in obj.items(): if key == 'botocore': service_and_type = value.split('/') service = '/'.join(service_and_type[:-1]) botocore_type = service_and_type[-1] if service not in botocore_cache: botocore_cache[service] = json.loads(get_url_content( 'https://raw.githubusercontent.com/boto/botocore/master/botocore/data/' + service + '/service-2.json')) new_obj['AllowedValues'] = sorted( botocore_cache[service]['shapes'][botocore_type]['enum']) else: new_obj[key] = search_and_replace_botocore_types(value) return new_obj if isinstance(obj, list): new_list = [] for item in obj: new_list.append(search_and_replace_botocore_types(item)) return new_list return obj spec = search_and_replace_botocore_types(spec) with open(filename, 'w') as f: json.dump(spec, f, indent=2, sort_keys=True, separators=(',', ': '))