def add_spec_missing_services_patch(region, services):
""" Go through spec and determine patching """
LOGGER.info('Create 07_ssm_service_addition patch for region %s', region)
spec_string = get_url_content(SPEC_REGIONS.get(region))
spec_string_standard = get_url_content(SPEC_REGIONS.get('us-east-1'))
spec = json.loads(spec_string)
spec_standard = json.loads(spec_string_standard)
patches = []
for spec_type in ['ResourceTypes']:
for service in services:
found = False
for resource in sorted(spec.get(spec_type).keys()):
for spec_name in service_map.get(service):
if resource.startswith(spec_name):
found = True
if found is False:
for standard_spec_type in ['ResourceTypes', 'PropertyTypes']:
for resource in sorted(
spec_standard.get(standard_spec_type).keys()):
for spec_name in service_map.get(service):
if resource.startswith(spec_name):
if spec_standard.get(standard_spec_type).get(
resource):
element = {
'op':
'add',
'path':
'/%s/%s' %
(standard_spec_type, resource),
'value':
spec_standard.get(
standard_spec_type).get(resource)
}
patches.append(element)
elif standard_spec_type == 'ResourceTypes':
print('patch for %s not found' % service)
if patches:
filename = 'src/cfnlint/data/ExtendedSpecs/%s/07_ssm_service_addition.json' % region
with open(filename, 'w+') as f:
json.dump(patches,
f,
indent=2,
sort_keys=True,
separators=(',', ': '))
def search_and_replace_botocore_types(obj):
if isinstance(obj, dict):
new_obj = {}
for key, value in obj.items():
if key == 'botocore':
service_and_type = value.split('/')
service = '/'.join(service_and_type[:-1])
botocore_type = service_and_type[-1]
if service not in botocore_cache:
botocore_cache[service] = json.loads(
get_url_content(
'https://raw.githubusercontent.com/boto/botocore/master/botocore/data/'
+ service + '/service-2.json'))
new_obj['AllowedValues'] = sorted(
botocore_cache[service]['shapes'][botocore_type]
['enum'])
else:
new_obj[key] = search_and_replace_botocore_types(value)
return new_obj
if isinstance(obj, list):
new_list = []
for item in obj:
new_list.append(search_and_replace_botocore_types(item))
return new_list
return obj
def update_resource_spec(region, url):
""" Update a single resource spec """
filename = os.path.join(os.path.dirname(cfnlint.__file__),
'data/CloudSpecs/%s.json' % region)
multiprocessing_logger = multiprocessing.log_to_stderr()
multiprocessing_logger.debug('Downloading template %s into %s', url,
filename)
# Check to see if we already have the latest version, and if so stop
if not url_has_newer_version(url):
return
spec_content = get_url_content(url, caching=True)
multiprocessing_logger.debug(
'A more recent version of %s was found, and will be downloaded to %s',
url, filename)
spec = json.loads(spec_content)
# Patch the files
spec = patch_spec(spec, 'all')
spec = patch_spec(spec, region)
with open(filename, 'w') as f:
json.dump(spec, f, indent=2, sort_keys=True, separators=(',', ': '))
def update_iam_policies():
"""update iam policies file"""
url = 'https://awspolicygen.s3.amazonaws.com/js/policies.js'
filename = os.path.join(
os.path.dirname(cfnlint.data.AdditionalSpecs.__file__),
'Policies.json')
LOGGER.debug('Downloading policies %s into %s', url, filename)
content = get_url_content(url)
content = content.split('app.PolicyEditorConfig=')[1]
content = json.loads(content)
actions = {
'Manage Amazon API Gateway': ['HEAD', 'OPTIONS'],
'Amazon API Gateway Management': ['HEAD', 'OPTIONS'],
'Amazon API Gateway Management V2': ['HEAD', 'OPTIONS'],
'Amazon Kinesis Video Streams': ['StartStreamEncryption'],
}
for k, v in actions.items():
if content.get('serviceMap').get(k):
content['serviceMap'][k]['Actions'].extend(v)
else:
LOGGER.debug('"%s" was not found in the policies file', k)
with open(filename, 'w') as f:
json.dump(content, f, indent=2, sort_keys=True, separators=(',', ': '))
def update_resource_spec(region, url):
""" Update a single resource spec """
filename = os.path.join(os.path.dirname(cfnlint.__file__),
'data/CloudSpecs/%s.json' % region)
multiprocessing_logger = multiprocessing.log_to_stderr()
multiprocessing_logger.debug('Downloading template %s into %s', url,
filename)
# Check to see if we already have the latest version, and if so stop
if not url_has_newer_version(url):
return
spec_content = get_url_content(url, caching=True)
multiprocessing_logger.debug(
'A more recent version of %s was found, and will be downloaded to %s',
url, filename)
spec = json.loads(spec_content)
# Patch the files
spec = patch_spec(spec, 'all')
spec = patch_spec(spec, region)
botocore_cache = {}
def search_and_replace_botocore_types(obj):
if isinstance(obj, dict):
new_obj = {}
for key, value in obj.items():
if key == 'botocore':
service_and_type = value.split('/')
service = '/'.join(service_and_type[:-1])
botocore_type = service_and_type[-1]
if service not in botocore_cache:
botocore_cache[service] = json.loads(
get_url_content(
'https://raw.githubusercontent.com/boto/botocore/master/botocore/data/'
+ service + '/service-2.json'))
new_obj['AllowedValues'] = sorted(
botocore_cache[service]['shapes'][botocore_type]
['enum'])
else:
new_obj[key] = search_and_replace_botocore_types(value)
return new_obj
if isinstance(obj, list):
new_list = []
for item in obj:
new_list.append(search_and_replace_botocore_types(item))
return new_list
return obj
spec = search_and_replace_botocore_types(spec)
with open(filename, 'w') as f:
json.dump(spec, f, indent=2, sort_keys=True, separators=(',', ': '))
def update_resource_specs():
""" Update Resource Specs """
for region, url in SPEC_REGIONS.items():
filename = pkg_resources.resource_filename(
__name__,
'/data/CloudSpecs/%s.json' % region,
)
LOGGER.debug('Downloading template %s into %s', url, filename)
spec = json.loads(get_url_content(url))
# Patch the files
spec = patch_spec(spec, 'all')
spec = patch_spec(spec, region)
with open(filename, 'w') as f:
json.dump(spec, f, indent=2, sort_keys=True, separators=(',', ': '))
def add_spec_patch(region, services):
""" Go through spec and determine patching """
LOGGER.info('Create 06_ssm_service_removal patch for region %s', region)
spec = json.loads(get_url_content(SPEC_REGIONS.get(region)))
patches = []
for spec_type in ['ResourceTypes', 'PropertyTypes']:
for resource in sorted(spec.get(spec_type).keys()):
for service in services:
for spec_name in service_map.get(service):
if resource.startswith(spec_name):
element = {
'op': 'remove',
'path': '/%s/%s' % (spec_type, resource)
}
patches.append(element)
filename = 'src/cfnlint/data/ExtendedSpecs/%s/06_ssm_service_removal.json' % region
with open(filename, 'w+') as f:
json.dump(patches, f, indent=2, sort_keys=True, separators=(',', ': '))
def update_iam_policies():
"""update iam policies file"""
url = 'https://awspolicygen.s3.amazonaws.com/js/policies.js'
filename = os.path.join(
os.path.dirname(cfnlint.data.AdditionalSpecs.__file__),
'Policies.json')
LOGGER.debug('Downloading policies %s into %s', url, filename)
content = get_url_content(url)
content = content.split('app.PolicyEditorConfig=')[1]
content = json.loads(content)
content['serviceMap']['Manage Amazon API Gateway']['Actions'].extend(
['HEAD', 'OPTIONS'])
content['serviceMap']['Amazon Kinesis Video Streams']['Actions'].append(
'StartStreamEncryption')
with open(filename, 'w') as f:
json.dump(content, f, indent=2, sort_keys=True, separators=(',', ': '))
def update_resource_spec(region, url, schema_cache):
""" Update a single resource spec """
filename = os.path.join(os.path.dirname(cfnlint.__file__), 'data/CloudSpecs/%s.json' % region)
multiprocessing_logger = multiprocessing.log_to_stderr()
multiprocessing_logger.debug('Downloading template %s into %s', url, filename)
# Check to see if we already have the latest version, and if so stop
if not url_has_newer_version(url):
return
spec_content = get_url_content(url, caching=True)
multiprocessing_logger.debug(
'A more recent version of %s was found, and will be downloaded to %s', url, filename)
spec = json.loads(spec_content)
# Patch the files
spec = patch_spec(spec, 'all')
spec = patch_spec(spec, region)
# do each patch individually so we can ignore errors
for patch in schema_cache:
try:
# since there could be patched in values to ValueTypes
# Ref/GetAtt as an example. So we want to add new
# ValueTypes that don't exist
for i_patch in patch:
path_details = i_patch.get('path').split('/')
if path_details[1] == 'ValueTypes':
if not spec.get('ValueTypes').get(path_details[2]):
spec['ValueTypes'][path_details[2]] = {}
# do the patch
jsonpatch.JsonPatch(patch).apply(spec, in_place=True)
except jsonpatch.JsonPatchConflict:
for i_patch in patch:
path_details = i_patch.get('path').split('/')
if path_details[1] == 'ValueTypes':
if not spec.get('ValueTypes').get(path_details[2]):
try:
del spec['ValueTypes'][path_details[2]]
except: # pylint: disable=bare-except
pass
LOGGER.debug('Patch (%s) not applied in region %s', patch, region)
except jsonpatch.JsonPointerException:
for i_patch in patch:
path_details = i_patch.get('path').split('/')
if path_details[1] == 'ValueTypes':
if not spec.get('ValueTypes').get(path_details[2]):
try:
del spec['ValueTypes'][path_details[2]]
except: # pylint: disable=bare-except
pass
# Debug as the parent element isn't supported in the region
LOGGER.debug('Parent element not found for patch (%s) in region %s',
patch, region)
botocore_cache = {}
def search_and_replace_botocore_types(obj):
if isinstance(obj, dict):
new_obj = {}
for key, value in obj.items():
if key == 'botocore':
service_and_type = value.split('/')
service = '/'.join(service_and_type[:-1])
botocore_type = service_and_type[-1]
if service not in botocore_cache:
botocore_cache[service] = json.loads(get_url_content(
'https://raw.githubusercontent.com/boto/botocore/master/botocore/data/' + service + '/service-2.json'))
new_obj['AllowedValues'] = sorted(
botocore_cache[service]['shapes'][botocore_type]['enum'])
else:
new_obj[key] = search_and_replace_botocore_types(value)
return new_obj
if isinstance(obj, list):
new_list = []
for item in obj:
new_list.append(search_and_replace_botocore_types(item))
return new_list
return obj
spec = search_and_replace_botocore_types(spec)
with open(filename, 'w') as f:
json.dump(spec, f, indent=2, sort_keys=True, separators=(',', ': '))
