def test_element_doc(self):
class A(object):
pass
class B(object):
"""Hello World
xyz
"""
class C(B):
pass
class D(ValueFilter):
pass
class E(ValueFilter):
"""Something"""
class F(D):
pass
class G(E):
pass
self.assertEqual(ElementSchema.doc(G), "Something")
self.assertEqual(ElementSchema.doc(D), "")
self.assertEqual(ElementSchema.doc(F), "")
self.assertEqual(
ElementSchema.doc(B), "Hello World\n\nxyz")
def test_element_resolve(self):
vocab = resource_vocabulary()
self.assertEqual(
ElementSchema.resolve(vocab, 'mode.periodic').type, 'periodic')
self.assertEqual(ElementSchema.resolve(vocab, 'aws.ec2').type, 'ec2')
self.assertEqual(
ElementSchema.resolve(vocab, 'aws.ec2.actions.stop').type, 'stop')
self.assertRaises(ValueError, ElementSchema.resolve, vocab,
'aws.ec2.actions.foo')
def render_resource(cls, resource_path):
resource_class = cls.resolve(resource_path)
provider_name, resource_name = resource_path.split('.', 1)
return cls._render('resource.rst',
variables=dict(
provider_name=provider_name,
resource_name="%s.%s" % (provider_name, resource_class.type),
filters=ElementSchema.elements(resource_class.filter_registry),
actions=ElementSchema.elements(resource_class.action_registry),
resource=resource_class))
def render_schema(cls, el):
return cls._render(
'schema.rst',
{'schema_yaml': yaml.dump(
ElementSchema.schema(cls.definitions, el),
Dumper=SafeNoAliasDumper,
default_flow_style=False)})
def test_schema_expand(self):
# refs should only ever exist in a dictionary by itself
test_schema = {
'$ref': '#/definitions/filters_common/value_from'
}
result = ElementSchema.schema(generate()['definitions'], test_schema)
self.assertEqual(result, ValuesFrom.schema)
def render_schema(cls, el):
return cls._render(
'schema.rst', {
'schema_yaml':
yaml.safe_dump(ElementSchema.schema(cls.definitions, el),
default_flow_style=False)
})
def run(self):
schema_path = self.arguments[0]
el = self.resolve(schema_path)
schema_yaml = yaml.safe_dump(
ElementSchema.schema(self.definitions, el), default_flow_style=False)
return self._nodify(
'schema.rst', '<c7n-schema>',
dict(name=schema_path, schema_yaml=schema_yaml))
def test_schema_multi_expand(self):
test_schema = {
'schema1': {
'$ref': '#/definitions/filters_common/value_from'
},
'schema2': {
'$ref': '#/definitions/filters_common/value_from'
}
}
expected = yaml_dump({
'schema1': {
'type': 'object',
'additionalProperties': 'False',
'required': ['url'],
'properties': {
'url': {
'type': 'string'
},
'format': {
'enum': ['csv', 'json', 'txt', 'csv2dict']
},
'expr': {
'oneOf': [{
'type': 'integer'
}, {
'type': 'string'
}]
}
}
},
'schema2': {
'type': 'object',
'additionalProperties': 'False',
'required': ['url'],
'properties': {
'url': {
'type': 'string'
},
'format': {
'enum': ['csv', 'json', 'txt', 'csv2dict']
},
'expr': {
'oneOf': [{
'type': 'integer'
}, {
'type': 'string'
}]
}
}
}
})
result = yaml_dump(
ElementSchema.schema(generate()['definitions'], test_schema))
self.assertEqual(result, expected)
def _print_cls_schema(cls):
# Print docstring
docstring = ElementSchema.doc(cls)
print("\nHelp\n----\n")
if docstring:
print(docstring)
else:
# Shouldn't ever hit this, so exclude from cover
print("No help is available for this item.") # pragma: no cover
# Print schema
print("\nSchema\n------\n")
if hasattr(cls, 'schema'):
definitions = generate()['definitions']
component_schema = ElementSchema.schema(definitions, cls)
print(yaml_dump(component_schema))
else:
# Shouldn't ever hit this, so exclude from cover
print("No schema is available for this item.", file=sys.sterr) # pragma: no cover
print('')
return
def get_doc_examples(resources):
policies = []
seen = set()
for resource_name, v in resources.items():
for k, cls in itertools.chain(v.filter_registry.items(), v.action_registry.items()):
if cls in seen:
continue
seen.add(cls)
doc = ElementSchema.doc(cls)
if not doc:
continue
# split on yaml and new lines
split_doc = [x.split('\n\n') for x in doc.split('yaml')]
for item in itertools.chain.from_iterable(split_doc):
if 'policies:\n' in item:
policies.append((item, resource_name, cls.type))
elif 'resource:' in item:
item = 'policies:\n' + item
policies.append((item, resource_name, cls.type))
return policies
def schema_cmd(options):
""" Print info about the resources, actions and filters available. """
from c7n import schema
if options.json:
schema.json_dump(options.resource)
return
if options.summary:
load_available()
resource_mapping = schema.resource_vocabulary()
schema.pprint_schema_summary(resource_mapping)
return
# Here are the formats for what we accept:
# - No argument
# - List all available RESOURCES
# - PROVIDER
# - List all available RESOURCES for supplied PROVIDER
# - RESOURCE
# - List all available actions and filters for supplied RESOURCE
# - MODE
# - List all available MODES
# - RESOURCE.actions
# - List all available actions for supplied RESOURCE
# - RESOURCE.actions.ACTION
# - Show class doc string and schema for supplied action
# - RESOURCE.filters
# - List all available filters for supplied RESOURCE
# - RESOURCE.filters.FILTER
# - Show class doc string and schema for supplied filter
if not options.resource:
load_available(resources=False)
resource_list = {'resources': sorted(itertools.chain(
*[clouds[p].resource_map.keys() for p in PROVIDER_NAMES]))}
print(yaml_dump(resource_list))
return
# Format is [PROVIDER].RESOURCE.CATEGORY.ITEM
# optional provider defaults to aws for compatibility
components = options.resource.lower().split('.')
if len(components) == 1 and components[0] in PROVIDER_NAMES:
load_providers((components[0]))
resource_list = {'resources': sorted(
clouds[components[0]].resource_map.keys())}
print(yaml_dump(resource_list))
return
if components[0] in PROVIDER_NAMES:
cloud_provider = components.pop(0)
components[0] = '%s.%s' % (cloud_provider, components[0])
load_resources((components[0],))
resource_mapping = schema.resource_vocabulary(
cloud_provider)
elif components[0] == 'mode':
load_available(resources=False)
resource_mapping = schema.resource_vocabulary()
else: # compatibility, aws is default for provider
components[0] = 'aws.%s' % components[0]
load_resources((components[0],))
resource_mapping = schema.resource_vocabulary('aws')
#
# Handle mode
#
if components[0] == "mode":
if len(components) == 1:
output = {components[0]: list(resource_mapping[components[0]].keys())}
print(yaml_dump(output))
return
if len(components) == 2:
if components[1] not in resource_mapping[components[0]]:
log.error('{} is not a valid mode'.format(components[1]))
sys.exit(1)
_print_cls_schema(resource_mapping[components[0]][components[1]])
return
# We received too much (e.g. mode.actions.foo)
log.error("Invalid selector '{}'. Valid options are 'mode' "
"or 'mode.TYPE'".format(options.resource))
sys.exit(1)
#
# Handle resource
#
resource = components[0]
if resource not in resource_mapping:
log.error('{} is not a valid resource'.format(resource))
sys.exit(1)
if len(components) == 1:
docstring = ElementSchema.doc(
resource_mapping[resource]['classes']['resource'])
del(resource_mapping[resource]['classes'])
if docstring:
print("\nHelp\n----\n")
print(docstring + '\n')
output = {resource: resource_mapping[resource]}
print(yaml_dump(output))
return
#
# Handle category
#
category = components[1]
if category not in ('actions', 'filters'):
log.error("Valid choices are 'actions' and 'filters'. You supplied '{}'".format(category))
sys.exit(1)
if len(components) == 2:
output = "No {} available for resource {}.".format(category, resource)
if category in resource_mapping[resource]:
output = {resource: {
category: resource_mapping[resource][category]}}
print(yaml_dump(output))
return
#
# Handle item
#
item = components[2]
if item not in resource_mapping[resource][category]:
log.error('{} is not in the {} list for resource {}'.format(item, category, resource))
sys.exit(1)
if len(components) == 3:
cls = resource_mapping[resource]['classes'][category][item]
_print_cls_schema(cls)
return
# We received too much (e.g. s3.actions.foo.bar)
log.error("Invalid selector '{}'. Max of 3 components in the "
"format RESOURCE.CATEGORY.ITEM".format(options.resource))
sys.exit(1)
def test_schema_expand_not_found(self):
test_schema = {'$ref': '#/definitions/filters_common/invalid_schema'}
result = ElementSchema.schema(generate()['definitions'], test_schema)
self.assertEqual(result, None)
def resolve(cls, schema_path):
return ElementSchema.resolve(cls.vocabulary, schema_path)
def _main(provider, output_dir, group_by):
"""Generate RST docs for a given cloud provider's resources
"""
env = init()
logging.basicConfig(level=logging.INFO)
output_dir = os.path.abspath(output_dir)
provider_class = clouds[provider]
# group by will be provider specific, supports nested attributes
group_by = operator.attrgetter(group_by or "type")
files = []
groups = {}
for r in provider_class.resources.values():
group = group_by(r)
if not isinstance(group, list):
group = [group]
for g in group:
groups.setdefault(g, []).append(r)
# Create individual resources pages
for r in provider_class.resources.values():
rpath = resource_file_name(output_dir, r)
with open(rpath, 'w') as fh:
t = env.get_template('provider-resource.rst')
fh.write(t.render(
provider_name=provider,
resource=r))
# Create files for all groups
for key, group in sorted(groups.items()):
group = sorted(group, key=operator.attrgetter('type'))
rpath = os.path.join(output_dir, "group-%s.rst" % key).replace(' ', '-').lower()
with open(rpath, 'w') as fh:
t = env.get_template('provider-group.rst')
fh.write(t.render(
provider_name=provider,
key=key,
resource_files=[os.path.basename(resource_file_name(output_dir, r)) for r in group],
resources=group))
files.append(os.path.basename(rpath))
# Write out common provider filters & actions
common_actions = {}
common_filters = {}
for r in provider_class.resources.values():
for f in ElementSchema.elements(r.filter_registry):
if not f.schema_alias:
continue
common_filters[ElementSchema.name(f)] = (f, r)
for a in ElementSchema.elements(r.action_registry):
if not a.schema_alias:
continue
common_actions[ElementSchema.name(a)] = (a, r)
fpath = os.path.join(
output_dir, "%s-common-filters.rst" % provider_class.type.lower())
with open(fpath, 'w') as fh:
t = env.get_template('provider-common-elements.rst')
fh.write(t.render(
provider_name=provider_class.display_name,
element_type='filters',
elements=[common_filters[k] for k in sorted(common_filters)]))
files.insert(0, os.path.basename(fpath))
fpath = os.path.join(
output_dir, "%s-common-actions.rst" % provider_class.type.lower())
with open(fpath, 'w') as fh:
t = env.get_template('provider-common-elements.rst')
fh.write(t.render(
provider_name=provider_class.display_name,
element_type='actions',
elements=[common_actions[k] for k in sorted(common_actions)]))
files.insert(0, os.path.basename(fpath))
log.info("%s Wrote %d resources groups", provider.title(), len(files))
# Write out the provider index
provider_path = os.path.join(output_dir, 'index.rst')
with open(provider_path, 'w') as fh:
log.info("Writing Provider Index to %s", provider_path)
t = env.get_template('provider-index.rst')
fh.write(t.render(provider_name=provider_class.display_name, files=files))
def _main(provider, output_dir, group_by):
"""Generate RST docs for a given cloud provider's resources
"""
env = init(provider)
logging.basicConfig(level=logging.INFO)
output_dir = os.path.abspath(output_dir)
provider_class = clouds[provider]
# group by will be provider specific, supports nested attributes
group_by = operator.attrgetter(group_by or "type")
files = []
written = 0
groups = {}
for r in provider_class.resources.values():
group = group_by(r)
if not isinstance(group, list):
group = [group]
for g in group:
groups.setdefault(g, []).append(r)
# Create individual resources pages
for r in provider_class.resources.values():
# FIXME / TODO: temporary work arounds for a few types that have recursion
# in jsonschema on these types.
if provider == 'awscc' and r.type in (
'wafv2_rulegroup', 'wafv2_webacl', 'amplifyuibuilder_theme',
'amplifyuibuilder_component', 'amplifybuilder_component'):
continue
rpath = resource_file_name(output_dir, r)
t = env.get_template('provider-resource.rst')
written += write_modified_file(rpath,
t.render(provider_name=provider,
resource=r),
diff_changes=not written)
# Create files for all groups
for key, group in sorted(groups.items()):
group = sorted(group, key=operator.attrgetter('type'))
rpath = os.path.join(output_dir,
("group-%s.rst" % key).replace(' ', '-').lower())
t = env.get_template('provider-group.rst')
written += write_modified_file(
rpath,
t.render(provider_name=provider,
key=key,
resource_files=[
os.path.basename(resource_file_name(output_dir, r))
for r in group
],
resources=group))
files.append(os.path.basename(rpath))
# Write out common provider filters & actions
common_actions = {}
common_filters = {}
for _, r in sorted(provider_class.resources.items()):
for f in ElementSchema.elements(r.filter_registry):
if not f.schema_alias:
continue
common_filters[ElementSchema.name(f)] = (f, r)
for a in ElementSchema.elements(r.action_registry):
if not a.schema_alias:
continue
common_actions[ElementSchema.name(a)] = (a, r)
fpath = os.path.join(
output_dir, ("%s-common-filters.rst" % provider_class.type.lower()))
t = env.get_template('provider-common-elements.rst')
written += write_modified_file(
fpath,
t.render(provider_name=provider_class.display_name,
element_type='filters',
elements=[common_filters[k] for k in sorted(common_filters)]))
files.insert(0, os.path.basename(fpath))
fpath = os.path.join(
output_dir, ("%s-common-actions.rst" % provider_class.type.lower()))
t = env.get_template('provider-common-elements.rst')
written += write_modified_file(
fpath,
t.render(provider_name=provider_class.display_name,
element_type='actions',
elements=[common_actions[k] for k in sorted(common_actions)]))
files.insert(0, os.path.basename(fpath))
# Write out provider modes
modes = get_provider_modes(provider)
mode_path = os.path.join(output_dir,
'%s-modes.rst' % provider_class.type.lower())
t = env.get_template('provider-mode.rst')
written += write_modified_file(
mode_path,
t.render(provider_name=provider_class.display_name, modes=modes))
files.insert(0, os.path.basename(mode_path))
# Write out the provider index
provider_path = os.path.join(output_dir, 'index.rst')
t = env.get_template('provider-index.rst')
written += write_modified_file(
provider_path,
t.render(provider_name=provider_class.display_name, files=files))
if written:
log.info("%s Wrote %d files", provider.title(), written)