def _create_argument_table(self):
argument_table = OrderedDict()
input_shape = self._operation_model.input_shape
required_arguments = []
arg_dict = {}
if input_shape is not None:
required_arguments = input_shape.required_members
arg_dict = input_shape.members
for arg_name, arg_shape in arg_dict.items():
cli_arg_name = xform_name(arg_name, '-')
arg_class = self.ARG_TYPES.get(arg_shape.type_name,
self.DEFAULT_ARG_CLASS)
is_token = arg_shape.metadata.get('idempotencyToken', False)
is_required = arg_name in required_arguments and not is_token
event_emitter = self._session.get_component('event_emitter')
arg_object = arg_class(name=cli_arg_name,
argument_model=arg_shape,
is_required=is_required,
operation_model=self._operation_model,
serialized_name=arg_name,
event_emitter=event_emitter)
arg_object.add_to_arg_table(argument_table)
LOG.debug(argument_table)
self._emit('building-argument-table.%s.%s' %
(self._parent_name, self._name),
operation_model=self._operation_model,
session=self._session,
command=self,
argument_table=argument_table)
return argument_table
def _build_argument_table(self):
argument_table = OrderedDict()
cli_data = self._get_cli_data()
cli_arguments = cli_data.get('options', None)
for option in cli_arguments:
option_params = copy_kwargs(cli_arguments[option])
# Special case the 'choices' param. Allows choices
# to reference a variable from the session.
if 'choices' in option_params:
choices = option_params['choices']
if not isinstance(choices, list):
# Assume it's a reference like
# "{provider}/_regions", so first resolve
# the provider.
provider = self.session.get_config_variable('provider')
# The grab the var from the session
choices_path = choices.format(provider=provider)
choices = list(self.session.get_data(choices_path))
option_params['choices'] = choices
argument_object = self._create_argument_object(
option, option_params)
argument_object.add_to_arg_table(argument_table)
# Then the final step is to send out an event so handlers
# can add extra arguments or modify existing arguments.
self.session.emit('building-top-level-params',
argument_table=argument_table)
return argument_table
def _generate_type_map(self, shape, stack):
key_shape = shape.key
value_shape = shape.value
assert key_shape.type_name == 'string'
return OrderedDict([
('KeyName', self._generate_skeleton(value_shape, stack)),
])
def report(config, output, use, output_dir, accounts, field, no_default_fields,
tags, region, debug, verbose, policy, format, resource):
"""report on a cross account policy execution."""
accounts_config, custodian_config, executor = init(config,
use,
debug,
verbose,
accounts,
tags,
policy,
resource=resource)
resource_types = set()
for p in custodian_config.get('policies'):
resource_types.add(p['resource'])
if len(resource_types) > 1:
raise ValueError("can only report on one resource type at a time")
records = []
with executor(max_workers=WORKER_COUNT) as w:
futures = {}
for a in accounts_config.get('accounts', ()):
account_regions = region or a['regions']
for r in account_regions:
futures[w.submit(report_account, a, r, custodian_config,
output_dir, debug)] = (a, r)
for f in as_completed(futures):
a, r = futures[f]
if f.exception():
if debug:
raise
log.warning("Error running policy in %s @ %s exception: %s",
a['name'], r, f.exception())
records.extend(f.result())
log.debug("Found %d records across %d accounts and %d policies",
len(records), len(accounts_config['accounts']),
len(custodian_config['policies']))
if format == 'json':
dumps(records, output, indent=2)
return
prefix_fields = OrderedDict(
(('Account', 'account'), ('Region', 'region'), ('Policy', 'policy')))
config = Bag.empty()
factory = resource_registry.get(list(resource_types)[0])
formatter = Formatter(factory.resource_type,
extra_fields=field,
include_default_fields=not (no_default_fields),
include_region=False,
include_policy=False,
fields=prefix_fields)
rows = formatter.to_csv(records, unique=False)
writer = csv.writer(output, formatter.headers())
writer.writerow(formatter.headers())
writer.writerows(rows)
def test_list_of_structures_with_triple_dots(self):
list_shape = {
'type': 'list',
'member': {
'shape': 'StructShape'
},
}
shapes = {
'Top': list_shape,
'String': {
'type': 'string'
},
'StructShape': {
'type':
'structure',
'members':
OrderedDict([
('A', {
'shape': 'String'
}),
('B', {
'shape': 'String'
}),
])
}
}
m = model.ListShape(shape_name='Top',
shape_model=list_shape,
shape_resolver=model.ShapeResolver(shapes))
generated_example = self.shorthand_documenter.generate_shorthand_example(
'--foo', m)
self.assertIn('A=string,B=string ...', generated_example)
def test_gen_structure_list_scalar_docs(self):
schema = {
"type": "object",
"properties": {
"Consistent": {
"type": "boolean",
},
"Args": {
"type": "array",
"items": {
"type": "string"
}
}
}
}
argument_model = create_argument_model_from_schema(schema)
m = model.DenormalizedStructureBuilder().with_members(
OrderedDict([
('Consistent', {
'type': 'boolean'
}),
('Args', {
'type': 'list',
'member': {
'type': 'string',
}
}),
])).build_model()
generated_example = self.shorthand_documenter.generate_shorthand_example(
'--foo', m)
self.assertIn('Consistent=boolean,Args=string,string',
generated_example)
def test_list_of_structures_with_triple_dots(self):
list_shape = {
'type': 'list',
'member': {
'shape': 'StructShape'
},
}
shapes = {
'Top': list_shape,
'String': {
'type': 'string'
},
'StructShape': {
'type':
'structure',
'members':
OrderedDict([
('A', {
'shape': 'String'
}),
('B', {
'shape': 'String'
}),
])
}
}
m = model.ListShape(shape_name='Top',
shape_model=list_shape,
shape_resolver=model.ShapeResolver(shapes))
argument = mock.Mock()
argument.argument_model = m
argument.name = 'foo'
argument.cli_name = '--foo'
generated_example = self.get_generated_example_for(argument)
self.assertIn('A=string,B=string ...', generated_example)
def test_can_document_recursive_struct(self):
# It's a little more work to set up a recursive
# shape because DenormalizedStructureBuilder cannot handle
# recursion.
struct_shape = {
'type':
'structure',
'members':
OrderedDict([
('Recurse', {
'shape': 'SubShape'
}),
('Scalar', {
'shape': 'String'
}),
]),
}
shapes = {
'Top': struct_shape,
'String': {
'type': 'string'
},
'SubShape': {
'type':
'structure',
'members':
OrderedDict([
('SubRecurse', {
'shape': 'Top'
}),
('Scalar', {
'shape': 'String'
}),
]),
}
}
m = model.StructureShape(shape_name='Top',
shape_model=struct_shape,
shape_resolver=model.ShapeResolver(shapes))
argument = mock.Mock()
argument.argument_model = m
argument.name = 'foo'
argument.cli_name = '--foo'
generated_example = self.get_generated_example_for(argument)
self.assertIn(
'Recurse={SubRecurse={( ... recursive ... ),Scalar=string},'
'Scalar=string},Scalar=string', generated_example)
def test_does_decode_template_body_in_order(self):
expected_ordering = OrderedDict([
('TemplateVersion', 1.0),
('APropertyOfSomeKind', 'a value'),
('list', [1, 2, 3]),
('nested', OrderedDict([('key', 'value'),
('foo', 'bar')]))
])
template_string = json.dumps(expected_ordering)
parsed_response = {'TemplateBody': template_string}
handlers.json_decode_template_body(parsed=parsed_response)
result = parsed_response['TemplateBody']
self.assertTrue(isinstance(result, OrderedDict))
for element, expected_element in zip(result, expected_ordering):
self.assertEqual(element, expected_element)
def test_gen_structure_list_scalar_docs(self):
argument = self.create_argument(OrderedDict([
('Consistent', {'type': 'boolean'}),
('Args', {'type': 'list', 'member': {'type': 'string'}}),
]), 'foo')
generated_example = self.get_generated_example_for(argument)
self.assertIn('Consistent=boolean,Args=string,string',
generated_example)
def test_list_objects_unicode_query_string_eu_central_1(self):
self.region_name = 'eu-central-1'
params = OrderedDict([('Bucket', 'safename'),
('Marker', u'\xe4\xf6\xfc-01.txt')])
prepared_request = self.get_prepared_request('list_objects', params)
self.assertEqual(prepared_request.url,
('https://s3.eu-central-1.amazonaws.com/safename'
'?marker=%C3%A4%C3%B6%C3%BC-01.txt'))
def _generate_type_structure(self, shape, stack):
if stack.count(shape.name) > 1:
return {}
skeleton = OrderedDict()
for member_name, member_shape in shape.members.items():
skeleton[member_name] = self._generate_skeleton(member_shape,
stack)
return skeleton
def _build_builtin_commands(self, session):
commands = OrderedDict()
services = session.get_available_services()
for service_name in services:
commands[service_name] = ServiceCommand(cli_name=service_name,
session=self.session,
service_name=service_name)
return commands
def _setup_models(self):
self.json_model = {
'metadata': {
'apiVersion': '2014-01-01',
'endpointPrefix': 'myservice',
'signatureVersion': 'v4',
'serviceFullName': 'AWS MyService',
'protocol': 'query'
},
'operations': {
'SampleOperation': {
'name': 'SampleOperation',
'input': {'shape': 'SampleOperationInputOutput'},
'output': {'shape': 'SampleOperationInputOutput'}
}
},
'shapes': {
'SampleOperationInputOutput': {
'type': 'structure',
'members': OrderedDict()
},
'String': {
'type': 'string'
}
}
}
self.waiter_json_model = {
"version": 2,
"waiters": {
"SampleOperationComplete": {
"delay": 15,
"operation": "SampleOperation",
"maxAttempts": 40,
"acceptors": [
{"expected": "complete",
"matcher": "pathAll",
"state": "success",
"argument": "Biz"},
{"expected": "failed",
"matcher": "pathAny",
"state": "failure",
"argument": "Biz"}
]
}
}
}
self.paginator_json_model = {
"pagination": {
"SampleOperation": {
"input_token": "NextResult",
"output_token": "NextResult",
"limit_key": "MaxResults",
"result_key": "Biz"
}
}
}
def test_update_existing(self):
initial = OrderedDict([
("apiVersion", "v1"),
("clusters", [
OrderedDict([
("cluster", OrderedDict([
("server", "endpoint")
])),
("name", "clustername")
])
]),
("contexts", []),
("current-context", None),
("kind", "Config"),
("preferences", OrderedDict()),
("users", [])
])
cluster = OrderedDict([
("cluster", OrderedDict([
("certificate-authority-data", "data"),
("server", "endpoint")
])),
("name", "clustername")
])
correct = OrderedDict([
("apiVersion", "v1"),
("clusters", [
OrderedDict([
("cluster", OrderedDict([
("certificate-authority-data", "data"),
("server", "endpoint")
])),
("name", "clustername")
])
]),
("contexts", []),
("current-context", None),
("kind", "Config"),
("preferences", OrderedDict()),
("users", [])
])
updated = self._appender.insert_entry(Kubeconfig(None, initial),
"clusters",
cluster)
self.assertDictEqual(updated.content, correct)
def test_write_directory(self):
content = OrderedDict([("current-context", "context"),
("apiVersion", "v1")])
containing_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, containing_dir)
config = Kubeconfig(containing_dir, content)
self.assertRaises(KubeconfigInaccessableError,
self._writer.write_kubeconfig, config)
def get_cluster_entry(self):
"""
Return a cluster entry generated using
the previously obtained description.
"""
cert_data = self._get_cluster_description().get("certificateAuthority",
{"data": ""})["data"]
endpoint = self._get_cluster_description().get("endpoint")
arn = self._get_cluster_description().get("arn")
return OrderedDict([
("cluster", OrderedDict([
("certificate-authority-data", cert_data),
("server", endpoint)
])),
("name", arn)
])
def _build_structure(self, model, shapes):
members = OrderedDict()
shape = self._build_initial_shape(model)
shape['members'] = members
for name, member_model in model['members'].items():
member_shape_name = self._get_shape_name(member_model)
members[name] = {'shape': member_shape_name}
self._build_model(member_model, shapes, member_shape_name)
return shape
def _build_subcommand_table(self):
subcommand_table = OrderedDict()
for subcommand in self.SUBCOMMANDS:
subcommand_name = subcommand['name']
subcommand_class = subcommand['command_class']
subcommand_table[subcommand_name] = subcommand_class(self._session)
self._session.emit('building-command-table.%s' % self.NAME,
command_table=subcommand_table,
session=self._session)
return subcommand_table
def build_policy(self,
resource,
date_less_than,
date_greater_than=None,
ip_address=None):
"""A helper to build policy.
:type resource: str
:param resource: The URL or the stream filename of the protected object
:type date_less_than: datetime
:param date_less_than: The URL will expire after the time has passed
:type date_greater_than: datetime
:param date_greater_than: The URL will not be valid until this time
:type ip_address: str
:param ip_address: Use 'x.x.x.x' for an IP, or 'x.x.x.x/x' for a subnet
:rtype: str
:return: The policy in a compact string.
"""
# Note:
# 1. Order in canned policy is significant. Special care has been taken
# to ensure the output will match the order defined by the document.
# There is also a test case to ensure that order.
# SEE: http://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/private-content-creating-signed-url-canned-policy.html#private-content-canned-policy-creating-policy-statement
# 2. Albeit the order in custom policy is not required by CloudFront,
# we still use OrderedDict internally to ensure the result is stable
# and also matches canned policy requirement.
# SEE: http://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/private-content-creating-signed-url-custom-policy.html
moment = int(datetime2timestamp(date_less_than))
condition = OrderedDict({"DateLessThan": {"AWS:EpochTime": moment}})
if ip_address:
if '/' not in ip_address:
ip_address += '/32'
condition["IpAddress"] = {"AWS:SourceIp": ip_address}
if date_greater_than:
moment = int(datetime2timestamp(date_greater_than))
condition["DateGreaterThan"] = {"AWS:EpochTime": moment}
ordered_payload = [('Resource', resource), ('Condition', condition)]
custom_policy = {"Statement": [OrderedDict(ordered_payload)]}
return json.dumps(custom_policy, separators=(',', ':'))
def test_profile(self):
self._session.profile = "profile"
self._correct_user_entry["user"]["exec"]["env"] = [
OrderedDict([("name", "AWS_PROFILE"), ("value", "profile")])
]
self.assertEqual(self._client.get_user_entry(),
self._correct_user_entry)
self._mock_client.describe_cluster.assert_called_once_with(
name="ExampleCluster")
self._session.create_client.assert_called_once_with("eks")
def _create_command_table(self):
command_table = OrderedDict()
service_object = self._get_service_object()
for operation_object in service_object.operations:
cli_name = xform_name(operation_object.name, '-')
command_table[cli_name] = ServiceOperation(
name=cli_name,
operation_object=operation_object,
operation_caller=CLIOperationCaller(self.session),
service_object=service_object)
return command_table
def __init__(self, resource):
self._resource = resource
self._client = self._resource.meta.client
self._resource_model = self._resource.meta.resource_model
self._service_model = self._client.meta.service_model
self._resource_name = self._resource.meta.resource_model.name
self._service_name = self._service_model.service_name
self._service_docs_name = self._client.__class__.__name__
self.member_map = OrderedDict()
self.represents_service_resource = (
self._service_name == self._resource_name)
def arg_table(self):
arg_table = OrderedDict()
for arg_data in self.ARG_TABLE:
custom_argument = CustomArgument(**arg_data)
# If a custom schema was passed in, create the argument object
# so that it can be validated and docs can be generated
if 'schema' in arg_data:
custom_argument.create_argument_object()
arg_table[arg_data['name']] = custom_argument
return arg_table
def test_key_not_array(self):
initial = OrderedDict([
("apiVersion", "v1"),
("contexts", []),
("current-context", None),
("kind", "Config"),
("preferences", OrderedDict()),
("users", [])
])
cluster = OrderedDict([
("cluster", OrderedDict([
("certificate-authority-data", "data"),
("server", "endpoint")
])),
("name", "clustername")
])
self.assertRaises(KubeconfigError,
self._appender.insert_entry,
Kubeconfig(None, initial),
"kind",
cluster)
def get_user_entry(self):
"""
Return a user entry generated using
the previously obtained description.
"""
region = self._get_cluster_description().get("arn").split(":")[3]
generated_user = OrderedDict([
("name", self._get_cluster_description().get("arn", "")),
("user", OrderedDict([
("exec", OrderedDict([
("apiVersion", API_VERSION),
("args",
[
"--region",
region,
"eks",
"get-token",
"--cluster-name",
self._cluster_name,
]),
("command", "aws")
]))
]))
])
if self._role_arn is not None:
generated_user["user"]["exec"]["args"].extend([
"--role",
self._role_arn
])
if self._session.profile:
generated_user["user"]["exec"]["env"] = [OrderedDict([
("name", "AWS_PROFILE"),
("value", self._session.profile)
])]
return generated_user
def __init__(self,
resource_manager,
extra_fields=(),
include_default_fields=True,
include_region=False,
include_policy=False):
self.resource_manager = resource_manager
# Lookup default fields for resource type.
model = resource_manager.resource_type
self._id_field = model.id
self._date_field = getattr(model, 'date', None)
mfields = getattr(model, 'default_report_fields', None)
if mfields is None:
mfields = [model.id]
if model.name != model.id:
mfields.append(model.name)
if getattr(model, 'date', None):
mfields.append(model.date)
if include_default_fields:
fields = OrderedDict(zip(mfields, mfields))
else:
fields = OrderedDict()
for index, field in enumerate(extra_fields):
# TODO this type coercion should be done at cli input, not here
h, cexpr = field.split('=', 1)
fields[h] = cexpr
# Add these at the end so that they are the last fields
if include_default_fields:
if include_region:
fields['Region'] = 'region'
if include_policy:
fields['Policy'] = 'policy'
self.fields = fields
def _build_argument_table(self):
argument_table = OrderedDict()
cli_data = self._get_cli_data()
cli_arguments = cli_data.get('options', None)
for option in cli_arguments:
option_params = copy_kwargs(cli_arguments[option])
cli_argument = self._create_cli_argument(option, option_params)
cli_argument.add_to_arg_table(argument_table)
# Then the final step is to send out an event so handlers
# can add extra arguments or modify existing arguments.
self.session.emit('building-top-level-params',
argument_table=argument_table)
return argument_table
def test_ordered_shape_builder(self):
b = model.DenormalizedStructureBuilder()
shape = b.with_members(
OrderedDict([('A', {
'type': 'string'
}),
('B', {
'type':
'structure',
'members':
OrderedDict([('C', {
'type': 'string'
}), ('D', {
'type': 'string'
})])
})])).build_model()
# Members should be in order
self.assertEqual(['A', 'B'], list(shape.members.keys()))
# Nested structure members should *also* stay ordered
self.assertEqual(['C', 'D'], list(shape.members['B'].members.keys()))
def test_parse_yaml_preserve_elements_order(self):
input_template = (
'B_Resource:\n'
' Key2:\n'
' Name: name2\n'
' Key1:\n'
' Name: name1\n'
'A_Resource:\n'
' Key2:\n'
' Name: name2\n'
' Key1:\n'
' Name: name1\n'
)
output_dict = yaml_parse(input_template)
expected_dict = OrderedDict([
('B_Resource', OrderedDict([('Key2', {'Name': 'name2'}), ('Key1', {'Name': 'name1'})])),
('A_Resource', OrderedDict([('Key2', {'Name': 'name2'}), ('Key1', {'Name': 'name1'})]))
])
self.assertEqual(expected_dict, output_dict)
output_template = yaml_dump(output_dict)
self.assertEqual(input_template, output_template)
