def _create_argument_table(self): argument_table = OrderedDict() input_shape = self._operation_model.input_shape required_arguments = [] arg_dict = {} if input_shape is not None: required_arguments = input_shape.required_members arg_dict = input_shape.members for arg_name, arg_shape in arg_dict.items(): cli_arg_name = xform_name(arg_name, '-') arg_class = self.ARG_TYPES.get(arg_shape.type_name, self.DEFAULT_ARG_CLASS) is_token = arg_shape.metadata.get('idempotencyToken', False) is_required = arg_name in required_arguments and not is_token event_emitter = self._session.get_component('event_emitter') arg_object = arg_class(name=cli_arg_name, argument_model=arg_shape, is_required=is_required, operation_model=self._operation_model, serialized_name=arg_name, event_emitter=event_emitter) arg_object.add_to_arg_table(argument_table) LOG.debug(argument_table) self._emit('building-argument-table.%s.%s' % (self._parent_name, self._name), operation_model=self._operation_model, session=self._session, command=self, argument_table=argument_table) return argument_table
def _build_argument_table(self): argument_table = OrderedDict() cli_data = self._get_cli_data() cli_arguments = cli_data.get('options', None) for option in cli_arguments: option_params = copy_kwargs(cli_arguments[option]) # Special case the 'choices' param. Allows choices # to reference a variable from the session. if 'choices' in option_params: choices = option_params['choices'] if not isinstance(choices, list): # Assume it's a reference like # "{provider}/_regions", so first resolve # the provider. provider = self.session.get_config_variable('provider') # The grab the var from the session choices_path = choices.format(provider=provider) choices = list(self.session.get_data(choices_path)) option_params['choices'] = choices argument_object = self._create_argument_object( option, option_params) argument_object.add_to_arg_table(argument_table) # Then the final step is to send out an event so handlers # can add extra arguments or modify existing arguments. self.session.emit('building-top-level-params', argument_table=argument_table) return argument_table
def _generate_type_map(self, shape, stack): key_shape = shape.key value_shape = shape.value assert key_shape.type_name == 'string' return OrderedDict([ ('KeyName', self._generate_skeleton(value_shape, stack)), ])
def report(config, output, use, output_dir, accounts, field, no_default_fields, tags, region, debug, verbose, policy, format, resource): """report on a cross account policy execution.""" accounts_config, custodian_config, executor = init(config, use, debug, verbose, accounts, tags, policy, resource=resource) resource_types = set() for p in custodian_config.get('policies'): resource_types.add(p['resource']) if len(resource_types) > 1: raise ValueError("can only report on one resource type at a time") records = [] with executor(max_workers=WORKER_COUNT) as w: futures = {} for a in accounts_config.get('accounts', ()): account_regions = region or a['regions'] for r in account_regions: futures[w.submit(report_account, a, r, custodian_config, output_dir, debug)] = (a, r) for f in as_completed(futures): a, r = futures[f] if f.exception(): if debug: raise log.warning("Error running policy in %s @ %s exception: %s", a['name'], r, f.exception()) records.extend(f.result()) log.debug("Found %d records across %d accounts and %d policies", len(records), len(accounts_config['accounts']), len(custodian_config['policies'])) if format == 'json': dumps(records, output, indent=2) return prefix_fields = OrderedDict( (('Account', 'account'), ('Region', 'region'), ('Policy', 'policy'))) config = Bag.empty() factory = resource_registry.get(list(resource_types)[0]) formatter = Formatter(factory.resource_type, extra_fields=field, include_default_fields=not (no_default_fields), include_region=False, include_policy=False, fields=prefix_fields) rows = formatter.to_csv(records, unique=False) writer = csv.writer(output, formatter.headers()) writer.writerow(formatter.headers()) writer.writerows(rows)
def test_list_of_structures_with_triple_dots(self): list_shape = { 'type': 'list', 'member': { 'shape': 'StructShape' }, } shapes = { 'Top': list_shape, 'String': { 'type': 'string' }, 'StructShape': { 'type': 'structure', 'members': OrderedDict([ ('A', { 'shape': 'String' }), ('B', { 'shape': 'String' }), ]) } } m = model.ListShape(shape_name='Top', shape_model=list_shape, shape_resolver=model.ShapeResolver(shapes)) generated_example = self.shorthand_documenter.generate_shorthand_example( '--foo', m) self.assertIn('A=string,B=string ...', generated_example)
def test_gen_structure_list_scalar_docs(self): schema = { "type": "object", "properties": { "Consistent": { "type": "boolean", }, "Args": { "type": "array", "items": { "type": "string" } } } } argument_model = create_argument_model_from_schema(schema) m = model.DenormalizedStructureBuilder().with_members( OrderedDict([ ('Consistent', { 'type': 'boolean' }), ('Args', { 'type': 'list', 'member': { 'type': 'string', } }), ])).build_model() generated_example = self.shorthand_documenter.generate_shorthand_example( '--foo', m) self.assertIn('Consistent=boolean,Args=string,string', generated_example)
def test_list_of_structures_with_triple_dots(self): list_shape = { 'type': 'list', 'member': { 'shape': 'StructShape' }, } shapes = { 'Top': list_shape, 'String': { 'type': 'string' }, 'StructShape': { 'type': 'structure', 'members': OrderedDict([ ('A', { 'shape': 'String' }), ('B', { 'shape': 'String' }), ]) } } m = model.ListShape(shape_name='Top', shape_model=list_shape, shape_resolver=model.ShapeResolver(shapes)) argument = mock.Mock() argument.argument_model = m argument.name = 'foo' argument.cli_name = '--foo' generated_example = self.get_generated_example_for(argument) self.assertIn('A=string,B=string ...', generated_example)
def test_can_document_recursive_struct(self): # It's a little more work to set up a recursive # shape because DenormalizedStructureBuilder cannot handle # recursion. struct_shape = { 'type': 'structure', 'members': OrderedDict([ ('Recurse', { 'shape': 'SubShape' }), ('Scalar', { 'shape': 'String' }), ]), } shapes = { 'Top': struct_shape, 'String': { 'type': 'string' }, 'SubShape': { 'type': 'structure', 'members': OrderedDict([ ('SubRecurse', { 'shape': 'Top' }), ('Scalar', { 'shape': 'String' }), ]), } } m = model.StructureShape(shape_name='Top', shape_model=struct_shape, shape_resolver=model.ShapeResolver(shapes)) argument = mock.Mock() argument.argument_model = m argument.name = 'foo' argument.cli_name = '--foo' generated_example = self.get_generated_example_for(argument) self.assertIn( 'Recurse={SubRecurse={( ... recursive ... ),Scalar=string},' 'Scalar=string},Scalar=string', generated_example)
def test_does_decode_template_body_in_order(self): expected_ordering = OrderedDict([ ('TemplateVersion', 1.0), ('APropertyOfSomeKind', 'a value'), ('list', [1, 2, 3]), ('nested', OrderedDict([('key', 'value'), ('foo', 'bar')])) ]) template_string = json.dumps(expected_ordering) parsed_response = {'TemplateBody': template_string} handlers.json_decode_template_body(parsed=parsed_response) result = parsed_response['TemplateBody'] self.assertTrue(isinstance(result, OrderedDict)) for element, expected_element in zip(result, expected_ordering): self.assertEqual(element, expected_element)
def test_gen_structure_list_scalar_docs(self): argument = self.create_argument(OrderedDict([ ('Consistent', {'type': 'boolean'}), ('Args', {'type': 'list', 'member': {'type': 'string'}}), ]), 'foo') generated_example = self.get_generated_example_for(argument) self.assertIn('Consistent=boolean,Args=string,string', generated_example)
def test_list_objects_unicode_query_string_eu_central_1(self): self.region_name = 'eu-central-1' params = OrderedDict([('Bucket', 'safename'), ('Marker', u'\xe4\xf6\xfc-01.txt')]) prepared_request = self.get_prepared_request('list_objects', params) self.assertEqual(prepared_request.url, ('https://s3.eu-central-1.amazonaws.com/safename' '?marker=%C3%A4%C3%B6%C3%BC-01.txt'))
def _generate_type_structure(self, shape, stack): if stack.count(shape.name) > 1: return {} skeleton = OrderedDict() for member_name, member_shape in shape.members.items(): skeleton[member_name] = self._generate_skeleton(member_shape, stack) return skeleton
def _build_builtin_commands(self, session): commands = OrderedDict() services = session.get_available_services() for service_name in services: commands[service_name] = ServiceCommand(cli_name=service_name, session=self.session, service_name=service_name) return commands
def _setup_models(self): self.json_model = { 'metadata': { 'apiVersion': '2014-01-01', 'endpointPrefix': 'myservice', 'signatureVersion': 'v4', 'serviceFullName': 'AWS MyService', 'protocol': 'query' }, 'operations': { 'SampleOperation': { 'name': 'SampleOperation', 'input': {'shape': 'SampleOperationInputOutput'}, 'output': {'shape': 'SampleOperationInputOutput'} } }, 'shapes': { 'SampleOperationInputOutput': { 'type': 'structure', 'members': OrderedDict() }, 'String': { 'type': 'string' } } } self.waiter_json_model = { "version": 2, "waiters": { "SampleOperationComplete": { "delay": 15, "operation": "SampleOperation", "maxAttempts": 40, "acceptors": [ {"expected": "complete", "matcher": "pathAll", "state": "success", "argument": "Biz"}, {"expected": "failed", "matcher": "pathAny", "state": "failure", "argument": "Biz"} ] } } } self.paginator_json_model = { "pagination": { "SampleOperation": { "input_token": "NextResult", "output_token": "NextResult", "limit_key": "MaxResults", "result_key": "Biz" } } }
def test_update_existing(self): initial = OrderedDict([ ("apiVersion", "v1"), ("clusters", [ OrderedDict([ ("cluster", OrderedDict([ ("server", "endpoint") ])), ("name", "clustername") ]) ]), ("contexts", []), ("current-context", None), ("kind", "Config"), ("preferences", OrderedDict()), ("users", []) ]) cluster = OrderedDict([ ("cluster", OrderedDict([ ("certificate-authority-data", "data"), ("server", "endpoint") ])), ("name", "clustername") ]) correct = OrderedDict([ ("apiVersion", "v1"), ("clusters", [ OrderedDict([ ("cluster", OrderedDict([ ("certificate-authority-data", "data"), ("server", "endpoint") ])), ("name", "clustername") ]) ]), ("contexts", []), ("current-context", None), ("kind", "Config"), ("preferences", OrderedDict()), ("users", []) ]) updated = self._appender.insert_entry(Kubeconfig(None, initial), "clusters", cluster) self.assertDictEqual(updated.content, correct)
def test_write_directory(self): content = OrderedDict([("current-context", "context"), ("apiVersion", "v1")]) containing_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, containing_dir) config = Kubeconfig(containing_dir, content) self.assertRaises(KubeconfigInaccessableError, self._writer.write_kubeconfig, config)
def get_cluster_entry(self): """ Return a cluster entry generated using the previously obtained description. """ cert_data = self._get_cluster_description().get("certificateAuthority", {"data": ""})["data"] endpoint = self._get_cluster_description().get("endpoint") arn = self._get_cluster_description().get("arn") return OrderedDict([ ("cluster", OrderedDict([ ("certificate-authority-data", cert_data), ("server", endpoint) ])), ("name", arn) ])
def _build_structure(self, model, shapes): members = OrderedDict() shape = self._build_initial_shape(model) shape['members'] = members for name, member_model in model['members'].items(): member_shape_name = self._get_shape_name(member_model) members[name] = {'shape': member_shape_name} self._build_model(member_model, shapes, member_shape_name) return shape
def _build_subcommand_table(self): subcommand_table = OrderedDict() for subcommand in self.SUBCOMMANDS: subcommand_name = subcommand['name'] subcommand_class = subcommand['command_class'] subcommand_table[subcommand_name] = subcommand_class(self._session) self._session.emit('building-command-table.%s' % self.NAME, command_table=subcommand_table, session=self._session) return subcommand_table
def build_policy(self, resource, date_less_than, date_greater_than=None, ip_address=None): """A helper to build policy. :type resource: str :param resource: The URL or the stream filename of the protected object :type date_less_than: datetime :param date_less_than: The URL will expire after the time has passed :type date_greater_than: datetime :param date_greater_than: The URL will not be valid until this time :type ip_address: str :param ip_address: Use 'x.x.x.x' for an IP, or 'x.x.x.x/x' for a subnet :rtype: str :return: The policy in a compact string. """ # Note: # 1. Order in canned policy is significant. Special care has been taken # to ensure the output will match the order defined by the document. # There is also a test case to ensure that order. # SEE: http://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/private-content-creating-signed-url-canned-policy.html#private-content-canned-policy-creating-policy-statement # 2. Albeit the order in custom policy is not required by CloudFront, # we still use OrderedDict internally to ensure the result is stable # and also matches canned policy requirement. # SEE: http://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/private-content-creating-signed-url-custom-policy.html moment = int(datetime2timestamp(date_less_than)) condition = OrderedDict({"DateLessThan": {"AWS:EpochTime": moment}}) if ip_address: if '/' not in ip_address: ip_address += '/32' condition["IpAddress"] = {"AWS:SourceIp": ip_address} if date_greater_than: moment = int(datetime2timestamp(date_greater_than)) condition["DateGreaterThan"] = {"AWS:EpochTime": moment} ordered_payload = [('Resource', resource), ('Condition', condition)] custom_policy = {"Statement": [OrderedDict(ordered_payload)]} return json.dumps(custom_policy, separators=(',', ':'))
def test_profile(self): self._session.profile = "profile" self._correct_user_entry["user"]["exec"]["env"] = [ OrderedDict([("name", "AWS_PROFILE"), ("value", "profile")]) ] self.assertEqual(self._client.get_user_entry(), self._correct_user_entry) self._mock_client.describe_cluster.assert_called_once_with( name="ExampleCluster") self._session.create_client.assert_called_once_with("eks")
def _create_command_table(self): command_table = OrderedDict() service_object = self._get_service_object() for operation_object in service_object.operations: cli_name = xform_name(operation_object.name, '-') command_table[cli_name] = ServiceOperation( name=cli_name, operation_object=operation_object, operation_caller=CLIOperationCaller(self.session), service_object=service_object) return command_table
def __init__(self, resource): self._resource = resource self._client = self._resource.meta.client self._resource_model = self._resource.meta.resource_model self._service_model = self._client.meta.service_model self._resource_name = self._resource.meta.resource_model.name self._service_name = self._service_model.service_name self._service_docs_name = self._client.__class__.__name__ self.member_map = OrderedDict() self.represents_service_resource = ( self._service_name == self._resource_name)
def arg_table(self): arg_table = OrderedDict() for arg_data in self.ARG_TABLE: custom_argument = CustomArgument(**arg_data) # If a custom schema was passed in, create the argument object # so that it can be validated and docs can be generated if 'schema' in arg_data: custom_argument.create_argument_object() arg_table[arg_data['name']] = custom_argument return arg_table
def test_key_not_array(self): initial = OrderedDict([ ("apiVersion", "v1"), ("contexts", []), ("current-context", None), ("kind", "Config"), ("preferences", OrderedDict()), ("users", []) ]) cluster = OrderedDict([ ("cluster", OrderedDict([ ("certificate-authority-data", "data"), ("server", "endpoint") ])), ("name", "clustername") ]) self.assertRaises(KubeconfigError, self._appender.insert_entry, Kubeconfig(None, initial), "kind", cluster)
def get_user_entry(self): """ Return a user entry generated using the previously obtained description. """ region = self._get_cluster_description().get("arn").split(":")[3] generated_user = OrderedDict([ ("name", self._get_cluster_description().get("arn", "")), ("user", OrderedDict([ ("exec", OrderedDict([ ("apiVersion", API_VERSION), ("args", [ "--region", region, "eks", "get-token", "--cluster-name", self._cluster_name, ]), ("command", "aws") ])) ])) ]) if self._role_arn is not None: generated_user["user"]["exec"]["args"].extend([ "--role", self._role_arn ]) if self._session.profile: generated_user["user"]["exec"]["env"] = [OrderedDict([ ("name", "AWS_PROFILE"), ("value", self._session.profile) ])] return generated_user
def __init__(self, resource_manager, extra_fields=(), include_default_fields=True, include_region=False, include_policy=False): self.resource_manager = resource_manager # Lookup default fields for resource type. model = resource_manager.resource_type self._id_field = model.id self._date_field = getattr(model, 'date', None) mfields = getattr(model, 'default_report_fields', None) if mfields is None: mfields = [model.id] if model.name != model.id: mfields.append(model.name) if getattr(model, 'date', None): mfields.append(model.date) if include_default_fields: fields = OrderedDict(zip(mfields, mfields)) else: fields = OrderedDict() for index, field in enumerate(extra_fields): # TODO this type coercion should be done at cli input, not here h, cexpr = field.split('=', 1) fields[h] = cexpr # Add these at the end so that they are the last fields if include_default_fields: if include_region: fields['Region'] = 'region' if include_policy: fields['Policy'] = 'policy' self.fields = fields
def _build_argument_table(self): argument_table = OrderedDict() cli_data = self._get_cli_data() cli_arguments = cli_data.get('options', None) for option in cli_arguments: option_params = copy_kwargs(cli_arguments[option]) cli_argument = self._create_cli_argument(option, option_params) cli_argument.add_to_arg_table(argument_table) # Then the final step is to send out an event so handlers # can add extra arguments or modify existing arguments. self.session.emit('building-top-level-params', argument_table=argument_table) return argument_table
def test_ordered_shape_builder(self): b = model.DenormalizedStructureBuilder() shape = b.with_members( OrderedDict([('A', { 'type': 'string' }), ('B', { 'type': 'structure', 'members': OrderedDict([('C', { 'type': 'string' }), ('D', { 'type': 'string' })]) })])).build_model() # Members should be in order self.assertEqual(['A', 'B'], list(shape.members.keys())) # Nested structure members should *also* stay ordered self.assertEqual(['C', 'D'], list(shape.members['B'].members.keys()))
def test_parse_yaml_preserve_elements_order(self): input_template = ( 'B_Resource:\n' ' Key2:\n' ' Name: name2\n' ' Key1:\n' ' Name: name1\n' 'A_Resource:\n' ' Key2:\n' ' Name: name2\n' ' Key1:\n' ' Name: name1\n' ) output_dict = yaml_parse(input_template) expected_dict = OrderedDict([ ('B_Resource', OrderedDict([('Key2', {'Name': 'name2'}), ('Key1', {'Name': 'name1'})])), ('A_Resource', OrderedDict([('Key2', {'Name': 'name2'}), ('Key1', {'Name': 'name1'})])) ]) self.assertEqual(expected_dict, output_dict) output_template = yaml_dump(output_dict) self.assertEqual(input_template, output_template)