def read_Table(self, ModelObj): #conn=self.connect(ModelObj) #response = conn.scan() data = [] #data = response['Items'] #while response.get('LastEvaluatedKey'): # response = self.dynamoTable.scan(ExclusiveStartKey=response['LastEvaluatedKey']) # data.extend(response['Items']) client = boto3.client( 'dynamodb', region_name=self.region_name, aws_access_key_id=settings.DYNAMO_AWS_ACCESS['AWS_ACCESS_KEY_ID'], aws_secret_access_key=settings. DYNAMO_AWS_ACCESS['AWS_SECRET_ACCESS_KEY'], endpoint_url=settings.DYNAMO['URL']) paginator = client.get_paginator('scan') service_model = client._service_model.operation_model('Scan') operation_parameters = { 'TableName': ModelObj.Meta.table_name, } trans = TransformationInjector(deserializer=TypeDeserializer()) for page in paginator.paginate(**operation_parameters): trans.inject_attribute_value_output(page, service_model) while page.get('LastEvaluatedKey'): page = self.dynamoTable.scan( ExclusiveStartKey=page['LastEvaluatedKey']) data.extend(page['Items']) else: data.extend(page['Items']) return data
def history(tableName: str = 'wcmc-data-stasis-ecs-dev'): import boto3 from boto3.dynamodb.types import TypeDeserializer from boto3.dynamodb.transform import TransformationInjector client = boto3.client('dynamodb') paginator = client.get_paginator('scan') operation_model = client._service_model.operation_model('Scan') trans = TransformationInjector(deserializer=TypeDeserializer()) operation_parameters = {'TableName': tableName} items = [] for page in paginator.paginate(**operation_parameters): has_last_key = 'LastEvaluatedKey' in page if has_last_key: last_key = page['LastEvaluatedKey'].copy() trans.inject_attribute_value_output(page, operation_model) if has_last_key: page['LastEvaluatedKey'] = last_key for x in page['Items']: items.append(x) return items
def setUp(self): super(TestTransformConditionExpression, self).setUp() self.add_shape({'ConditionExpression': {'type': 'string'}}) self.add_shape({'KeyExpression': {'type': 'string'}}) shapes = self.json_model['shapes'] input_members = shapes['SampleOperationInputOutput']['members'] input_members['KeyCondition'] = {'shape': 'KeyExpression'} input_members['AttrCondition'] = {'shape': 'ConditionExpression'} self.injector = TransformationInjector() self.build_models()
def setUp(self): self.target_shape = 'AttributeValue' self.setup_models() self.build_models() self.python_value = 'mystring' self.dynamodb_value = {'S': self.python_value} self.injector = TransformationInjector() self.add_shape({self.target_shape: {'type': 'string'}})
def scan_last(): client = get_dynamo_client() paginator = client.get_paginator('scan') operation_model = client._service_model.operation_model('Scan') trans = TransformationInjector(deserializer=TypeDeserializer()) operation_parameters = {'TableName': ACCOUNT_TABLE} items = [] for page in paginator.paginate(**operation_parameters): has_last_key = 'LastEvaluatedKey' in page if has_last_key: last_key = page['LastEvaluatedKey'].copy() trans.inject_attribute_value_output(page, operation_model) if has_last_key: page['LastEvaluatedKey'] = last_key items.extend(page['Items']) return items
def get_account_role_map(boto_session, region_name): """Fetch the ARNs of all the IAM Roles which people have created in other AWS accounts which are inserted into DynamoDB with http://github.com/gene1wood/cloudformation-cross-account-outputs :return: dict with account ID keys and IAM Role ARN values """ client = boto_session.client('dynamodb', region_name=region_name) paginator = client.get_paginator('scan') service_model = client._service_model.operation_model('Scan') trans = TransformationInjector(deserializer=TypeDeserializer()) items = [] for page in paginator.paginate(TableName=DYNAMODB_TABLE_NAME): trans.inject_attribute_value_output(page, service_model) items.extend([x for x in page['Items']]) return { x['aws-account-id']: x['GuardDutyMemberAccountIAMRoleArn'] for x in items if x.get('category') == DB_CATEGORY and {'aws-account-id', 'GuardDutyMemberAccountIAMRoleArn'} <= set(x) }
class TestTransformConditionExpression(BaseTransformationTest): def setUp(self): super(TestTransformConditionExpression, self).setUp() self.add_shape({'ConditionExpression': {'type': 'string'}}) self.add_shape({'KeyExpression': {'type': 'string'}}) shapes = self.json_model['shapes'] input_members = shapes['SampleOperationInputOutput']['members'] input_members['KeyCondition'] = {'shape': 'KeyExpression'} input_members['AttrCondition'] = {'shape': 'ConditionExpression'} self.injector = TransformationInjector() self.build_models() def test_non_condition_input(self): params = { 'KeyCondition': 'foo', 'AttrCondition': 'bar' } self.injector.inject_condition_expressions( params, self.operation_model) assert params == {'KeyCondition': 'foo', 'AttrCondition': 'bar'} def test_single_attr_condition_expression(self): params = { 'AttrCondition': Attr('foo').eq('bar') } self.injector.inject_condition_expressions( params, self.operation_model) assert params == { 'AttrCondition': '#n0 = :v0', 'ExpressionAttributeNames': {'#n0': 'foo'}, 'ExpressionAttributeValues': {':v0': 'bar'}} def test_single_key_conditon_expression(self): params = { 'KeyCondition': Key('foo').eq('bar') } self.injector.inject_condition_expressions( params, self.operation_model) assert params == { 'KeyCondition': '#n0 = :v0', 'ExpressionAttributeNames': {'#n0': 'foo'}, 'ExpressionAttributeValues': {':v0': 'bar'}} def test_key_and_attr_conditon_expression(self): params = { 'KeyCondition': Key('foo').eq('bar'), 'AttrCondition': Attr('biz').eq('baz') } self.injector.inject_condition_expressions( params, self.operation_model) assert params == { 'KeyCondition': '#n1 = :v1', 'AttrCondition': '#n0 = :v0', 'ExpressionAttributeNames': {'#n0': 'biz', '#n1': 'foo'}, 'ExpressionAttributeValues': {':v0': 'baz', ':v1': 'bar'}} def test_key_and_attr_conditon_expression_with_placeholders(self): params = { 'KeyCondition': Key('foo').eq('bar'), 'AttrCondition': Attr('biz').eq('baz'), 'ExpressionAttributeNames': {'#a': 'b'}, 'ExpressionAttributeValues': {':c': 'd'} } self.injector.inject_condition_expressions( params, self.operation_model) assert params == { 'KeyCondition': '#n1 = :v1', 'AttrCondition': '#n0 = :v0', 'ExpressionAttributeNames': { '#n0': 'biz', '#n1': 'foo', '#a': 'b'}, 'ExpressionAttributeValues': { ':v0': 'baz', ':v1': 'bar', ':c': 'd'}}
class TestTransformConditionExpression(BaseTransformationTest): def setUp(self): super(TestTransformConditionExpression, self).setUp() self.add_shape({'ConditionExpression': {'type': 'string'}}) self.add_shape({'KeyExpression': {'type': 'string'}}) shapes = self.json_model['shapes'] input_members = shapes['SampleOperationInputOutput']['members'] input_members['KeyCondition'] = {'shape': 'KeyExpression'} input_members['AttrCondition'] = {'shape': 'ConditionExpression'} self.injector = TransformationInjector() self.build_models() def test_non_condition_input(self): params = { 'KeyCondition': 'foo', 'AttrCondition': 'bar' } self.injector.inject_condition_expressions( params, self.operation_model) self.assertEqual( params, {'KeyCondition': 'foo', 'AttrCondition': 'bar'}) def test_single_attr_condition_expression(self): params = { 'AttrCondition': Attr('foo').eq('bar') } self.injector.inject_condition_expressions( params, self.operation_model) self.assertEqual( params, {'AttrCondition': '#n0 = :v0', 'ExpressionAttributeNames': {'#n0': 'foo'}, 'ExpressionAttributeValues': {':v0': 'bar'}} ) def test_single_key_conditon_expression(self): params = { 'KeyCondition': Key('foo').eq('bar') } self.injector.inject_condition_expressions( params, self.operation_model) self.assertEqual( params, {'KeyCondition': '#n0 = :v0', 'ExpressionAttributeNames': {'#n0': 'foo'}, 'ExpressionAttributeValues': {':v0': 'bar'}} ) def test_key_and_attr_conditon_expression(self): params = { 'KeyCondition': Key('foo').eq('bar'), 'AttrCondition': Attr('biz').eq('baz') } self.injector.inject_condition_expressions( params, self.operation_model) self.assertEqual( params, {'KeyCondition': '#n1 = :v1', 'AttrCondition': '#n0 = :v0', 'ExpressionAttributeNames': {'#n0': 'biz', '#n1': 'foo'}, 'ExpressionAttributeValues': {':v0': 'baz', ':v1': 'bar'}} ) def test_key_and_attr_conditon_expression_with_placeholders(self): params = { 'KeyCondition': Key('foo').eq('bar'), 'AttrCondition': Attr('biz').eq('baz'), 'ExpressionAttributeNames': {'#a': 'b'}, 'ExpressionAttributeValues': {':c': 'd'} } self.injector.inject_condition_expressions( params, self.operation_model) self.assertEqual( params, {'KeyCondition': '#n1 = :v1', 'AttrCondition': '#n0 = :v0', 'ExpressionAttributeNames': { '#n0': 'biz', '#n1': 'foo', '#a': 'b'}, 'ExpressionAttributeValues': { ':v0': 'baz', ':v1': 'bar', ':c': 'd'}} )