Example #1
0
    def __init__(self, config):
        assert isinstance(config, dict), "Config must be provided during DynamoDbClient initialization"

        # If this is a test, make sure the table is a test table
        if os.environ.get('STAGE') == 'test' and 'table_name' in config:
            assert config['table_name'].startswith('autotest_') or config['table_name'] == 'config', \
                f"Bad table name {config['table_name']} in autotest"

        self.config = config

        if not str(config.get('table_name')).startswith('autotest_mock_'):
            self.dynamo_client = boto3.client('dynamodb')
        else:
            logger.info(f"Initialized DynamoClient without boto3 client for table {config.get('table_name')}")

        # storage for table description(s)
        self._table_descriptions: Optional[Dict[str, Dict]] = {}

        # initialize table store
        self._table_capacity = {}
        self.identify_dynamo_capacity(table_name=self.config['table_name'])

        self.stats = defaultdict(int)
        if not hasattr(self, 'row_mapper'):
            self.row_mapper = self.config.get('row_mapper')

        self.type_serializer = TypeSerializer()
        self.type_deserializer = TypeDeserializer()
Example #2
0
    def updateData(self, numRows, start_key, field_key, field_value):
        client = self.dynamodb_client()
        deserializer = TypeDeserializer()
        serializer = TypeSerializer()

        table_configs = self.expected_table_config()

        for table in table_configs:
            LOGGER.info('Updating %s Items by setting field with key %s to the value %s, with start_key %s, for table %s', numRows, field_key, field_value, start_key, table['TableName'])
            for item in table['generator'](numRows, start_key):
                record = deserializer.deserialize(item)
                hashKey = table['HashKey']
                key = {
                    hashKey: serializer.serialize(record[hashKey])
                }
                serializedFieldValue = serializer.serialize(field_value)
                # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Client.update_item
                client.update_item(
                    TableName=table['TableName'],
                    Key=key,
                    UpdateExpression='set {}=:v'.format(field_key),
                    ExpressionAttributeValues={
                        ':v': serializedFieldValue,
                    },
                )
Example #3
0
    def _load_dice_pools(self):
        if self.pools is not None:
            return

        response = self._client.query(
            TableName='DicePools',
            Select='ALL_ATTRIBUTES',
            Limit=1,
            ScanIndexForward=False,
            KeyConditionExpression='game = :game_name',
            ExpressionAttributeValues={':game_name': {
                "S": "Shadowrun"
            }})
        logger.debug("DB Response: {}".format(response))
        des = TypeDeserializer()
        if 'Items' in response and len(
                response['Items']) > 0 and 'pools' in response['Items'][0]:
            self.pools = des.deserialize(response['Items'][0]['pools'])
        else:
            logger.error("Dice Pools not found in DynamoDB.")
            logger.error(response)
            raise Exception("No values found in DynamoDB.")

        logger.debug("Pools: {}".format(self.pools))
        if self._gm not in self.pools:
            self.pools[self._gm] = 0
        return self.pools
Example #4
0
def get_start_over(intent, session):
    """
    Go back to the beginning of the instruction set
    """
    sesh_attr = persist_attributes(session)
    userID = session['user']['userId']

    if 'last_step' in sesh_attr and 'recipe' in sesh_attr:
        recipe = sesh_attr['recipe']
    else:
        ds = TypeDeserializer()
        full_last_step = db_get_last_step(userID)
        recipe = ds.deserialize(full_last_step['recipe'])

    next_step = recipe['recipe'][0]

    # log to database for cross-session use
    db_log_step(userID, recipe, next_step)
    sesh_attr.update({'last_step': next_step, 'recipe': recipe})

    return build_response(
        sesh_attr,
        build_speechlet_response("Starting Over",
                                 step_to_ssml(next_step),
                                 reprompt_text=done_this_step(),
                                 should_end_session=False,
                                 show_card=False))
Example #5
0
def get_previous(intent, session):
    """
    Go back to previous step in instructions
    """
    sesh_attr = persist_attributes(session)
    userID = session['user']['userId']

    if 'last_step' in sesh_attr and 'recipe' in sesh_attr:
        recipe = sesh_attr['recipe']
        last_step = sesh_attr['last_step']
    else:
        ds = TypeDeserializer()
        full_last_step = db_get_last_step(userID)
        recipe = ds.deserialize(full_last_step['recipe'])
        last_step = ds.deserialize(full_last_step['step'])

    next_step = recipe_prior_step(recipe, last_step)

    # log to database for cross-session use
    db_log_step(userID, recipe, next_step)
    sesh_attr.update({'last_step': next_step, 'recipe': recipe})

    return build_response(
        sesh_attr,
        build_speechlet_response("Going Back",
                                 step_to_ssml(next_step),
                                 reprompt_text=done_this_step(),
                                 should_end_session=False,
                                 show_card=False))
Example #6
0
def deserialize_dynamo_db(dynamo_format_dict):
    deserializer = TypeDeserializer()
    python_data = {
        k: deserializer.deserialize(v)
        for k, v in dynamo_format_dict.items()
    }
    return replace_decimals(python_data)
    def _deserialize(dynamodb_data):
        """Static method to convert dynamodb data type to python data type
        Types convention between DynamoDB and Python.
        Reference link: http://boto3.readthedocs.io/en/latest/_modules/boto3/dynamodb/types.html
            DynamoDB                                Python
            --------                                ------
            {'NULL': True}                          None
            {'BOOL': True/False}                    True/False
            {'N': str(value)}                       Decimal(str(value))
            {'S': string}                           string
            {'B': bytes}                            Binary(bytes)
            {'NS': [str(value)]}                    set([Decimal(str(value))])
            {'SS': [string]}                        set([string])
            {'BS': [bytes]}                         set([bytes])
            {'L': list}                             list
            {'M': dict}                             dict

        Args:
            dynamodb_data (list): Contains IOC info with DynamoDB types

        Returns:
            list: A list of Python dictionary type containing ioc_value and ioc_type
        """
        result = []
        if not dynamodb_data:
            return result

        deserializer = TypeDeserializer()
        for raw_data in dynamodb_data:
            python_data = {}
            for key, val in raw_data.iteritems():
                python_data[key] = deserializer.deserialize(val)
            result.append(python_data)
        return result
Example #8
0
def lambda_handler(event, context):
    deserializer = TypeDeserializer()

    try:
        body = []
        resources = (
            boto3.client("dynamodb")
            .scan(TableName=os.environ.get("WHITELISTTABLE"))
            .get("Items")
        )
        for resource in resources:
            item = {}
            for key, value in resource.items():
                item[key] = str(deserializer.deserialize(value))

            body.append(item)

        return get_return(
            200,
            "Whitelist retrieved",
            None,
            {"whitelist": sorted(body, key=itemgetter("resource_id", "expiration"))},
        )
    except Exception as error:
        print(f"[ERROR] {error}")
        return get_return(400, "Could not retrieve whitelist", None, None)
Example #9
0
def lambda_handler(event, context):
    client = boto3.client("dynamodb")
    paginator = client.get_paginator("scan")
    deserializer = TypeDeserializer()

    try:
        body = []
        page_iterator = paginator.paginate(
            TableName=os.environ.get("WHITELISTTABLE"), )

        for page in page_iterator:
            for item in page["Items"]:
                record = {}
                for key, value in item.items():
                    record[key] = str(deserializer.deserialize(value))

                body.append(record)

        return get_return(
            200,
            "Whitelist retrieved",
            None,
            {
                "whitelist":
                sorted(body, key=itemgetter("resource_id", "expiration"))
            },
        )
    except Exception as error:
        print(f"[ERROR] {error}")
        return get_return(400, "Could not retrieve whitelist", None, None)
def deconstruct_from_dynamo_event(event):
    """
        {'Records':
        [
            {
            'eventID': 'f2552522f15ac49bae72cc751973ac34',
            'eventName': 'MODIFY',
            'eventVersion': '1.1',
            'eventSource': 'aws:dynamodb',
            'awsRegion': 'us-east-2',
            'dynamodb': {
                'ApproximateCreatedDateTime': 1559833090.0,
                'Keys': {
                    'id': {'S': 'b722c9e0-839e-11e9-8ff3-3a9ce2fecb55'}
                    },
                'NewImage': {... object of the new image to be "deserialized"...},
                'SequenceNumber': '27489800000000001219353382',
                'SizeBytes': 304,
                'StreamViewType': 'NEW_IMAGE'
                },
            'eventSourceARN': 'arn:aws:dynamodb:us-east-2:475285711284:table/Tab_DEV_AddFundReferenceTable/stream/2019-06-03T12:14:21.372'
            }
        ]
    }
    :param event:
    :return:
    """
    deserializer = TypeDeserializer()
    res = []
    for r in event['Records']:
        if 'dynamodb' in r.keys():
            res.append(
                deserializer.deserialize({'M': r['dynamodb']['NewImage']}))
    return res
Example #11
0
 def test_fake_dynamo_flat_dict_output(self):
     # This profile must have ldap groups and staff data
     p = fake_profile.FakeUser(seed=44)
     ddb = p.as_dynamo_flat_dict()
     deserializer = TypeDeserializer()
     res = {k: deserializer.deserialize(v) for k, v in ddb.items()}
     assert len(res["access_information"]["ldap"]) > 0
     assert len(res["staff_information"]["office_location"]) > 0
Example #12
0
def deseralize(ddb_record):
    # This is probablt a semi-dangerous hack.
    # https://github.com/boto/boto3/blob/e353ecc219497438b955781988ce7f5cf7efae25/boto3/dynamodb/types.py#L233
    ds = TypeDeserializer()
    output = {}
    for k, v in ddb_record.items():
        output[k] = ds.deserialize(v)
    return (output)
Example #13
0
 def deserialize_output(value):
     try:
         td = TypeDeserializer()
         for k, v in dict(value).items():
             value[k] = td.deserialize(v)
     except BaseException:
         pass
     return value
Example #14
0
 def deserialize_ddb_data(self, ddb_data):
     """
     Convert a DynamoDB item to a regular dictionary
     """
     
     deserializer = TypeDeserializer()
     deserialized_ddb_data = {k: deserializer.deserialize(v) for k, v in ddb_data.items()}
     return deserialized_ddb_data
Example #15
0
def dump_table(table, profile, **opts):
    limit = opts.get('limit')
    delay = opts.get('delay', 30)  # seconds

    session = botocore.session.Session(profile=profile)
    client = session.create_client('dynamodb')

    deser = TypeDeserializer()

    more_data = True
    i = 0
    n = 0
    start_key = None
    while more_data:
        if debug:
            print(
                f'{bcolors.GREY20}fetch {i} n:{n} with delay:{delay}{bcolors.ENDC}',
                file=sys.stderr)

        scan_opts = {}
        if limit: scan_opts['Limit'] = limit - n
        if start_key: scan_opts['ExclusiveStartKey'] = start_key

        try:
            r = client.scan(TableName=table, **scan_opts)

            # pp.pprint(r)
            # print("=====")

            for d in r['Items']:
                n += 1
                # print(f'\n{bcolors.GREY10}record {n}:{bcolors.ENDC}')
                # pp.pprint(json.loads(d['Data']))
                # pp.pprint(d)
                # pp.pprint(deser.deserialize({'M': d}))
                print(
                    json.dumps(deser.deserialize({'M': d}),
                               default=defaultencode))

            start_key = r.get('LastEvaluatedKey', {})

        except client.exceptions.ProvisionedThroughputExceededException as e:
            print(f'{bcolors.WARNING}caught {e}{bcolors.ENDC}',
                  file=sys.stderr)
            delay = delay * 2

        if not start_key:
            more_data = False
        elif limit and n >= limit:
            more_data = False
        else:
            # Sleep to avoid exceeding 5-calls-per-second limit.  This
            # rate-limit is applied across all consumers, and clients will
            # receive ProvisionedThroughputExceededException if it is
            # exceeded.
            time.sleep(delay)
        i += 1
Example #16
0
File: user.py Project: romanom/cis
 def __init__(self,
              dynamodb_table_resource=None,
              dynamodb_client=None,
              transactions=True):
     """Take a dynamodb table resource to use for operations."""
     self.table = dynamodb_table_resource
     self.client = dynamodb_client
     self.transactions = transactions
     self.deserializer = TypeDeserializer()
Example #17
0
    def deserializeItem(record):
        print(record)
        dynamodb = AwsHelper().getResource('dynamodb')

        deserializer = TypeDeserializer()
        deserializedItem = {
            k: deserializer.deserialize(v)
            for k, v in record.items()
        }
        return deserializedItem
Example #18
0
def convert_dynamo_json_to_py_data(input_json: list) -> list:
    """

    :param input_json: DynamoDB compatible JSON with included attributes
    :return: py_data: DynamoDB incompatible JSON with removed attributes
    """
    py_data = []
    deserializer = TypeDeserializer()
    for item in input_json:
        json_data = {k: deserializer.deserialize(v) for k, v in item.items()}
        py_data.append(json_data)

    logger.debug(f"Stripped attributes result: {py_data}")
    return py_data
Example #19
0
def lambda_handler(event, context):

    deserializer = TypeDeserializer()

    for record in event['Records']:

        dynamodb_item = record['dynamodb']['NewImage']

        deserialized = {
            k: deserializer.deserialize(v)
            for k, v in dynamodb_item.items()
        }

        print(deserialized)
Example #20
0
def is_item_exists(table, item_to_get):
    print("getting item:", item_to_get)
    deserializer = TypeDeserializer()
    response = table.get_item(Key={
        k: deserializer.deserialize(v)
        for k, v in item_to_get.iteritems()
    },
                              AttributesToGet=['id'],
                              ConsistentRead=False)
    print(response)
    if 'Item' in response:
        return True
    print("not found  {} in restored table".format(item_to_get))
    return False
Example #21
0
def ddb_to_dict(item):
    # type: (Dict[str, Any]) -> Dict[str, Any]
    # TODO: narrow these types down
    """Converts a raw DynamoDB item to a native Python dictionary.

    :param dict item: DynamoDB item
    :returns: Native item
    :rtype: dict
    """
    deserializer = TypeDeserializer()
    return {
        key: deserializer.deserialize(value)
        for key, value in item.items()
    }
Example #22
0
def ddb_to_dict(item):
    # type: (Dict[str, Any]) -> Dict[str, Any]
    # narrow these types down
    # https://github.com/aws/aws-dynamodb-encryption-python/issues/66
    """Converts a raw DynamoDB item to a native Python dictionary.

    :param dict item: DynamoDB item
    :returns: Native item
    :rtype: dict
    """
    deserializer = TypeDeserializer()
    return {
        key: deserializer.deserialize(value)
        for key, value in item.items()
    }
Example #23
0
    def store_item_hash(self, item):
        """
        Store the item hash in the metadata.
        :param item: The item to store the hash for.
        """
        deserializer = TypeDeserializer()

        # Try to deserialize the data in order to remove dynamoDB data types.
        for key in item:
            try:
                item[key] = deserializer.deserialize(item[key])
            except (TypeError, AttributeError):
                break
        self.resource['metadata']['item_hash'] = hashlib.md5(
            json.dumps(item, sort_keys=True).encode('utf-8')).hexdigest()
    def get_items(self, table_name, item_ids):
        keys = list([{'id': {'S': item_id}} for item_id in item_ids])
        response = self.client.batch_get_item(
            RequestItems={table_name: {
                'Keys': keys,
                'ConsistentRead': True
            }})
        type_deserializer = TypeDeserializer()
        items = response['Responses'][table_name]
        for item in items:
            for key, value in item.items():
                value = type_deserializer.deserialize(value)
                item[key] = value

        return {'Items': items}
def __get_dynamo_schedule(ctx, table_name):

    cmd = "aws {} dynamodb scan --table-name {}".format(profile_arg(), table_name)
    res = ctx.run(cmd, hide=True).stdout
    res = json.loads(res)

    from boto3.dynamodb.types import TypeDeserializer
    tds = TypeDeserializer()
    current_schedule = {}

    for item in res["Items"]:
        item = { k: tds.deserialize(v) for k, v in item.items() }
        current_schedule[item["zoom_series_id"]] = item

    return current_schedule
Example #26
0
def get_all_object_links_from_db(paginator, url: str) -> dict:
    """
    Gets all objects from a db
    :param paginator: a dynamodb scan paginator
    :param url: the base url for an api get request
    :return: the object
    """
    database = paginator  # type: botostubs.DynamoDB.ScanOutput
    items = []
    deserializer = TypeDeserializer()
    for page in paginator:
        for item in page["Items"]:
            item = {k: deserializer.deserialize(v) for k, v in item.items()}
            items.append({"url": f"{url}/{item['uid']}"})
    return items
Example #27
0
    def _validate_keys(dynamodb_data):
        """Helper method to check if query key empty or duplicated"""
        result = []
        if not dynamodb_data:
            err_msg = {'Error': {'Code': 403, 'Message': 'Empty query keys'}}
            raise ParamValidationError(report=err_msg)

        deserializer = TypeDeserializer()
        for raw_data in dynamodb_data:
            for _, val in raw_data.iteritems():
                python_data = deserializer.deserialize(val).lower()
                if not python_data or python_data in result:
                    err_msg = {'Error': {'Code': 403, 'Message': 'Parameter Validation Error'}}
                    raise ParamValidationError(report=err_msg)
                result.append(python_data)
Example #28
0
def quest_intent_handler(handler_input):
    slots = handler_input.request_envelope.request.intent.slots
    slot = slots['name']
    quest_name = slot.value.title()
    response = client.get_item(
        Key={
            'name': {
                'S': quest_name.title(),
            }
        },
        TableName='Runescape_Quests'
    )

    if 'Item' not in response:
        speech_text = 'This quest is currently not supported. Please try again later.'
        handler_input.response_builder.speak(speech_text).set_card(
            SimpleCard('Unknown Quest', speech_text)).set_should_end_session(True)
        return handler_input.response_builder.response

    quest = response['Item']

    deserializer = TypeDeserializer()

    if 'requiredItems' in quest or 'requiredSkills' in quest:
        speech_text = "For %s, you will need" % quest_name
        if 'requiredItems' in quest:
            speech_text = speech_text + ' ' + deserializer.deserialize(quest['requiredItems'])
        
        if 'requiredItems' in quest and 'requiredSkills' in quest:
            speech_text = speech_text + ' and '

        if 'requiredSkills' in quest:
            speech_text = speech_text + ' ' + deserializer.deserialize(quest['requiredSkills'])
        
        speech_text = speech_text + '. '
    else:
        speech_text = "%s does not require any item or skill. " % quest_name

    speech_text = speech_text + 'Do you want to start or cancel?'

    handler_input.response_builder.speak(speech_text).set_card(
        SimpleCard(quest_name, speech_text)).set_should_end_session(False)

    handler_input.attributes_manager.session_attributes['step'] = -1
    handler_input.attributes_manager.session_attributes['steps'] = deserializer.deserialize(quest['steps'])
    handler_input.attributes_manager.session_attributes['quest_name'] = quest_name
    
    return handler_input.response_builder.response
Example #29
0
    def read_Table(self, ModelObj):

        #conn=self.connect(ModelObj)
        #response = conn.scan()
        data = []
        #data = response['Items']
        #while response.get('LastEvaluatedKey'):
        #   response = self.dynamoTable.scan(ExclusiveStartKey=response['LastEvaluatedKey'])
        #   data.extend(response['Items'])

        client = boto3.client(
            'dynamodb',
            region_name=self.region_name,
            aws_access_key_id=settings.DYNAMO_AWS_ACCESS['AWS_ACCESS_KEY_ID'],
            aws_secret_access_key=settings.
            DYNAMO_AWS_ACCESS['AWS_SECRET_ACCESS_KEY'],
            endpoint_url=settings.DYNAMO['URL'])
        paginator = client.get_paginator('scan')
        service_model = client._service_model.operation_model('Scan')
        operation_parameters = {
            'TableName': ModelObj.Meta.table_name,
        }
        trans = TransformationInjector(deserializer=TypeDeserializer())
        for page in paginator.paginate(**operation_parameters):
            trans.inject_attribute_value_output(page, service_model)
            while page.get('LastEvaluatedKey'):
                page = self.dynamoTable.scan(
                    ExclusiveStartKey=page['LastEvaluatedKey'])
                data.extend(page['Items'])
            else:
                data.extend(page['Items'])
        return data
Example #30
0
def deserialize_dynamo_data(input_data):
    """
    Given a dict containing the "serialized" data format used by the low-level
    dynamodb APIs, convert it to a standard python dictionary

    Args:
        input_data: (dict) - "serialized" form of a dynamodb record

    Returns:
        (dict) - the "deserialized" form of the input data
    """
    deserializer = TypeDeserializer()
    output_data = {}
    for k, v in input_data.items():
        output_data[k] = deserializer.deserialize(v)
    return output_data
Example #31
0
 def setUp(self):
     self.deserializer = TypeDeserializer()
Example #32
0
class TestDeserializer(unittest.TestCase):
    def setUp(self):
        self.deserializer = TypeDeserializer()

    def test_deserialize_invalid_type(self):
        with self.assertRaisesRegexp(TypeError, 'FOO is not supported'):
            self.deserializer.deserialize({'FOO': 'bar'})

    def test_deserialize_empty_structure(self):
        with self.assertRaisesRegexp(TypeError, 'Value must be a nonempty'):
            self.assertEqual(self.deserializer.deserialize({}), {})

    def test_deserialize_null(self):
        self.assertEqual(self.deserializer.deserialize({"NULL": True}), None)

    def test_deserialize_boolean(self):
        self.assertEqual(self.deserializer.deserialize({"BOOL": False}), False)

    def test_deserialize_integer(self):
        self.assertEqual(
            self.deserializer.deserialize({'N': '1'}), Decimal('1'))

    def test_deserialize_decimal(self):
        self.assertEqual(
            self.deserializer.deserialize({'N': '1.25'}), Decimal('1.25'))

    def test_deserialize_string(self):
        self.assertEqual(
            self.deserializer.deserialize({'S': 'foo'}), 'foo')

    def test_deserialize_binary(self):
        self.assertEqual(
            self.deserializer.deserialize({'B': b'\x00'}), Binary(b'\x00'))

    def test_deserialize_number_set(self):
        self.assertEqual(
            self.deserializer.deserialize(
                {'NS': ['1', '1.25']}), set([Decimal('1'), Decimal('1.25')]))

    def test_deserialize_string_set(self):
        self.assertEqual(
            self.deserializer.deserialize(
                {'SS': ['foo', 'bar']}), set(['foo', 'bar']))

    def test_deserialize_binary_set(self):
        self.assertEqual(
            self.deserializer.deserialize(
                {'BS': [b'\x00', b'\x01']}),
            set([Binary(b'\x00'), Binary(b'\x01')]))

    def test_deserialize_list(self):
        self.assertEqual(
            self.deserializer.deserialize(
                {'L': [{'N': '1'}, {'S': 'foo'}, {'L': [{'N': '1.25'}]}]}),
            [Decimal('1'), 'foo', [Decimal('1.25')]])

    def test_deserialize_map(self):
        self.assertEqual(
            self.deserializer.deserialize(
                {'M': {'foo': {'S': 'mystring'},
                       'bar': {'M': {'baz': {'N': '1'}}}}}),
            {'foo': 'mystring', 'bar': {'baz': Decimal('1')}}
        )