def put_metrics(self, build_time_metrics):
        if not build_time_metrics:
            return None

        try:
            item = self.table.get_item(instance_type=_get_instance_type(self.localrun), config=self.benchmark_config)
            self.logger.debug("Found existing entity in dynamodb")
        except ItemNotFound:
            self.logger.debug("No existing entity found in dynamodb, creating new one")
            item = Item(self.table, data={'instance_type': _get_instance_type(self.localrun),
                                          'config': self.benchmark_config})

        build_time_json = item['build_time']

        if build_time_json:
            self.logger.debug("Extending existing metric list for build_time")
            # extend existing list
            build_time = json.loads(build_time_json)
            build_time.extend(build_time_metrics)
            item['build_time'] = json.dumps(build_time)
        else:
            item['build_time'] = json.dumps(build_time_metrics)

        if item.needs_save():
            item.partial_save()
            self.logger.debug("Saved item to dynamodb")
def activate(table, project, playbook, progress_str):
    print('.', end='')

    try:
        state = table.get_item(project=project, consistent=True)
    except ItemNotFound:
        state = Item(table, data={
            'project': project,
            'state': 'idle',
        })

    if state['state'] == 'blocked':
        raise ProjectBlockedException()

    if state['state'] == 'active':
        raise ProjectActiveException()

    state['state'] = 'active'
    state.partial_save()
    return state
def activate(table, project, playbook, progress_str):
	print('.', end='')

	try:
		state = table.get_item(project=project, consistent=True)
	except ItemNotFound:
		state = Item(table, data={
			'project': project,
			'state': 'idle',
		})

	if state['state'] == 'blocked':
		raise ProjectBlockedException()

	if state['state'] == 'active':
		raise ProjectActiveException()

	state['state'] = 'active'
	state.partial_save()
	return state
Exemple #4
0
    def test_integration(self):
        # Test creating a full table with all options specified.
        users = Table.create('users', schema=[
            HashKey('username'),
            RangeKey('friend_count', data_type=NUMBER)
        ], throughput={
            'read': 5,
            'write': 5,
        }, indexes=[
            KeysOnlyIndex('LastNameIndex', parts=[
                HashKey('username'),
                RangeKey('last_name')
            ]),
        ])
        self.addCleanup(users.delete)

        self.assertEqual(len(users.schema), 2)
        self.assertEqual(users.throughput['read'], 5)

        # Wait for it.
        time.sleep(60)

        # Make sure things line up if we're introspecting the table.
        users_hit_api = Table('users')
        users_hit_api.describe()
        self.assertEqual(len(users.schema), len(users_hit_api.schema))
        self.assertEqual(users.throughput, users_hit_api.throughput)
        self.assertEqual(len(users.indexes), len(users_hit_api.indexes))

        # Test putting some items individually.
        users.put_item(data={
            'username': '******',
            'first_name': 'John',
            'last_name': 'Doe',
            'friend_count': 4
        })

        users.put_item(data={
            'username': '******',
            'first_name': 'Alice',
            'last_name': 'Expert',
            'friend_count': 2
        })

        time.sleep(5)

        # Test batch writing.
        with users.batch_write() as batch:
            batch.put_item({
                'username': '******',
                'first_name': 'Jane',
                'last_name': 'Doe',
                'friend_count': 3
            })
            batch.delete_item(username='******', friend_count=2)
            batch.put_item({
                'username': '******',
                'first_name': 'Bob',
                'last_name': 'Smith',
                'friend_count': 1
            })

        time.sleep(5)

        # Test getting an item & updating it.
        # This is the "safe" variant (only write if there have been no
        # changes).
        jane = users.get_item(username='******', friend_count=3)
        self.assertEqual(jane['first_name'], 'Jane')
        jane['last_name'] = 'Doh'
        self.assertTrue(jane.save())

        # Test strongly consistent getting of an item.
        # Additionally, test the overwrite behavior.
        client_1_jane = users.get_item(
            username='******',
            friend_count=3,
            consistent=True
        )
        self.assertEqual(jane['first_name'], 'Jane')
        client_2_jane = users.get_item(
            username='******',
            friend_count=3,
            consistent=True
        )
        self.assertEqual(jane['first_name'], 'Jane')

        # Write & assert the ``first_name`` is gone, then...
        del client_1_jane['first_name']
        self.assertTrue(client_1_jane.save())
        check_name = users.get_item(
            username='******',
            friend_count=3,
            consistent=True
        )
        self.assertEqual(check_name['first_name'], None)

        # ...overwrite the data with what's in memory.
        client_2_jane['first_name'] = 'Joan'
        # Now a write that fails due to default expectations...
        self.assertRaises(exceptions.JSONResponseError, client_2_jane.save)
        # ... so we force an overwrite.
        self.assertTrue(client_2_jane.save(overwrite=True))
        check_name_again = users.get_item(
            username='******',
            friend_count=3,
            consistent=True
        )
        self.assertEqual(check_name_again['first_name'], 'Joan')

        # Reset it.
        jane['username'] = '******'
        jane['first_name'] = 'Jane'
        jane['last_name'] = 'Doe'
        jane['friend_count'] = 3
        self.assertTrue(jane.save(overwrite=True))

        # Test the partial update behavior.
        client_3_jane = users.get_item(
            username='******',
            friend_count=3,
            consistent=True
        )
        client_4_jane = users.get_item(
            username='******',
            friend_count=3,
            consistent=True
        )
        client_3_jane['favorite_band'] = 'Feed Me'
        # No ``overwrite`` needed due to new data.
        self.assertTrue(client_3_jane.save())
        # Expectations are only checked on the ``first_name``, so what wouldn't
        # have succeeded by default does succeed here.
        client_4_jane['first_name'] = 'Jacqueline'
        self.assertTrue(client_4_jane.partial_save())
        partial_jane = users.get_item(
            username='******',
            friend_count=3,
            consistent=True
        )
        self.assertEqual(partial_jane['favorite_band'], 'Feed Me')
        self.assertEqual(partial_jane['first_name'], 'Jacqueline')

        # Reset it.
        jane['username'] = '******'
        jane['first_name'] = 'Jane'
        jane['last_name'] = 'Doe'
        jane['friend_count'] = 3
        self.assertTrue(jane.save(overwrite=True))

        # Ensure that partial saves of a brand-new object work.
        sadie = Item(users, data={
            'username': '******',
            'first_name': 'Sadie',
            'favorite_band': 'Zedd',
            'friend_count': 7
        })
        self.assertTrue(sadie.partial_save())
        serverside_sadie = users.get_item(
            username='******',
            friend_count=7,
            consistent=True
        )
        self.assertEqual(serverside_sadie['first_name'], 'Sadie')

        # Test the eventually consistent query.
        results = users.query(
            username__eq='johndoe',
            last_name__eq='Doe',
            index='LastNameIndex',
            attributes=('username',),
            reverse=True
        )

        for res in results:
            self.assertTrue(res['username'] in ['johndoe',])
            self.assertEqual(res.keys(), ['username'])


        # Test the strongly consistent query.
        c_results = users.query(
            username__eq='johndoe',
            last_name__eq='Doe',
            index='LastNameIndex',
            reverse=True,
            consistent=True
        )

        for res in c_results:
            self.assertTrue(res['username'] in ['johndoe',])

        # Test scans without filters.
        all_users = users.scan(limit=7)
        self.assertEqual(all_users.next()['username'], 'bob')
        self.assertEqual(all_users.next()['username'], 'jane')
        self.assertEqual(all_users.next()['username'], 'johndoe')

        # Test scans with a filter.
        filtered_users = users.scan(limit=2, username__beginswith='j')
        self.assertEqual(filtered_users.next()['username'], 'jane')
        self.assertEqual(filtered_users.next()['username'], 'johndoe')

        # Test deleting a single item.
        johndoe = users.get_item(username='******', friend_count=4)
        johndoe.delete()

        # Test the eventually consistent batch get.
        results = users.batch_get(keys=[
            {'username': '******', 'friend_count': 1},
            {'username': '******', 'friend_count': 3}
        ])
        batch_users = []

        for res in results:
            batch_users.append(res)
            self.assertTrue(res['first_name'] in ['Bob', 'Jane'])

        self.assertEqual(len(batch_users), 2)

        # Test the strongly consistent batch get.
        c_results = users.batch_get(keys=[
            {'username': '******', 'friend_count': 1},
            {'username': '******', 'friend_count': 3}
        ], consistent=True)
        c_batch_users = []

        for res in c_results:
            c_batch_users.append(res)
            self.assertTrue(res['first_name'] in ['Bob', 'Jane'])

        self.assertEqual(len(c_batch_users), 2)

        # Test count, but in a weak fashion. Because lag time.
        self.assertTrue(users.count() > -1)

        # Test query count
        count = users.query_count(
            username__eq='bob',
        )

        self.assertEqual(count, 1)

        # Test without LSIs (describe calls shouldn't fail).
        admins = Table.create('admins', schema=[
            HashKey('username')
        ])
        self.addCleanup(admins.delete)
        time.sleep(60)
        admins.describe()
        self.assertEqual(admins.throughput['read'], 5)
        self.assertEqual(admins.indexes, [])

        # A single query term should fail on a table with *ONLY* a HashKey.
        self.assertRaises(
            exceptions.QueryError,
            admins.query,
            username__eq='johndoe'
        )
        # But it shouldn't break on more complex tables.
        res = users.query(username__eq='johndoe')

        # Test putting with/without sets.
        mau5_created = users.put_item(data={
            'username': '******',
            'first_name': 'dead',
            'last_name': 'mau5',
            'friend_count': 2,
            'friends': set(['skrill', 'penny']),
        })
        self.assertTrue(mau5_created)

        penny_created = users.put_item(data={
            'username': '******',
            'first_name': 'Penny',
            'friend_count': 0,
            'friends': set([]),
        })
        self.assertTrue(penny_created)
Exemple #5
0
    def test_integration(self):
        # Test creating a full table with all options specified.
        users = Table.create('users', schema=[
            HashKey('username'),
            RangeKey('friend_count', data_type=NUMBER)
        ], throughput={
            'read': 5,
            'write': 5,
        }, indexes={
            KeysOnlyIndex('LastNameIndex', parts=[
                HashKey('username'),
                RangeKey('last_name')
            ]),
        })
        self.addCleanup(users.delete)

        self.assertEqual(len(users.schema), 2)
        self.assertEqual(users.throughput['read'], 5)

        # Wait for it.
        time.sleep(60)

        # Make sure things line up if we're introspecting the table.
        users_hit_api = Table('users')
        users_hit_api.describe()
        self.assertEqual(len(users.schema), len(users_hit_api.schema))
        self.assertEqual(users.throughput, users_hit_api.throughput)
        self.assertEqual(len(users.indexes), len(users_hit_api.indexes))

        # Test putting some items individually.
        users.put_item(data={
            'username': '******',
            'first_name': 'John',
            'last_name': 'Doe',
            'friend_count': 4
        })

        users.put_item(data={
            'username': '******',
            'first_name': 'Alice',
            'last_name': 'Expert',
            'friend_count': 2
        })

        time.sleep(5)

        # Test batch writing.
        with users.batch_write() as batch:
            batch.put_item({
                'username': '******',
                'first_name': 'Jane',
                'last_name': 'Doe',
                'friend_count': 3
            })
            batch.delete_item(username='******', friend_count=2)
            batch.put_item({
                'username': '******',
                'first_name': 'Bob',
                'last_name': 'Smith',
                'friend_count': 1
            })

        time.sleep(5)

        # Test getting an item & updating it.
        # This is the "safe" variant (only write if there have been no
        # changes).
        jane = users.get_item(username='******', friend_count=3)
        self.assertEqual(jane['first_name'], 'Jane')
        jane['last_name'] = 'Doh'
        self.assertTrue(jane.save())

        # Test strongly consistent getting of an item.
        # Additionally, test the overwrite behavior.
        client_1_jane = users.get_item(
            username='******',
            friend_count=3,
            consistent=True
        )
        self.assertEqual(jane['first_name'], 'Jane')
        client_2_jane = users.get_item(
            username='******',
            friend_count=3,
            consistent=True
        )
        self.assertEqual(jane['first_name'], 'Jane')

        # Write & assert the ``first_name`` is gone, then...
        del client_1_jane['first_name']
        self.assertTrue(client_1_jane.save())
        check_name = users.get_item(
            username='******',
            friend_count=3,
            consistent=True
        )
        self.assertEqual(check_name['first_name'], None)

        # ...overwrite the data with what's in memory.
        client_2_jane['first_name'] = 'Joan'
        # Now a write that fails due to default expectations...
        self.assertRaises(exceptions.JSONResponseError, client_2_jane.save)
        # ... so we force an overwrite.
        self.assertTrue(client_2_jane.save(overwrite=True))
        check_name_again = users.get_item(
            username='******',
            friend_count=3,
            consistent=True
        )
        self.assertEqual(check_name_again['first_name'], 'Joan')

        # Reset it.
        jane['username'] = '******'
        jane['first_name'] = 'Jane'
        jane['last_name'] = 'Doe'
        jane['friend_count'] = 3
        self.assertTrue(jane.save(overwrite=True))

        # Test the partial update behavior.
        client_3_jane = users.get_item(
            username='******',
            friend_count=3,
            consistent=True
        )
        client_4_jane = users.get_item(
            username='******',
            friend_count=3,
            consistent=True
        )
        client_3_jane['favorite_band'] = 'Feed Me'
        # No ``overwrite`` needed due to new data.
        self.assertTrue(client_3_jane.save())
        # Expectations are only checked on the ``first_name``, so what wouldn't
        # have succeeded by default does succeed here.
        client_4_jane['first_name'] = 'Jacqueline'
        self.assertTrue(client_4_jane.partial_save())
        partial_jane = users.get_item(
            username='******',
            friend_count=3,
            consistent=True
        )
        self.assertEqual(partial_jane['favorite_band'], 'Feed Me')
        self.assertEqual(partial_jane['first_name'], 'Jacqueline')

        # Reset it.
        jane['username'] = '******'
        jane['first_name'] = 'Jane'
        jane['last_name'] = 'Doe'
        jane['friend_count'] = 3
        self.assertTrue(jane.save(overwrite=True))

        # Ensure that partial saves of a brand-new object work.
        sadie = Item(users, data={
            'username': '******',
            'first_name': 'Sadie',
            'favorite_band': 'Zedd',
            'friend_count': 7
        })
        self.assertTrue(sadie.partial_save())
        serverside_sadie = users.get_item(
            username='******',
            friend_count=7,
            consistent=True
        )
        self.assertEqual(serverside_sadie['first_name'], 'Sadie')

        # Test the eventually consistent query.
        results = users.query(
            username__eq='johndoe',
            last_name__eq='Doe',
            index='LastNameIndex',
            attributes=('username',),
            reverse=True
        )

        for res in results:
            self.assertTrue(res['username'] in ['johndoe',])
            self.assertEqual(res.keys(), ['username'])


        # Test the strongly consistent query.
        c_results = users.query(
            username__eq='johndoe',
            last_name__eq='Doe',
            index='LastNameIndex',
            reverse=True,
            consistent=True
        )

        for res in c_results:
            self.assertTrue(res['username'] in ['johndoe',])

        # Test scans without filters.
        all_users = users.scan(limit=7)
        self.assertEqual(all_users.next()['username'], 'bob')
        self.assertEqual(all_users.next()['username'], 'jane')
        self.assertEqual(all_users.next()['username'], 'johndoe')

        # Test scans with a filter.
        filtered_users = users.scan(limit=2, username__beginswith='j')
        self.assertEqual(filtered_users.next()['username'], 'jane')
        self.assertEqual(filtered_users.next()['username'], 'johndoe')

        # Test deleting a single item.
        johndoe = users.get_item(username='******', friend_count=4)
        johndoe.delete()

        # Test the eventually consistent batch get.
        results = users.batch_get(keys=[
            {'username': '******', 'friend_count': 1},
            {'username': '******', 'friend_count': 3}
        ])
        batch_users = []

        for res in results:
            batch_users.append(res)
            self.assertTrue(res['first_name'] in ['Bob', 'Jane'])

        self.assertEqual(len(batch_users), 2)

        # Test the strongly consistent batch get.
        c_results = users.batch_get(keys=[
            {'username': '******', 'friend_count': 1},
            {'username': '******', 'friend_count': 3}
        ], consistent=True)
        c_batch_users = []

        for res in c_results:
            c_batch_users.append(res)
            self.assertTrue(res['first_name'] in ['Bob', 'Jane'])

        self.assertEqual(len(c_batch_users), 2)

        # Test count, but in a weak fashion. Because lag time.
        self.assertTrue(users.count() > -1)

        # Test query count
        count = users.query_count(
            username__eq='bob',
        )

        self.assertEqual(count, 1)

        # Test without LSIs (describe calls shouldn't fail).
        admins = Table.create('admins', schema=[
            HashKey('username')
        ])
        self.addCleanup(admins.delete)
        time.sleep(60)
        admins.describe()
        self.assertEqual(admins.throughput['read'], 5)
        self.assertEqual(admins.indexes, [])

        # A single query term should fail on a table with *ONLY* a HashKey.
        self.assertRaises(
            exceptions.QueryError,
            admins.query,
            username__eq='johndoe'
        )
        # But it shouldn't break on more complex tables.
        res = users.query(username__eq='johndoe')

        # Test putting with/without sets.
        mau5_created = users.put_item(data={
            'username': '******',
            'first_name': 'dead',
            'last_name': 'mau5',
            'friend_count': 2,
            'friends': set(['skrill', 'penny']),
        })
        self.assertTrue(mau5_created)

        penny_created = users.put_item(data={
            'username': '******',
            'first_name': 'Penny',
            'friend_count': 0,
            'friends': set([]),
        })
        self.assertTrue(penny_created)
Exemple #6
0
    def test_integration(self):
        # Test creating a full table with all options specified.
        users = Table.create(
            "users",
            schema=[HashKey("username"), RangeKey("friend_count", data_type=NUMBER)],
            throughput={"read": 5, "write": 5},
            indexes=[KeysOnlyIndex("LastNameIndex", parts=[HashKey("username"), RangeKey("last_name")])],
        )
        self.addCleanup(users.delete)

        self.assertEqual(len(users.schema), 2)
        self.assertEqual(users.throughput["read"], 5)

        # Wait for it.
        time.sleep(60)

        # Make sure things line up if we're introspecting the table.
        users_hit_api = Table("users")
        users_hit_api.describe()
        self.assertEqual(len(users.schema), len(users_hit_api.schema))
        self.assertEqual(users.throughput, users_hit_api.throughput)
        self.assertEqual(len(users.indexes), len(users_hit_api.indexes))

        # Test putting some items individually.
        users.put_item(data={"username": "******", "first_name": "John", "last_name": "Doe", "friend_count": 4})

        users.put_item(data={"username": "******", "first_name": "Alice", "last_name": "Expert", "friend_count": 2})

        time.sleep(5)

        # Test batch writing.
        with users.batch_write() as batch:
            batch.put_item({"username": "******", "first_name": "Jane", "last_name": "Doe", "friend_count": 3})
            batch.delete_item(username="******", friend_count=2)
            batch.put_item({"username": "******", "first_name": "Bob", "last_name": "Smith", "friend_count": 1})

        time.sleep(5)

        # Test getting an item & updating it.
        # This is the "safe" variant (only write if there have been no
        # changes).
        jane = users.get_item(username="******", friend_count=3)
        self.assertEqual(jane["first_name"], "Jane")
        jane["last_name"] = "Doh"
        self.assertTrue(jane.save())

        # Test strongly consistent getting of an item.
        # Additionally, test the overwrite behavior.
        client_1_jane = users.get_item(username="******", friend_count=3, consistent=True)
        self.assertEqual(jane["first_name"], "Jane")
        client_2_jane = users.get_item(username="******", friend_count=3, consistent=True)
        self.assertEqual(jane["first_name"], "Jane")

        # Write & assert the ``first_name`` is gone, then...
        del client_1_jane["first_name"]
        self.assertTrue(client_1_jane.save())
        check_name = users.get_item(username="******", friend_count=3, consistent=True)
        self.assertEqual(check_name["first_name"], None)

        # ...overwrite the data with what's in memory.
        client_2_jane["first_name"] = "Joan"
        # Now a write that fails due to default expectations...
        self.assertRaises(exceptions.JSONResponseError, client_2_jane.save)
        # ... so we force an overwrite.
        self.assertTrue(client_2_jane.save(overwrite=True))
        check_name_again = users.get_item(username="******", friend_count=3, consistent=True)
        self.assertEqual(check_name_again["first_name"], "Joan")

        # Reset it.
        jane["username"] = "******"
        jane["first_name"] = "Jane"
        jane["last_name"] = "Doe"
        jane["friend_count"] = 3
        self.assertTrue(jane.save(overwrite=True))

        # Test the partial update behavior.
        client_3_jane = users.get_item(username="******", friend_count=3, consistent=True)
        client_4_jane = users.get_item(username="******", friend_count=3, consistent=True)
        client_3_jane["favorite_band"] = "Feed Me"
        # No ``overwrite`` needed due to new data.
        self.assertTrue(client_3_jane.save())
        # Expectations are only checked on the ``first_name``, so what wouldn't
        # have succeeded by default does succeed here.
        client_4_jane["first_name"] = "Jacqueline"
        self.assertTrue(client_4_jane.partial_save())
        partial_jane = users.get_item(username="******", friend_count=3, consistent=True)
        self.assertEqual(partial_jane["favorite_band"], "Feed Me")
        self.assertEqual(partial_jane["first_name"], "Jacqueline")

        # Reset it.
        jane["username"] = "******"
        jane["first_name"] = "Jane"
        jane["last_name"] = "Doe"
        jane["friend_count"] = 3
        self.assertTrue(jane.save(overwrite=True))

        # Ensure that partial saves of a brand-new object work.
        sadie = Item(
            users, data={"username": "******", "first_name": "Sadie", "favorite_band": "Zedd", "friend_count": 7}
        )
        self.assertTrue(sadie.partial_save())
        serverside_sadie = users.get_item(username="******", friend_count=7, consistent=True)
        self.assertEqual(serverside_sadie["first_name"], "Sadie")

        # Test the eventually consistent query.
        results = users.query(
            username__eq="johndoe", last_name__eq="Doe", index="LastNameIndex", attributes=("username",), reverse=True
        )

        for res in results:
            self.assertTrue(res["username"] in ["johndoe"])
            self.assertEqual(res.keys(), ["username"])

        # Test the strongly consistent query.
        c_results = users.query(
            username__eq="johndoe", last_name__eq="Doe", index="LastNameIndex", reverse=True, consistent=True
        )

        for res in c_results:
            self.assertTrue(res["username"] in ["johndoe"])

        # Test scans without filters.
        all_users = users.scan(limit=7)
        self.assertEqual(all_users.next()["username"], "bob")
        self.assertEqual(all_users.next()["username"], "jane")
        self.assertEqual(all_users.next()["username"], "johndoe")

        # Test scans with a filter.
        filtered_users = users.scan(limit=2, username__beginswith="j")
        self.assertEqual(filtered_users.next()["username"], "jane")
        self.assertEqual(filtered_users.next()["username"], "johndoe")

        # Test deleting a single item.
        johndoe = users.get_item(username="******", friend_count=4)
        johndoe.delete()

        # Test the eventually consistent batch get.
        results = users.batch_get(
            keys=[{"username": "******", "friend_count": 1}, {"username": "******", "friend_count": 3}]
        )
        batch_users = []

        for res in results:
            batch_users.append(res)
            self.assertTrue(res["first_name"] in ["Bob", "Jane"])

        self.assertEqual(len(batch_users), 2)

        # Test the strongly consistent batch get.
        c_results = users.batch_get(
            keys=[{"username": "******", "friend_count": 1}, {"username": "******", "friend_count": 3}], consistent=True
        )
        c_batch_users = []

        for res in c_results:
            c_batch_users.append(res)
            self.assertTrue(res["first_name"] in ["Bob", "Jane"])

        self.assertEqual(len(c_batch_users), 2)

        # Test count, but in a weak fashion. Because lag time.
        self.assertTrue(users.count() > -1)

        # Test query count
        count = users.query_count(username__eq="bob")

        self.assertEqual(count, 1)

        # Test without LSIs (describe calls shouldn't fail).
        admins = Table.create("admins", schema=[HashKey("username")])
        self.addCleanup(admins.delete)
        time.sleep(60)
        admins.describe()
        self.assertEqual(admins.throughput["read"], 5)
        self.assertEqual(admins.indexes, [])

        # A single query term should fail on a table with *ONLY* a HashKey.
        self.assertRaises(exceptions.QueryError, admins.query, username__eq="johndoe")
        # But it shouldn't break on more complex tables.
        res = users.query(username__eq="johndoe")

        # Test putting with/without sets.
        mau5_created = users.put_item(
            data={
                "username": "******",
                "first_name": "dead",
                "last_name": "mau5",
                "friend_count": 2,
                "friends": set(["skrill", "penny"]),
            }
        )
        self.assertTrue(mau5_created)

        penny_created = users.put_item(
            data={"username": "******", "first_name": "Penny", "friend_count": 0, "friends": set([])}
        )
        self.assertTrue(penny_created)
			shard_iterator = k_conn.get_shard_iterator(stream_name, shard_id, shard_iterator_type)
			next_iterator = shard_iterator['ShardIterator']
			shard_ids.append({'shard_id' : shard_id ,'shard_iterator' : shard_iterator['ShardIterator'] })
	tries = 0
	result = []
	while tries < 100:
		tries += 1
		try:
			response = k_conn.get_records(next_iterator, limit=1)
			next_iterator = response['NextShardIterator']
			bookmark= Table(tableName)
			if len(response['Records'])> 0:
				for res in response['Records']:
					dbrecord = bookmark.get_item(redShiftEndpoint=dbHost)
					dbrecord['next_iterator'] = next_iterator
					print res['Data']
					dbrecord['sequenceNumber'] = res['SequenceNumber']
					dbrecord.partial_save()
					try:
						with psycopg2.connect(**db_args) as conn:
							with conn.cursor() as curs:
								curs.execute(res['Data'])
					except:
						pass	
			else :
				print tries
		except ProvisionedThroughputExceededException as ptee:
			print (ptee.message)
			time.sleep(5)