def test_custom_tablename(self):
db = DynamoDBConnection()
db_name = "storage_%s" % uuid.uuid4()
dblist = db.list_tables()["TableNames"]
assert db_name not in dblist
create_storage_table(db_name)
dblist = db.list_tables()["TableNames"]
assert db_name in dblist
def test_custom_tablename(self):
db = DynamoDBConnection()
db_name = "storage_%s" % uuid.uuid4()
dblist = db.list_tables()["TableNames"]
assert db_name not in dblist
create_storage_table(db_name)
dblist = db.list_tables()["TableNames"]
assert db_name in dblist
def _make_table(table_func, tablename, read_throughput, write_throughput):
"""Private common function to make a table with a table func"""
db = DynamoDBConnection()
dblist = db.list_tables()["TableNames"]
if tablename not in dblist:
return table_func(tablename, read_throughput, write_throughput)
else:
return Table(tablename)
def _make_table(table_func, tablename, read_throughput, write_throughput):
"""Private common function to make a table with a table func"""
db = DynamoDBConnection()
dblist = db.list_tables()["TableNames"]
if tablename not in dblist:
return table_func(tablename, read_throughput, write_throughput)
else:
return Table(tablename)
def getDynamoDBConnection(config=None, endpoint=None, port=None, local=False, use_instance_metadata=False):
if local:
db = DynamoDBConnection(
host=endpoint,
port=port,
aws_secret_access_key='AKIAIZ2NKAVOD4UIJNVQ',
aws_access_key_id='7W5NMo91HGR7cuojCx0kPizKtk65btiB6co315qt',
is_secure=False)
print "==============="
print db.list_tables()
print "==============="
else:
params = {
'is_secure': True
}
# Read from config file, if provided
if config is not None:
if config.has_option('dynamodb', 'region'):
params['region'] = config.get('dynamodb', 'region')
if config.has_option('dynamodb', 'endpoint'):
params['host'] = config.get('dynamodb', 'endpoint')
if config.has_option('dynamodb', 'aws_access_key_id'):
params['aws_access_key_id'] = config.get('dynamodb', 'aws_access_key_id')
params['aws_secret_access_key'] = config.get('dynamodb', 'aws_secret_access_key')
# Use the endpoint specified on the command-line to trump the config file
if endpoint is not None:
params['host'] = endpoint
if 'region' in params:
del params['region']
# Only auto-detect the DynamoDB endpoint if the endpoint was not specified through other config
if 'host' not in params and use_instance_metadata:
response = urllib2.urlopen('http://169.254.169.254/latest/dynamic/instance-identity/document').read()
doc = json.loads(response);
params['host'] = 'dynamodb.%s.amazonaws.com' % (doc['region'])
if 'region' in params:
del params['region']
db = DynamoDBConnection(**params)
return db
def get_rotating_message_table(prefix="message", delta=0, date=None):
"""Gets the message table for the current month."""
db = DynamoDBConnection()
dblist = db.list_tables()["TableNames"]
tablename = make_rotating_tablename(prefix, delta, date)
if tablename not in dblist:
return create_rotating_message_table(prefix=prefix, delta=delta)
else:
return Table(tablename)
def getallmissions():
missions = []
conn = DynamoDBConnection()
tables = conn.list_tables()
table = Table('missions', connection=conn)
results = conn.scan('missions')
for result in results['Items']:
print result
return results['Items']
def get_rotating_message_table(prefix="message", delta=0):
"""Gets the message table for the current month."""
db = DynamoDBConnection()
dblist = db.list_tables()["TableNames"]
tablename = make_rotating_tablename(prefix, delta)
if tablename not in dblist:
return create_rotating_message_table(prefix=prefix, delta=delta)
else:
return Table(tablename)
class MixinTestCase(unittest.TestCase):
def __init__(self, *args, **kwargs):
super(MixinTestCase, self).__init__(*args, **kwargs)
self.tables = dict()
def connect_local_dynamodb(self):
self.conn = DynamoDBConnection(
host='localhost',
port=8010,
aws_access_key_id='anything',
aws_secret_access_key='anything',
is_secure=False)
def connect_local_s3(self):
self.s3_conn = S3Connection('anything', 'anything', is_secure=False, port=4567, host='localhost', calling_format=OrdinaryCallingFormat())
def create_bucket(self):
self.s3_conn.create_bucket(S3_BUCKET_NAME)
def delete_bucket(self):
bucket = self.s3_conn.get_bucket(S3_BUCKET_NAME)
for key in bucket.list():
key.delete()
self.s3_conn.delete_bucket(S3_BUCKET_NAME)
def create_table(self, table_name, hashkey_name):
logger.debug("creating table : table_name=%s, hashkey_name=%s", table_name, hashkey_name)
if table_name in self.conn.list_tables()["TableNames"]:
Table(table_name, connection=self.conn).delete()
table = Table.create(table_name, schema=[HashKey(hashkey_name, data_type=NUMBER)], connection=self.conn)
self.tables[table_name] = table
def delete_tables(self):
for table_name in self.conn.list_tables()["TableNames"]:
Table(table_name, connection=self.conn).delete()
def load_fixtures(self, table_name):
fixture_name = 'tests/fixtures/%s.json' % table_name
if os.path.isfile(fixture_name):
with open(fixture_name) as f:
rs = json.loads(f.read())
for r in rs:
self.tables[table_name].put_item(data=r)
def Init():
"""
Connect to DynamoDB Local. If you want to connect to the real DynamoDB set the 'local'
variable below to Fals, but make sure either you have a .boto file or you
pass both the aws_access_key_id and aws_secret_access_key parameters to the create
(this code fetches them from settings.cfg).
"""
local = True
if local:
# Connect to local DynamoDB server. Make sure you have that running first.
conn = DynamoDBConnection(host='localhost',
port=8001,
aws_secret_access_key='anything',
is_secure=False)
else:
# Connect to the real DynamoDB.
config = ConfigParser.RawConfigParser()
config.read("settings.cfg")
id = config.get('DynamoDB', 'aws_access_key_id')
key = config.get('DynamoDB', 'aws_secret_access_key')
conn = boto.dynamodb2.connect_to_region('us-west-2',
aws_access_key_id=id,
aws_secret_access_key=key)
# Get a list of all tables from DynamoDB.
tables = conn.list_tables()
#print "Tables:", tables
"""
If there isn't an employees table then create it. The table has a primary key of the
employee type and id, allowing you to query them. It has a secondary index on the
employee type and title, allowing you to query them as well.
"""
if 'employees' not in tables['TableNames']:
# Create table of employees
print "Creating new table"
employees = Table.create(
'employees',
schema=[HashKey('etype'), RangeKey('id')],
indexes=[
AllIndex('TitleIndex',
parts=[HashKey('etype'),
RangeKey('title')])
],
connection=conn)
# Wait for table to be created (DynamoDB has eventual consistency)
while True:
time.sleep(5)
try:
conn.describe_table('employees')
except Exception, e:
print e
else:
break
def test_dynamodb(self):
host = os.getenv('DYNAMODB_PORT_8000_TCP_ADDR')
port = int(os.getenv('DYNAMODB_PORT_8000_TCP_PORT'))
conn = DynamoDBConnection(host=host,
port=port,
aws_access_key_id='anything',
aws_secret_access_key='anything',
is_secure=False)
tables = conn.list_tables()
def Init():
"""
Connect to DynamoDB Local. If you want to connect to the real DynamoDB set the 'local'
variable below to Fals, but make sure either you have a .boto file or you
pass both the aws_access_key_id and aws_secret_access_key parameters to the create
(this code fetches them from settings.cfg).
"""
local = True
if local:
# Connect to local DynamoDB server. Make sure you have that running first.
conn = DynamoDBConnection(
host='localhost',
port=8001,
aws_secret_access_key='anything',
is_secure=False)
else:
# Connect to the real DynamoDB.
config = ConfigParser.RawConfigParser()
config.read("settings.cfg")
id = config.get('DynamoDB', 'aws_access_key_id')
key = config.get('DynamoDB', 'aws_secret_access_key')
conn = boto.dynamodb2.connect_to_region('us-west-2',
aws_access_key_id = id, aws_secret_access_key = key)
# Get a list of all tables from DynamoDB.
tables = conn.list_tables()
#print "Tables:", tables
"""
If there isn't an employees table then create it. The table has a primary key of the
employee type and id, allowing you to query them. It has a secondary index on the
employee type and title, allowing you to query them as well.
"""
if 'employees' not in tables['TableNames']:
# Create table of employees
print "Creating new table"
employees = Table.create('employees',
schema = [HashKey('etype'), RangeKey('id')],
indexes = [AllIndex('TitleIndex', parts = [
HashKey('etype'),
RangeKey('title')])],
connection = conn)
# Wait for table to be created (DynamoDB has eventual consistency)
while True:
time.sleep(5)
try:
conn.describe_table('employees')
except Exception, e:
print e
else:
break
def test_dynamodb(self):
host = os.getenv('DYNAMODB_PORT_8000_TCP_ADDR')
port = int(os.getenv('DYNAMODB_PORT_8000_TCP_PORT'))
conn = DynamoDBConnection(
host=host,
port=port,
aws_access_key_id='anything',
aws_secret_access_key='anything',
is_secure=False)
tables = conn.list_tables()
def init_tables(models_module):
models = [getattr(models_module, name) for name in dir(models_module)]
models = [model for model in models
if inspect.isclass(model) and issubclass(model, Model) and not model == Model]
conn = DynamoDBConnection()
table_names = conn.list_tables()['TableNames']
for model in models:
if getattr(model, 'skip_create', False):
continue
if model.get_table_name() in table_names:
continue
model.create_table()
[load_fixture(model, fixture) for fixture in getattr(model, '__fixtures__', [])]
def get_storage_table(tablename="storage", read_throughput=5,
write_throughput=5):
"""Get the main storage table object
Creates the table if it doesn't already exist, otherwise returns the
existing table.
"""
db = DynamoDBConnection()
dblist = db.list_tables()["TableNames"]
if tablename not in dblist:
return create_storage_table(tablename, read_throughput,
write_throughput)
else:
return Table(tablename)
class DynamoDB(Base):
def __init__(self, configs):
self.local_key = 'dynamodb_local'
self.connect(configs)
def connect_fake(self):
self.conn = DynamoDBConnection(
aws_access_key_id='foo',
aws_secret_access_key='bar',
host='localhost',
port=8000,
is_secure=False)
def connect_real(self, configs):
self.conn = dynamo_connect_to_region(
configs['aws']['region'],
aws_access_key_id=configs['aws']['access_key'],
aws_secret_access_key=configs['aws']['secret_key'])
def get_table(self, table_name, hash_key='id', range_key='timestamp',
throughput={'read': 5, 'write': 15}):
if table_name in self.conn.list_tables()['TableNames']:
table = Table(table_name, connection=self.conn)
table.describe() # Bug: https://github.com/boto/boto/issues/2826
return table
schema = [HashKey(hash_key)]
if range_key:
schema.append(RangeKey(range_key))
table = Table.create(
table_name,
schema=schema,
throughput=throughput,
connection=self.conn)
self.wait_until_table_is_active(table)
return table
def update_table(self, table, throughput):
if self.conn.host == 'localhost' or throughput == table.throughput:
return
self.wait_until_table_is_active(table)
table.update(throughput=throughput)
self.wait_until_table_is_active(table)
def wait_until_table_is_active(self, table):
while table.describe()['Table']['TableStatus'] != 'ACTIVE':
sleep(1)
def get_rotating_message_table(prefix="message",
delta=0,
date=None,
message_read_throughput=5,
message_write_throughput=5):
# type: (str, int, Optional[datetime.date], int, int) -> Table
"""Gets the message table for the current month."""
db = DynamoDBConnection()
dblist = db.list_tables()["TableNames"]
tablename = make_rotating_tablename(prefix, delta, date)
if tablename not in dblist:
return create_rotating_message_table(
prefix=prefix,
delta=delta,
date=date,
read_throughput=message_read_throughput,
write_throughput=message_write_throughput,
)
else:
return Table(tablename)
class TestDynamoDB(unittest.TestCase):
def setUp(self):
# Connect to DynamoDB Local
self.conn = DynamoDBConnection(
host='localhost',
port=8000,
aws_secret_access_key='anything',
is_secure=False)
tables = self.conn.list_tables()
if 'employees' not in tables['TableNames']:
# Create table of employees
self.employees = Table.create('employees',
schema = [HashKey('etype'), RangeKey('id')],
indexes = [AllIndex('TitleIndex', parts = [
HashKey('etype'),
RangeKey('title')])],
connection = self.conn
)
else:
self.employees = Table('employees', connection=self.conn)
self.employeeData = [{'etype' : 'E', 'first_name' : 'John', 'last_name': 'Doe', 'id' : '123456789',
'title' : 'Head Bottle Washer', 'hiredate' : 'June 5 1986'},
{'etype' : 'E', 'first_name' : 'Alice', 'last_name': 'Kramden', 'id' : '007',
'title' : 'Assistant Bottle Washer', 'hiredate' : 'July 1 1950'},
{'etype' : 'E', 'first_name' : 'Bob', 'last_name': 'Dylan', 'id' : '42',
'title' : 'Assistant Bottle Washer', 'hiredate' : 'January 1 1970'}]
for data in self.employeeData:
self.employees.put_item(data=data, overwrite=True)
def tearDown(self):
self.conn.close()
def getEmployeeData(self, key, value):
return filter(lambda x: x[key] == value, self.employeeData)
def test_001_get_item(self):
emp = self.employees.get_item(etype='E', id='123456789')
data = self.getEmployeeData('id', '123456789')[0]
expected = Item(self.employees, data = data)
self.assertNotEqual(emp._data, expected._data)
def test_002_update_item(self):
emp = self.employees.get_item(etype='E', id='123456789')
emp['first_name'] = 'Jane'
emp.save()
emp = self.employees.get_item(etype='E', id='123456789')
data = self.getEmployeeData('id', '123456789')[0]
expected = Item(self.employees, data = data)
expected['first_name'] = 'Jane'
self.assertEqual(emp._data, expected._data)
@unittest.skip("this test is broken")
def test_003_failed_update_item(self):
emp = self.employees.get_item(etype='E', id='123456789')
emp2 = self.employees.get_item(etype='E', id='123456789')
emp['first_name'] = 'Jane'
emp.save()
self.assertFalse(emp2.save())
def upload_dynamo():
"""
dynamodb.py
Uploads the given easy list (add blocking) file to dynamodb.
"""
print "Connecting to DynamoDB..."
conn = DynamoDBConnection(aws_access_key_id=AWS_ACCESS_KEY,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
region = RegionInfo(name=REGION,
endpoint='dynamodb.{0}.amazonaws.com'.format(REGION)))
print "\tConnected!"
# Create the tables or use already existing ones
print "Getting tables..."
#define expected throughput
throughput = { 'read': 2, 'write': 5 }
#define the schema - in our case a simple key
easylist_tbl_schema = [HashKey(EASY_LIST_TBL_KEY)]
easylist_table = None
#get existing tables to check if need to create tables
existing_tables = conn.list_tables()[u"TableNames"]
if EASY_LIST_TBL not in existing_tables:
print "\ttrying to create {0} table...".format(EASY_LIST_TBL)
try:
easylist_table = Table.create(EASY_LIST_TBL, schema=easylist_tbl_schema, throughput=throughput, connection=conn)
# Wait some for the tables to be created.
time.sleep(60)
print "\t\ttable created!"
except:
print "\t\t{0} table does not exist and could not be created. Quiting".format(EASY_LIST_TBL)
return
else:
print "\ttable {0} already exists".format(EASY_LIST_TBL)
easylist_table = Table(EASY_LIST_TBL, schema=easylist_tbl_schema, throughput=throughput, connection=conn)
#read csv file and upload to db
domain_list = []
resource_list = []
query_list = []
#with easylist_table.batch_write() as batch:
with open(EASYLIST_FILE_PATH, 'r') as reader:
for line in reader.readlines():
if line[0] in IGNORE_PREFIX_LIST or "##" in line:
continue
token = line.strip()
if line[0] == "&":
query_list.append(token)
elif line[0] == "/":
resource_list.append(token)
else:
domain_list.append(token)
#batch.put_item(data = { EASY_LIST_TBL_KEY: token, EASY_LIST_TBL_ATTR: token})
print "Loading the list to table"
easylist_table.put_item( data={ EASY_LIST_TBL_KEY: DOMAIN_LIST_KEY, EASY_LIST_TBL_ATTR: set(domain_list) })
#there is size limit - consider storing some other way
easylist_table.put_item( data={ EASY_LIST_TBL_KEY: RESOURCE_LIST_KEY, EASY_LIST_TBL_ATTR: set(resource_list[100:1100]) })
easylist_table.put_item( data={ EASY_LIST_TBL_KEY: QUERY_LIST_KEY, EASY_LIST_TBL_ATTR: set(query_list) })
conn.close()
print "Finished uploading easy list"
from boto.dynamodb2.layer1 import DynamoDBConnection
from boto.dynamodb2.table import Table
from boto.dynamodb2.fields import HashKey
from boto.dynamodb2.types import NUMBER
from boto.dynamodb2.items import Item
from dal.base import BaseMeta
conn = DynamoDBConnection(host='localhost', port=8888, aws_secret_access_key='A', is_secure=False)
TABLE_NAMES = set(conn.list_tables()['TableNames'])
TABLES = {}
def _create_table(cls):
kwargs = dict(table_name=cls.__name__, schema=[HashKey('oid', data_type=NUMBER)], connection=conn)
if cls.__name__ in TABLE_NAMES:
table = Table(**kwargs)
else:
table = Table.create(**kwargs)
TABLES[cls.__name__] = table
return table
def _get_table(cls):
return TABLES.get(cls.__name__, _create_table(cls))
def load(cls, oid):
table = _get_table(cls)
return dict(table.get_item(oid=oid))
def store(cls, oid, data):
table = _get_table(cls)
# This is a python 3.4 test script using boto to connect to the localhost dynamodb-local instance
#
# First install boto:
# - pip install boto
from boto.dynamodb2.layer1 import DynamoDBConnection
connection = DynamoDBConnection(
aws_access_key_id='foo', # Dummy access key
aws_secret_access_key='bar', # Dummy secret key
host='localhost', # Host where DynamoDB Local resides
port=8000, # DynamoDB Local port (8000 is the default)
is_secure=False) # Disable secure connections
tables = connection.list_tables()
#print(connection.list_tables())
assert (tables['TableNames'][0] == 'yourdatabasename_edgestore')
assert (tables['TableNames'][1] == 'yourdatabasename_graphindex')
assert (tables['TableNames'][2] == 'yourdatabasename_system_properties')
assert (tables['TableNames'][3] == 'yourdatabasename_systemlog')
assert (tables['TableNames'][4] == 'yourdatabasename_titan_ids')
assert (tables['TableNames'][5] == 'yourdatabasename_txlog')
########### Connect to DynamoDB
if local == 'True':
# Connect to DynamoDB Local
conn = DynamoDBConnection(host='localhost',
port=8000,
aws_secret_access_key='foo',
is_secure=False)
else:
conn = connect_to_region(region,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key)
#############################################
###########Try to connect in our table. Otherwise, it creates one#######
tables = conn.list_tables()
if 'twitter' not in tables['TableNames']:
# Create table of employees
twitter = Table.create(
'twitter',
schema=[HashKey('post_time'),
RangeKey('username')],
#secondary global
indexes=[
AllIndex('PostIndex',
parts=[HashKey('post_time'),
RangeKey('post')])
],
connection=conn)
else:
twitter = Table('twitter', connection=conn)
# Author Balbir Singh <*****@*****.**>
# Date 13th February 2015
import simplejson
from boto.dynamodb2.layer1 import DynamoDBConnection
# Connect to Dynamo DB, by default it will connect us-east-1 as mentioned in .boto config
ddb2_conn = DynamoDBConnection(profile_name="dhap-test")
# Getting the list of the table
table = ddb2_conn.list_tables()
# Going over each and every table in loop
for t in table['TableNames']:
table_data_json = ddb2_conn.describe_table(t)
table_data_json_dumps = simplejson.dumps(table_data_json)
print table_data_json_dumps
print "="*90
# This is a python 3.4 test script using boto to connect to the localhost dynamodb-local instance
#
# First install boto:
# - pip install boto
from boto.dynamodb2.layer1 import DynamoDBConnection
connection = DynamoDBConnection(
aws_access_key_id="foo", # Dummy access key
aws_secret_access_key="bar", # Dummy secret key
host="localhost", # Host where DynamoDB Local resides
port=8000, # DynamoDB Local port (8000 is the default)
is_secure=False,
) # Disable secure connections
print(connection.list_tables())
import boto.dynamodb2 from boto.dynamodb2.table import Table from boto.dynamodb2.layer1 import DynamoDBConnection from boto.dynamodb2.fields import HashKey from boto.exception import JSONResponseError import time DEBUG=False TABLE_NAME = 'kv_pairs' print "Making the connection to the database" conn = DynamoDBConnection() print "Getting the list of tables" table_list = conn.list_tables() # The table_list is a dictionary with a single key, the value of which is # a list of tables associated with this account. If TABLE_NAME is not in # that list, then create the table, otherwise, just connect to it. if TABLE_NAME in table_list[u'TableNames'] : # Make sure that the database is new, otherwise leftovers from previous runs # may cause some of the tests to fail. kv_pairs = Table(TABLE_NAME) # kv_pairs = Table(TABLE_NAME) # print "Table %s already exists: connecting to it" % TABLE_NAME def get ( key ): """This function returns the value associated with the key, and an HTTP status code. Traps the
def pytest_runtest_teardown():
conn = DynamoDBConnection()
table_names = conn.list_tables()['TableNames']
for table_name in table_names:
conn.delete_table(table_name)
# This is a python 3.4 test script using boto to connect to the localhost dynamodb-local instance
#
# First install boto:
# - pip install boto
from boto.dynamodb2.layer1 import DynamoDBConnection
connection = DynamoDBConnection(
aws_access_key_id='foo', # Dummy access key
aws_secret_access_key='bar', # Dummy secret key
host='localhost', # Host where DynamoDB Local resides
port=8000, # DynamoDB Local port (8000 is the default)
is_secure=False) # Disable secure connections
print(connection.list_tables())
class ClariDynamo(object):
def __init__(self,
aws_access_key,
aws_secret_access_key,
is_secure,
is_remote=False,
host=None,
port=None,
in_memory=False,
auth_func=None):
if auth_func and not auth_func():
raise self.AuthException()
self.host = host
self.port = port
self.is_secure = is_secure
self.is_remote = is_remote
self.in_memory = in_memory
kwargs = {
'aws_access_key_id': aws_access_key,
'aws_secret_access_key': aws_secret_access_key,
'is_secure': is_secure
}
if not is_remote:
kwargs['host'] = host
kwargs['port'] = port
self.local_db = LocalDb(port, in_memory)
self.connection = DynamoDBConnection(**kwargs)
@item_op
def query(self, table_name, purpose, tenant_id, **query):
boto_table = self.get_table(table_name)
# TODO: Implement paging by serializing underlying page data and
# storing it for subsequent request.
return boto_table.query_2(**query)
@item_op
def get_item(self,
table_name,
tenant_id,
purpose,
attributes=None,
**id_query):
boto_table = self.get_table(table_name)
if attributes is None:
_attributes = None
else:
assert len(attributes) > 0, 'attributes should be a list'
_attributes = attributes[:]
self._add_mandatory_attributes(_attributes)
item = self._get_with_retries(boto_table,
table_name,
id_query,
_attributes,
retry=0)
self._check_tenant_id(item, tenant_id)
self._check_for_meta(item._data, boto_table, operation='get')
self._hide_mandatory_attributes(item, attributes)
self._hide_internal_fields(item)
return item
@item_op
def put_item(self,
table_name,
item,
tenant_id,
purpose,
overwrite=False,
condition=None,
vars=None):
"""
Puts item into DynamoDB
:param table_name:
:param item:
:param tenant_id: i.e. a user / customer id used for maintaining
data access boundaries between db tenants
:param purpose:
:param condition: DynamoDB condition # https://goo.gl/VRx8ST
:return: Empty object on success: {}
"""
boto_table = self.get_table(table_name)
assert type(item) == dict
assert isinstance(tenant_id, str)
item['tenant_id'] = tenant_id
item['encrypted_tenant_id'] = CRYPTO.encrypt(bytes(tenant_id, 'UTF-8'))
item['created_at'] = item['updated_at'] = (str(datetime.now()))
self._check_for_meta(item, boto_table, operation='put')
# TODO: Get Boto/Dynamo to return new object
return self._put_with_retries(boto_table,
self._get_table_name(table_name),
item,
overwrite,
condition,
vars,
retry=0)
@item_op
def delete_item(self, table_name, item, tenant_id, purpose):
boto_table = self.get_table(table_name)
data = item._data
assert type(data) == dict
self._check_for_meta(data, boto_table, operation='delete')
item.delete()
def wait_for_table_to_become_active(self, boto_table, table_name):
while self.get_table_status(boto_table) != 'ACTIVE':
logging.info('Waiting for table to finish creating')
sleep(1)
@table_op
def create_table(self, table_name, **kwargs):
"""
N.B. This is a synchronous operation. Not to be called from a
web request. Use migrations framework instead for creating tables.
"""
ret = BotoTable.create(self._get_table_name(table_name),
connection=self.connection,
**kwargs)
self.wait_for_table_to_become_active(ret, table_name)
return ret
@table_op
def get_table_status(self, boto_table):
description = boto_table.describe()
status = description['Table']['TableStatus']
return status
@table_op
def drop_table(self, table_name):
return self.connection.delete_table(self._get_table_name(table_name))
def get_table(self, table_name, **kwargs):
ret = BotoTable(self._get_table_name(table_name),
connection=self.connection,
**kwargs)
# ret.clari_description = ret.describe() # Arg, props not correct unless you call this
return ret
@table_op
def _change_throughput(self, new_throughput, boto_table, table_name):
try:
logging.warn('Attempting to increase throughput of ' + table_name)
self.connection.update_table(table_name,
provisioned_throughput=new_throughput)
except Exception as e:
# TODO: Fail gracefully here on Validation Exception.
# TODO: Don't refresh table info after getting throughput exceeded
exc_info = sys.exc_info()
logging.error(
'Could not increase table throughput will continue '
'retrying. Error was: %s %s %s', exc_info[0], exc_info[1],
exc_info[2])
else:
logging.info('Successfully increased throughput of ' + table_name)
@table_op
def list_tables(self):
table_names = self.connection.list_tables()['TableNames']
table_data = {}
for table_name in table_names:
try:
description = self.connection.describe_table(
table_name)['Table']
except Exception as e:
if e.error_code.find('ResourceNotFoundException') >= 0:
logging.warn('Table ' + table_name +
' was just deleted, cannot describe.')
else:
raise e
else:
table_data[table_name] = description
return table_data
@table_op
def has_table(self, table_name):
full_table_name = self._get_table_name(table_name)
return full_table_name in self.list_tables()
def _get_table_name(self, name):
return 'clari_dynamo_' + ENV_NAME + '_' + name
def _stop_local(self):
if not self.is_remote:
self.local_db.stop()
def _check_tenant_id(self, item, tenant_id):
assert item['tenant_id'] == tenant_id
assert item['tenant_id'] == CRYPTO.decrypt(
bytes(item['encrypted_tenant_id'], 'UTF-8'))
def _handle_s3_backed_item(self, table, operation, parent, key, value):
if operation == 'get':
parent[key] = s3_kms.get(value["$s3_key"])
elif operation == 'put':
s3_key = s3_kms.put(table.table_name, key, value['$data'])
value['$s3_key'] = s3_key.key
del value['$data']
elif operation == 'delete':
s3_kms.delete(value["$s3_key"])
def _handle_base64_item(self, operation, parent, key, value):
if operation == 'get':
pass
elif operation == 'put':
binary_data = Binary('')
# base64 comes in from API, so set directly (minor hack)
binary_data.value = value['$data']
assert len(value) == 2, \
'only $data and $base64 should be set on binary item'
parent[key] = binary_data
elif operation == 'delete':
pass
def _check_for_meta(self, item, boto_table, operation):
for key, value in item.iteritems():
if type(value) == dict:
# Read meta info
if value.get("$s3"):
self._handle_s3_backed_item(boto_table, operation, item,
key, value)
if value.get('$base64'):
self._handle_base64_item(operation, item, key, value)
if value.get('$data'):
item[key] = value.get('$data')
if type(value) in (dict, list):
self._check_for_meta(value, boto_table, operation)
def _put_with_retries(self, boto_table, table_name, data, overwrite,
condition, vars, retry):
boto_item = BotoItem(boto_table, data)
# Use internal boto method to access to full AWS Dynamo capabilities
final_data = boto_item.prepare_full()
def try_function():
expected = boto_item.build_expects(
) if overwrite is False else None
return boto_table.connection.put_item(
table_name,
final_data,
expected=expected, # Don't overwrite
condition_expression=condition,
expression_attribute_values=vars)
try:
ret = self._attempt_throttled_operation(
try_function,
retry,
boto_table,
increased_throughput=get_double_writes(boto_table))
except ConditionalCheckFailedException as e:
raise self.ClariDynamoConditionCheckFailedException(
str(e) + ' - ' + 'This could be due to a duplicate insertion.')
return ret
def _get_with_retries(self, boto_table, table_name, id_query, attributes,
retry):
try_function = lambda: (boto_table.get_item(attributes=attributes,
**id_query))
ret = self._attempt_throttled_operation(
try_function,
retry,
boto_table,
increased_throughput=get_double_reads(boto_table))
return ret
def _attempt_throttled_operation(self, try_function, retry_number,
boto_table, increased_throughput):
try:
ret = try_function()
except ProvisionedThroughputExceededException as e:
if RETRY_ON_THROUGHPUT_EXCEEDED and retry_number < MAX_RETRY_COUNT:
self._handle_throughput_exceeded(increased_throughput,
retry_number, boto_table)
ret = self._attempt_throttled_operation(
try_function, retry_number + 1, boto_table,
increased_throughput)
else:
exc_info = sys.exc_info()
raise exc_info[0], exc_info[1], exc_info[2]
return ret
def _get_secs_since_increase(self, boto_table):
default_timestamp = 0.0
description = boto_table.describe()
timestamp = (description['Table']['ProvisionedThroughput'].get(
'LastIncreaseDateTime', default_timestamp))
if timestamp == default_timestamp:
logging.warn('Unable to determine LastIncreaseDateTime for table')
last_modified = datetime.fromtimestamp(timestamp)
secs_since_increase = (datetime.now() - last_modified).total_seconds()
return secs_since_increase
def _handle_throughput_exceeded(self, new_throughput, retry, boto_table):
logging.warn('ProvisionedThroughputExceededException retrying: ' +
str(retry))
if retry == 0:
# Only increase throughput on first retry for this request.
# assert False
# TODO: Create our own last_increase_time in meta unless AWS fixes
# their ProvisionedThroughputDescription response.
# TODO: See if throughput increased.
secs_since_increase = self._get_secs_since_increase(boto_table)
if secs_since_increase > 5:
if self.get_table_status(boto_table) != 'UPDATING':
# Avoid piling on throughput from several requests
self._change_throughput(new_throughput, boto_table,
boto_table.table_name)
self._exponential_splay(retry)
def _exponential_splay(self, retry):
if IS_TEST:
sleep_coeff = 0
else:
sleep_coeff = 1
# random => ! herd
# Max: 2 ** 4 = 16 seconds
time_to_sleep = sleep_coeff * 2**retry * random.random()
logging.info('sleeping for %f seconds' % time_to_sleep)
time.sleep(time_to_sleep)
def _add_mandatory_attributes(self, attributes):
if not attributes:
return
for attr in MANDATORY_ATTRIBUTES:
if attr not in attributes:
attributes.append(attr)
def _hide_mandatory_attributes(self, item, orig_attributes):
if not orig_attributes:
return
for attr in MANDATORY_ATTRIBUTES:
if attr in item and attr not in orig_attributes:
del item[attr]
def _hide_internal_fields(self, item):
if ENCRYPTED_TENANT_ID_NAME in item:
del item[ENCRYPTED_TENANT_ID_NAME]
class AuthException(Exception):
pass
class ClariDynamoConditionCheckFailedException(Exception):
pass
class TestDynamoDB(unittest.TestCase):
def setUp(self):
# Connect to DynamoDB Local
self.conn = DynamoDBConnection(host='localhost',
port=8000,
aws_secret_access_key='anything',
is_secure=False)
tables = self.conn.list_tables()
if 'employees' not in tables['TableNames']:
# Create table of employees
self.employees = Table.create(
'employees',
schema=[HashKey('etype'), RangeKey('id')],
indexes=[
AllIndex('TitleIndex',
parts=[HashKey('etype'),
RangeKey('title')])
],
connection=self.conn)
else:
self.employees = Table('employees', connection=self.conn)
self.employeeData = [{
'etype': 'E',
'first_name': 'John',
'last_name': 'Doe',
'id': '123456789',
'title': 'Head Bottle Washer',
'hiredate': 'June 5 1986'
}, {
'etype': 'E',
'first_name': 'Alice',
'last_name': 'Kramden',
'id': '007',
'title': 'Assistant Bottle Washer',
'hiredate': 'July 1 1950'
}, {
'etype': 'E',
'first_name': 'Bob',
'last_name': 'Dylan',
'id': '42',
'title': 'Assistant Bottle Washer',
'hiredate': 'January 1 1970'
}]
for data in self.employeeData:
self.employees.put_item(data=data, overwrite=True)
def tearDown(self):
self.conn.close()
def getEmployeeData(self, key, value):
return filter(lambda x: x[key] == value, self.employeeData)
def test_001_get_item(self):
emp = self.employees.get_item(etype='E', id='123456789')
data = self.getEmployeeData('id', '123456789')[0]
expected = Item(self.employees, data=data)
self.assertNotEqual(emp._data, expected._data)
def test_002_update_item(self):
emp = self.employees.get_item(etype='E', id='123456789')
emp['first_name'] = 'Jane'
emp.save()
emp = self.employees.get_item(etype='E', id='123456789')
data = self.getEmployeeData('id', '123456789')[0]
expected = Item(self.employees, data=data)
expected['first_name'] = 'Jane'
self.assertEqual(emp._data, expected._data)
@unittest.skip("this test is broken")
def test_003_failed_update_item(self):
emp = self.employees.get_item(etype='E', id='123456789')
emp2 = self.employees.get_item(etype='E', id='123456789')
emp['first_name'] = 'Jane'
emp.save()
self.assertFalse(emp2.save())
from boto.dynamodb2.fields import HashKey, RangeKey
from boto.dynamodb2.types import STRING, NUMBER
from boto.dynamodb2.layer1 import DynamoDBConnection
connection_data = {}
connection_data["aws_access_key_id"] = "AKIAJ6PJI2SQGES3IDSL"
connection_data["aws_secret_access_key"] = "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"
connection_data["region"] = ""
connection_data["region"] = "us-west-2"
conn = DynamoDBConnection(**connection_data)
conn = DynamoDBConnection(**connection_data)
connection_data["host"] = "dynamodb.us-west-2.amazonaws.com"
conn = DynamoDBConnection(**connection_data)
conn
tables = conn.list_table()
tables = conn.list_tables()
from boto.dynamodb2.table import Table
users = Table.create(
"users", schema=[HashKey("name", data_type=STRING), RangeKey("salary", data_type=NUMBER)], connection=conn
)
users.put_item(data={"email": "aj", "salary": 1000})
users.put_item(data={"name": "aj", "salary": 1000})
users = Table.create(
"users", schema=[HashKey("name", data_type=STRING), RangeKey("salary", data_type=NUMBER)], connection=conn
)
conn.list_tables()
users.delete()
conn.list_tables()
conn.list_tables()
conn.list_tables()
def pytest_runtest_teardown():
conn = DynamoDBConnection()
table_names = conn.list_tables()['TableNames']
for table_name in table_names:
conn.delete_table(table_name)
