def test_custom_tablename(self): db = DynamoDBConnection() db_name = "storage_%s" % uuid.uuid4() dblist = db.list_tables()["TableNames"] assert db_name not in dblist create_storage_table(db_name) dblist = db.list_tables()["TableNames"] assert db_name in dblist
def _make_table(table_func, tablename, read_throughput, write_throughput): """Private common function to make a table with a table func""" db = DynamoDBConnection() dblist = db.list_tables()["TableNames"] if tablename not in dblist: return table_func(tablename, read_throughput, write_throughput) else: return Table(tablename)
def getDynamoDBConnection(config=None, endpoint=None, port=None, local=False, use_instance_metadata=False): if local: db = DynamoDBConnection( host=endpoint, port=port, aws_secret_access_key='AKIAIZ2NKAVOD4UIJNVQ', aws_access_key_id='7W5NMo91HGR7cuojCx0kPizKtk65btiB6co315qt', is_secure=False) print "===============" print db.list_tables() print "===============" else: params = { 'is_secure': True } # Read from config file, if provided if config is not None: if config.has_option('dynamodb', 'region'): params['region'] = config.get('dynamodb', 'region') if config.has_option('dynamodb', 'endpoint'): params['host'] = config.get('dynamodb', 'endpoint') if config.has_option('dynamodb', 'aws_access_key_id'): params['aws_access_key_id'] = config.get('dynamodb', 'aws_access_key_id') params['aws_secret_access_key'] = config.get('dynamodb', 'aws_secret_access_key') # Use the endpoint specified on the command-line to trump the config file if endpoint is not None: params['host'] = endpoint if 'region' in params: del params['region'] # Only auto-detect the DynamoDB endpoint if the endpoint was not specified through other config if 'host' not in params and use_instance_metadata: response = urllib2.urlopen('http://169.254.169.254/latest/dynamic/instance-identity/document').read() doc = json.loads(response); params['host'] = 'dynamodb.%s.amazonaws.com' % (doc['region']) if 'region' in params: del params['region'] db = DynamoDBConnection(**params) return db
def get_rotating_message_table(prefix="message", delta=0, date=None): """Gets the message table for the current month.""" db = DynamoDBConnection() dblist = db.list_tables()["TableNames"] tablename = make_rotating_tablename(prefix, delta, date) if tablename not in dblist: return create_rotating_message_table(prefix=prefix, delta=delta) else: return Table(tablename)
def getallmissions(): missions = [] conn = DynamoDBConnection() tables = conn.list_tables() table = Table('missions', connection=conn) results = conn.scan('missions') for result in results['Items']: print result return results['Items']
def get_rotating_message_table(prefix="message", delta=0): """Gets the message table for the current month.""" db = DynamoDBConnection() dblist = db.list_tables()["TableNames"] tablename = make_rotating_tablename(prefix, delta) if tablename not in dblist: return create_rotating_message_table(prefix=prefix, delta=delta) else: return Table(tablename)
class MixinTestCase(unittest.TestCase): def __init__(self, *args, **kwargs): super(MixinTestCase, self).__init__(*args, **kwargs) self.tables = dict() def connect_local_dynamodb(self): self.conn = DynamoDBConnection( host='localhost', port=8010, aws_access_key_id='anything', aws_secret_access_key='anything', is_secure=False) def connect_local_s3(self): self.s3_conn = S3Connection('anything', 'anything', is_secure=False, port=4567, host='localhost', calling_format=OrdinaryCallingFormat()) def create_bucket(self): self.s3_conn.create_bucket(S3_BUCKET_NAME) def delete_bucket(self): bucket = self.s3_conn.get_bucket(S3_BUCKET_NAME) for key in bucket.list(): key.delete() self.s3_conn.delete_bucket(S3_BUCKET_NAME) def create_table(self, table_name, hashkey_name): logger.debug("creating table : table_name=%s, hashkey_name=%s", table_name, hashkey_name) if table_name in self.conn.list_tables()["TableNames"]: Table(table_name, connection=self.conn).delete() table = Table.create(table_name, schema=[HashKey(hashkey_name, data_type=NUMBER)], connection=self.conn) self.tables[table_name] = table def delete_tables(self): for table_name in self.conn.list_tables()["TableNames"]: Table(table_name, connection=self.conn).delete() def load_fixtures(self, table_name): fixture_name = 'tests/fixtures/%s.json' % table_name if os.path.isfile(fixture_name): with open(fixture_name) as f: rs = json.loads(f.read()) for r in rs: self.tables[table_name].put_item(data=r)
def Init(): """ Connect to DynamoDB Local. If you want to connect to the real DynamoDB set the 'local' variable below to Fals, but make sure either you have a .boto file or you pass both the aws_access_key_id and aws_secret_access_key parameters to the create (this code fetches them from settings.cfg). """ local = True if local: # Connect to local DynamoDB server. Make sure you have that running first. conn = DynamoDBConnection(host='localhost', port=8001, aws_secret_access_key='anything', is_secure=False) else: # Connect to the real DynamoDB. config = ConfigParser.RawConfigParser() config.read("settings.cfg") id = config.get('DynamoDB', 'aws_access_key_id') key = config.get('DynamoDB', 'aws_secret_access_key') conn = boto.dynamodb2.connect_to_region('us-west-2', aws_access_key_id=id, aws_secret_access_key=key) # Get a list of all tables from DynamoDB. tables = conn.list_tables() #print "Tables:", tables """ If there isn't an employees table then create it. The table has a primary key of the employee type and id, allowing you to query them. It has a secondary index on the employee type and title, allowing you to query them as well. """ if 'employees' not in tables['TableNames']: # Create table of employees print "Creating new table" employees = Table.create( 'employees', schema=[HashKey('etype'), RangeKey('id')], indexes=[ AllIndex('TitleIndex', parts=[HashKey('etype'), RangeKey('title')]) ], connection=conn) # Wait for table to be created (DynamoDB has eventual consistency) while True: time.sleep(5) try: conn.describe_table('employees') except Exception, e: print e else: break
def test_dynamodb(self): host = os.getenv('DYNAMODB_PORT_8000_TCP_ADDR') port = int(os.getenv('DYNAMODB_PORT_8000_TCP_PORT')) conn = DynamoDBConnection(host=host, port=port, aws_access_key_id='anything', aws_secret_access_key='anything', is_secure=False) tables = conn.list_tables()
def Init(): """ Connect to DynamoDB Local. If you want to connect to the real DynamoDB set the 'local' variable below to Fals, but make sure either you have a .boto file or you pass both the aws_access_key_id and aws_secret_access_key parameters to the create (this code fetches them from settings.cfg). """ local = True if local: # Connect to local DynamoDB server. Make sure you have that running first. conn = DynamoDBConnection( host='localhost', port=8001, aws_secret_access_key='anything', is_secure=False) else: # Connect to the real DynamoDB. config = ConfigParser.RawConfigParser() config.read("settings.cfg") id = config.get('DynamoDB', 'aws_access_key_id') key = config.get('DynamoDB', 'aws_secret_access_key') conn = boto.dynamodb2.connect_to_region('us-west-2', aws_access_key_id = id, aws_secret_access_key = key) # Get a list of all tables from DynamoDB. tables = conn.list_tables() #print "Tables:", tables """ If there isn't an employees table then create it. The table has a primary key of the employee type and id, allowing you to query them. It has a secondary index on the employee type and title, allowing you to query them as well. """ if 'employees' not in tables['TableNames']: # Create table of employees print "Creating new table" employees = Table.create('employees', schema = [HashKey('etype'), RangeKey('id')], indexes = [AllIndex('TitleIndex', parts = [ HashKey('etype'), RangeKey('title')])], connection = conn) # Wait for table to be created (DynamoDB has eventual consistency) while True: time.sleep(5) try: conn.describe_table('employees') except Exception, e: print e else: break
def test_dynamodb(self): host = os.getenv('DYNAMODB_PORT_8000_TCP_ADDR') port = int(os.getenv('DYNAMODB_PORT_8000_TCP_PORT')) conn = DynamoDBConnection( host=host, port=port, aws_access_key_id='anything', aws_secret_access_key='anything', is_secure=False) tables = conn.list_tables()
def init_tables(models_module): models = [getattr(models_module, name) for name in dir(models_module)] models = [model for model in models if inspect.isclass(model) and issubclass(model, Model) and not model == Model] conn = DynamoDBConnection() table_names = conn.list_tables()['TableNames'] for model in models: if getattr(model, 'skip_create', False): continue if model.get_table_name() in table_names: continue model.create_table() [load_fixture(model, fixture) for fixture in getattr(model, '__fixtures__', [])]
def get_storage_table(tablename="storage", read_throughput=5, write_throughput=5): """Get the main storage table object Creates the table if it doesn't already exist, otherwise returns the existing table. """ db = DynamoDBConnection() dblist = db.list_tables()["TableNames"] if tablename not in dblist: return create_storage_table(tablename, read_throughput, write_throughput) else: return Table(tablename)
class DynamoDB(Base): def __init__(self, configs): self.local_key = 'dynamodb_local' self.connect(configs) def connect_fake(self): self.conn = DynamoDBConnection( aws_access_key_id='foo', aws_secret_access_key='bar', host='localhost', port=8000, is_secure=False) def connect_real(self, configs): self.conn = dynamo_connect_to_region( configs['aws']['region'], aws_access_key_id=configs['aws']['access_key'], aws_secret_access_key=configs['aws']['secret_key']) def get_table(self, table_name, hash_key='id', range_key='timestamp', throughput={'read': 5, 'write': 15}): if table_name in self.conn.list_tables()['TableNames']: table = Table(table_name, connection=self.conn) table.describe() # Bug: https://github.com/boto/boto/issues/2826 return table schema = [HashKey(hash_key)] if range_key: schema.append(RangeKey(range_key)) table = Table.create( table_name, schema=schema, throughput=throughput, connection=self.conn) self.wait_until_table_is_active(table) return table def update_table(self, table, throughput): if self.conn.host == 'localhost' or throughput == table.throughput: return self.wait_until_table_is_active(table) table.update(throughput=throughput) self.wait_until_table_is_active(table) def wait_until_table_is_active(self, table): while table.describe()['Table']['TableStatus'] != 'ACTIVE': sleep(1)
def get_rotating_message_table(prefix="message", delta=0, date=None, message_read_throughput=5, message_write_throughput=5): # type: (str, int, Optional[datetime.date], int, int) -> Table """Gets the message table for the current month.""" db = DynamoDBConnection() dblist = db.list_tables()["TableNames"] tablename = make_rotating_tablename(prefix, delta, date) if tablename not in dblist: return create_rotating_message_table( prefix=prefix, delta=delta, date=date, read_throughput=message_read_throughput, write_throughput=message_write_throughput, ) else: return Table(tablename)
class TestDynamoDB(unittest.TestCase): def setUp(self): # Connect to DynamoDB Local self.conn = DynamoDBConnection( host='localhost', port=8000, aws_secret_access_key='anything', is_secure=False) tables = self.conn.list_tables() if 'employees' not in tables['TableNames']: # Create table of employees self.employees = Table.create('employees', schema = [HashKey('etype'), RangeKey('id')], indexes = [AllIndex('TitleIndex', parts = [ HashKey('etype'), RangeKey('title')])], connection = self.conn ) else: self.employees = Table('employees', connection=self.conn) self.employeeData = [{'etype' : 'E', 'first_name' : 'John', 'last_name': 'Doe', 'id' : '123456789', 'title' : 'Head Bottle Washer', 'hiredate' : 'June 5 1986'}, {'etype' : 'E', 'first_name' : 'Alice', 'last_name': 'Kramden', 'id' : '007', 'title' : 'Assistant Bottle Washer', 'hiredate' : 'July 1 1950'}, {'etype' : 'E', 'first_name' : 'Bob', 'last_name': 'Dylan', 'id' : '42', 'title' : 'Assistant Bottle Washer', 'hiredate' : 'January 1 1970'}] for data in self.employeeData: self.employees.put_item(data=data, overwrite=True) def tearDown(self): self.conn.close() def getEmployeeData(self, key, value): return filter(lambda x: x[key] == value, self.employeeData) def test_001_get_item(self): emp = self.employees.get_item(etype='E', id='123456789') data = self.getEmployeeData('id', '123456789')[0] expected = Item(self.employees, data = data) self.assertNotEqual(emp._data, expected._data) def test_002_update_item(self): emp = self.employees.get_item(etype='E', id='123456789') emp['first_name'] = 'Jane' emp.save() emp = self.employees.get_item(etype='E', id='123456789') data = self.getEmployeeData('id', '123456789')[0] expected = Item(self.employees, data = data) expected['first_name'] = 'Jane' self.assertEqual(emp._data, expected._data) @unittest.skip("this test is broken") def test_003_failed_update_item(self): emp = self.employees.get_item(etype='E', id='123456789') emp2 = self.employees.get_item(etype='E', id='123456789') emp['first_name'] = 'Jane' emp.save() self.assertFalse(emp2.save())
def upload_dynamo(): """ dynamodb.py Uploads the given easy list (add blocking) file to dynamodb. """ print "Connecting to DynamoDB..." conn = DynamoDBConnection(aws_access_key_id=AWS_ACCESS_KEY, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, region = RegionInfo(name=REGION, endpoint='dynamodb.{0}.amazonaws.com'.format(REGION))) print "\tConnected!" # Create the tables or use already existing ones print "Getting tables..." #define expected throughput throughput = { 'read': 2, 'write': 5 } #define the schema - in our case a simple key easylist_tbl_schema = [HashKey(EASY_LIST_TBL_KEY)] easylist_table = None #get existing tables to check if need to create tables existing_tables = conn.list_tables()[u"TableNames"] if EASY_LIST_TBL not in existing_tables: print "\ttrying to create {0} table...".format(EASY_LIST_TBL) try: easylist_table = Table.create(EASY_LIST_TBL, schema=easylist_tbl_schema, throughput=throughput, connection=conn) # Wait some for the tables to be created. time.sleep(60) print "\t\ttable created!" except: print "\t\t{0} table does not exist and could not be created. Quiting".format(EASY_LIST_TBL) return else: print "\ttable {0} already exists".format(EASY_LIST_TBL) easylist_table = Table(EASY_LIST_TBL, schema=easylist_tbl_schema, throughput=throughput, connection=conn) #read csv file and upload to db domain_list = [] resource_list = [] query_list = [] #with easylist_table.batch_write() as batch: with open(EASYLIST_FILE_PATH, 'r') as reader: for line in reader.readlines(): if line[0] in IGNORE_PREFIX_LIST or "##" in line: continue token = line.strip() if line[0] == "&": query_list.append(token) elif line[0] == "/": resource_list.append(token) else: domain_list.append(token) #batch.put_item(data = { EASY_LIST_TBL_KEY: token, EASY_LIST_TBL_ATTR: token}) print "Loading the list to table" easylist_table.put_item( data={ EASY_LIST_TBL_KEY: DOMAIN_LIST_KEY, EASY_LIST_TBL_ATTR: set(domain_list) }) #there is size limit - consider storing some other way easylist_table.put_item( data={ EASY_LIST_TBL_KEY: RESOURCE_LIST_KEY, EASY_LIST_TBL_ATTR: set(resource_list[100:1100]) }) easylist_table.put_item( data={ EASY_LIST_TBL_KEY: QUERY_LIST_KEY, EASY_LIST_TBL_ATTR: set(query_list) }) conn.close() print "Finished uploading easy list"
from boto.dynamodb2.layer1 import DynamoDBConnection from boto.dynamodb2.table import Table from boto.dynamodb2.fields import HashKey from boto.dynamodb2.types import NUMBER from boto.dynamodb2.items import Item from dal.base import BaseMeta conn = DynamoDBConnection(host='localhost', port=8888, aws_secret_access_key='A', is_secure=False) TABLE_NAMES = set(conn.list_tables()['TableNames']) TABLES = {} def _create_table(cls): kwargs = dict(table_name=cls.__name__, schema=[HashKey('oid', data_type=NUMBER)], connection=conn) if cls.__name__ in TABLE_NAMES: table = Table(**kwargs) else: table = Table.create(**kwargs) TABLES[cls.__name__] = table return table def _get_table(cls): return TABLES.get(cls.__name__, _create_table(cls)) def load(cls, oid): table = _get_table(cls) return dict(table.get_item(oid=oid)) def store(cls, oid, data): table = _get_table(cls)
# This is a python 3.4 test script using boto to connect to the localhost dynamodb-local instance # # First install boto: # - pip install boto from boto.dynamodb2.layer1 import DynamoDBConnection connection = DynamoDBConnection( aws_access_key_id='foo', # Dummy access key aws_secret_access_key='bar', # Dummy secret key host='localhost', # Host where DynamoDB Local resides port=8000, # DynamoDB Local port (8000 is the default) is_secure=False) # Disable secure connections tables = connection.list_tables() #print(connection.list_tables()) assert (tables['TableNames'][0] == 'yourdatabasename_edgestore') assert (tables['TableNames'][1] == 'yourdatabasename_graphindex') assert (tables['TableNames'][2] == 'yourdatabasename_system_properties') assert (tables['TableNames'][3] == 'yourdatabasename_systemlog') assert (tables['TableNames'][4] == 'yourdatabasename_titan_ids') assert (tables['TableNames'][5] == 'yourdatabasename_txlog')
########### Connect to DynamoDB if local == 'True': # Connect to DynamoDB Local conn = DynamoDBConnection(host='localhost', port=8000, aws_secret_access_key='foo', is_secure=False) else: conn = connect_to_region(region, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) ############################################# ###########Try to connect in our table. Otherwise, it creates one####### tables = conn.list_tables() if 'twitter' not in tables['TableNames']: # Create table of employees twitter = Table.create( 'twitter', schema=[HashKey('post_time'), RangeKey('username')], #secondary global indexes=[ AllIndex('PostIndex', parts=[HashKey('post_time'), RangeKey('post')]) ], connection=conn) else: twitter = Table('twitter', connection=conn)
# Author Balbir Singh <*****@*****.**> # Date 13th February 2015 import simplejson from boto.dynamodb2.layer1 import DynamoDBConnection # Connect to Dynamo DB, by default it will connect us-east-1 as mentioned in .boto config ddb2_conn = DynamoDBConnection(profile_name="dhap-test") # Getting the list of the table table = ddb2_conn.list_tables() # Going over each and every table in loop for t in table['TableNames']: table_data_json = ddb2_conn.describe_table(t) table_data_json_dumps = simplejson.dumps(table_data_json) print table_data_json_dumps print "="*90
# This is a python 3.4 test script using boto to connect to the localhost dynamodb-local instance # # First install boto: # - pip install boto from boto.dynamodb2.layer1 import DynamoDBConnection connection = DynamoDBConnection( aws_access_key_id="foo", # Dummy access key aws_secret_access_key="bar", # Dummy secret key host="localhost", # Host where DynamoDB Local resides port=8000, # DynamoDB Local port (8000 is the default) is_secure=False, ) # Disable secure connections print(connection.list_tables())
import boto.dynamodb2 from boto.dynamodb2.table import Table from boto.dynamodb2.layer1 import DynamoDBConnection from boto.dynamodb2.fields import HashKey from boto.exception import JSONResponseError import time DEBUG=False TABLE_NAME = 'kv_pairs' print "Making the connection to the database" conn = DynamoDBConnection() print "Getting the list of tables" table_list = conn.list_tables() # The table_list is a dictionary with a single key, the value of which is # a list of tables associated with this account. If TABLE_NAME is not in # that list, then create the table, otherwise, just connect to it. if TABLE_NAME in table_list[u'TableNames'] : # Make sure that the database is new, otherwise leftovers from previous runs # may cause some of the tests to fail. kv_pairs = Table(TABLE_NAME) # kv_pairs = Table(TABLE_NAME) # print "Table %s already exists: connecting to it" % TABLE_NAME def get ( key ): """This function returns the value associated with the key, and an HTTP status code. Traps the
def pytest_runtest_teardown(): conn = DynamoDBConnection() table_names = conn.list_tables()['TableNames'] for table_name in table_names: conn.delete_table(table_name)
# This is a python 3.4 test script using boto to connect to the localhost dynamodb-local instance # # First install boto: # - pip install boto from boto.dynamodb2.layer1 import DynamoDBConnection connection = DynamoDBConnection( aws_access_key_id='foo', # Dummy access key aws_secret_access_key='bar', # Dummy secret key host='localhost', # Host where DynamoDB Local resides port=8000, # DynamoDB Local port (8000 is the default) is_secure=False) # Disable secure connections print(connection.list_tables())
class ClariDynamo(object): def __init__(self, aws_access_key, aws_secret_access_key, is_secure, is_remote=False, host=None, port=None, in_memory=False, auth_func=None): if auth_func and not auth_func(): raise self.AuthException() self.host = host self.port = port self.is_secure = is_secure self.is_remote = is_remote self.in_memory = in_memory kwargs = { 'aws_access_key_id': aws_access_key, 'aws_secret_access_key': aws_secret_access_key, 'is_secure': is_secure } if not is_remote: kwargs['host'] = host kwargs['port'] = port self.local_db = LocalDb(port, in_memory) self.connection = DynamoDBConnection(**kwargs) @item_op def query(self, table_name, purpose, tenant_id, **query): boto_table = self.get_table(table_name) # TODO: Implement paging by serializing underlying page data and # storing it for subsequent request. return boto_table.query_2(**query) @item_op def get_item(self, table_name, tenant_id, purpose, attributes=None, **id_query): boto_table = self.get_table(table_name) if attributes is None: _attributes = None else: assert len(attributes) > 0, 'attributes should be a list' _attributes = attributes[:] self._add_mandatory_attributes(_attributes) item = self._get_with_retries(boto_table, table_name, id_query, _attributes, retry=0) self._check_tenant_id(item, tenant_id) self._check_for_meta(item._data, boto_table, operation='get') self._hide_mandatory_attributes(item, attributes) self._hide_internal_fields(item) return item @item_op def put_item(self, table_name, item, tenant_id, purpose, overwrite=False, condition=None, vars=None): """ Puts item into DynamoDB :param table_name: :param item: :param tenant_id: i.e. a user / customer id used for maintaining data access boundaries between db tenants :param purpose: :param condition: DynamoDB condition # https://goo.gl/VRx8ST :return: Empty object on success: {} """ boto_table = self.get_table(table_name) assert type(item) == dict assert isinstance(tenant_id, str) item['tenant_id'] = tenant_id item['encrypted_tenant_id'] = CRYPTO.encrypt(bytes(tenant_id, 'UTF-8')) item['created_at'] = item['updated_at'] = (str(datetime.now())) self._check_for_meta(item, boto_table, operation='put') # TODO: Get Boto/Dynamo to return new object return self._put_with_retries(boto_table, self._get_table_name(table_name), item, overwrite, condition, vars, retry=0) @item_op def delete_item(self, table_name, item, tenant_id, purpose): boto_table = self.get_table(table_name) data = item._data assert type(data) == dict self._check_for_meta(data, boto_table, operation='delete') item.delete() def wait_for_table_to_become_active(self, boto_table, table_name): while self.get_table_status(boto_table) != 'ACTIVE': logging.info('Waiting for table to finish creating') sleep(1) @table_op def create_table(self, table_name, **kwargs): """ N.B. This is a synchronous operation. Not to be called from a web request. Use migrations framework instead for creating tables. """ ret = BotoTable.create(self._get_table_name(table_name), connection=self.connection, **kwargs) self.wait_for_table_to_become_active(ret, table_name) return ret @table_op def get_table_status(self, boto_table): description = boto_table.describe() status = description['Table']['TableStatus'] return status @table_op def drop_table(self, table_name): return self.connection.delete_table(self._get_table_name(table_name)) def get_table(self, table_name, **kwargs): ret = BotoTable(self._get_table_name(table_name), connection=self.connection, **kwargs) # ret.clari_description = ret.describe() # Arg, props not correct unless you call this return ret @table_op def _change_throughput(self, new_throughput, boto_table, table_name): try: logging.warn('Attempting to increase throughput of ' + table_name) self.connection.update_table(table_name, provisioned_throughput=new_throughput) except Exception as e: # TODO: Fail gracefully here on Validation Exception. # TODO: Don't refresh table info after getting throughput exceeded exc_info = sys.exc_info() logging.error( 'Could not increase table throughput will continue ' 'retrying. Error was: %s %s %s', exc_info[0], exc_info[1], exc_info[2]) else: logging.info('Successfully increased throughput of ' + table_name) @table_op def list_tables(self): table_names = self.connection.list_tables()['TableNames'] table_data = {} for table_name in table_names: try: description = self.connection.describe_table( table_name)['Table'] except Exception as e: if e.error_code.find('ResourceNotFoundException') >= 0: logging.warn('Table ' + table_name + ' was just deleted, cannot describe.') else: raise e else: table_data[table_name] = description return table_data @table_op def has_table(self, table_name): full_table_name = self._get_table_name(table_name) return full_table_name in self.list_tables() def _get_table_name(self, name): return 'clari_dynamo_' + ENV_NAME + '_' + name def _stop_local(self): if not self.is_remote: self.local_db.stop() def _check_tenant_id(self, item, tenant_id): assert item['tenant_id'] == tenant_id assert item['tenant_id'] == CRYPTO.decrypt( bytes(item['encrypted_tenant_id'], 'UTF-8')) def _handle_s3_backed_item(self, table, operation, parent, key, value): if operation == 'get': parent[key] = s3_kms.get(value["$s3_key"]) elif operation == 'put': s3_key = s3_kms.put(table.table_name, key, value['$data']) value['$s3_key'] = s3_key.key del value['$data'] elif operation == 'delete': s3_kms.delete(value["$s3_key"]) def _handle_base64_item(self, operation, parent, key, value): if operation == 'get': pass elif operation == 'put': binary_data = Binary('') # base64 comes in from API, so set directly (minor hack) binary_data.value = value['$data'] assert len(value) == 2, \ 'only $data and $base64 should be set on binary item' parent[key] = binary_data elif operation == 'delete': pass def _check_for_meta(self, item, boto_table, operation): for key, value in item.iteritems(): if type(value) == dict: # Read meta info if value.get("$s3"): self._handle_s3_backed_item(boto_table, operation, item, key, value) if value.get('$base64'): self._handle_base64_item(operation, item, key, value) if value.get('$data'): item[key] = value.get('$data') if type(value) in (dict, list): self._check_for_meta(value, boto_table, operation) def _put_with_retries(self, boto_table, table_name, data, overwrite, condition, vars, retry): boto_item = BotoItem(boto_table, data) # Use internal boto method to access to full AWS Dynamo capabilities final_data = boto_item.prepare_full() def try_function(): expected = boto_item.build_expects( ) if overwrite is False else None return boto_table.connection.put_item( table_name, final_data, expected=expected, # Don't overwrite condition_expression=condition, expression_attribute_values=vars) try: ret = self._attempt_throttled_operation( try_function, retry, boto_table, increased_throughput=get_double_writes(boto_table)) except ConditionalCheckFailedException as e: raise self.ClariDynamoConditionCheckFailedException( str(e) + ' - ' + 'This could be due to a duplicate insertion.') return ret def _get_with_retries(self, boto_table, table_name, id_query, attributes, retry): try_function = lambda: (boto_table.get_item(attributes=attributes, **id_query)) ret = self._attempt_throttled_operation( try_function, retry, boto_table, increased_throughput=get_double_reads(boto_table)) return ret def _attempt_throttled_operation(self, try_function, retry_number, boto_table, increased_throughput): try: ret = try_function() except ProvisionedThroughputExceededException as e: if RETRY_ON_THROUGHPUT_EXCEEDED and retry_number < MAX_RETRY_COUNT: self._handle_throughput_exceeded(increased_throughput, retry_number, boto_table) ret = self._attempt_throttled_operation( try_function, retry_number + 1, boto_table, increased_throughput) else: exc_info = sys.exc_info() raise exc_info[0], exc_info[1], exc_info[2] return ret def _get_secs_since_increase(self, boto_table): default_timestamp = 0.0 description = boto_table.describe() timestamp = (description['Table']['ProvisionedThroughput'].get( 'LastIncreaseDateTime', default_timestamp)) if timestamp == default_timestamp: logging.warn('Unable to determine LastIncreaseDateTime for table') last_modified = datetime.fromtimestamp(timestamp) secs_since_increase = (datetime.now() - last_modified).total_seconds() return secs_since_increase def _handle_throughput_exceeded(self, new_throughput, retry, boto_table): logging.warn('ProvisionedThroughputExceededException retrying: ' + str(retry)) if retry == 0: # Only increase throughput on first retry for this request. # assert False # TODO: Create our own last_increase_time in meta unless AWS fixes # their ProvisionedThroughputDescription response. # TODO: See if throughput increased. secs_since_increase = self._get_secs_since_increase(boto_table) if secs_since_increase > 5: if self.get_table_status(boto_table) != 'UPDATING': # Avoid piling on throughput from several requests self._change_throughput(new_throughput, boto_table, boto_table.table_name) self._exponential_splay(retry) def _exponential_splay(self, retry): if IS_TEST: sleep_coeff = 0 else: sleep_coeff = 1 # random => ! herd # Max: 2 ** 4 = 16 seconds time_to_sleep = sleep_coeff * 2**retry * random.random() logging.info('sleeping for %f seconds' % time_to_sleep) time.sleep(time_to_sleep) def _add_mandatory_attributes(self, attributes): if not attributes: return for attr in MANDATORY_ATTRIBUTES: if attr not in attributes: attributes.append(attr) def _hide_mandatory_attributes(self, item, orig_attributes): if not orig_attributes: return for attr in MANDATORY_ATTRIBUTES: if attr in item and attr not in orig_attributes: del item[attr] def _hide_internal_fields(self, item): if ENCRYPTED_TENANT_ID_NAME in item: del item[ENCRYPTED_TENANT_ID_NAME] class AuthException(Exception): pass class ClariDynamoConditionCheckFailedException(Exception): pass
class TestDynamoDB(unittest.TestCase): def setUp(self): # Connect to DynamoDB Local self.conn = DynamoDBConnection(host='localhost', port=8000, aws_secret_access_key='anything', is_secure=False) tables = self.conn.list_tables() if 'employees' not in tables['TableNames']: # Create table of employees self.employees = Table.create( 'employees', schema=[HashKey('etype'), RangeKey('id')], indexes=[ AllIndex('TitleIndex', parts=[HashKey('etype'), RangeKey('title')]) ], connection=self.conn) else: self.employees = Table('employees', connection=self.conn) self.employeeData = [{ 'etype': 'E', 'first_name': 'John', 'last_name': 'Doe', 'id': '123456789', 'title': 'Head Bottle Washer', 'hiredate': 'June 5 1986' }, { 'etype': 'E', 'first_name': 'Alice', 'last_name': 'Kramden', 'id': '007', 'title': 'Assistant Bottle Washer', 'hiredate': 'July 1 1950' }, { 'etype': 'E', 'first_name': 'Bob', 'last_name': 'Dylan', 'id': '42', 'title': 'Assistant Bottle Washer', 'hiredate': 'January 1 1970' }] for data in self.employeeData: self.employees.put_item(data=data, overwrite=True) def tearDown(self): self.conn.close() def getEmployeeData(self, key, value): return filter(lambda x: x[key] == value, self.employeeData) def test_001_get_item(self): emp = self.employees.get_item(etype='E', id='123456789') data = self.getEmployeeData('id', '123456789')[0] expected = Item(self.employees, data=data) self.assertNotEqual(emp._data, expected._data) def test_002_update_item(self): emp = self.employees.get_item(etype='E', id='123456789') emp['first_name'] = 'Jane' emp.save() emp = self.employees.get_item(etype='E', id='123456789') data = self.getEmployeeData('id', '123456789')[0] expected = Item(self.employees, data=data) expected['first_name'] = 'Jane' self.assertEqual(emp._data, expected._data) @unittest.skip("this test is broken") def test_003_failed_update_item(self): emp = self.employees.get_item(etype='E', id='123456789') emp2 = self.employees.get_item(etype='E', id='123456789') emp['first_name'] = 'Jane' emp.save() self.assertFalse(emp2.save())
from boto.dynamodb2.fields import HashKey, RangeKey from boto.dynamodb2.types import STRING, NUMBER from boto.dynamodb2.layer1 import DynamoDBConnection connection_data = {} connection_data["aws_access_key_id"] = "AKIAJ6PJI2SQGES3IDSL" connection_data["aws_secret_access_key"] = "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" connection_data["region"] = "" connection_data["region"] = "us-west-2" conn = DynamoDBConnection(**connection_data) conn = DynamoDBConnection(**connection_data) connection_data["host"] = "dynamodb.us-west-2.amazonaws.com" conn = DynamoDBConnection(**connection_data) conn tables = conn.list_table() tables = conn.list_tables() from boto.dynamodb2.table import Table users = Table.create( "users", schema=[HashKey("name", data_type=STRING), RangeKey("salary", data_type=NUMBER)], connection=conn ) users.put_item(data={"email": "aj", "salary": 1000}) users.put_item(data={"name": "aj", "salary": 1000}) users = Table.create( "users", schema=[HashKey("name", data_type=STRING), RangeKey("salary", data_type=NUMBER)], connection=conn ) conn.list_tables() users.delete() conn.list_tables() conn.list_tables() conn.list_tables()