def sendtodynamo_cnn(cnnjson): ''' Send json to DynamoDB Assumes that article timestamps have been deduped to avoid collisions ''' conn = connect_to_region('us-west-2', aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY) hashkey = "CNN" # primary key to be used for DynamoDB table try: table = Table('CNN', connection=conn) table.describe() except boto.exception.JSONResponseError: print "Creating table" table = Table.create('CNN', schema=[HashKey('source'), RangeKey('tstamp',data_type=NUMBER)], throughput={'read':25, 'write':25}, indexes=[GlobalAllIndex('showidx',parts=[HashKey('show')],throughput={'read':10,'write':5})]) iteration = 0 for article in cnnjson: # Iterate through list of articles and upload to table rangekey = float(article['timestamp']) rowdata = {'source':hashkey,'tstamp':rangekey, 'cnnShow':article['show']} for key in article.keys(): rowdata[key]=article[key] item = table.put_item(data = rowdata) iteration += 1 if iteration%100==0: print "Uploaded "+iteration+" articles" return None
def bind(self): """Bind to existing tables. """ if self.data_points_name and self.index_name: data_points_table = Table(self.data_points_name, connection=self.connection) try: s1 = data_points_table.describe()['Table']['TableStatus'] except: raise else: self.data_points_table = data_points_table self.dp_writer = TimedBatchTable( self.data_points_table.batch_write()) index_table = Table(self.index_name, connection=self.connection) try: s2 = index_table.describe()['Table']['TableStatus'] except: raise else: self.index_table = index_table if s1 == s2: self.item['state'] = s1 else: self.item['state'] = 'UNDEFINED' return self.state
def table_exists(name): t = Table(name) try: t.describe() return True except: return False
def table_exists(name): t = Table(name) try: t.describe() return True except: return False
def migrate(self, args): # lets get the origin table try: origin = Table(args['origin_table'], connection=self.connection) origin.describe() except Exception as e: return "table {0} could not be found in {1}".format( args['origin_table'], self.region) # now, create the destination_table (using create) destination = Table(args['destination_table'], connection=self.connection) print "creating table {0}".format(destination.table_name) if self.create(args) != 'CREATING': print " table {0} exists".format(destination.table_name) else: while destination.describe()['Table']['TableStatus'] != 'ACTIVE': print " ..." time.sleep(5) print " table {0} created".format(destination.table_name) print "copying items from {0} to {1}".format(origin.table_name, destination.table_name) for item in origin.scan(): # be sure to mold the fields into their proper shapes item = self._mold(item) destination.put_item(item, overwrite=True)
def _table_exists(table_name): table = Table(table_name) try: table.describe() except JSONResponseError as exc: if exc.error_code == 'ResourceNotFoundException': return False return True
def _table_exists(table_name): table = Table(table_name) try: table.describe() except JSONResponseError as exc: if exc.error_code == "ResourceNotFoundException": return False return True
def get_table(self, table_name): """ Fetch a table from DynamoDB. NOTE: this is a somewhat expensive operation, which must query dynamo for the current state of the table """ table = Table(table_name, connection=self.dynamo_cx) # must describe the table, or it doesn't have the correct throughput values table.describe() return table
def create_missing_indexes(m_region, r_region, instance): # Skip all DynamoDB metadata tables. master_tables = [ i for i in db_connection(region=m_region).list_tables()['TableNames'] if i.startswith('DynamoDB') is False ] # Main db diff loop, just global indexes. for m1 in master_tables: # Set the region before selecting the table to describe. DynamoDBConnection.DefaultRegionName = m_region t = Table(m1) t.describe() # Set replica region to get table information. DynamoDBConnection.DefaultRegionName = r_region try: s = Table(m1) s.describe() except boto.exception.JSONResponseError: print colored("!!! CREATE TABLE '{0}' table doesn't exist in '{1}' region. ".format(m1, r_region), "red") # raise SystemExit("Ensure table exists in replica region and re-run.", 1) if len(t.global_indexes) > 0: print colored('{0} has globlal indexes.'.format(m1), 'green') # Get destination table global index names. d_index_names = [ i.schema()['IndexName'] for i in s.global_indexes ] # Check if any global indexes exist in replica table prior to trying # to create them and skip them if they do exist. for index in t.global_indexes: if index.schema()['IndexName'] in d_index_names: print "Index '{0}' exists in replica, skipping.".format(index.schema()['IndexName']) continue else: print "\tCreating '{0}' index in replica table.".format(index.schema()['IndexName']) for retry_count in range(0, 6): try: if s.create_global_secondary_index(index) is True: print colored("Successfully created index '{0}' on '{1}' table.".format(index.schema()['IndexName'], s.table_name), "green") break else: print colored("Failed to create index '{0}' on '{1}' table.".format(index.schema()['IndexName'], s.table_name), "red") except boto.exception.JSONResponseError: print "Index creation is finishing up, sleeping 1 minute before proceeding." time.sleep(60) continue else: print colored("!!! Retry count exceeded! Manually create '{0}' index.".format(index.schema()['IndexName']), "red") else: print colored('{0} has no global indexes. Skipping table.'.format(m1), 'red') print "Index creation complete."
def copy(self, args): # lets get the origin table try: origin = Table(args['origin_table'], connection=self.connection) origin.describe() except Exception as e: return "table {0} could not be found in {1}".format(args['origin_table'], self.region) # now, get the destination_table destination = Table(args['destination_table'], connection=self.connection) print "copying items from {0} to {1}".format(origin.table_name, destination.table_name) for item in origin.scan(): destination.put_item(dict(item))
def main(): if len(sys.argv) == 2 and sys.argv[1] == 'check': print "*** Checking the table in dynamoDB, create one if not exist..." try: ddbc = DynamoDBConnection() src = ddbc.describe_table(iperf_table_name)['Table'] logs = Table(iperf_table_name, schema=[HashKey('path'),RangeKey('datetime'),]) logs.describe() sys.exit(0) except JSONResponseError: logs = Table.create(iperf_table_name, schema=[HashKey('path'),RangeKey('datetime'),]) while ddbc.describe_table(iperf_table_name)['Table']['TableStatus'] != 'ACTIVE': sleep(3) sys.exit(1) if len(sys.argv) != 4: print "usage: %s <iperf_client_name> <datetime> <iperf_server_name>" % sys.argv[0] sys.exit(2) # Store arg lists iperf_client_name = sys.argv[1] datetime = sys.argv[2] iperf_server_name = sys.argv[3] path = iperf_client_name + '-' + iperf_server_name # Retrieve dynamoDB object try: logs = Table(iperf_table_name, schema=[HashKey('path'),RangeKey('datetime'),]) tmp = logs.describe() except JSONResponseError: print "The table %s doesn't exist!" % iperf_table_name sys.exit(1) # Parse iperf log iperf = {} iperf['path'] = path iperf['datetime'] = datetime line = open(os.path.dirname(os.path.abspath(__file__))+'/log/'+datetime+'.log','r').readlines()[6] m = re.search(r"sec\s+(\d+\s+\w+)\s+(\d+\s+[\w/]+)", line) transfer = m.group(1) bandwidth = m.group(2) iperf['transfer'] = transfer iperf['bandwidth'] = bandwidth # Put the log to the dynamoDB table try: logs.put_item(data=iperf, overwrite=True) except ValidationException: pprint(iperf) except JSONResponseError: pass
def sendtodynamo_cnn(cnnjson): ''' Send json to DynamoDB Assumes that article timestamps have been deduped to avoid collisions ''' conn = connect_to_region('us-west-2', aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY) hashkey = "CNN" # primary key to be used for DynamoDB table try: table = Table('CNN', connection=conn) table.describe() except boto.exception.JSONResponseError: print "Creating table" table = Table.create( 'CNN', schema=[HashKey('source'), RangeKey('tstamp', data_type=NUMBER)], throughput={ 'read': 25, 'write': 25 }, indexes=[ GlobalAllIndex('showidx', parts=[HashKey('show')], throughput={ 'read': 10, 'write': 5 }) ]) iteration = 0 for article in cnnjson: # Iterate through list of articles and upload to table rangekey = float(article['timestamp']) rowdata = { 'source': hashkey, 'tstamp': rangekey, 'cnnShow': article['show'] } for key in article.keys(): rowdata[key] = article[key] item = table.put_item(data=rowdata) iteration += 1 if iteration % 100 == 0: print "Uploaded " + iteration + " articles" return None
def archive(self, args): # lets get the origin table try: origin = Table(args['origin_table'], connection=self.connection) origin.describe() except Exception as e: return "table {0} could not be found in {1}".format(args['origin_table'], self.region) for item in origin.scan(): item = dict(item) for key in item.keys(): if type(item[key]) is set: item[key] = list(item[key]) print json.dumps(item, use_decimal=True)
class TestCustomThroughput(unittest.TestCase): def setUp(self): # Configuration options self.table_name = 'conf' self.store_name = 'test' self.read_units = 10 self.write_units = 8 # Instanciate the store self.store = DynamoDBConfigStore(connection, self.table_name, self.store_name, read_units=self.read_units, write_units=self.write_units) # Get an Table instance for validation self.table = Table(self.table_name, connection=connection) def test_custom_throughput(self): """ Test that we can set custom thoughput for new tables """ throughput = self.table.describe()[u'Table'][u'ProvisionedThroughput'] self.assertEqual(throughput[u'ReadCapacityUnits'], self.read_units) self.assertEqual(throughput[u'WriteCapacityUnits'], self.write_units) def tearDown(self): """ Tear down the test case """ self.table.delete()
def populate_tables(connection, table_data): try: table = Table(table_data['table_name'], connection=connection) table.describe() except JSONResponseError, e: if e.error_code == 'ResourceNotFoundException': table = Table.create( table_name=table_data['table_name'], schema=DynamoDB_Schema, connection=connection ) else: raise e
def get_table(self, table_name, hash_key='id', range_key='timestamp', throughput={'read': 5, 'write': 15}): if table_name in self.conn.list_tables()['TableNames']: table = Table(table_name, connection=self.conn) table.describe() # Bug: https://github.com/boto/boto/issues/2826 return table schema = [HashKey(hash_key)] if range_key: schema.append(RangeKey(range_key)) table = Table.create( table_name, schema=schema, throughput=throughput, connection=self.conn) self.wait_until_table_is_active(table) return table
def test_describe_table(self): self.storage_mocker.StubOutWithMock(storage, 'describe_table') storage.describe_table(IgnoreArg(), 'test_table').AndReturn( models.TableMeta( models.TableSchema( { 'city1': models.ATTRIBUTE_TYPE_STRING, 'id': models.ATTRIBUTE_TYPE_STRING, 'name': models.ATTRIBUTE_TYPE_STRING }, ['id', 'name'], {'index_name': models.IndexDefinition('city1')}), models.TableMeta.TABLE_STATUS_ACTIVE)) self.storage_mocker.ReplayAll() table = Table('test_table', connection=self.DYNAMODB_CON) table_description = table.describe() self.storage_mocker.VerifyAll() self.assertEquals('test_table', table_description['Table']['TableName']) self.assertItemsEqual([{ "AttributeName": "city1", "AttributeType": "S" }, { "AttributeName": "id", "AttributeType": "S" }, { "AttributeName": "name", "AttributeType": "S" }], table_description['Table']['AttributeDefinitions'])
def archive(self, args): # lets get the origin table try: origin = Table(args['origin_table'], connection=self.connection) origin.describe() except Exception as e: return "table {0} could not be found in {1}".format( args['origin_table'], self.region) for item in origin.scan(): item = dict(item) for key in item.keys(): if type(item[key]) is set: item[key] = list(item[key]) print json.dumps(item, use_decimal=True)
class TestCustomThroughput(unittest.TestCase): def setUp(self): # Configuration options self.table_name = 'conf' self.store_name = 'test' self.read_units = 10 self.write_units = 8 # Instanciate the store self.store = DynamoDBConfigStore( connection, self.table_name, self.store_name, read_units=self.read_units, write_units=self.write_units) # Get an Table instance for validation self.table = Table(self.table_name, connection=connection) def test_custom_throughput(self): """ Test that we can set custom thoughput for new tables """ throughput = self.table.describe()[u'Table'][u'ProvisionedThroughput'] self.assertEqual(throughput[u'ReadCapacityUnits'], self.read_units) self.assertEqual(throughput[u'WriteCapacityUnits'], self.write_units) def tearDown(self): """ Tear down the test case """ self.table.delete()
def setUp(self): logging.getLogger('boto').setLevel(logging.CRITICAL) err = 'Use a real DynamoDB %s. Add datastore/dynamo/test_settings.py.' assert aws_access_key != '<aws access key>', err % 'access key.' assert aws_secret_key != '<aws secret key>', err % 'secret key.' self.conn = boto.dynamodb2.connect_to_region(aws_region, aws_access_key_id=aws_access_key,aws_secret_access_key=aws_secret_key) # Create an indexed table table = Table(self.INDEXED_TABLE, connection=self.conn) try: status = table.describe() except: table = Table.create(self.INDEXED_TABLE, schema=[ HashKey('department', data_type=STRING), RangeKey('name', data_type=STRING) ], indexes=[ AllIndex('ScoreIndex', parts=[ HashKey('department'), RangeKey('score', data_type=NUMBER) ]) ], global_indexes=[ GlobalAllIndex('GroupIndex', parts=[ HashKey('group'), RangeKey('age', data_type=NUMBER) ]) ], connection=self.conn) # make sure we're clean :) self._delete_keys_from_table(self.SIMPLE_TABLE) self._delete_keys_from_table(self.INDEXED_TABLE) self._delete_keys_from_table(self.RANGEKEY_TABLE)
def _table(self, name, schema, read, write): table = Table(name, connection=self._dynamo) try: table.describe() return table except Exception as e: if e.error_code != 'ResourceNotFoundException': raise e logger.debug('Creating table %s', name) table = Table.create(name, schema=schema, throughput={ 'read': read, 'write': write }, connection=self._dynamo) return table
def copy(self, args): # lets get the origin table try: origin = Table(args['origin_table'], connection=self.connection) origin.describe() except Exception as e: return "table {0} could not be found in {1}".format( args['origin_table'], self.region) # now, get the destination_table destination = Table(args['destination_table'], connection=self.connection) print "copying items from {0} to {1}".format(origin.table_name, destination.table_name) for item in origin.scan(): destination.put_item(dict(item))
def apply(message, dynamodb_connection, indexer_queue, statsd): """ Applies the specific change. New Table behaviour: If a DynamoDB Table doesn't already exist for the configured prefix, day and time resolution combination, then a Table will be created. New Item behaviour: If a DynamoDB Item doesn't already exist for the metric and the configured time period, then one will be created. Update Item behaviour: If a DynamoDB Item already exists for the metric and configured time period, then the new data points will be merged with the existing data as per the aggregation type specified. Aggregation Type behaviour: - sum: the values for each measurement will be added together - average: the values for each measurement will be added together with the data point already stored and divided by 2 - minimum: the minimum value will be stored for each measuremenet out of the new value and the existing value - maximum: the maximum value will be stored for each measuremenet out of the new value and the existing value """ metric = message.metric start_time = message.start_time resolution = message.resolution datapoints = message.datapoints aggregation_type = message.aggregation_type table_name = get_table_name(resolution, start_time) try: table = Table(table_name, connection=dynamodb_connection) statsd.incr('apply.dynamodb.table.describe') table.describe() except JSONResponseError, e: if e.error_code == 'ResourceNotFoundException': statsd.incr('apply.dynamodb.table.create') table = Table.create(table_name, schema=DynamoDB_Schema, connection=dynamodb_connection) else: raise e
def desc_table(name): # inputs must be strings try: table = Table(name, connection=client_dynamo) out = table.describe() print json.dumps(out, indent=4, sort_keys=True) return out except KeyboardInterrupt: exit
def get_table(self, table_name): """ Fetch a Table from DynamoDB. NOTE: this is a somewhat expensive operation, which must query dynamo for the current state of the table. :type table_name: str :param table_name: Name of Table to load :rtype: boto.dynamodb2.table.Table: :returns: Requested Table """ table = Table(table_name, connection=self.dynamo_cx) # must describe the table, or it doesn't have the correct throughput values table.describe() return table
def get_table(self, table_name): """ Fetch a Table from DynamoDB. NOTE: this is a somewhat expensive operation, which must query dynamo for the current state of the table. :type table_name: str :param table_name: Name of Table to load :rtype: boto.dynamodb2.table.Table: :returns: Requested Table """ table = Table(table_name, connection=self.dynamo_cx) # must describe the table, or it doesn't have the correct throughput values table.describe() return table
def describe(table_name, region=None, key=None, keyid=None, profile=None): ''' Describe a DynamoDB table. CLI example:: salt myminion boto_dynamodb.describe table_name region=us-east-1 ''' conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) table = Table(table_name, connection=conn) return table.describe()
def describe(table_name, region=None, key=None, keyid=None, profile=None): ''' Describe a DynamoDB table. CLI example:: salt myminion boto_dynamodb.describe table_name region=us-east-1 ''' conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) table = Table(table_name, connection=conn) return table.describe()
def createTable( tname, primary_key, secondary_key=None ): #check that table does not already exist table = Table(tname) try: table.describe() print "Table", tname, "already exists." return table except JSONResponseError as jse: if jse.error_code == 'ResourceNotFoundException': sc = [HashKey('src_id')] if secondary_key: sc.append(RangeKey(secondary_key)) net_data = Table.create(tname, schema=sc, throughput={'read': 5, 'write': 15, } ) return net_data else: #k, something else (not table not existing) happened #propagate raise(JSONResponseError(jse))
def create_or_update_dynamo_table(connection, module): table_name = module.params.get('name') hash_key_name = module.params.get('hash_key_name') hash_key_type = module.params.get('hash_key_type') range_key_name = module.params.get('range_key_name') range_key_type = module.params.get('range_key_type') read_capacity = module.params.get('read_capacity') write_capacity = module.params.get('write_capacity') all_indexes = module.params.get('indexes') for index in all_indexes: validate_index(index, module) schema = get_schema_param(hash_key_name, hash_key_type, range_key_name, range_key_type) throughput = { 'read': read_capacity, 'write': write_capacity } indexes, global_indexes = get_indexes(all_indexes) result = dict( region=module.params.get('region'), table_name=table_name, hash_key_name=hash_key_name, hash_key_type=hash_key_type, range_key_name=range_key_name, range_key_type=range_key_type, read_capacity=read_capacity, write_capacity=write_capacity, indexes=all_indexes, ) try: table = Table(table_name, connection=connection) if dynamo_table_exists(table): result['changed'] = update_dynamo_table(table, throughput=throughput, check_mode=module.check_mode, global_indexes=global_indexes) else: if not module.check_mode: Table.create(table_name, connection=connection, schema=schema, throughput=throughput, indexes=indexes, global_indexes=global_indexes) result['changed'] = True if not module.check_mode: result['table_status'] = table.describe()['Table']['TableStatus'] except BotoServerError: result['msg'] = 'Failed to create/update dynamo table due to error: ' + traceback.format_exc() module.fail_json(**result) else: module.exit_json(**result)
def test_describe_unexisting_table(self): self.storage_mocker.StubOutWithMock(storage, 'describe_table') storage.describe_table(IgnoreArg(), 'test_table1').AndRaise(TableNotExistsException) self.storage_mocker.ReplayAll() table = Table('test_table1', connection=self.DYNAMODB_CON) try: table.describe() except JSONResponseError as e: self.assertEqual( e.body['__type'], 'com.amazonaws.dynamodb.v20111205#ResourceNotFoundException') self.assertEqual( e.body['message'], 'The resource which is being requested does not exist.')
def create_or_update_dynamo_table(connection, module): table_name = module.params.get('name') hash_key_name = module.params.get('hash_key_name') hash_key_type = module.params.get('hash_key_type') range_key_name = module.params.get('range_key_name') range_key_type = module.params.get('range_key_type') read_capacity = module.params.get('read_capacity') write_capacity = module.params.get('write_capacity') if range_key_name: schema = [ HashKey(hash_key_name, DYNAMO_TYPE_MAP.get(hash_key_type)), RangeKey(range_key_name, DYNAMO_TYPE_MAP.get(range_key_type)) ] else: schema = [HashKey(hash_key_name, DYNAMO_TYPE_MAP.get(hash_key_type))] throughput = {'read': read_capacity, 'write': write_capacity} result = dict( region=module.params.get('region'), table_name=table_name, hash_key_name=hash_key_name, hash_key_type=hash_key_type, range_key_name=range_key_name, range_key_type=range_key_type, read_capacity=read_capacity, write_capacity=write_capacity, ) try: table = Table(table_name, connection=connection) if dynamo_table_exists(table): result['changed'] = update_dynamo_table( table, throughput=throughput, check_mode=module.check_mode) else: if not module.check_mode: Table.create(table_name, connection=connection, schema=schema, throughput=throughput) result['changed'] = True if not module.check_mode: result['table_status'] = table.describe()['Table']['TableStatus'] except BotoServerError: result[ 'msg'] = 'Failed to create/update dynamo table due to error: ' + traceback.format_exc( ) module.fail_json(**result) else: module.exit_json(**result)
def create_or_update_dynamo_table(connection, module): table_name = module.params.get('name') hash_key_name = module.params.get('hash_key_name') hash_key_type = module.params.get('hash_key_type') range_key_name = module.params.get('range_key_name') range_key_type = module.params.get('range_key_type') read_capacity = module.params.get('read_capacity') write_capacity = module.params.get('write_capacity') all_indexes = module.params.get('indexes') for index in all_indexes: validate_index(index, module) schema = get_schema_param(hash_key_name, hash_key_type, range_key_name, range_key_type) throughput = { 'read': read_capacity, 'write': write_capacity } indexes, global_indexes = get_indexes(all_indexes) result = dict( region=module.params.get('region'), table_name=table_name, hash_key_name=hash_key_name, hash_key_type=hash_key_type, range_key_name=range_key_name, range_key_type=range_key_type, read_capacity=read_capacity, write_capacity=write_capacity, indexes=all_indexes, ) try: table = Table(table_name, connection=connection) if dynamo_table_exists(table): result['changed'] = update_dynamo_table(table, throughput=throughput, check_mode=module.check_mode, global_indexes=global_indexes) else: if not module.check_mode: Table.create(table_name, connection=connection, schema=schema, throughput=throughput, indexes=indexes, global_indexes=global_indexes) result['changed'] = True if not module.check_mode: result['table_status'] = table.describe()['Table']['TableStatus'] except BotoServerError: result['msg'] = 'Failed to create/update dynamo table due to error: ' + traceback.format_exc() module.fail_json(**result) else: module.exit_json(**result)
def create_or_update_dynamo_table(connection, module): table_name = module.params.get('name') hash_key_name = module.params.get('hash_key_name') hash_key_type = module.params.get('hash_key_type') range_key_name = module.params.get('range_key_name') range_key_type = module.params.get('range_key_type') read_capacity = module.params.get('read_capacity') write_capacity = module.params.get('write_capacity') if range_key_name: schema = [ HashKey(hash_key_name, DYNAMO_TYPE_MAP.get(hash_key_type)), RangeKey(range_key_name, DYNAMO_TYPE_MAP.get(range_key_type)) ] else: schema = [ HashKey(hash_key_name, DYNAMO_TYPE_MAP.get(hash_key_type)) ] throughput = { 'read': read_capacity, 'write': write_capacity } result = dict( region=module.params.get('region'), table_name=table_name, hash_key_name=hash_key_name, hash_key_type=hash_key_type, range_key_name=range_key_name, range_key_type=range_key_type, read_capacity=read_capacity, write_capacity=write_capacity, ) try: table = Table(table_name, connection=connection) if dynamo_table_exists(table): result['changed'] = update_dynamo_table(table, throughput=throughput, check_mode=module.check_mode) else: if not module.check_mode: Table.create(table_name, connection=connection, schema=schema, throughput=throughput) result['changed'] = True if not module.check_mode: result['table_status'] = table.describe()['Table']['TableStatus'] except BotoServerError: result['msg'] = 'Failed to create/update dynamo table due to error: ' + traceback.format_exc() module.fail_json(**result) else: module.exit_json(**result)
def describe(table_name, region=None, key=None, keyid=None, profile=None): """ Describe a DynamoDB table. CLI Example: .. code-block:: bash salt myminion boto_dynamodb.describe table_name region=us-east-1 """ conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) table = Table(table_name, connection=conn) return table.describe()
def _create_table(self): ''' Create the underlying dynamodb table for writing locks to if it does not exist, otherwise uses the existing table. We use the `describe` method call to verify if the table exists or not. :returns: A handle to the underlying dynamodb table ''' try: table = Table(self.schema.table_name) _logger.debug("current table description:\n%s", table.describe()) except JSONResponseError, ex: _logger.exception("table %s does not exist, creating it", self.schema.table_name) table = Table.create( self.schema.table_name, schema=[HashKey(self.schema.name, data_type=STRING)], throughput={ 'read': self.schema.read_capacity, 'write': self.schema.write_capacity, }) _logger.debug("current table description:\n%s", table.describe())
def test_describe_unexisting_table(self): self.storage_mocker.StubOutWithMock(storage, 'describe_table') storage.describe_table( IgnoreArg(), 'test_table1' ).AndRaise(TableNotExistsException) self.storage_mocker.ReplayAll() table = Table('test_table1', connection=self.DYNAMODB_CON) try: table.describe() except JSONResponseError as e: self.assertEqual( e.body['__type'], 'com.amazonaws.dynamodb.v20111205#ResourceNotFoundException') self.assertEqual( e.body['message'], 'The resource which is being requested does not exist.')
def migrate(self, args): # lets get the origin table try: origin = Table(args['origin_table'], connection=self.connection) origin.describe() except Exception as e: return "table {0} could not be found in {1}".format(args['origin_table'], self.region) # now, create the destination_table (using create) destination = Table(args['destination_table'], connection=self.connection) print "creating table {0}".format(destination.table_name) if self.create(args) != 'CREATING': print " table {0} exists".format(destination.table_name) else: while destination.describe()['Table']['TableStatus'] != 'ACTIVE': print " ..." time.sleep(5) print " table {0} created".format(destination.table_name) print "copying items from {0} to {1}".format(origin.table_name, destination.table_name) for item in origin.scan(): # be sure to mold the fields into their proper shapes item = self._mold(item) destination.put_item(item, overwrite=True)
def make_table(ddb, name, **kwargs): table = Table(table_name=name, connection=ddb, **kwargs) while True: try: if table.describe()["Table"]["TableStatus"] == "ACTIVE": return table else: time.sleep(1) except JSONResponseError as exc: if exc.error_code == "ResourceNotFoundException": table = Table.create(table_name=name, connection=ddb, **kwargs) else: raise
def test_describe_table(self): self.storage_mocker.StubOutWithMock(storage, 'describe_table') storage.describe_table(IgnoreArg(), 'test_table').AndReturn( models.TableMeta( '00000000-0000-0000-0000-000000000000', models.TableSchema( { 'city1': models.AttributeType('S'), 'id': models.AttributeType('S'), 'name': models.AttributeType('S') }, ['id', 'name'], {'index_name': models.IndexDefinition('id', 'city1')} ), models.TableMeta.TABLE_STATUS_ACTIVE, None ) ) self.storage_mocker.ReplayAll() table = Table('test_table', connection=self.DYNAMODB_CON) table_description = table.describe() self.storage_mocker.VerifyAll() self.assertEqual('test_table', table_description['Table']['TableName']) self.assertItemsEqual( [ { "AttributeName": "city1", "AttributeType": "S" }, { "AttributeName": "id", "AttributeType": "S" }, { "AttributeName": "name", "AttributeType": "S" } ], table_description['Table']['AttributeDefinitions'])
def restore(self, args): # create the destination_table (using create) destination = Table(args['destination_table'], connection=self.connection) print "creating table {0}".format(destination.table_name) if self.create(args) != 'CREATING': print " table {0} exists".format(destination.table_name) else: while destination.describe()['Table']['TableStatus'] != 'ACTIVE': print " ..." time.sleep(5) print " table {0} created".format(destination.table_name) print "reading items from stdin to {0}".format(destination.table_name) for line in sys.stdin: # be sure to mold the fields into their proper shapes item = self._mold(json.loads(line)) destination.put_item(item, overwrite=True)
def restore(self, args): # create the destination_table (using create) destination = Table(args['destination_table'], connection=self.connection) print "creating table {0}".format(destination.table_name) if self.create(args) != 'CREATING': print " table {0} exists".format(destination.table_name) else: while destination.describe()['Table']['TableStatus'] != 'ACTIVE': print " ..." time.sleep(5) print " table {0} created".format(destination.table_name) print "reading items from stdin to {0}".format(destination.table_name) for line in sys.stdin: # be sure to mold the fields into their proper shapes item = self._mold(json.loads(line)) destination.put_item(item, overwrite=True)
def test_describe_table(self): self.storage_mocker.StubOutWithMock(storage, 'describe_table') storage.describe_table(IgnoreArg(), 'test_table').AndReturn( models.TableSchema( 'test_table', { 'city1': models.ATTRIBUTE_TYPE_STRING, 'id': models.ATTRIBUTE_TYPE_STRING, 'name': models.ATTRIBUTE_TYPE_STRING }, ['id', 'name'], {'index_name': models.IndexDefinition('city1')} ) ) self.storage_mocker.ReplayAll() table = Table('test_table', connection=self.DYNAMODB_CON) table_description = table.describe() self.storage_mocker.VerifyAll() self.assertEquals('test_table', table_description['Table']['TableName']) self.assertItemsEqual( [ { "AttributeName": "city1", "AttributeType": "S" }, { "AttributeName": "id", "AttributeType": "S" }, { "AttributeName": "name", "AttributeType": "S" } ], table_description['Table']['AttributeDefinitions'])
def getMessageTable(): conn = None if os.environ.get('DEVELOPER_MODE'): host, port = 'localhost', 8000 confirmDynamoDbLocalIsRunning(host, port) conn = DynamoDBConnection(host=host, port=port, aws_access_key_id='unit_test', aws_secret_access_key='unit_test', is_secure=False) else: conn = DynamoDBConnection() try: msg_table_desc = conn.describe_table(_message_table) msg_table = Table(_message_table, connection=conn) except JSONResponseError as e: # Only handle the ResourceNotFoundException here if e.error_code != 'ResourceNotFoundException': raise e msg_table = Table.create( _message_table, schema=[HashKey('date_string'), RangeKey('date')], throughput={ 'read': 5, 'write': 5 }, connection=conn) while not msg_table.describe()['Table']['TableStatus'] == "ACTIVE": from time import sleep sleep(1) return msg_table
def main(): # TODO: Need argument parsing unit_flag = False instance_name = sys.argv[1] trial = sys.argv[2] try: logs = Table(instance_name, schema=[HashKey('trial'),RangeKey('parallel'),]) tmp = logs.describe() except JSONResponseError: sys.exit() # Create units table if unit_flag: try: units_t = Table('unixbench_unit', schema=[HashKey('test_name'),]) tmp = units_t.describe() except JSONResponseError: units_t = Table.create('unixbench_unit', schema=[HashKey('test_name'),]) sleep(15) # Wait for the new db becomes available units = {} multi_flag = False result_flag = False test_index = 0 b = {} single = {} multi = {} for line in open(os.path.dirname(os.path.abspath(__file__))+'/log/'+instance_name+'_'+trial+'.log','r'): if "Benchmark Run" in line: result_flag = True elif not result_flag: continue if Tests[test_index] in line: m = re.search(r"\s+(\d+\.\d)\s(\w+)", line) if test_index+1 != len(Tests): if unit_flag: units[Tests[test_index]] = m.group(2) b[Tests[test_index]] = m.group(1) test_index += 1 else: m = re.search(r"\s+(\d+\.\d)", line) result_flag = False if not multi_flag: multi_flag = True single = copy.deepcopy(b) single[Tests[test_index]] = m.group(1) single['parallel'] = 'single' single['trial'] = trial test_index = 0 b.clear() else: multi = copy.deepcopy(b) multi[Tests[test_index]] = m.group(1) multi['trial'] = trial multi['parallel'] = 'multi' logs.put_item(data=single, overwrite=True) if len(multi) != 0: logs.put_item(data=multi, overwrite=True) # Upload units for tests if unit_flag: for t, u in units.iteritems(): print t, u units_t.put_item(data={ 'test_name': t, 'unit': u })
def create_or_update_dynamo_table(connection, module, boto3_dynamodb=None, boto3_sts=None, region=None): table_name = module.params.get('name') hash_key_name = module.params.get('hash_key_name') hash_key_type = module.params.get('hash_key_type') range_key_name = module.params.get('range_key_name') range_key_type = module.params.get('range_key_type') read_capacity = module.params.get('read_capacity') write_capacity = module.params.get('write_capacity') all_indexes = module.params.get('indexes') tags = module.params.get('tags') wait_for_active_timeout = module.params.get('wait_for_active_timeout') for index in all_indexes: validate_index(index, module) schema = get_schema_param(hash_key_name, hash_key_type, range_key_name, range_key_type) throughput = { 'read': read_capacity, 'write': write_capacity } indexes, global_indexes = get_indexes(all_indexes) result = dict( region=region, table_name=table_name, hash_key_name=hash_key_name, hash_key_type=hash_key_type, range_key_name=range_key_name, range_key_type=range_key_type, read_capacity=read_capacity, write_capacity=write_capacity, indexes=all_indexes, ) try: table = Table(table_name, connection=connection) if dynamo_table_exists(table): result['changed'] = update_dynamo_table(table, throughput=throughput, check_mode=module.check_mode, global_indexes=global_indexes) else: if not module.check_mode: Table.create(table_name, connection=connection, schema=schema, throughput=throughput, indexes=indexes, global_indexes=global_indexes) result['changed'] = True if not module.check_mode: result['table_status'] = table.describe()['Table']['TableStatus'] if tags: # only tables which are active can be tagged wait_until_table_active(module, table, wait_for_active_timeout) account_id = get_account_id(boto3_sts) boto3_dynamodb.tag_resource( ResourceArn='arn:aws:dynamodb:' + region + ':' + account_id + ':table/' + table_name, Tags=ansible_dict_to_boto3_tag_list(tags)) result['tags'] = tags except BotoServerError: result['msg'] = 'Failed to create/update dynamo table due to error: ' + traceback.format_exc() module.fail_json(**result) else: module.exit_json(**result)
def main(): n_arg = len(sys.argv) if 1 < n_arg: if sys.argv[1] == 'unixbench': u_data_model = 'unixbench/unixbench_ec2_userscript_model.dat' elif sys.argv[1] == 'x264': u_data_model = 'x264/x264_userscript_model.dat' else: print "usage: %s [unixbench|x264]" % sys.argv[0] sys.exit(0) # Block device storage size if n_arg == 3: dev_sda1 = BlockDeviceType() dev_sda1.size = int(sys.argv[2]) bdm = BlockDeviceMapping() bdm['/dev/sda1'] = dev_sda1 else: bdm = None # Lists of instance types to be benchmarked and already completed instances = [] completed = [] try: instances_dict = json.load(open("web/data/instances.json", "r")) except IOError: print "*** web/data/instances.json not found! Try ./update_instances.py ***" sys.exit(1) for k, v in instances_dict.iteritems(): if v['cloud'] == 'EC2': if u_data_model == 'unixbench/unixbench_userscript_model.dat': try: instance_logs = Table(k) instance_logs.describe() completed.append(k) except JSONResponseError: instances.append(k) elif u_data_model == 'x264/x264_userscript_model.dat': instances.append(k) else: print 'Nothing to do' sys.exit(0) # Start all the benchmark at once will most likely exceeds the quota limit per user # Better to execute the benchmark on a category to category basis conn = boto.ec2.connect_to_region(region) #instances = [] #completed = ['t1.micro_paravirtual'] num_instances = len(instances) while 0 < len(instances): for i in instances: print '%s is waiting for launch' % i for i in completed: if i in instances: instances.remove(i) for i in instances: # Generate an user-script per instance userscript = '' if u_data_model == 'unixbench/unixbench_userscript_model.dat': userscript = "#!/bin/sh\nTRIAL=%d\nINSTANCE_NAME=%s\n"%(trial,i) + open(u_data_model,'r').read() elif u_data_model == 'x264/x264_userscript_model.dat': userscript = "#!/bin/sh\nTRIAL=%d\necho %s > /var/local/instance_name\n"%(trial,i) + open(u_data_model,'r').read() u_data = base64.b64encode(userscript) res = start_benchmark_instance(conn, i, u_data, bdm) if res is not None and not res in completed: completed.append(res) sleep(5) if len(completed) == num_instances: break else: print '*** Cooling down...' # 30 mins interval sleep(60*30)
def test_query_after_describe_with_gsi(self): # Create a table to using gsi to reproduce the error mentioned on issue # https://github.com/boto/boto/issues/2828 users = Table.create('more_gsi_query_users', schema=[HashKey('user_id')], throughput={ 'read': 5, 'write': 5 }, global_indexes=[ GlobalAllIndex('EmailGSIIndex', parts=[HashKey('email')], throughput={ 'read': 1, 'write': 1 }) ]) # Add this function to be called after tearDown() self.addCleanup(users.delete) # Wait for it. time.sleep(60) # populate a couple of items in it users.put_item( data={ 'user_id': '7', 'username': '******', 'first_name': 'John', 'last_name': 'Doe', 'email': '*****@*****.**', }) users.put_item( data={ 'user_id': '24', 'username': '******', 'first_name': 'Alice', 'last_name': 'Expert', 'email': '*****@*****.**', }) users.put_item( data={ 'user_id': '35', 'username': '******', 'first_name': 'Jane', 'last_name': 'Doe', 'email': '*****@*****.**', }) # Try the GSI. it should work. rs = users.query_2(email__eq='*****@*****.**', index='EmailGSIIndex') for rs_item in rs: self.assertEqual(rs_item['username'], ['johndoe']) # The issue arises when we're introspecting the table and try to # query_2 after call describe method users_hit_api = Table('more_gsi_query_users') users_hit_api.describe() # Try the GSI. This is what os going wrong on #2828 issue. It should # work fine now. rs = users_hit_api.query_2(email__eq='*****@*****.**', index='EmailGSIIndex') for rs_item in rs: self.assertEqual(rs_item['username'], ['johndoe'])
#!/usr/bin/python # -*- coding: utf-8 -*- from time import sleep import sys try: from boto.dynamodb2.fields import HashKey, RangeKey from boto.dynamodb2.table import Table from boto.exception import JSONResponseError except ImportError: sys.exit(0) instance_name = open('/var/local/instance_name').read().strip() # Check if the table exists or not, create one if not print "*** Connecting to DynamoDB '%s' table..." % instance_name table_struct = None try: logs = Table(instance_name, schema=[HashKey('trial'),RangeKey('parallel'),]) table_struct = logs.describe() except JSONResponseError: logs = Table.create(instance_name, schema=[HashKey('trial'),RangeKey('parallel'),]) sys.stdout.write("*** DynamoDB is creating a new table...") while table_struct is None: try: logs = Table(instance_name, schema=[HashKey('trial'),RangeKey('parallel'),]) table_struct = logs.describe() print "\tDone" except JSONResponseError: sleep(5)
def main(): n_arg = len(sys.argv) if 1 < n_arg: option = sys.argv[1] if option == 'unixbench': u_data_model = 'unixbench/unixbench_ec2_userscript_model.dat' trial = int(sys.argv[2]) elif option == 'x264': u_data_model = 'x264/x264_userscript_model.dat' trial = int(sys.argv[2]) elif option == 'iperf': u_data_model = 'iperf/iperf_userscript_model.dat' minute = int(sys.argv[2]) else: print "unrecognized option: %s" % option print "usage: %s [unixbench|x264] [trial]" % sys.argv[0] print "usage: %s [iperf] [iperf-server] [minute]" % sys.argv[0] sys.exit(1) else: print "usage: %s [unixbench|x264] [trial]" % sys.argv[0] print "usage: %s [iperf] [iperf-server] [minute]" % sys.argv[0] sys.exit(1) ''' # Block device storage size if n_arg == 3: dev_sda1 = BlockDeviceType() dev_sda1.size = int(sys.argv[2]) bdm = BlockDeviceMapping() bdm['/dev/sda1'] = dev_sda1 else: bdm = None ''' bdm = None # Lists of instance types to be benchmarked and already completed instances = [] completed = [] try: instances_dict = json.load(open("web/data/instances.json", "r")) except IOError: print "*** web/data/instances.json not found! Try ./update_instances.py ***" sys.exit(1) # For EC2 unixbench, ignore the instance with results already created for k, v in instances_dict.iteritems(): if v['cloud'] == 'EC2': if option == 'unixbench': try: instance_logs = Table(k) instance_logs.describe() completed.append(k) except JSONResponseError: instances.append(k) elif option == 'x264' or option == 'iperf': instances.append(k) else: print 'Nothing to do' sys.exit(0) # Start all the benchmark at once will most likely exceeds the quota limit per user # Better to execute the benchmark on a category to category basis conn = boto.ec2.connect_to_region(region) # For manual list of instances, modify here instances = ['m3.medium_hvm','m3.medium_paravirtual','m3.large_hvm','m3.large_paravirtual'] #completed = ['t1.micro_paravirtual'] num_instances = len(instances) if option == 'iperf': wait_until_next(0, minute) while True: runnings = [] iperf_servers = 'declare -a arr=(' # Start Iperf servers here for sid, sregion in iperfs.iteritems(): co = boto.ec2.connect_to_region(sregion) co.start_instances(sid) print '+Stating instance %s...'%sid while co.get_only_instances(sid)[0].state_code != 16: # 16 (running) sleep(3) iperf_servers += "\"" + co.get_only_instances(sid)[0].public_dns_name + "\" " iperf_servers += ")\n" for i in instances: userscript = "#!/bin/sh\nINSTANCE_NAME=%s\n"%(i) + iperf_servers + open(u_data_model,'r').read() u_data = base64.b64encode(userscript) res, i_id = start_benchmark_instance(conn, i, u_data, bdm) if res is not None: runnings.append(i_id) # Sleep 6 mins to give interval between instances sleep(60*6) print '*** Waiting for the instances to complete iperf...' while conn.get_only_instances(i_id)[0].state_code != 80: # 80 (stopped) sleep(30) conn.terminate_instances(runnings) for i in runnings: print "-Instances %s terminated" % i # Stop Iperf servers here for sid, sregion in iperfs.iteritems(): co = boto.ec2.connect_to_region(sregion) co.stop_instances(sid) print '-Stopping instance %s...'%sid while co.get_only_instances(sid)[0].state_code != 80: # 80 (stopped) sleep(3) # Wait until next hour wait_until_next(1, minute) else: while 0 < len(instances): for i in instances: print '%s is waiting for launch' % i for i in completed: if i in instances: instances.remove(i) for i in instances: # Generate an user-script per instance userscript = '' if option == 'unixbench': userscript = "#!/bin/sh\nTRIAL=%d\nINSTANCE_NAME=%s\n"%(trial,i) + open(u_data_model,'r').read() elif option == 'x264': userscript = "#!/bin/sh\nTRIAL=%d\necho %s > /var/local/instance_name\n"%(trial,i) + open(u_data_model,'r').read() u_data = base64.b64encode(userscript) res, i_id = start_benchmark_instance(conn, i, u_data, bdm) if res is not None and not res in completed: completed.append(res) sleep(5) if len(completed) == num_instances: break else: print '*** Cooling down...' # 30 mins interval sleep(60*30)
def test_integration(self): # Test creating a full table with all options specified. users = Table.create('users', schema=[ HashKey('username'), RangeKey('friend_count', data_type=NUMBER) ], throughput={ 'read': 5, 'write': 5, }, indexes=[ KeysOnlyIndex('LastNameIndex', parts=[ HashKey('username'), RangeKey('last_name') ]), ]) self.addCleanup(users.delete) self.assertEqual(len(users.schema), 2) self.assertEqual(users.throughput['read'], 5) # Wait for it. time.sleep(60) # Make sure things line up if we're introspecting the table. users_hit_api = Table('users') users_hit_api.describe() self.assertEqual(len(users.schema), len(users_hit_api.schema)) self.assertEqual(users.throughput, users_hit_api.throughput) self.assertEqual(len(users.indexes), len(users_hit_api.indexes)) # Test putting some items individually. users.put_item(data={ 'username': '******', 'first_name': 'John', 'last_name': 'Doe', 'friend_count': 4 }) users.put_item(data={ 'username': '******', 'first_name': 'Alice', 'last_name': 'Expert', 'friend_count': 2 }) time.sleep(5) # Test batch writing. with users.batch_write() as batch: batch.put_item({ 'username': '******', 'first_name': 'Jane', 'last_name': 'Doe', 'friend_count': 3 }) batch.delete_item(username='******', friend_count=2) batch.put_item({ 'username': '******', 'first_name': 'Bob', 'last_name': 'Smith', 'friend_count': 1 }) time.sleep(5) # Test getting an item & updating it. # This is the "safe" variant (only write if there have been no # changes). jane = users.get_item(username='******', friend_count=3) self.assertEqual(jane['first_name'], 'Jane') jane['last_name'] = 'Doh' self.assertTrue(jane.save()) # Test strongly consistent getting of an item. # Additionally, test the overwrite behavior. client_1_jane = users.get_item( username='******', friend_count=3, consistent=True ) self.assertEqual(jane['first_name'], 'Jane') client_2_jane = users.get_item( username='******', friend_count=3, consistent=True ) self.assertEqual(jane['first_name'], 'Jane') # Write & assert the ``first_name`` is gone, then... del client_1_jane['first_name'] self.assertTrue(client_1_jane.save()) check_name = users.get_item( username='******', friend_count=3, consistent=True ) self.assertEqual(check_name['first_name'], None) # ...overwrite the data with what's in memory. client_2_jane['first_name'] = 'Joan' # Now a write that fails due to default expectations... self.assertRaises(exceptions.JSONResponseError, client_2_jane.save) # ... so we force an overwrite. self.assertTrue(client_2_jane.save(overwrite=True)) check_name_again = users.get_item( username='******', friend_count=3, consistent=True ) self.assertEqual(check_name_again['first_name'], 'Joan') # Reset it. jane['username'] = '******' jane['first_name'] = 'Jane' jane['last_name'] = 'Doe' jane['friend_count'] = 3 self.assertTrue(jane.save(overwrite=True)) # Test the partial update behavior. client_3_jane = users.get_item( username='******', friend_count=3, consistent=True ) client_4_jane = users.get_item( username='******', friend_count=3, consistent=True ) client_3_jane['favorite_band'] = 'Feed Me' # No ``overwrite`` needed due to new data. self.assertTrue(client_3_jane.save()) # Expectations are only checked on the ``first_name``, so what wouldn't # have succeeded by default does succeed here. client_4_jane['first_name'] = 'Jacqueline' self.assertTrue(client_4_jane.partial_save()) partial_jane = users.get_item( username='******', friend_count=3, consistent=True ) self.assertEqual(partial_jane['favorite_band'], 'Feed Me') self.assertEqual(partial_jane['first_name'], 'Jacqueline') # Reset it. jane['username'] = '******' jane['first_name'] = 'Jane' jane['last_name'] = 'Doe' jane['friend_count'] = 3 self.assertTrue(jane.save(overwrite=True)) # Ensure that partial saves of a brand-new object work. sadie = Item(users, data={ 'username': '******', 'first_name': 'Sadie', 'favorite_band': 'Zedd', 'friend_count': 7 }) self.assertTrue(sadie.partial_save()) serverside_sadie = users.get_item( username='******', friend_count=7, consistent=True ) self.assertEqual(serverside_sadie['first_name'], 'Sadie') # Test the eventually consistent query. results = users.query( username__eq='johndoe', last_name__eq='Doe', index='LastNameIndex', attributes=('username',), reverse=True ) for res in results: self.assertTrue(res['username'] in ['johndoe',]) self.assertEqual(res.keys(), ['username']) # Test the strongly consistent query. c_results = users.query( username__eq='johndoe', last_name__eq='Doe', index='LastNameIndex', reverse=True, consistent=True ) for res in c_results: self.assertTrue(res['username'] in ['johndoe',]) # Test scans without filters. all_users = users.scan(limit=7) self.assertEqual(all_users.next()['username'], 'bob') self.assertEqual(all_users.next()['username'], 'jane') self.assertEqual(all_users.next()['username'], 'johndoe') # Test scans with a filter. filtered_users = users.scan(limit=2, username__beginswith='j') self.assertEqual(filtered_users.next()['username'], 'jane') self.assertEqual(filtered_users.next()['username'], 'johndoe') # Test deleting a single item. johndoe = users.get_item(username='******', friend_count=4) johndoe.delete() # Test the eventually consistent batch get. results = users.batch_get(keys=[ {'username': '******', 'friend_count': 1}, {'username': '******', 'friend_count': 3} ]) batch_users = [] for res in results: batch_users.append(res) self.assertTrue(res['first_name'] in ['Bob', 'Jane']) self.assertEqual(len(batch_users), 2) # Test the strongly consistent batch get. c_results = users.batch_get(keys=[ {'username': '******', 'friend_count': 1}, {'username': '******', 'friend_count': 3} ], consistent=True) c_batch_users = [] for res in c_results: c_batch_users.append(res) self.assertTrue(res['first_name'] in ['Bob', 'Jane']) self.assertEqual(len(c_batch_users), 2) # Test count, but in a weak fashion. Because lag time. self.assertTrue(users.count() > -1) # Test query count count = users.query_count( username__eq='bob', ) self.assertEqual(count, 1) # Test without LSIs (describe calls shouldn't fail). admins = Table.create('admins', schema=[ HashKey('username') ]) self.addCleanup(admins.delete) time.sleep(60) admins.describe() self.assertEqual(admins.throughput['read'], 5) self.assertEqual(admins.indexes, []) # A single query term should fail on a table with *ONLY* a HashKey. self.assertRaises( exceptions.QueryError, admins.query, username__eq='johndoe' ) # But it shouldn't break on more complex tables. res = users.query(username__eq='johndoe') # Test putting with/without sets. mau5_created = users.put_item(data={ 'username': '******', 'first_name': 'dead', 'last_name': 'mau5', 'friend_count': 2, 'friends': set(['skrill', 'penny']), }) self.assertTrue(mau5_created) penny_created = users.put_item(data={ 'username': '******', 'first_name': 'Penny', 'friend_count': 0, 'friends': set([]), }) self.assertTrue(penny_created)
return '%s_%s' % (TABLE_SPACE, table) def table_names(tables): """Given a list of table base names, return a dictionary mapping from base name to full name. @param tables: a list of base table names. @return: a dictionary mapping from base bame to full name. """ return dict(map(lambda name: (name, table_name(name)), tables)) config_table = Table(table_name('config'), connection=connection) desc = None try: desc = config_table.describe() except: config_table = Table.create(table_name('config'), schema=[HashKey('name')], throughput={'read': 1, 'write': 1}, connection=connection) desc = config_table.describe() while MAX_WAIT and desc['Table']['TableStatus'] != 'ACTIVE': MAX_WAIT -= 1 time.sleep(1) desc = config_table.describe() if desc['Table']['TableStatus'] != 'ACTIVE': print 'error accessing', table_name('config'), 'table in region', REGION sys.exit(1)
if len(sys.argv) != 3: print("Usage: %s <SOURCE_TABLE> <DESTNATION_TABLE>" % sys.argv[0]) sys.exit(1) src_name = sys.argv[1] dst_name = sys.argv[2] region = os.getenv("AWS_DEFAULT_REGION", "ap-northeast-2") DynamoDBConnection.DefaultRegionName = region ddbc = DynamoDBConnection() # source table try: src_logs = Table(src_name, connection=ddbc) src_logs.describe() except JSONResponseError: print("Table [%s] does not exist." % src_name) sys.exit(1) print("# Read from [%s]." % src_name) src_table = ddbc.describe_table(src_name)["Table"] hash_key = "" range_key = "" for schema in src_table["KeySchema"]: attr_name = schema["AttributeName"] key_type = schema["KeyType"] if key_type == "HASH": hash_key = attr_name elif key_type == "RANGE":
def test_integration(self): # Test creating a full table with all options specified. users = Table.create('users', schema=[ HashKey('username'), RangeKey('friend_count', data_type=NUMBER) ], throughput={ 'read': 5, 'write': 5, }, indexes={ KeysOnlyIndex('LastNameIndex', parts=[ HashKey('username'), RangeKey('last_name') ]), }) self.addCleanup(users.delete) self.assertEqual(len(users.schema), 2) self.assertEqual(users.throughput['read'], 5) # Wait for it. time.sleep(60) # Make sure things line up if we're introspecting the table. users_hit_api = Table('users') users_hit_api.describe() self.assertEqual(len(users.schema), len(users_hit_api.schema)) self.assertEqual(users.throughput, users_hit_api.throughput) self.assertEqual(len(users.indexes), len(users_hit_api.indexes)) # Test putting some items individually. users.put_item(data={ 'username': '******', 'first_name': 'John', 'last_name': 'Doe', 'friend_count': 4 }) users.put_item(data={ 'username': '******', 'first_name': 'Alice', 'last_name': 'Expert', 'friend_count': 2 }) time.sleep(5) # Test batch writing. with users.batch_write() as batch: batch.put_item({ 'username': '******', 'first_name': 'Jane', 'last_name': 'Doe', 'friend_count': 3 }) batch.delete_item(username='******', friend_count=2) batch.put_item({ 'username': '******', 'first_name': 'Bob', 'last_name': 'Smith', 'friend_count': 1 }) time.sleep(5) # Test getting an item & updating it. # This is the "safe" variant (only write if there have been no # changes). jane = users.get_item(username='******', friend_count=3) self.assertEqual(jane['first_name'], 'Jane') jane['last_name'] = 'Doh' self.assertTrue(jane.save()) # Test strongly consistent getting of an item. # Additionally, test the overwrite behavior. client_1_jane = users.get_item( username='******', friend_count=3, consistent=True ) self.assertEqual(jane['first_name'], 'Jane') client_2_jane = users.get_item( username='******', friend_count=3, consistent=True ) self.assertEqual(jane['first_name'], 'Jane') # Write & assert the ``first_name`` is gone, then... del client_1_jane['first_name'] self.assertTrue(client_1_jane.save()) check_name = users.get_item( username='******', friend_count=3, consistent=True ) self.assertEqual(check_name['first_name'], None) # ...overwrite the data with what's in memory. client_2_jane['first_name'] = 'Joan' # Now a write that fails due to default expectations... self.assertRaises(exceptions.JSONResponseError, client_2_jane.save) # ... so we force an overwrite. self.assertTrue(client_2_jane.save(overwrite=True)) check_name_again = users.get_item( username='******', friend_count=3, consistent=True ) self.assertEqual(check_name_again['first_name'], 'Joan') # Reset it. jane['username'] = '******' jane['first_name'] = 'Jane' jane['last_name'] = 'Doe' jane['friend_count'] = 3 self.assertTrue(jane.save(overwrite=True)) # Test the partial update behavior. client_3_jane = users.get_item( username='******', friend_count=3, consistent=True ) client_4_jane = users.get_item( username='******', friend_count=3, consistent=True ) client_3_jane['favorite_band'] = 'Feed Me' # No ``overwrite`` needed due to new data. self.assertTrue(client_3_jane.save()) # Expectations are only checked on the ``first_name``, so what wouldn't # have succeeded by default does succeed here. client_4_jane['first_name'] = 'Jacqueline' self.assertTrue(client_4_jane.partial_save()) partial_jane = users.get_item( username='******', friend_count=3, consistent=True ) self.assertEqual(partial_jane['favorite_band'], 'Feed Me') self.assertEqual(partial_jane['first_name'], 'Jacqueline') # Reset it. jane['username'] = '******' jane['first_name'] = 'Jane' jane['last_name'] = 'Doe' jane['friend_count'] = 3 self.assertTrue(jane.save(overwrite=True)) # Ensure that partial saves of a brand-new object work. sadie = Item(users, data={ 'username': '******', 'first_name': 'Sadie', 'favorite_band': 'Zedd', 'friend_count': 7 }) self.assertTrue(sadie.partial_save()) serverside_sadie = users.get_item( username='******', friend_count=7, consistent=True ) self.assertEqual(serverside_sadie['first_name'], 'Sadie') # Test the eventually consistent query. results = users.query( username__eq='johndoe', last_name__eq='Doe', index='LastNameIndex', attributes=('username',), reverse=True ) for res in results: self.assertTrue(res['username'] in ['johndoe',]) self.assertEqual(res.keys(), ['username']) # Test the strongly consistent query. c_results = users.query( username__eq='johndoe', last_name__eq='Doe', index='LastNameIndex', reverse=True, consistent=True ) for res in c_results: self.assertTrue(res['username'] in ['johndoe',]) # Test scans without filters. all_users = users.scan(limit=7) self.assertEqual(all_users.next()['username'], 'bob') self.assertEqual(all_users.next()['username'], 'jane') self.assertEqual(all_users.next()['username'], 'johndoe') # Test scans with a filter. filtered_users = users.scan(limit=2, username__beginswith='j') self.assertEqual(filtered_users.next()['username'], 'jane') self.assertEqual(filtered_users.next()['username'], 'johndoe') # Test deleting a single item. johndoe = users.get_item(username='******', friend_count=4) johndoe.delete() # Test the eventually consistent batch get. results = users.batch_get(keys=[ {'username': '******', 'friend_count': 1}, {'username': '******', 'friend_count': 3} ]) batch_users = [] for res in results: batch_users.append(res) self.assertTrue(res['first_name'] in ['Bob', 'Jane']) self.assertEqual(len(batch_users), 2) # Test the strongly consistent batch get. c_results = users.batch_get(keys=[ {'username': '******', 'friend_count': 1}, {'username': '******', 'friend_count': 3} ], consistent=True) c_batch_users = [] for res in c_results: c_batch_users.append(res) self.assertTrue(res['first_name'] in ['Bob', 'Jane']) self.assertEqual(len(c_batch_users), 2) # Test count, but in a weak fashion. Because lag time. self.assertTrue(users.count() > -1) # Test query count count = users.query_count( username__eq='bob', ) self.assertEqual(count, 1) # Test without LSIs (describe calls shouldn't fail). admins = Table.create('admins', schema=[ HashKey('username') ]) self.addCleanup(admins.delete) time.sleep(60) admins.describe() self.assertEqual(admins.throughput['read'], 5) self.assertEqual(admins.indexes, []) # A single query term should fail on a table with *ONLY* a HashKey. self.assertRaises( exceptions.QueryError, admins.query, username__eq='johndoe' ) # But it shouldn't break on more complex tables. res = users.query(username__eq='johndoe') # Test putting with/without sets. mau5_created = users.put_item(data={ 'username': '******', 'first_name': 'dead', 'last_name': 'mau5', 'friend_count': 2, 'friends': set(['skrill', 'penny']), }) self.assertTrue(mau5_created) penny_created = users.put_item(data={ 'username': '******', 'first_name': 'Penny', 'friend_count': 0, 'friends': set([]), }) self.assertTrue(penny_created)
src_table = sys.argv[1] dst_table = sys.argv[2] create_dst_table = False if sys.argv[3] in ['False', 'false'] else True if len( sys.argv) == 4 else True region = os.getenv('AWS_DEFAULT_REGION', 'us-west-2') # host = 'dynamodb.%s.amazonaws.com' % region # ddbc = DynamoDBConnection(is_secure=False, region=region, host=host) DynamoDBConnection.DefaultRegionName = region ddbc = DynamoDBConnection() # 1. Read and copy the target table to be copied table_struct = None try: logs = Table(src_table, connection=ddbc) table_struct = logs.describe() except JSONResponseError: print "Table %s does not exist" % src_table sys.exit(1) print '*** Reading key schema from %s table' % src_table src = ddbc.describe_table(src_table)['Table'] hash_key = '' range_key = '' for schema in src['KeySchema']: attr_name = schema['AttributeName'] key_type = schema['KeyType'] if key_type == 'HASH': hash_key = attr_name elif key_type == 'RANGE': range_key = attr_name
def check(self, args): self._validate() try: table = Table(args['origin_table'], connection=self.connection) table.describe() except Exception as e: return "table {0} could not be found in {1}".format( args['origin_table'], self.region) # first check hash- and rangekey (optional) hk_type = self.table_spec['schema']['hashkey']['type'] hk_type = eval(Meister.types.format(hk_type)) hk_name = self.table_spec['schema']['hashkey']['name'] if table.schema[0].name != hk_name or table.schema[ 0].data_type != hk_type: return "{0}'s hashkey should be {1}, and type should be {2}".format( table.table_name, self.table_spec['schema']['hashkey']['name'], self.table_spec['schema']['hashkey']['type']) # beware, rangekey is optional if 'rangekey' in self.table_spec['schema']: rk_type = self.table_spec['schema']['rangekey']['type'] rk_type = eval(Meister.types.format(rk_type)) rk_name = self.table_spec['schema']['rangekey']['name'] if len(table.schema) < 2 or \ table.schema[1].name != rk_name or \ table.schema[1].data_type != rk_type: return "{0}'s rangekey should be {1}, of type {2}".format( table.table_name, self.table_spec['schema']['rangekey']['name'], self.table_spec['schema']['rangekey']['type']) elif len(table.schema) >= 2: return "{0} has a rangekey, specification not".format( table.table_name) # indexes indexes = {} for index in table.indexes: indexes[index.name] = index if 'indexes' in self.table_spec: for index in self.table_spec['indexes']: if index['name'] in indexes: i_type = eval( Meister.types.format(index['attribute']['type'])) i_name = index['attribute']['name'] if i_name != indexes[index['name']].parts[1].name or \ i_type != indexes[index['name']].parts[1].data_type: return "index {0} in table {1} has type or name mismatch".format( index['name'], table.table_name) del (indexes[index['name']]) else: return "index {0} in table {1} not found in specification".format( index['name'], table.table_name) if len(indexes) > 0: return "table {0} has indexes not found in specification".format( table.table_name) # and last, indexes global_indexes = {} for index in table.global_indexes: global_indexes[index.name] = index if 'global_indexes' in self.table_spec: for index in self.table_spec['global_indexes']: if index['name'] in global_indexes: i_type = eval( Meister.types.format(index['attribute']['type'])) i_name = index['attribute']['name'] if i_name != global_indexes[index['name']].parts[1].name or \ i_type != global_indexes[index['name']].parts[1].data_type: return "index {0} in table {1} has type or name mismatch".format( index['name'], table.table_name) del (global_indexes[index['name']]) else: return "index {0} in table {1} not found in specification".format( index['name'], table.table_name) if len(global_indexes) > 0: return "table {0} has indexes not found in specification".format( table.table_name) return "no mismatch or errors found"
def create_or_update_dynamo_table(connection, module, boto3_dynamodb=None, boto3_sts=None): table_name = module.params.get('name') hash_key_name = module.params.get('hash_key_name') hash_key_type = module.params.get('hash_key_type') range_key_name = module.params.get('range_key_name') range_key_type = module.params.get('range_key_type') read_capacity = module.params.get('read_capacity') write_capacity = module.params.get('write_capacity') all_indexes = module.params.get('indexes') region = module.params.get('region') tags = module.params.get('tags') wait_for_active_timeout = module.params.get('wait_for_active_timeout') for index in all_indexes: validate_index(index, module) schema = get_schema_param(hash_key_name, hash_key_type, range_key_name, range_key_type) throughput = {'read': read_capacity, 'write': write_capacity} indexes, global_indexes = get_indexes(all_indexes) result = dict( region=region, table_name=table_name, hash_key_name=hash_key_name, hash_key_type=hash_key_type, range_key_name=range_key_name, range_key_type=range_key_type, read_capacity=read_capacity, write_capacity=write_capacity, indexes=all_indexes, ) try: table = Table(table_name, connection=connection) if dynamo_table_exists(table): result['changed'] = update_dynamo_table( table, throughput=throughput, check_mode=module.check_mode, global_indexes=global_indexes) else: if not module.check_mode: Table.create(table_name, connection=connection, schema=schema, throughput=throughput, indexes=indexes, global_indexes=global_indexes) result['changed'] = True if not module.check_mode: result['table_status'] = table.describe()['Table']['TableStatus'] if tags: # only tables which are active can be tagged wait_until_table_active(module, table, wait_for_active_timeout) account_id = get_account_id(boto3_sts) boto3_dynamodb.tag_resource( ResourceArn='arn:aws:dynamodb:' + region + ':' + account_id + ':table/' + table_name, Tags=ansible_dict_to_boto3_tag_list(tags)) result['tags'] = tags except BotoServerError: result[ 'msg'] = 'Failed to create/update dynamo table due to error: ' + traceback.format_exc( ) module.fail_json(**result) else: module.exit_json(**result)
class MyDatabase(object): def __init__(self, region): self.schema = [ HashKey(args.hashkey, data_type=STRING), RangeKey(args.indexkey, data_type=STRING), ] self.indexes = [ AllIndex('FirstName', parts=[ HashKey(args.hashkey, data_type=STRING), RangeKey('FirstName', data_type=STRING), ]), AllIndex('LastName', parts=[ HashKey(args.hashkey, data_type=STRING), RangeKey('LastName', data_type=STRING), ]) ] self.conn = boto.dynamodb2.connect_to_region( region, aws_access_key_id=os.environ['AWS_KEY'], aws_secret_access_key=os.environ['AWS_SECRET'] ) return def __del__(self): return def list(self): sys.stderr.write('Tables:\n') tables = self.conn.list_tables() for table in tables['TableNames']: sys.stdout.write('%s\n'%table) return def create(self, name): self.table = Table.create( name, schema=self.schema, indexes=self.indexes, connection=self.conn ) sys.stdout.write('%s\n'%self.table) return def all(self): query = { '%s__eq'%args.hashkey : args.hashval } if args.verbose: prettyPrint(query, output=sys.stderr,colour=True) records = [] #query['document_id__eq'] = '3e272906-a5df-4a70-b19e-b4e2504525f2' for row in self.table.query_2(**query): record = {} for key in row.keys(): record[key] = row[key] records.append(record) return records def connect(self, name): self.table = Table( name, connection=self.conn ) sys.stderr.write('Connect: %s\n'%self.table) return def dbschema(self): sys.stderr.write('Schema:\n') schema = self.table.describe() return schema def insert(self, dictstring): guid = '%s'%uuid.uuid4() data = { args.hashkey : args.hashval, args.indexkey : guid } for key in dictstring.keys(): data[key] = dictstring[key] if args.verbose: prettyPrint(data,colour=True,output=sys.stderr) self.table.put_item(data=data) return guid def query(self): return def get(self, id): sys.stderr.write('Get(%s):\n'%id) get = { args.hashkey : args.hashval, args.indexkey : id } if args.verbose: prettyPrint(get,output=sys.stderr,colour=True) doc = self.table.get_item(**get) got = {} for key in doc.keys(): got[key] = '%s'%doc.get(key) return got def update(self): return def delete(self, name): return def drop(self, name): table = Table(name, connection=self.conn) sys.stderr.write('drop=%s\n'%table) table.delete() return