class TestMisconfiguredSchemaException(unittest.TestCase): def setUp(self): # Configuration options self.table_name = 'conf' self.store_name = 'test' # Instanciate the store DynamoDBConfigStore(connection, self.table_name, self.store_name) # Get an Table instance for validation self.table = Table(self.table_name, connection=connection) def test_misconfigured_schema_store_key(self): """ Test that an exception is raised if the store key is not an hash """ with self.assertRaises(MisconfiguredSchemaException): DynamoDBConfigStore( connection, self.table_name, self.store_name, store_key='test') def test_misconfigured_schema_option_key(self): """ Test that an exception is raised if the option key isn't a range """ with self.assertRaises(MisconfiguredSchemaException): DynamoDBConfigStore( connection, self.table_name, self.store_name, option_key='test') def tearDown(self): """ Tear down the test case """ self.table.delete()
class TestCustomThroughput(unittest.TestCase): def setUp(self): # Configuration options self.table_name = 'conf' self.store_name = 'test' self.read_units = 10 self.write_units = 8 # Instanciate the store self.store = DynamoDBConfigStore(connection, self.table_name, self.store_name, read_units=self.read_units, write_units=self.write_units) # Get an Table instance for validation self.table = Table(self.table_name, connection=connection) def test_custom_throughput(self): """ Test that we can set custom thoughput for new tables """ throughput = self.table.describe()[u'Table'][u'ProvisionedThroughput'] self.assertEqual(throughput[u'ReadCapacityUnits'], self.read_units) self.assertEqual(throughput[u'WriteCapacityUnits'], self.write_units) def tearDown(self): """ Tear down the test case """ self.table.delete()
class TestMisconfiguredSchemaException(unittest.TestCase): def setUp(self): # Configuration options self.table_name = 'conf' self.store_name = 'test' # Instanciate the store DynamoDBConfigStore(connection, self.table_name, self.store_name) # Get an Table instance for validation self.table = Table(self.table_name, connection=connection) def test_misconfigured_schema_store_key(self): """ Test that an exception is raised if the store key is not an hash """ with self.assertRaises(MisconfiguredSchemaException): DynamoDBConfigStore(connection, self.table_name, self.store_name, store_key='test') def test_misconfigured_schema_option_key(self): """ Test that an exception is raised if the option key isn't a range """ with self.assertRaises(MisconfiguredSchemaException): DynamoDBConfigStore(connection, self.table_name, self.store_name, option_key='test') def tearDown(self): """ Tear down the test case """ self.table.delete()
def delete_dynamo_table(connection, module): table_name = module.params.get('name') result = dict( region=module.params.get('region'), table_name=table_name, ) try: table = Table(table_name, connection=connection) if dynamo_table_exists(table): if not module.check_mode: table.delete() result['changed'] = True else: result['changed'] = False except BotoServerError: result[ 'msg'] = 'Failed to delete dynamo table due to error: ' + traceback.format_exc( ) module.fail_json(**result) else: module.exit_json(**result)
def debug_delete_table(): try: users = Table('users', connection=boto.dynamodb2.connect_to_region('us-west-2')) Table.delete(users) print "Deleting users!" except Exception, c: print c
class TestCustomThroughput(unittest.TestCase): def setUp(self): # Configuration options self.table_name = 'conf' self.store_name = 'test' self.read_units = 10 self.write_units = 8 # Instanciate the store self.store = DynamoDBConfigStore( connection, self.table_name, self.store_name, read_units=self.read_units, write_units=self.write_units) # Get an Table instance for validation self.table = Table(self.table_name, connection=connection) def test_custom_throughput(self): """ Test that we can set custom thoughput for new tables """ throughput = self.table.describe()[u'Table'][u'ProvisionedThroughput'] self.assertEqual(throughput[u'ReadCapacityUnits'], self.read_units) self.assertEqual(throughput[u'WriteCapacityUnits'], self.write_units) def tearDown(self): """ Tear down the test case """ self.table.delete()
def delete_table(name): # input must be string try: table = Table(name, connection=client_dynamo) table.delete() return except KeyboardInterrupt: exit
def deleteTables(): for tableName in config.TABLES: table = Table( tableName, connection=dbconn, ) table.delete() print 'Table %s deleted' % tableName
def _delete_table_if_exists(conn, name): try: table = Table(name, connection=conn) table.delete() except JSONResponseError as e: if e.status == 400 and e.error_code == 'ResourceNotFoundException': return raise e
def pf_ddb_delete_item(self, name): johndoe = Table(name).get_item(username='******', friend_count=4) johndoe.delete() # Wait for it. time.sleep(5) return json.dumps(name + " delete_item")
class TestTimeBasedConfigStore(unittest.TestCase): def setUp(self): # Configuration options self.table_name = 'conf' self.store_name = 'test' # Instanciate the store self.store = DynamoDBConfigStore( connection, self.table_name, self.store_name, config_store='TimeBasedConfigStore', config_store_kwargs={'update_interval': 5}) # Get an Table instance for validation self.table = Table(self.table_name, connection=connection) def test_time_based_config_store(self): """ Test inserting and updating in time based config stores """ obj = { 'host': '127.0.0.1', 'port': 27017 } # Insert the object self.store.set('db', obj) with self.assertRaises(AttributeError): # We do not expect the attribute to exist until the # config has been reloaded self.store.config.db # Force config reload self.store.reload() self.assertEqual(self.store.config.db['host'], obj['host']) self.assertEqual(self.store.config.db['port'], obj['port']) # Update the object updatedObj = { 'host': '127.0.0.1', 'port': 8000 } self.store.set('db', updatedObj) self.assertEqual(self.store.config.db['host'], obj['host']) self.assertEqual(self.store.config.db['port'], obj['port']) time.sleep(5) self.assertEqual(self.store.config.db['host'], updatedObj['host']) self.assertEqual(self.store.config.db['port'], updatedObj['port']) def tearDown(self): """ Tear down the test case """ self.table.delete()
class TestGetFullStore(unittest.TestCase): def setUp(self): # Configuration options self.table_name = 'conf' self.store_name = 'test' # Instanciate the store self.store = DynamoDBConfigStore( connection, self.table_name, self.store_name) # Get an Table instance for validation self.table = Table(self.table_name, connection=connection) def test_get_of_full_store(self): """ Test that we can retrieve all objects in the store """ objApi = { 'endpoint': 'http://test.com', 'port': 80, 'username': '******', 'password': '******' } objUser = { 'username': '******', 'password': '******' } # Insert the object self.store.set('api', objApi) self.store.set('user', objUser) # Retrieve all objects options = self.store.config.get() self.assertEquals(len(options), 2) optApi = options['api'] optUser = options['user'] self.assertNotIn('_store', optApi) self.assertNotIn('_option', optApi) self.assertEqual(optApi['endpoint'], objApi['endpoint']) self.assertEqual(optApi['port'], objApi['port']) self.assertEqual(optApi['username'], objApi['username']) self.assertEqual(optApi['password'], objApi['password']) self.assertNotIn('_store', optUser) self.assertNotIn('_option', optUser) self.assertEqual(optUser['username'], objUser['username']) self.assertEqual(optUser['password'], objUser['password']) def tearDown(self): """ Tear down the test case """ self.table.delete()
def name_uni_delete_table(): # input must be string try: table = Table('AssignmentTwo', connection=client_dynamo) table.delete() return except KeyboardInterrupt: exit except: print "Error" return False
def delete_table(name): # input must be string try: table = Table(name, connection=client_dynamo) table.delete() print 'waiting for deletion...' time.sleep(12) return except KeyboardInterrupt: exit except: print "Error in delete_table()" return False
class TestTimeBasedConfigStore(unittest.TestCase): def setUp(self): # Configuration options self.table_name = 'conf' self.store_name = 'test' # Instanciate the store self.store = DynamoDBConfigStore( connection, self.table_name, self.store_name, config_store='TimeBasedConfigStore', config_store_kwargs={'update_interval': 5}) # Get an Table instance for validation self.table = Table(self.table_name, connection=connection) def test_time_based_config_store(self): """ Test inserting and updating in time based config stores """ obj = {'host': '127.0.0.1', 'port': 27017} # Insert the object self.store.set('db', obj) with self.assertRaises(AttributeError): # We do not expect the attribute to exist until the # config has been reloaded self.store.config.db # Force config reload self.store.reload() self.assertEqual(self.store.config.db['host'], obj['host']) self.assertEqual(self.store.config.db['port'], obj['port']) # Update the object updatedObj = {'host': '127.0.0.1', 'port': 8000} self.store.set('db', updatedObj) self.assertEqual(self.store.config.db['host'], obj['host']) self.assertEqual(self.store.config.db['port'], obj['port']) time.sleep(5) self.assertEqual(self.store.config.db['host'], updatedObj['host']) self.assertEqual(self.store.config.db['port'], updatedObj['port']) def tearDown(self): """ Tear down the test case """ self.table.delete()
class TestGetFullStore(unittest.TestCase): def setUp(self): # Configuration options self.table_name = 'conf' self.store_name = 'test' # Instanciate the store self.store = DynamoDBConfigStore(connection, self.table_name, self.store_name) # Get an Table instance for validation self.table = Table(self.table_name, connection=connection) def test_get_of_full_store(self): """ Test that we can retrieve all objects in the store """ objApi = { 'endpoint': 'http://test.com', 'port': 80, 'username': '******', 'password': '******' } objUser = {'username': '******', 'password': '******'} # Insert the object self.store.set('api', objApi) self.store.set('user', objUser) # Retrieve all objects options = self.store.config.get() self.assertEquals(len(options), 2) optApi = options['api'] optUser = options['user'] self.assertNotIn('_store', optApi) self.assertNotIn('_option', optApi) self.assertEqual(optApi['endpoint'], objApi['endpoint']) self.assertEqual(optApi['port'], objApi['port']) self.assertEqual(optApi['username'], objApi['username']) self.assertEqual(optApi['password'], objApi['password']) self.assertNotIn('_store', optUser) self.assertNotIn('_option', optUser) self.assertEqual(optUser['username'], objUser['username']) self.assertEqual(optUser['password'], objUser['password']) def tearDown(self): """ Tear down the test case """ self.table.delete()
def test_delete_table(self): self.storage_mocker.StubOutWithMock(storage, 'delete_table') self.storage_mocker.StubOutWithMock(storage, 'describe_table') storage.delete_table(IgnoreArg(), 'test_table') storage.describe_table(IgnoreArg(), 'test_table').AndReturn( models.TableMeta( models.TableSchema( { 'city1': models.AttributeType('S'), 'id': models.AttributeType('S'), 'name': models.AttributeType('S') }, ['id', 'name'], {'index_name': models.IndexDefinition('id', 'city1')} ), models.TableMeta.TABLE_STATUS_ACTIVE ) ) self.storage_mocker.ReplayAll() table = Table('test_table', connection=self.DYNAMODB_CON) self.assertTrue(table.delete()) self.storage_mocker.VerifyAll()
def test_delete_table(self): self.storage_mocker.StubOutWithMock(storage, 'delete_table') self.storage_mocker.StubOutWithMock(storage, 'describe_table') storage.delete_table(IgnoreArg(), 'test_table') storage.describe_table(IgnoreArg(), 'test_table').AndReturn( models.TableMeta( models.TableSchema( { 'city1': models.ATTRIBUTE_TYPE_STRING, 'id': models.ATTRIBUTE_TYPE_STRING, 'name': models.ATTRIBUTE_TYPE_STRING }, ['id', 'name'], {'index_name': models.IndexDefinition('city1')} ), models.TableMeta.TABLE_STATUS_ACTIVE ) ) self.storage_mocker.ReplayAll() table = Table('test_table', connection=self.DYNAMODB_CON) self.assertTrue(table.delete()) self.storage_mocker.VerifyAll()
def create_table(self, table_name, schema, throughput=None, indexes=None, global_indexes=None): ''' (str, list, dict, list, list) -> boto.dynamodb2.table.Table table_name: str con el nombre de la tabla a crear. schema: list de "BaseSchemaField" que representa el esquema de la tabla throughput: dict con 'read' & 'write' key y values enteros. indexes: list de "BaseIndexField" que define los indices de la tabla. global_indexes: list de "GlobalBaseIndexField" que define los indices globales para la tabla. Permite crear una tabla. Retorna la tabla que se ha creado. ''' tables = self.cnn.list_tables() table = Table(table_name, connection=self.cnn) #verifica si se debe eliminar una tabla antes de crearla, por ejemplo #si la aplicacion esta ejecuntado un entorno de tests if table_name in tables['TableNames'] and table_name.startswith('_'): if table.delete(): tables = self.cnn.list_tables() #valida si la tabla ya se encuentra creada. if not table_name in tables['TableNames']: Table.create(table_name, schema=schema, throughput=throughput, global_indexes=global_indexes, indexes=indexes, connection=self.cnn) return table
class TestGetOption(unittest.TestCase): def setUp(self): # Configuration options self.table_name = 'conf' self.store_name = 'test' # Instanciate the store self.store = DynamoDBConfigStore( connection, self.table_name, self.store_name) # Get an Table instance for validation self.table = Table(self.table_name, connection=connection) def test_get(self): """ Test that we can retrieve an object from the store """ obj = { 'endpoint': 'http://test.com', 'port': 80, 'username': '******', 'password': '******' } # Insert the object self.store.set('api', obj) # Retrieve the object option = self.store.config.get('api') self.assertNotIn('_store', option) self.assertNotIn('_option', option) self.assertEqual(option['endpoint'], obj['endpoint']) self.assertEqual(option['port'], obj['port']) self.assertEqual(option['username'], obj['username']) self.assertEqual(option['password'], obj['password']) def test_get_item_not_found(self): """ Test that we can't retrieve non-existing items """ with self.assertRaises(ItemNotFound): self.store.config.get('doesnotexist') def tearDown(self): """ Tear down the test case """ self.table.delete()
class TestCustomStoreAndOptionKeys(unittest.TestCase): def setUp(self): # Configuration options self.table_name = 'conf' self.store_name = 'test' self.store_key = '_s' self.option_key = '_o' # Instanciate the store self.store = DynamoDBConfigStore( connection, self.table_name, self.store_name, store_key=self.store_key, option_key=self.option_key) # Get an Table instance for validation self.table = Table(self.table_name, connection=connection) def test_custom_store_and_option_keys(self): """ Test that we can set custom store and option keys """ obj = { 'host': '127.0.0.1', 'port': 27017 } # Insert the object self.store.set('db', obj) # Fetch the object directly from DynamoDB kwargs = { '_s': self.store_name, '_o': 'db' } item = self.table.get_item(**kwargs) self.assertEqual(item['_s'], self.store_name) self.assertEqual(item['_o'], 'db') self.assertEqual(item['host'], '127.0.0.1') self.assertEqual(item['port'], 27017) def tearDown(self): """ Tear down the test case """ self.table.delete()
class TestGetOption(unittest.TestCase): def setUp(self): # Configuration options self.table_name = 'conf' self.store_name = 'test' # Instanciate the store self.store = DynamoDBConfigStore(connection, self.table_name, self.store_name) # Get an Table instance for validation self.table = Table(self.table_name, connection=connection) def test_get(self): """ Test that we can retrieve an object from the store """ obj = { 'endpoint': 'http://test.com', 'port': 80, 'username': '******', 'password': '******' } # Insert the object self.store.set('api', obj) # Retrieve the object option = self.store.config.get('api') self.assertNotIn('_store', option) self.assertNotIn('_option', option) self.assertEqual(option['endpoint'], obj['endpoint']) self.assertEqual(option['port'], obj['port']) self.assertEqual(option['username'], obj['username']) self.assertEqual(option['password'], obj['password']) def test_get_item_not_found(self): """ Test that we can't retrieve non-existing items """ with self.assertRaises(ItemNotFound): self.store.config.get('doesnotexist') def tearDown(self): """ Tear down the test case """ self.table.delete()
class TestGetOptionAndKeysSubset(unittest.TestCase): def setUp(self): # Configuration options self.table_name = 'conf' self.store_name = 'test' # Instanciate the store self.store = DynamoDBConfigStore( connection, self.table_name, self.store_name) # Get an Table instance for validation self.table = Table(self.table_name, connection=connection) def test_get(self): """ Test that we can retrieve an object from the store """ obj = { 'endpoint': 'http://test.com', 'port': 80, 'username': '******', 'password': '******' } # Insert the object self.store.set('api', obj) # Retrieve the object option = self.store.config.get('api', keys=['endpoint', 'port']) self.assertNotIn('_store', option) self.assertNotIn('_option', option) self.assertNotIn('username', option) self.assertNotIn('password', option) self.assertEqual(option['endpoint'], obj['endpoint']) self.assertEqual(option['port'], obj['port']) def tearDown(self): """ Tear down the test case """ self.table.delete()
def insertData(): conn = boto.dynamodb.connect_to_region('us-east-1'); try: tdescr=conn.describe_table('consumer_complaint') consumer_complaint=Table('consumer_complaint') except: consumer_complaint = createTable() try: isTableActive='false' while isTableActive=='false': tdescr=conn.describe_table('consumer_complaint') if(((tdescr['Table'])['TableStatus']) == 'ACTIVE'): #consumer_complaint=Table('consumer_complaint') start_time = time.time() reader=downloadData() i=0 for row in reader: if i==0: i=i+1 else: with consumer_complaint.batch_write() as batch: batch.put_item(data={ 'Complaint_ID' : row[0], 'Product' : row[1], 'Sub-product' : row[2], 'Issue' : row[3], 'State' : row[4], 'ZIP_code' : row[5], 'Company' : row[6], 'Company_response' : row[7], 'Timely_response?' : row[8], 'Consumer_disputed': row[9], }) print("--- Time %s in seconds for Insert Query ---" % (time.time() - start_time)) time=time.time() - start_time isTableActive='true' except: consumer_complaint.delete() return render_template('form_submit.html',tableStatus='false') #consumer_complaint.delete() return render_template('form_submit.html',tableStatus='true')
class TestNotImplementedConfigStore(unittest.TestCase): def test_not_implemented_config_store(self): # Configuration options self.table_name = 'conf' self.store_name = 'test' with self.assertRaises(NotImplementedError): # Instanciate the store self.store = DynamoDBConfigStore( connection, self.table_name, self.store_name, config_store='NotExistingConfigStore') # Get an Table instance for validation self.table = Table(self.table_name, connection=connection) def tearDown(self): """ Tear down the test case """ self.table.delete()
def delete(table_name, region=None, key=None, keyid=None, profile=None): ''' Delete a DynamoDB table. CLI example:: salt myminion boto_dynamodb.delete table_name region=us-east-1 ''' conn = _create_connection(region, key, keyid, profile) table = Table(table_name, connection=conn) table.delete() # Table deletion can take several seconds to propagate. # We will retry MAX_ATTEMPTS times. MAX_ATTEMPTS = 30 for i in range(MAX_ATTEMPTS): if not exists(table_name, region, key, keyid, profile): return True else: time.sleep(1) # sleep for one second and try again return False
class TestGetOptionAndKeysSubset(unittest.TestCase): def setUp(self): # Configuration options self.table_name = 'conf' self.store_name = 'test' # Instanciate the store self.store = DynamoDBConfigStore(connection, self.table_name, self.store_name) # Get an Table instance for validation self.table = Table(self.table_name, connection=connection) def test_get(self): """ Test that we can retrieve an object from the store """ obj = { 'endpoint': 'http://test.com', 'port': 80, 'username': '******', 'password': '******' } # Insert the object self.store.set('api', obj) # Retrieve the object option = self.store.config.get('api', keys=['endpoint', 'port']) self.assertNotIn('_store', option) self.assertNotIn('_option', option) self.assertNotIn('username', option) self.assertNotIn('password', option) self.assertEqual(option['endpoint'], obj['endpoint']) self.assertEqual(option['port'], obj['port']) def tearDown(self): """ Tear down the test case """ self.table.delete()
class TestCustomStoreAndOptionKeys(unittest.TestCase): def setUp(self): # Configuration options self.table_name = 'conf' self.store_name = 'test' self.store_key = '_s' self.option_key = '_o' # Instanciate the store self.store = DynamoDBConfigStore(connection, self.table_name, self.store_name, store_key=self.store_key, option_key=self.option_key) # Get an Table instance for validation self.table = Table(self.table_name, connection=connection) def test_custom_store_and_option_keys(self): """ Test that we can set custom store and option keys """ obj = {'host': '127.0.0.1', 'port': 27017} # Insert the object self.store.set('db', obj) # Fetch the object directly from DynamoDB kwargs = {'_s': self.store_name, '_o': 'db'} item = self.table.get_item(**kwargs) self.assertEqual(item['_s'], self.store_name) self.assertEqual(item['_o'], 'db') self.assertEqual(item['host'], '127.0.0.1') self.assertEqual(item['port'], 27017) def tearDown(self): """ Tear down the test case """ self.table.delete()
def delete_dynamo_table(connection, module): table_name = module.params.get('name') result = dict( region=module.params.get('region'), table_name=table_name, ) try: table = Table(table_name, connection=connection) if dynamo_table_exists(table): if not module.check_mode: table.delete() result['changed'] = True else: result['changed'] = False except BotoServerError: result['msg'] = 'Failed to delete dynamo table due to error: ' + traceback.format_exc() module.fail_json(**result) else: module.exit_json(**result)
class TableOpt(object): def __init__(self, conn, table_name, if_create_table): ''' Constructor ''' self.TABLE_NAME = "table" # Max number of threads self.MAX_THREAD_COUNT = 40 # Min number of tasks for each thread self.MIN_NUMBER_TASK_PER_THERAD = 500 # default record size self.record_size = 200 # util object self.util = BaseUtil() self.conn = conn self.table = None self.table_name = table_name # boto.set_file_logger('boto', './logs/logLogFile', 'DEBUG') if (if_create_table) : self.create_table() else: self._get_table() def _scan_records(self): items = self.table.scan() return items def _query_pkg_idx(self, pkg_id, partition_idx): log_files = self.table.query( partitionIndex__eq = partition_idx, pkgId__eq = pkg_id, index = 'LogPackageIdIndex') return log_files def _get_table(self): if(self.table_name in self.conn.list_tables()['TableNames']): self.table = Table(self.table_name, connection = self.conn) else: raise Exception("Do create table before other operations") def create_table(self): try: self.table = Table.create(self.table_name, schema = [ HashKey('PartitionID', data_type=NUMBER), RangeKey('FileName', data_type = STRING), ], throughput = { 'read' : 50, 'write': 30, }, connection = self.conn ) print "Create " + self.TABLE_NAME + " succeed\n" except Exception as e: print e print "Create " + self.TABLE_NAME + " failed\n" def insert_records(self, numberOfRecords, record_size): if not record_size: record_size = self.record_size for i in range(0, numberOfRecords): record = RecordInfo(record_size, i) item_data = record.get_record_info() self.table.put_item(item_data) print "done" def get(self, range_key): key = self.util.get_PartitionID(range_key) record = self.table.get_item(PartitionID=key, FileName=range_key) print 'FileName:' + record['FileName'] + "\tSize:" + str(record["Size"]) def range_get(self, key, ranges): # required to provide hash key and range key for range search records = self.table.query_2(PartitionID__eq=int(key), FileName__beginswith=ranges) count = 0 for record in records: print 'FileName:' + record['FileName'] + "\tSize:" + str(record["Size"]) count += 1 print "# Records got: " + str(count) def update(self, range_key, record_size): key = self.util.get_PartitionID(range_key) if not record_size: record_size = self.record_size new_item = RecordInfo(record_size, 0) record = self.table.get_item(PartitionID=key, FileName=range_key) record["Size"] = new_item.get_record_info()['Size'] record["Data"] = new_item.get_record_info()['Data'] # new_item.set_PartitionID(key) # new_item.set_FileName(range_key) # new_record = new_item.get_record_info() # record = Item(self.table, data=new_record) if record.partial_save(): print "Done" def scan(self, record_size): records = self.table.scan(Size__eq=record_size, ) count = 0 for record in records: print 'FileName:' + record['FileName'] + "\tSize:" + str(record["Size"]) count += 1 print "# Records got: " + str(count) def batch_insert_records(self, number_of_records, record_size): if record_size == 0: record_size = self.record_size with BatchTable(self.table) as batch: for i in range(0, number_of_records): record = RecordInfo(record_size, i) item_data = record.get_record_info() batch.put_item(data = item_data) print "done" def multithread_insert(self, number_of_records): threads = [] # Balance tasks; make sure each task at least has min number tasks except the last one if ((self.MIN_NUMBER_TASK_PER_THERAD * self.MAX_THREAD_COUNT) > number_of_records): number_of_task_per_thread = self.MIN_NUMBER_TASK_PER_THERAD else: number_of_task_per_thread = number_of_records / self.MAX_THREAD_COUNT i = 0 while i < number_of_records: inserting_task = None if ((number_of_records - i) > self.MIN_NUMBER_TASK_PER_THERAD): inserting_task = Thread(target = self.batch_insert_records, args = (number_of_task_per_thread, )) else: inserting_task = Thread(target = self.batch_insert_records, args = (number_of_records - i, )) threads.append(inserting_task) inserting_task.start() i += number_of_task_per_thread for thread in threads: thread.join() print "Successed insert " + str(number_of_records) + " items" def list_records(self): files = self._scan_records() for record in files: print record['FileName'] def count_records(self): count = 0 kwargs = {} kwargs['select'] = 'COUNT' raw_results = self.table.connection.scan( self.table_name, **kwargs ) count += raw_results.get('Count', int) last_key_seen = raw_results.get('LastEvaluatedKey', None) while last_key_seen: kwargs['exclusive_start_key'] = last_key_seen raw_results = self.table.connection.scan( self.table_name, **kwargs ) count += raw_results.get('Count', int) last_key_seen = raw_results.get('LastEvaluatedKey', None) print "total number of records: " + str(count) def count_one_page(self): kwargs = {} kwargs['select'] = 'COUNT' raw_results = self.table.connection.scan( self.table.table_name, **kwargs ) count = raw_results.get('Count', int) print "Number of records in on page: " + str(count) def delete_records(self): records = self._scan_records() count = 0 for record in records: record.delete() count += 1 print "Delete " + str(count) + " items\n" def batch_delete_records(self): records = self._scan_records() count = 0 with BatchTable(self.table) as batch: for record in records: batch.delete_item( PartitionID = record["PartitionID"], FileName = record["FileName"] ) count += 1 print "Delete " + str(count) + " items\n" # Used by packageDao def pkg_delete_helper(self, pkg): count = 0 with BatchTable(self.table) as batch: for idx in pkg["logFilePartitionIndexSet"]: log_files = self._query_pkg_idx(pkg["pkgId"], idx) for log in log_files: batch.delete_item( partitionIndex = log["partitionIndex"], logFileUrl = log["logFileUrl"] ) count += 1 print "Delete " + str(count) + " logs in pkg:" + pkg["pkgId"] + "\n" def delete_table(self): self.table.delete()
assert value[0] == check_dict[key] assert value[1] == 200 else : print "Key %s is not in check_dict" % key # A little consistency check to make sure that the database and the dictionary # really are in sync. value = get ( key ) assert value[0] == None assert value[1] == 403 ########################################################## check_dict = {} # This dictionary is used to verify that the database # is working correctly if DEBUG : print "Deleting the table %s" % TABLE_NAME kv_pairs.delete() # now that the table is gone, recreate it while True : time.sleep(10) # it takes some time for the table to delete try: kv_pairs = Table.create(TABLE_NAME, schema=[ HashKey('key')], throughput={ 'read': 5, 'write': 15, }) except JSONResponseError: print "The table %s still isn't deleted.... waiting" % TABLE_NAME else: break print "Created table %s" % TABLE_NAME time.sleep(10) while True: try: test_post("Dillon", 17)
name = i['Instance Name'] print "Deleteing %s" % name #s = name.split('_') #if 'Instance' in name: # Standard # itype = s[0].lower() + '_' + s[1].lower() #else: # Performance # itype = s[0] + "gb_" + s[2].lower() #virt = 'paravirtual' #new_name = itype + '_' + virt table_struct = None try: logs = Table(name, schema=[HashKey('trial'),RangeKey('parallel'),]) table_struct = logs.describe() logs.delete() except JSONResponseError: print "%s not existing" % name sleep(5) #table_struct = None #try: # new_logs = Table(new_name, schema=[HashKey('trial'),RangeKey('parallel'),]) # table_struct = new_logs.describe() #except JSONResponseError: # new_logs = Table.create(new_name, schema=[HashKey('trial'),RangeKey('parallel'),]) # sleep(5) #while table_struct is None: # try: # new_logs = Table(new_name, schema=[HashKey('trial'),RangeKey('parallel'),]) # table_struct = new_logs.describe()
def delete_table(self, name): try: table = Table(name, connection=self.conn) table.delete() except Exception as e: print 'Exception deleting table %s [%s]\n' % (name, e)
from boto.dynamodb2.fields import HashKey from boto.dynamodb2.table import Table table = Table('video') table.delete()
def drop(self, name): table = Table(name, connection=self.conn) sys.stderr.write('drop=%s\n'%table) table.delete() return
class TestSet(unittest.TestCase): def setUp(self): # Configuration options self.table_name = 'conf' self.store_name = 'test' # Instanciate the store self.store = DynamoDBConfigStore( connection, self.table_name, self.store_name) # Get an Table instance for validation self.table = Table(self.table_name, connection=connection) def test_set(self): """ Test that we can insert an object """ obj = { 'host': '127.0.0.1', 'port': 27017 } # Insert the object self.store.set('db', obj) # Fetch the object directly from DynamoDB kwargs = { '_store': self.store_name, '_option': 'db' } item = self.table.get_item(**kwargs) self.assertEqual(item['_store'], self.store_name) self.assertEqual(item['_option'], 'db') self.assertEqual(item['host'], '127.0.0.1') self.assertEqual(item['port'], 27017) def test_update(self): """ Test that we can change values in an option """ obj = { 'username': '******', 'password': '******' } # Insert the object self.store.set('user', obj) # Get the option option = self.store.config.get('user') self.assertEqual(option['username'], obj['username']) self.assertEqual(option['password'], obj['password']) # Updated version of the object updatedObj = { 'username': '******', 'password': '******' } # Insert the object self.store.set('user', updatedObj) # Get the option option = self.store.config.get('user') self.assertEqual(option['username'], updatedObj['username']) self.assertEqual(option['password'], updatedObj['password']) def test_update_with_new_keys(self): """ Test that we can completely change the keys """ obj = { 'username': '******', 'password': '******' } # Insert the object self.store.set('credentials', obj) # Get the option option = self.store.config.get('credentials') self.assertEqual(option['username'], obj['username']) self.assertEqual(option['password'], obj['password']) # Updated version of the object updatedObj = { 'access_key': 'anakin', 'secret_key': 'skywalker' } # Insert the object self.store.set('credentials', updatedObj) # Get the option option = self.store.config.get('credentials') self.assertEqual(option['access_key'], updatedObj['access_key']) self.assertEqual(option['secret_key'], updatedObj['secret_key']) self.assertNotIn('username', option) self.assertNotIn('password', option) def test_instert_too_large_object(self): """ Test of inserting an object larger than 64 kb """ with self.assertRaises(ValidationException): self.store.set( 'large', {x: int(random()*100000000000000) for x in xrange(1, 9999)}) def tearDown(self): """ Tear down the test case """ self.table.delete()
class TestSet(unittest.TestCase): def setUp(self): # Configuration options self.table_name = 'conf' self.store_name = 'test' # Instanciate the store self.store = DynamoDBConfigStore(connection, self.table_name, self.store_name) # Get an Table instance for validation self.table = Table(self.table_name, connection=connection) def test_set(self): """ Test that we can insert an object """ obj = {'host': '127.0.0.1', 'port': 27017} # Insert the object self.store.set('db', obj) # Fetch the object directly from DynamoDB kwargs = {'_store': self.store_name, '_option': 'db'} item = self.table.get_item(**kwargs) self.assertEqual(item['_store'], self.store_name) self.assertEqual(item['_option'], 'db') self.assertEqual(item['host'], '127.0.0.1') self.assertEqual(item['port'], 27017) def test_update(self): """ Test that we can change values in an option """ obj = {'username': '******', 'password': '******'} # Insert the object self.store.set('user', obj) # Get the option option = self.store.config.get('user') self.assertEqual(option['username'], obj['username']) self.assertEqual(option['password'], obj['password']) # Updated version of the object updatedObj = {'username': '******', 'password': '******'} # Insert the object self.store.set('user', updatedObj) # Get the option option = self.store.config.get('user') self.assertEqual(option['username'], updatedObj['username']) self.assertEqual(option['password'], updatedObj['password']) def test_update_with_new_keys(self): """ Test that we can completely change the keys """ obj = {'username': '******', 'password': '******'} # Insert the object self.store.set('credentials', obj) # Get the option option = self.store.config.get('credentials') self.assertEqual(option['username'], obj['username']) self.assertEqual(option['password'], obj['password']) # Updated version of the object updatedObj = {'access_key': 'anakin', 'secret_key': 'skywalker'} # Insert the object self.store.set('credentials', updatedObj) # Get the option option = self.store.config.get('credentials') self.assertEqual(option['access_key'], updatedObj['access_key']) self.assertEqual(option['secret_key'], updatedObj['secret_key']) self.assertNotIn('username', option) self.assertNotIn('password', option) def test_instert_too_large_object(self): """ Test of inserting an object larger than 64 kb """ with self.assertRaises(ValidationException): self.store.set( 'large', {x: int(random() * 100000000000000) for x in xrange(1, 9999)}) def tearDown(self): """ Tear down the test case """ self.table.delete()
class DBTable(object): item_class = None simple_schema = [HashKey('_id')] def __init__(self, table_name, schema, global_indexes, dbconn): self.table_name = table_name self._table = Table(table_name, connection=dbconn.conn) self.schema = schema self.global_indexes = global_indexes self.dbconn = dbconn.conn return self def get_table(self): return self._table def exists(self): return self._table.table_name in self.dbconn.list_tables( )['TableNames'] def create(self): try: # print self.schema # print self.table_name self._table = Table.create(self.table_name, schema=self.schema, global_indexes=self.global_indexes, connection=self.dbconn) except JSONResponseError as e: logging.warn("Error creating table " + self.table_name) log_boto_error(e) if 'message' in e.body and e.body['message'].startswith( 'Table already exists'): self._table = Table(self.table_name, connection=self.dbconn) elif 'Message' in e.body and e.body['Message'].startswith( "Cannot create preexisting"): self._table = Table(self.table_name, connection=self.dbconn) else: raise e return self def delete(self): try: self._table.delete() except JSONResponseError as e: logging.warn("Error deleting table " + self.table_name) log_boto_error(e) if e.body['Message'].startswith( 'Cannot do operations on a non-existent tabl'): return else: raise e return def insert(self, data): self._table.put_item(data=data, overwrite=True) return def get_item(self, **kwargs): item = self._table.get_item(**kwargs) # TODO return item or class?? # new_item = self.item_class(self) # new_item.__dict__ = item._data # return new_item return item def query_2(self, **kwargs): return self._table.query_2(**kwargs) def scan(self, **kwargs): return self._table.scan(**kwargs) def remove(self): return self._table.remove()
class StateDynamo(object): def __init__(self, region, prefix = 'cicada_'): self.prefix = prefix self.connection= boto.dynamodb2.connect_to_region(region) self.project_job = Table(prefix + 'project_job', connection=self.connection, **dynamo_schema['project_job']) self.job_history = Table(prefix + 'job_history', connection=self.connection, **dynamo_schema['job_history']) self.worker = Table(prefix + 'worker', connection=self.connection, **dynamo_schema['workers']) self.poll = Table(prefix + 'poll', connection=self.connection, **dynamo_schema['poll']) def tables_create(self): """ Creates a new table.. throws exception if tables already exist """ for table_name, schema in dynamo_schema.iteritems(): logging.info("Creating " + self.prefix + table_name) Table.create(self.prefix + table_name, **schema) def tables_destroy(self): """ """ self.project_job.delete() self.job_history.delete() self.worker.delete() self.poll.delete() def tables_list(self): print self.connection.list_tables()['TableNames'] def workers_list_all(self): return self.worker.scan() def workers_put(self, workerx, project, job, started): return self.worker.put_item(data={ worker: workerx, job:"{0}.{1}".format(project, job), started:started}, overwrite=True) def projectjobs_list_all(self): """ dumps all project/job states. """ return self.project_job.scan() def projectjobs_list(self, projectname): """ dumps all states for a given project """ return self.project_job.scan(project__eq=projectname) def projectjobs_put(self, project, job, started, state, updated): """ Designed for fast lookups of all projects. hash/range = (project, job) """ self.project_job.put_item(data={ 'project': project, 'job': job, 'started': started, 'state': state, 'updated': updated }, overwrite=True) def jobhistory_put(self, project, job, started, state, updated): """ combines project.job into one field for looking up history of a particular job. """ self.job_history.put_item(data={ 'projectjob': '{0}.{1}'.format(project, job), 'started': started, 'state': state, 'updated': updated }, overwrite=True) def poll_list_all(self): polls = self.poll.scan() return polls def poll_put(self, name, current, now): return self.poll.put_item(data={ 'name': name, 'current': current, 'now': now }, overwrite=True) def poll_get(self, name): lastpoll = self.poll.get_item(name=name) if lastpoll is None: return None else: return lastpoll['current']