class TestMisconfiguredSchemaException(unittest.TestCase):
def setUp(self):
# Configuration options
self.table_name = 'conf'
self.store_name = 'test'
# Instanciate the store
DynamoDBConfigStore(connection, self.table_name, self.store_name)
# Get an Table instance for validation
self.table = Table(self.table_name, connection=connection)
def test_misconfigured_schema_store_key(self):
""" Test that an exception is raised if the store key is not an hash """
with self.assertRaises(MisconfiguredSchemaException):
DynamoDBConfigStore(
connection,
self.table_name,
self.store_name,
store_key='test')
def test_misconfigured_schema_option_key(self):
""" Test that an exception is raised if the option key isn't a range """
with self.assertRaises(MisconfiguredSchemaException):
DynamoDBConfigStore(
connection,
self.table_name,
self.store_name,
option_key='test')
def tearDown(self):
""" Tear down the test case """
self.table.delete()
class TestCustomThroughput(unittest.TestCase):
def setUp(self):
# Configuration options
self.table_name = 'conf'
self.store_name = 'test'
self.read_units = 10
self.write_units = 8
# Instanciate the store
self.store = DynamoDBConfigStore(connection,
self.table_name,
self.store_name,
read_units=self.read_units,
write_units=self.write_units)
# Get an Table instance for validation
self.table = Table(self.table_name, connection=connection)
def test_custom_throughput(self):
""" Test that we can set custom thoughput for new tables """
throughput = self.table.describe()[u'Table'][u'ProvisionedThroughput']
self.assertEqual(throughput[u'ReadCapacityUnits'], self.read_units)
self.assertEqual(throughput[u'WriteCapacityUnits'], self.write_units)
def tearDown(self):
""" Tear down the test case """
self.table.delete()
class TestMisconfiguredSchemaException(unittest.TestCase):
def setUp(self):
# Configuration options
self.table_name = 'conf'
self.store_name = 'test'
# Instanciate the store
DynamoDBConfigStore(connection, self.table_name, self.store_name)
# Get an Table instance for validation
self.table = Table(self.table_name, connection=connection)
def test_misconfigured_schema_store_key(self):
""" Test that an exception is raised if the store key is not an hash """
with self.assertRaises(MisconfiguredSchemaException):
DynamoDBConfigStore(connection,
self.table_name,
self.store_name,
store_key='test')
def test_misconfigured_schema_option_key(self):
""" Test that an exception is raised if the option key isn't a range """
with self.assertRaises(MisconfiguredSchemaException):
DynamoDBConfigStore(connection,
self.table_name,
self.store_name,
option_key='test')
def tearDown(self):
""" Tear down the test case """
self.table.delete()
def delete_dynamo_table(connection, module):
table_name = module.params.get('name')
result = dict(
region=module.params.get('region'),
table_name=table_name,
)
try:
table = Table(table_name, connection=connection)
if dynamo_table_exists(table):
if not module.check_mode:
table.delete()
result['changed'] = True
else:
result['changed'] = False
except BotoServerError:
result[
'msg'] = 'Failed to delete dynamo table due to error: ' + traceback.format_exc(
)
module.fail_json(**result)
else:
module.exit_json(**result)
def debug_delete_table():
try:
users = Table('users', connection=boto.dynamodb2.connect_to_region('us-west-2'))
Table.delete(users)
print "Deleting users!"
except Exception, c:
print c
class TestCustomThroughput(unittest.TestCase):
def setUp(self):
# Configuration options
self.table_name = 'conf'
self.store_name = 'test'
self.read_units = 10
self.write_units = 8
# Instanciate the store
self.store = DynamoDBConfigStore(
connection,
self.table_name,
self.store_name,
read_units=self.read_units,
write_units=self.write_units)
# Get an Table instance for validation
self.table = Table(self.table_name, connection=connection)
def test_custom_throughput(self):
""" Test that we can set custom thoughput for new tables """
throughput = self.table.describe()[u'Table'][u'ProvisionedThroughput']
self.assertEqual(throughput[u'ReadCapacityUnits'], self.read_units)
self.assertEqual(throughput[u'WriteCapacityUnits'], self.write_units)
def tearDown(self):
""" Tear down the test case """
self.table.delete()
def delete_table(name):
# input must be string
try:
table = Table(name, connection=client_dynamo)
table.delete()
return
except KeyboardInterrupt:
exit
def deleteTables():
for tableName in config.TABLES:
table = Table(
tableName,
connection=dbconn,
)
table.delete()
print 'Table %s deleted' % tableName
def _delete_table_if_exists(conn, name):
try:
table = Table(name, connection=conn)
table.delete()
except JSONResponseError as e:
if e.status == 400 and e.error_code == 'ResourceNotFoundException':
return
raise e
def debug_delete_table():
try:
users = Table('users',
connection=boto.dynamodb2.connect_to_region('us-west-2'))
Table.delete(users)
print "Deleting users!"
except Exception, c:
print c
def pf_ddb_delete_item(self, name):
johndoe = Table(name).get_item(username='******', friend_count=4)
johndoe.delete()
# Wait for it.
time.sleep(5)
return json.dumps(name + " delete_item")
class TestTimeBasedConfigStore(unittest.TestCase):
def setUp(self):
# Configuration options
self.table_name = 'conf'
self.store_name = 'test'
# Instanciate the store
self.store = DynamoDBConfigStore(
connection,
self.table_name,
self.store_name,
config_store='TimeBasedConfigStore',
config_store_kwargs={'update_interval': 5})
# Get an Table instance for validation
self.table = Table(self.table_name, connection=connection)
def test_time_based_config_store(self):
""" Test inserting and updating in time based config stores """
obj = {
'host': '127.0.0.1',
'port': 27017
}
# Insert the object
self.store.set('db', obj)
with self.assertRaises(AttributeError):
# We do not expect the attribute to exist until the
# config has been reloaded
self.store.config.db
# Force config reload
self.store.reload()
self.assertEqual(self.store.config.db['host'], obj['host'])
self.assertEqual(self.store.config.db['port'], obj['port'])
# Update the object
updatedObj = {
'host': '127.0.0.1',
'port': 8000
}
self.store.set('db', updatedObj)
self.assertEqual(self.store.config.db['host'], obj['host'])
self.assertEqual(self.store.config.db['port'], obj['port'])
time.sleep(5)
self.assertEqual(self.store.config.db['host'], updatedObj['host'])
self.assertEqual(self.store.config.db['port'], updatedObj['port'])
def tearDown(self):
""" Tear down the test case """
self.table.delete()
class TestGetFullStore(unittest.TestCase):
def setUp(self):
# Configuration options
self.table_name = 'conf'
self.store_name = 'test'
# Instanciate the store
self.store = DynamoDBConfigStore(
connection,
self.table_name,
self.store_name)
# Get an Table instance for validation
self.table = Table(self.table_name, connection=connection)
def test_get_of_full_store(self):
""" Test that we can retrieve all objects in the store """
objApi = {
'endpoint': 'http://test.com',
'port': 80,
'username': '******',
'password': '******'
}
objUser = {
'username': '******',
'password': '******'
}
# Insert the object
self.store.set('api', objApi)
self.store.set('user', objUser)
# Retrieve all objects
options = self.store.config.get()
self.assertEquals(len(options), 2)
optApi = options['api']
optUser = options['user']
self.assertNotIn('_store', optApi)
self.assertNotIn('_option', optApi)
self.assertEqual(optApi['endpoint'], objApi['endpoint'])
self.assertEqual(optApi['port'], objApi['port'])
self.assertEqual(optApi['username'], objApi['username'])
self.assertEqual(optApi['password'], objApi['password'])
self.assertNotIn('_store', optUser)
self.assertNotIn('_option', optUser)
self.assertEqual(optUser['username'], objUser['username'])
self.assertEqual(optUser['password'], objUser['password'])
def tearDown(self):
""" Tear down the test case """
self.table.delete()
def name_uni_delete_table():
# input must be string
try:
table = Table('AssignmentTwo', connection=client_dynamo)
table.delete()
return
except KeyboardInterrupt:
exit
except:
print "Error"
return False
def delete_table(name):
# input must be string
try:
table = Table(name, connection=client_dynamo)
table.delete()
print 'waiting for deletion...'
time.sleep(12)
return
except KeyboardInterrupt:
exit
except:
print "Error in delete_table()"
return False
class TestTimeBasedConfigStore(unittest.TestCase):
def setUp(self):
# Configuration options
self.table_name = 'conf'
self.store_name = 'test'
# Instanciate the store
self.store = DynamoDBConfigStore(
connection,
self.table_name,
self.store_name,
config_store='TimeBasedConfigStore',
config_store_kwargs={'update_interval': 5})
# Get an Table instance for validation
self.table = Table(self.table_name, connection=connection)
def test_time_based_config_store(self):
""" Test inserting and updating in time based config stores """
obj = {'host': '127.0.0.1', 'port': 27017}
# Insert the object
self.store.set('db', obj)
with self.assertRaises(AttributeError):
# We do not expect the attribute to exist until the
# config has been reloaded
self.store.config.db
# Force config reload
self.store.reload()
self.assertEqual(self.store.config.db['host'], obj['host'])
self.assertEqual(self.store.config.db['port'], obj['port'])
# Update the object
updatedObj = {'host': '127.0.0.1', 'port': 8000}
self.store.set('db', updatedObj)
self.assertEqual(self.store.config.db['host'], obj['host'])
self.assertEqual(self.store.config.db['port'], obj['port'])
time.sleep(5)
self.assertEqual(self.store.config.db['host'], updatedObj['host'])
self.assertEqual(self.store.config.db['port'], updatedObj['port'])
def tearDown(self):
""" Tear down the test case """
self.table.delete()
class TestGetFullStore(unittest.TestCase):
def setUp(self):
# Configuration options
self.table_name = 'conf'
self.store_name = 'test'
# Instanciate the store
self.store = DynamoDBConfigStore(connection, self.table_name,
self.store_name)
# Get an Table instance for validation
self.table = Table(self.table_name, connection=connection)
def test_get_of_full_store(self):
""" Test that we can retrieve all objects in the store """
objApi = {
'endpoint': 'http://test.com',
'port': 80,
'username': '******',
'password': '******'
}
objUser = {'username': '******', 'password': '******'}
# Insert the object
self.store.set('api', objApi)
self.store.set('user', objUser)
# Retrieve all objects
options = self.store.config.get()
self.assertEquals(len(options), 2)
optApi = options['api']
optUser = options['user']
self.assertNotIn('_store', optApi)
self.assertNotIn('_option', optApi)
self.assertEqual(optApi['endpoint'], objApi['endpoint'])
self.assertEqual(optApi['port'], objApi['port'])
self.assertEqual(optApi['username'], objApi['username'])
self.assertEqual(optApi['password'], objApi['password'])
self.assertNotIn('_store', optUser)
self.assertNotIn('_option', optUser)
self.assertEqual(optUser['username'], objUser['username'])
self.assertEqual(optUser['password'], objUser['password'])
def tearDown(self):
""" Tear down the test case """
self.table.delete()
def test_delete_table(self):
self.storage_mocker.StubOutWithMock(storage, 'delete_table')
self.storage_mocker.StubOutWithMock(storage, 'describe_table')
storage.delete_table(IgnoreArg(), 'test_table')
storage.describe_table(IgnoreArg(), 'test_table').AndReturn(
models.TableMeta(
models.TableSchema(
{
'city1': models.AttributeType('S'),
'id': models.AttributeType('S'),
'name': models.AttributeType('S')
},
['id', 'name'],
{'index_name': models.IndexDefinition('id', 'city1')}
),
models.TableMeta.TABLE_STATUS_ACTIVE
)
)
self.storage_mocker.ReplayAll()
table = Table('test_table', connection=self.DYNAMODB_CON)
self.assertTrue(table.delete())
self.storage_mocker.VerifyAll()
def test_delete_table(self):
self.storage_mocker.StubOutWithMock(storage, 'delete_table')
self.storage_mocker.StubOutWithMock(storage, 'describe_table')
storage.delete_table(IgnoreArg(), 'test_table')
storage.describe_table(IgnoreArg(), 'test_table').AndReturn(
models.TableMeta(
models.TableSchema(
{
'city1': models.ATTRIBUTE_TYPE_STRING,
'id': models.ATTRIBUTE_TYPE_STRING,
'name': models.ATTRIBUTE_TYPE_STRING
},
['id', 'name'],
{'index_name': models.IndexDefinition('city1')}
),
models.TableMeta.TABLE_STATUS_ACTIVE
)
)
self.storage_mocker.ReplayAll()
table = Table('test_table', connection=self.DYNAMODB_CON)
self.assertTrue(table.delete())
self.storage_mocker.VerifyAll()
def create_table(self, table_name, schema, throughput=None, indexes=None,
global_indexes=None):
'''
(str, list, dict, list, list) -> boto.dynamodb2.table.Table
table_name: str con el nombre de la tabla a crear.
schema: list de "BaseSchemaField" que representa el esquema de la tabla
throughput: dict con 'read' & 'write' key y values enteros.
indexes: list de "BaseIndexField" que define los indices de la tabla.
global_indexes: list de "GlobalBaseIndexField" que define los indices
globales para la tabla.
Permite crear una tabla. Retorna la tabla que se ha creado.
'''
tables = self.cnn.list_tables()
table = Table(table_name, connection=self.cnn)
#verifica si se debe eliminar una tabla antes de crearla, por ejemplo
#si la aplicacion esta ejecuntado un entorno de tests
if table_name in tables['TableNames'] and table_name.startswith('_'):
if table.delete():
tables = self.cnn.list_tables()
#valida si la tabla ya se encuentra creada.
if not table_name in tables['TableNames']:
Table.create(table_name,
schema=schema,
throughput=throughput,
global_indexes=global_indexes,
indexes=indexes,
connection=self.cnn)
return table
class TestGetOption(unittest.TestCase):
def setUp(self):
# Configuration options
self.table_name = 'conf'
self.store_name = 'test'
# Instanciate the store
self.store = DynamoDBConfigStore(
connection,
self.table_name,
self.store_name)
# Get an Table instance for validation
self.table = Table(self.table_name, connection=connection)
def test_get(self):
""" Test that we can retrieve an object from the store """
obj = {
'endpoint': 'http://test.com',
'port': 80,
'username': '******',
'password': '******'
}
# Insert the object
self.store.set('api', obj)
# Retrieve the object
option = self.store.config.get('api')
self.assertNotIn('_store', option)
self.assertNotIn('_option', option)
self.assertEqual(option['endpoint'], obj['endpoint'])
self.assertEqual(option['port'], obj['port'])
self.assertEqual(option['username'], obj['username'])
self.assertEqual(option['password'], obj['password'])
def test_get_item_not_found(self):
""" Test that we can't retrieve non-existing items """
with self.assertRaises(ItemNotFound):
self.store.config.get('doesnotexist')
def tearDown(self):
""" Tear down the test case """
self.table.delete()
class TestCustomStoreAndOptionKeys(unittest.TestCase):
def setUp(self):
# Configuration options
self.table_name = 'conf'
self.store_name = 'test'
self.store_key = '_s'
self.option_key = '_o'
# Instanciate the store
self.store = DynamoDBConfigStore(
connection,
self.table_name,
self.store_name,
store_key=self.store_key,
option_key=self.option_key)
# Get an Table instance for validation
self.table = Table(self.table_name, connection=connection)
def test_custom_store_and_option_keys(self):
""" Test that we can set custom store and option keys """
obj = {
'host': '127.0.0.1',
'port': 27017
}
# Insert the object
self.store.set('db', obj)
# Fetch the object directly from DynamoDB
kwargs = {
'_s': self.store_name,
'_o': 'db'
}
item = self.table.get_item(**kwargs)
self.assertEqual(item['_s'], self.store_name)
self.assertEqual(item['_o'], 'db')
self.assertEqual(item['host'], '127.0.0.1')
self.assertEqual(item['port'], 27017)
def tearDown(self):
""" Tear down the test case """
self.table.delete()
class TestGetOption(unittest.TestCase):
def setUp(self):
# Configuration options
self.table_name = 'conf'
self.store_name = 'test'
# Instanciate the store
self.store = DynamoDBConfigStore(connection, self.table_name,
self.store_name)
# Get an Table instance for validation
self.table = Table(self.table_name, connection=connection)
def test_get(self):
""" Test that we can retrieve an object from the store """
obj = {
'endpoint': 'http://test.com',
'port': 80,
'username': '******',
'password': '******'
}
# Insert the object
self.store.set('api', obj)
# Retrieve the object
option = self.store.config.get('api')
self.assertNotIn('_store', option)
self.assertNotIn('_option', option)
self.assertEqual(option['endpoint'], obj['endpoint'])
self.assertEqual(option['port'], obj['port'])
self.assertEqual(option['username'], obj['username'])
self.assertEqual(option['password'], obj['password'])
def test_get_item_not_found(self):
""" Test that we can't retrieve non-existing items """
with self.assertRaises(ItemNotFound):
self.store.config.get('doesnotexist')
def tearDown(self):
""" Tear down the test case """
self.table.delete()
class TestGetOptionAndKeysSubset(unittest.TestCase):
def setUp(self):
# Configuration options
self.table_name = 'conf'
self.store_name = 'test'
# Instanciate the store
self.store = DynamoDBConfigStore(
connection,
self.table_name,
self.store_name)
# Get an Table instance for validation
self.table = Table(self.table_name, connection=connection)
def test_get(self):
""" Test that we can retrieve an object from the store """
obj = {
'endpoint': 'http://test.com',
'port': 80,
'username': '******',
'password': '******'
}
# Insert the object
self.store.set('api', obj)
# Retrieve the object
option = self.store.config.get('api', keys=['endpoint', 'port'])
self.assertNotIn('_store', option)
self.assertNotIn('_option', option)
self.assertNotIn('username', option)
self.assertNotIn('password', option)
self.assertEqual(option['endpoint'], obj['endpoint'])
self.assertEqual(option['port'], obj['port'])
def tearDown(self):
""" Tear down the test case """
self.table.delete()
def insertData():
conn = boto.dynamodb.connect_to_region('us-east-1');
try:
tdescr=conn.describe_table('consumer_complaint')
consumer_complaint=Table('consumer_complaint')
except:
consumer_complaint = createTable()
try:
isTableActive='false'
while isTableActive=='false':
tdescr=conn.describe_table('consumer_complaint')
if(((tdescr['Table'])['TableStatus']) == 'ACTIVE'):
#consumer_complaint=Table('consumer_complaint')
start_time = time.time()
reader=downloadData()
i=0
for row in reader:
if i==0:
i=i+1
else:
with consumer_complaint.batch_write() as batch:
batch.put_item(data={
'Complaint_ID' : row[0],
'Product' : row[1],
'Sub-product' : row[2],
'Issue' : row[3],
'State' : row[4],
'ZIP_code' : row[5],
'Company' : row[6],
'Company_response' : row[7],
'Timely_response?' : row[8],
'Consumer_disputed': row[9],
})
print("--- Time %s in seconds for Insert Query ---" % (time.time() - start_time))
time=time.time() - start_time
isTableActive='true'
except:
consumer_complaint.delete()
return render_template('form_submit.html',tableStatus='false')
#consumer_complaint.delete()
return render_template('form_submit.html',tableStatus='true')
class TestNotImplementedConfigStore(unittest.TestCase):
def test_not_implemented_config_store(self):
# Configuration options
self.table_name = 'conf'
self.store_name = 'test'
with self.assertRaises(NotImplementedError):
# Instanciate the store
self.store = DynamoDBConfigStore(
connection,
self.table_name,
self.store_name,
config_store='NotExistingConfigStore')
# Get an Table instance for validation
self.table = Table(self.table_name, connection=connection)
def tearDown(self):
""" Tear down the test case """
self.table.delete()
def delete(table_name, region=None, key=None, keyid=None, profile=None):
'''
Delete a DynamoDB table.
CLI example::
salt myminion boto_dynamodb.delete table_name region=us-east-1
'''
conn = _create_connection(region, key, keyid, profile)
table = Table(table_name, connection=conn)
table.delete()
# Table deletion can take several seconds to propagate.
# We will retry MAX_ATTEMPTS times.
MAX_ATTEMPTS = 30
for i in range(MAX_ATTEMPTS):
if not exists(table_name, region, key, keyid, profile):
return True
else:
time.sleep(1) # sleep for one second and try again
return False
def delete(table_name, region=None, key=None, keyid=None, profile=None):
'''
Delete a DynamoDB table.
CLI example::
salt myminion boto_dynamodb.delete table_name region=us-east-1
'''
conn = _create_connection(region, key, keyid, profile)
table = Table(table_name, connection=conn)
table.delete()
# Table deletion can take several seconds to propagate.
# We will retry MAX_ATTEMPTS times.
MAX_ATTEMPTS = 30
for i in range(MAX_ATTEMPTS):
if not exists(table_name, region, key, keyid, profile):
return True
else:
time.sleep(1) # sleep for one second and try again
return False
class TestGetOptionAndKeysSubset(unittest.TestCase):
def setUp(self):
# Configuration options
self.table_name = 'conf'
self.store_name = 'test'
# Instanciate the store
self.store = DynamoDBConfigStore(connection, self.table_name,
self.store_name)
# Get an Table instance for validation
self.table = Table(self.table_name, connection=connection)
def test_get(self):
""" Test that we can retrieve an object from the store """
obj = {
'endpoint': 'http://test.com',
'port': 80,
'username': '******',
'password': '******'
}
# Insert the object
self.store.set('api', obj)
# Retrieve the object
option = self.store.config.get('api', keys=['endpoint', 'port'])
self.assertNotIn('_store', option)
self.assertNotIn('_option', option)
self.assertNotIn('username', option)
self.assertNotIn('password', option)
self.assertEqual(option['endpoint'], obj['endpoint'])
self.assertEqual(option['port'], obj['port'])
def tearDown(self):
""" Tear down the test case """
self.table.delete()
class TestNotImplementedConfigStore(unittest.TestCase):
def test_not_implemented_config_store(self):
# Configuration options
self.table_name = 'conf'
self.store_name = 'test'
with self.assertRaises(NotImplementedError):
# Instanciate the store
self.store = DynamoDBConfigStore(
connection,
self.table_name,
self.store_name,
config_store='NotExistingConfigStore')
# Get an Table instance for validation
self.table = Table(self.table_name, connection=connection)
def tearDown(self):
""" Tear down the test case """
self.table.delete()
class TestCustomStoreAndOptionKeys(unittest.TestCase):
def setUp(self):
# Configuration options
self.table_name = 'conf'
self.store_name = 'test'
self.store_key = '_s'
self.option_key = '_o'
# Instanciate the store
self.store = DynamoDBConfigStore(connection,
self.table_name,
self.store_name,
store_key=self.store_key,
option_key=self.option_key)
# Get an Table instance for validation
self.table = Table(self.table_name, connection=connection)
def test_custom_store_and_option_keys(self):
""" Test that we can set custom store and option keys """
obj = {'host': '127.0.0.1', 'port': 27017}
# Insert the object
self.store.set('db', obj)
# Fetch the object directly from DynamoDB
kwargs = {'_s': self.store_name, '_o': 'db'}
item = self.table.get_item(**kwargs)
self.assertEqual(item['_s'], self.store_name)
self.assertEqual(item['_o'], 'db')
self.assertEqual(item['host'], '127.0.0.1')
self.assertEqual(item['port'], 27017)
def tearDown(self):
""" Tear down the test case """
self.table.delete()
def delete_dynamo_table(connection, module):
table_name = module.params.get('name')
result = dict(
region=module.params.get('region'),
table_name=table_name,
)
try:
table = Table(table_name, connection=connection)
if dynamo_table_exists(table):
if not module.check_mode:
table.delete()
result['changed'] = True
else:
result['changed'] = False
except BotoServerError:
result['msg'] = 'Failed to delete dynamo table due to error: ' + traceback.format_exc()
module.fail_json(**result)
else:
module.exit_json(**result)
class TableOpt(object):
def __init__(self, conn, table_name, if_create_table):
'''
Constructor
'''
self.TABLE_NAME = "table"
# Max number of threads
self.MAX_THREAD_COUNT = 40
# Min number of tasks for each thread
self.MIN_NUMBER_TASK_PER_THERAD = 500
# default record size
self.record_size = 200
# util object
self.util = BaseUtil()
self.conn = conn
self.table = None
self.table_name = table_name
# boto.set_file_logger('boto', './logs/logLogFile', 'DEBUG')
if (if_create_table) :
self.create_table()
else:
self._get_table()
def _scan_records(self):
items = self.table.scan()
return items
def _query_pkg_idx(self, pkg_id, partition_idx):
log_files = self.table.query(
partitionIndex__eq = partition_idx,
pkgId__eq = pkg_id,
index = 'LogPackageIdIndex')
return log_files
def _get_table(self):
if(self.table_name in self.conn.list_tables()['TableNames']):
self.table = Table(self.table_name,
connection = self.conn)
else:
raise Exception("Do create table before other operations")
def create_table(self):
try:
self.table = Table.create(self.table_name,
schema = [
HashKey('PartitionID', data_type=NUMBER),
RangeKey('FileName', data_type = STRING),
], throughput = {
'read' : 50,
'write': 30,
},
connection = self.conn
)
print "Create " + self.TABLE_NAME + " succeed\n"
except Exception as e:
print e
print "Create " + self.TABLE_NAME + " failed\n"
def insert_records(self, numberOfRecords, record_size):
if not record_size:
record_size = self.record_size
for i in range(0, numberOfRecords):
record = RecordInfo(record_size, i)
item_data = record.get_record_info()
self.table.put_item(item_data)
print "done"
def get(self, range_key):
key = self.util.get_PartitionID(range_key)
record = self.table.get_item(PartitionID=key, FileName=range_key)
print 'FileName:' + record['FileName'] + "\tSize:" + str(record["Size"])
def range_get(self, key, ranges):
# required to provide hash key and range key for range search
records = self.table.query_2(PartitionID__eq=int(key), FileName__beginswith=ranges)
count = 0
for record in records:
print 'FileName:' + record['FileName'] + "\tSize:" + str(record["Size"])
count += 1
print "# Records got: " + str(count)
def update(self, range_key, record_size):
key = self.util.get_PartitionID(range_key)
if not record_size:
record_size = self.record_size
new_item = RecordInfo(record_size, 0)
record = self.table.get_item(PartitionID=key, FileName=range_key)
record["Size"] = new_item.get_record_info()['Size']
record["Data"] = new_item.get_record_info()['Data']
# new_item.set_PartitionID(key)
# new_item.set_FileName(range_key)
# new_record = new_item.get_record_info()
# record = Item(self.table, data=new_record)
if record.partial_save():
print "Done"
def scan(self, record_size):
records = self.table.scan(Size__eq=record_size, )
count = 0
for record in records:
print 'FileName:' + record['FileName'] + "\tSize:" + str(record["Size"])
count += 1
print "# Records got: " + str(count)
def batch_insert_records(self, number_of_records, record_size):
if record_size == 0:
record_size = self.record_size
with BatchTable(self.table) as batch:
for i in range(0, number_of_records):
record = RecordInfo(record_size, i)
item_data = record.get_record_info()
batch.put_item(data = item_data)
print "done"
def multithread_insert(self, number_of_records):
threads = []
# Balance tasks; make sure each task at least has min number tasks except the last one
if ((self.MIN_NUMBER_TASK_PER_THERAD * self.MAX_THREAD_COUNT) > number_of_records):
number_of_task_per_thread = self.MIN_NUMBER_TASK_PER_THERAD
else:
number_of_task_per_thread = number_of_records / self.MAX_THREAD_COUNT
i = 0
while i < number_of_records:
inserting_task = None
if ((number_of_records - i) > self.MIN_NUMBER_TASK_PER_THERAD):
inserting_task = Thread(target = self.batch_insert_records, args = (number_of_task_per_thread, ))
else:
inserting_task = Thread(target = self.batch_insert_records, args = (number_of_records - i, ))
threads.append(inserting_task)
inserting_task.start()
i += number_of_task_per_thread
for thread in threads:
thread.join()
print "Successed insert " + str(number_of_records) + " items"
def list_records(self):
files = self._scan_records()
for record in files:
print record['FileName']
def count_records(self):
count = 0
kwargs = {}
kwargs['select'] = 'COUNT'
raw_results = self.table.connection.scan(
self.table_name,
**kwargs
)
count += raw_results.get('Count', int)
last_key_seen = raw_results.get('LastEvaluatedKey', None)
while last_key_seen:
kwargs['exclusive_start_key'] = last_key_seen
raw_results = self.table.connection.scan(
self.table_name,
**kwargs
)
count += raw_results.get('Count', int)
last_key_seen = raw_results.get('LastEvaluatedKey', None)
print "total number of records: " + str(count)
def count_one_page(self):
kwargs = {}
kwargs['select'] = 'COUNT'
raw_results = self.table.connection.scan(
self.table.table_name,
**kwargs
)
count = raw_results.get('Count', int)
print "Number of records in on page: " + str(count)
def delete_records(self):
records = self._scan_records()
count = 0
for record in records:
record.delete()
count += 1
print "Delete " + str(count) + " items\n"
def batch_delete_records(self):
records = self._scan_records()
count = 0
with BatchTable(self.table) as batch:
for record in records:
batch.delete_item(
PartitionID = record["PartitionID"],
FileName = record["FileName"]
)
count += 1
print "Delete " + str(count) + " items\n"
# Used by packageDao
def pkg_delete_helper(self, pkg):
count = 0
with BatchTable(self.table) as batch:
for idx in pkg["logFilePartitionIndexSet"]:
log_files = self._query_pkg_idx(pkg["pkgId"], idx)
for log in log_files:
batch.delete_item(
partitionIndex = log["partitionIndex"],
logFileUrl = log["logFileUrl"]
)
count += 1
print "Delete " + str(count) + " logs in pkg:" + pkg["pkgId"] + "\n"
def delete_table(self):
self.table.delete()
assert value[0] == check_dict[key]
assert value[1] == 200
else :
print "Key %s is not in check_dict" % key
# A little consistency check to make sure that the database and the dictionary
# really are in sync.
value = get ( key )
assert value[0] == None
assert value[1] == 403
##########################################################
check_dict = {} # This dictionary is used to verify that the database
# is working correctly
if DEBUG :
print "Deleting the table %s" % TABLE_NAME
kv_pairs.delete()
# now that the table is gone, recreate it
while True :
time.sleep(10) # it takes some time for the table to delete
try:
kv_pairs = Table.create(TABLE_NAME, schema=[ HashKey('key')],
throughput={ 'read': 5, 'write': 15, })
except JSONResponseError:
print "The table %s still isn't deleted.... waiting" % TABLE_NAME
else:
break
print "Created table %s" % TABLE_NAME
time.sleep(10)
while True:
try:
test_post("Dillon", 17)
name = i['Instance Name']
print "Deleteing %s" % name
#s = name.split('_')
#if 'Instance' in name: # Standard
# itype = s[0].lower() + '_' + s[1].lower()
#else: # Performance
# itype = s[0] + "gb_" + s[2].lower()
#virt = 'paravirtual'
#new_name = itype + '_' + virt
table_struct = None
try:
logs = Table(name, schema=[HashKey('trial'),RangeKey('parallel'),])
table_struct = logs.describe()
logs.delete()
except JSONResponseError:
print "%s not existing" % name
sleep(5)
#table_struct = None
#try:
# new_logs = Table(new_name, schema=[HashKey('trial'),RangeKey('parallel'),])
# table_struct = new_logs.describe()
#except JSONResponseError:
# new_logs = Table.create(new_name, schema=[HashKey('trial'),RangeKey('parallel'),])
# sleep(5)
#while table_struct is None:
# try:
# new_logs = Table(new_name, schema=[HashKey('trial'),RangeKey('parallel'),])
# table_struct = new_logs.describe()
def delete_table(self, name):
try:
table = Table(name, connection=self.conn)
table.delete()
except Exception as e:
print 'Exception deleting table %s [%s]\n' % (name, e)
from boto.dynamodb2.fields import HashKey
from boto.dynamodb2.table import Table
table = Table('video')
table.delete()
def drop(self, name):
table = Table(name, connection=self.conn)
sys.stderr.write('drop=%s\n'%table)
table.delete()
return
class TestSet(unittest.TestCase):
def setUp(self):
# Configuration options
self.table_name = 'conf'
self.store_name = 'test'
# Instanciate the store
self.store = DynamoDBConfigStore(
connection,
self.table_name,
self.store_name)
# Get an Table instance for validation
self.table = Table(self.table_name, connection=connection)
def test_set(self):
""" Test that we can insert an object """
obj = {
'host': '127.0.0.1',
'port': 27017
}
# Insert the object
self.store.set('db', obj)
# Fetch the object directly from DynamoDB
kwargs = {
'_store': self.store_name,
'_option': 'db'
}
item = self.table.get_item(**kwargs)
self.assertEqual(item['_store'], self.store_name)
self.assertEqual(item['_option'], 'db')
self.assertEqual(item['host'], '127.0.0.1')
self.assertEqual(item['port'], 27017)
def test_update(self):
""" Test that we can change values in an option """
obj = {
'username': '******',
'password': '******'
}
# Insert the object
self.store.set('user', obj)
# Get the option
option = self.store.config.get('user')
self.assertEqual(option['username'], obj['username'])
self.assertEqual(option['password'], obj['password'])
# Updated version of the object
updatedObj = {
'username': '******',
'password': '******'
}
# Insert the object
self.store.set('user', updatedObj)
# Get the option
option = self.store.config.get('user')
self.assertEqual(option['username'], updatedObj['username'])
self.assertEqual(option['password'], updatedObj['password'])
def test_update_with_new_keys(self):
""" Test that we can completely change the keys """
obj = {
'username': '******',
'password': '******'
}
# Insert the object
self.store.set('credentials', obj)
# Get the option
option = self.store.config.get('credentials')
self.assertEqual(option['username'], obj['username'])
self.assertEqual(option['password'], obj['password'])
# Updated version of the object
updatedObj = {
'access_key': 'anakin',
'secret_key': 'skywalker'
}
# Insert the object
self.store.set('credentials', updatedObj)
# Get the option
option = self.store.config.get('credentials')
self.assertEqual(option['access_key'], updatedObj['access_key'])
self.assertEqual(option['secret_key'], updatedObj['secret_key'])
self.assertNotIn('username', option)
self.assertNotIn('password', option)
def test_instert_too_large_object(self):
""" Test of inserting an object larger than 64 kb """
with self.assertRaises(ValidationException):
self.store.set(
'large',
{x: int(random()*100000000000000) for x in xrange(1, 9999)})
def tearDown(self):
""" Tear down the test case """
self.table.delete()
class TestSet(unittest.TestCase):
def setUp(self):
# Configuration options
self.table_name = 'conf'
self.store_name = 'test'
# Instanciate the store
self.store = DynamoDBConfigStore(connection, self.table_name,
self.store_name)
# Get an Table instance for validation
self.table = Table(self.table_name, connection=connection)
def test_set(self):
""" Test that we can insert an object """
obj = {'host': '127.0.0.1', 'port': 27017}
# Insert the object
self.store.set('db', obj)
# Fetch the object directly from DynamoDB
kwargs = {'_store': self.store_name, '_option': 'db'}
item = self.table.get_item(**kwargs)
self.assertEqual(item['_store'], self.store_name)
self.assertEqual(item['_option'], 'db')
self.assertEqual(item['host'], '127.0.0.1')
self.assertEqual(item['port'], 27017)
def test_update(self):
""" Test that we can change values in an option """
obj = {'username': '******', 'password': '******'}
# Insert the object
self.store.set('user', obj)
# Get the option
option = self.store.config.get('user')
self.assertEqual(option['username'], obj['username'])
self.assertEqual(option['password'], obj['password'])
# Updated version of the object
updatedObj = {'username': '******', 'password': '******'}
# Insert the object
self.store.set('user', updatedObj)
# Get the option
option = self.store.config.get('user')
self.assertEqual(option['username'], updatedObj['username'])
self.assertEqual(option['password'], updatedObj['password'])
def test_update_with_new_keys(self):
""" Test that we can completely change the keys """
obj = {'username': '******', 'password': '******'}
# Insert the object
self.store.set('credentials', obj)
# Get the option
option = self.store.config.get('credentials')
self.assertEqual(option['username'], obj['username'])
self.assertEqual(option['password'], obj['password'])
# Updated version of the object
updatedObj = {'access_key': 'anakin', 'secret_key': 'skywalker'}
# Insert the object
self.store.set('credentials', updatedObj)
# Get the option
option = self.store.config.get('credentials')
self.assertEqual(option['access_key'], updatedObj['access_key'])
self.assertEqual(option['secret_key'], updatedObj['secret_key'])
self.assertNotIn('username', option)
self.assertNotIn('password', option)
def test_instert_too_large_object(self):
""" Test of inserting an object larger than 64 kb """
with self.assertRaises(ValidationException):
self.store.set(
'large',
{x: int(random() * 100000000000000)
for x in xrange(1, 9999)})
def tearDown(self):
""" Tear down the test case """
self.table.delete()
class DBTable(object):
item_class = None
simple_schema = [HashKey('_id')]
def __init__(self, table_name, schema, global_indexes, dbconn):
self.table_name = table_name
self._table = Table(table_name, connection=dbconn.conn)
self.schema = schema
self.global_indexes = global_indexes
self.dbconn = dbconn.conn
return self
def get_table(self):
return self._table
def exists(self):
return self._table.table_name in self.dbconn.list_tables(
)['TableNames']
def create(self):
try:
# print self.schema
# print self.table_name
self._table = Table.create(self.table_name,
schema=self.schema,
global_indexes=self.global_indexes,
connection=self.dbconn)
except JSONResponseError as e:
logging.warn("Error creating table " + self.table_name)
log_boto_error(e)
if 'message' in e.body and e.body['message'].startswith(
'Table already exists'):
self._table = Table(self.table_name, connection=self.dbconn)
elif 'Message' in e.body and e.body['Message'].startswith(
"Cannot create preexisting"):
self._table = Table(self.table_name, connection=self.dbconn)
else:
raise e
return self
def delete(self):
try:
self._table.delete()
except JSONResponseError as e:
logging.warn("Error deleting table " + self.table_name)
log_boto_error(e)
if e.body['Message'].startswith(
'Cannot do operations on a non-existent tabl'):
return
else:
raise e
return
def insert(self, data):
self._table.put_item(data=data, overwrite=True)
return
def get_item(self, **kwargs):
item = self._table.get_item(**kwargs)
# TODO return item or class??
# new_item = self.item_class(self)
# new_item.__dict__ = item._data
# return new_item
return item
def query_2(self, **kwargs):
return self._table.query_2(**kwargs)
def scan(self, **kwargs):
return self._table.scan(**kwargs)
def remove(self):
return self._table.remove()
class StateDynamo(object):
def __init__(self, region, prefix = 'cicada_'):
self.prefix = prefix
self.connection= boto.dynamodb2.connect_to_region(region)
self.project_job = Table(prefix + 'project_job', connection=self.connection, **dynamo_schema['project_job'])
self.job_history = Table(prefix + 'job_history', connection=self.connection, **dynamo_schema['job_history'])
self.worker = Table(prefix + 'worker', connection=self.connection, **dynamo_schema['workers'])
self.poll = Table(prefix + 'poll', connection=self.connection, **dynamo_schema['poll'])
def tables_create(self):
"""
Creates a new table.. throws exception if tables already exist
"""
for table_name, schema in dynamo_schema.iteritems():
logging.info("Creating " + self.prefix + table_name)
Table.create(self.prefix + table_name, **schema)
def tables_destroy(self):
"""
"""
self.project_job.delete()
self.job_history.delete()
self.worker.delete()
self.poll.delete()
def tables_list(self):
print self.connection.list_tables()['TableNames']
def workers_list_all(self):
return self.worker.scan()
def workers_put(self, workerx, project, job, started):
return self.worker.put_item(data={
worker: workerx,
job:"{0}.{1}".format(project, job),
started:started}, overwrite=True)
def projectjobs_list_all(self):
"""
dumps all project/job states.
"""
return self.project_job.scan()
def projectjobs_list(self, projectname):
"""
dumps all states for a given project
"""
return self.project_job.scan(project__eq=projectname)
def projectjobs_put(self, project, job, started, state, updated):
"""
Designed for fast lookups of all projects.
hash/range = (project, job)
"""
self.project_job.put_item(data={
'project': project,
'job': job,
'started': started,
'state': state,
'updated': updated
}, overwrite=True)
def jobhistory_put(self, project, job, started, state, updated):
"""
combines project.job into one field for looking up history
of a particular job.
"""
self.job_history.put_item(data={
'projectjob': '{0}.{1}'.format(project, job),
'started': started,
'state': state,
'updated': updated
}, overwrite=True)
def poll_list_all(self):
polls = self.poll.scan()
return polls
def poll_put(self, name, current, now):
return self.poll.put_item(data={
'name': name,
'current': current,
'now': now
}, overwrite=True)
def poll_get(self, name):
lastpoll = self.poll.get_item(name=name)
if lastpoll is None:
return None
else:
return lastpoll['current']
