def test_1_basic(self):
print '--- running SDBConnection tests ---'
c = SDBConnection()
rs = c.get_all_domains()
num_domains = len(rs)
# try illegal name
try:
domain = c.create_domain('bad:domain:name')
except SDBResponseError:
pass
# now create one that should work and should be unique (i.e. a new one)
domain_name = 'test%d' % int(time.time())
domain = c.create_domain(domain_name)
rs = c.get_all_domains()
assert len(rs) == num_domains + 1
# now let's a couple of items and attributes
item_1 = 'item1'
same_value = 'same_value'
attrs_1 = {'name1': same_value, 'name2': 'diff_value_1'}
domain.put_attributes(item_1, attrs_1)
item_2 = 'item2'
attrs_2 = {'name1': same_value, 'name2': 'diff_value_2'}
domain.put_attributes(item_2, attrs_2)
time.sleep(10)
# try to get the attributes and see if they match
item = domain.get_attributes(item_1)
assert len(item.keys()) == len(attrs_1.keys())
assert item['name1'] == attrs_1['name1']
assert item['name2'] == attrs_1['name2']
# try a search or two
rs = domain.query("['name1'='%s']" % same_value)
n = 0
for item in rs:
n += 1
assert n == 2
rs = domain.query("['name2'='diff_value_2']")
n = 0
for item in rs:
n += 1
assert n == 1
# delete all attributes associated with item_1
stat = domain.delete_attributes(item_1)
assert stat
# now delete the domain
stat = c.delete_domain(domain)
assert stat
print '--- tests completed ---'
def test_1_basic(self):
print '--- running SDBConnection tests ---'
c = SDBConnection()
rs = c.get_all_domains()
num_domains = len(rs)
# try illegal name
try:
domain = c.create_domain('bad:domain:name')
except SDBResponseError:
pass
# now create one that should work and should be unique (i.e. a new one)
domain_name = 'test%d' % int(time.time())
domain = c.create_domain(domain_name)
rs = c.get_all_domains()
assert len(rs) == num_domains+1
# now let's a couple of items and attributes
item_1 = 'item1'
same_value = 'same_value'
attrs_1 = {'name1' : same_value, 'name2' : 'diff_value_1'}
domain.put_attributes(item_1, attrs_1)
item_2 = 'item2'
attrs_2 = {'name1' : same_value, 'name2' : 'diff_value_2'}
domain.put_attributes(item_2, attrs_2)
time.sleep(10)
# try to get the attributes and see if they match
item = domain.get_attributes(item_1)
assert len(item.keys()) == len(attrs_1.keys())
assert item['name1'] == attrs_1['name1']
assert item['name2'] == attrs_1['name2']
# try a search or two
rs = domain.query("['name1'='%s']" % same_value)
n = 0
for item in rs:
n += 1
assert n == 2
rs = domain.query("['name2'='diff_value_2']")
n = 0
for item in rs:
n += 1
assert n == 1
# delete all attributes associated with item_1
stat = domain.delete_attributes(item_1)
assert stat
# now delete the domain
stat = c.delete_domain(domain)
assert stat
print '--- tests completed ---'
def __init__(self, key, access, cluster):
try:
url = "http://169.254.169.254/latest/meta-data/"
public_hostname = urlopen(url + "public-hostname").read()
zone = urlopen(url + "placement/availability-zone").read()
region = zone[:-1]
except:
sys.exit("We should be getting user-data here...")
#us-east-1 breaks the convention. See http://docs.amazonwebservices.com/general/latest/gr/rande.html#sdb_region
endpoint = "sdb.{0}.amazonaws.com".format(region) if region != "us-east-1" \
else "sdb.amazonaws.com"
region_info = RegionInfo(name=region, endpoint=endpoint)
sdb = SDBConnection(key, access, region=region_info)
self.domain = sdb.create_domain(cluster)
self.metadata = self.domain.get_item('metadata', consistent_read=True)
if None == self.metadata:
self.metadata = self.domain.new_item('metadata')
self.metadata.add_value('master', '')
self.metadata.add_value('slave', '')
self.metadata.save()
def get_latest_snapshot(key, access, name):
sdb = SDBConnection(key, access, region=region_info)
domain = sdb.lookup(name, True)
if domain == None:
domain = sdb.create_domain(name)
return domain.get_item('snapshot', True)['snapshot']
def delete_snapshot(key, access, name, snapshot_id): sdb = SDBConnection(key, access, region=region_info) domain = sdb.lookup(name, True) if domain == None: domain = sdb.create_domain(name) return domain.delete_item(domain.get_item(snapshot_id))
def get_latest_snapshot(key, access, name):
sdb = SDBConnection(key, access, region=region_info)
domain = sdb.lookup(name, True)
if domain == None:
domain = sdb.create_domain(name)
return domain.get_item('snapshot', True)['snapshot']
def delete_snapshot(key, access, name, snapshot_id):
sdb = SDBConnection(key, access, region=region_info)
domain = sdb.lookup(name, True)
if domain == None:
domain = sdb.create_domain(name)
return domain.delete_item(domain.get_item(snapshot_id))
def get_all_snapshots(key, access, cluster):
sdb = SDBConnection(key, access, region=region_info)
domain = sdb.create_domain(cluster)
now = strftime("%Y-%m-%d %H:%M:%S", gmtime())
select = "select * from `{0}` where itemName() > 'snap-' and itemName() != 'snapshot'".format(cluster)
snapshots = domain.select(select, consistent_read=True)
return snapshots
def get_latest_snapshot(key, access, cluster):
sdb = SDBConnection(key, access, region=region_info)
domain = sdb.create_domain(cluster)
try:
return domain.get_item('snapshot', True)['snapshot']
except:
return None
def set_RDB(key, access, cluster, location):
sdb = SDBConnection(key, access, region=region_info)
domain = sdb.create_domain(cluster)
# add the latest rdb (for automatic restores)
latest = domain.new_item('rdb')
latest.add_value('rdb', location)
# get the expiration date from the object name (for comparison)
latest.add_value('created', strftime("%Y-%m-%d %H:%M:%S", gmtime()))
latest.save()
def SDB(domain_name):
"""
Create connection to SimpleDB and return the specifed domain.
domain_name - the SimpleDB domain you wish to return
"""
conn = SDBConnection(ACCESS_KEY_ID,SECRET_ACCESS_KEY)
domain = conn.lookup(domain_name)
if not domain:
domain = conn.create_domain(domain_name)
return domain
def get_expired_snapshots(key, access, name):
sdb = SDBConnection(key, access, region=region_info)
domain = sdb.lookup(name, True)
if domain == None:
domain = sdb.create_domain(name)
now = strftime("%Y-%m-%d %H:%M:%S", gmtime())
select = "select * from `{0}` where itemName() > 'snap-' and itemName() != 'snapshot' and expires < '{1}'".format(name, now)
print select
snapshots = domain.select(select, consistent_read=True)
return snapshots
def get_expired_snapshots(key, access, cluster):
sdb = SDBConnection(key, access, region=region_info)
domain = sdb.lookup(cluster, True)
if domain == None:
domain = sdb.create_domain(cluster)
now = strftime("%Y-%m-%d %H:%M:%S", gmtime())
select = "select * from `{0}` where itemName() > 'snap-' and itemName() != 'snapshot' and expires < '{1}'".format(
cluster, now)
snapshots = domain.select(select, consistent_read=True)
return snapshots
def get_all_snapshots(key, access, name):
sdb = SDBConnection(key, access, region=region_info)
domain = sdb.lookup(name, True)
if domain == None:
domain = sdb.create_domain(name)
now = strftime("%Y-%m-%d %H:%M:%S", gmtime())
select = "select * from `{0}` where itemName() > 'snap-' and itemName() != 'snapshot'".format(
name)
snapshots = domain.select(select, consistent_read=True)
return snapshots
def get_identity(key, access, cluster):
sdb = SDBConnection(key, access, region=region_info)
domain = sdb.create_domain(cluster)
slave_id = hashlib.md5(public_hostname).hexdigest()[:8]
slave_fqdn = "{0}.{1}".format(slave_id, cluster)
slave = domain.new_item(slave_fqdn)
slave.add_value('id', slave_id)
slave.add_value('endpoint', public_hostname)
slave.save()
return slave_fqdn
class SimpleDBBackend:
def __init__(self):
self.connection = SDBConnection(aws_credentials.accessKey,
aws_credentials.secretKey)
self.domain = self.connection.get_domain(TABLE_NAME)
def put(self, key, value):
sampler.begin()
try:
self.domain.put_attributes(key, {VALUE:value})
finally:
sampler.end()
def get(self, key):
sampler.begin()
try:
# First try an eventually consistent read.
result = self.domain.get_attributes(key, consistent_read=False)
return result[VALUE]
except KeyError:
# The eventually consistent read failed. Try a strongly consistent
# read.
result = self.domain.get_attributes(key, consistent_read=True)
return result[VALUE]
finally:
sampler.end()
def incRefCount(self, key):
# Not implemented.
pass
def decRefCount(self, key):
# Not implemented.
pass
def nuke(self):
# Delete and re-create the table.
self.connection.delete_domain(TABLE_NAME)
self.domain = self.connection.create_domain(TABLE_NAME)
def flush(self):
pass # No-op.
def __init__(self, key, access, cluster):
try:
url = "http://169.254.169.254/latest/meta-data/"
zone = urlopen(url + "placement/availability-zone").read()
region = zone[:-1]
except:
sys.exit("We should be getting user-data here...")
endpoint = "sdb.{0}.amazonaws.com".format(region)
region_info = RegionInfo(name=region, endpoint=endpoint)
sdb = SDBConnection(key, access, region=region_info)
self.domain = sdb.create_domain(cluster)
self.metadata = self.domain.get_item('metadata', consistent_read=True)
def add_snapshot(key, access, cluster, name, snapshot):
sdb = SDBConnection(key, access, region=region_info)
domain = sdb.lookup(cluster, True)
if domain == None:
domain = sdb.create_domain(cluster)
now = strftime("%Y-%m-%d %H:%M:%S", gmtime())
# add the snapshot for expiration
backup = domain.new_item(snapshot[0])
backup.add_value("name", name)
backup.add_value("snapshot", snapshot[0])
backup.add_value("created", now)
backup.add_value("expires", snapshot[1])
backup.save()
def add_snapshot(key, access, cluster, name, snapshot):
sdb = SDBConnection(key, access, region=region_info)
domain = sdb.lookup(cluster, True)
if domain == None:
domain = sdb.create_domain(cluster)
now = strftime("%Y-%m-%d %H:%M:%S", gmtime())
# add the snapshot for expiration
backup = domain.new_item(snapshot[0])
backup.add_value('name', name)
backup.add_value('snapshot', snapshot[0])
backup.add_value('created', now)
backup.add_value('expires', snapshot[1])
backup.save()
def get_latest_snapshot(key, access, cluster, name):
sdb = SDBConnection(key, access, region=region_info)
now = strftime("%Y-%m-%d %H:%M:%S", gmtime())
domain = sdb.lookup(cluster, True)
if domain == None:
domain = sdb.create_domain(cluster)
select = "select * from `{0}` where name = '{1}' and created < '{2}' order by created desc limit 1".format(
cluster, name, now)
snapshots = domain.select(select, consistent_read=True)
try:
snapshot = snapshots.next()
return snapshot['snapshot']
except:
return None
def set_cluster_metadata(key, access, cluster):
sdb = SDBConnection(key, access, region=region_info)
domain = sdb.create_domain(cluster)
# set the basic values in the 'master' record
metadata = domain.new_item('metadata')
#master = "{0}.{1}".format(
# os.environ['REDIS_NAME'].strip(),
# os.environ['HOSTED_ZONE_NAME'].rstrip('.'))
#metadata.add_value('master', master)
try:
if "" != os.environ['REDIS_SIZE'].strip():
metadata.add_value('size', os.environ['REDIS_SIZE'])
except:
pass
try:
if "" != os.environ['REDIS_PERSISTENCE'].strip():
metadata.add_value('persistence', os.environ['REDIS_PERSISTENCE'])
else:
metadata.add_value('persistence', 'no')
except:
metadata.add_value('persistence', 'no')
try:
if "" != os.environ['REDIS_SNAPSHOT'].strip():
metadata.add_value('snapshot', os.environ['REDIS_SNAPSHOT'])
except:
pass
try:
if "" != os.environ['REDIS_RDB'].strip():
metadata.add_value('rdb', os.environ['REDIS_RDB'])
except:
pass
try:
if "" != os.environ['REDIS_AOF'].strip():
metadata.add_value('aof', os.environ['REDIS_AOF'])
except:
pass
metadata.save()
def get_latest_snapshot(key, access, cluster, name):
sdb = SDBConnection(key, access, region=region_info)
now = strftime("%Y-%m-%d %H:%M:%S", gmtime())
domain = sdb.lookup(cluster, True)
if domain == None:
domain = sdb.create_domain(cluster)
select = "select * from `{0}` where name = '{1}' and created < '{2}' order by created desc limit 1".format(
cluster, name, now
)
snapshots = domain.select(select, consistent_read=True)
try:
snapshot = snapshots.next()
return snapshot["snapshot"]
except:
return None
def add_snapshot(key, access, name, snapshot):
sdb = SDBConnection(key, access, region=region_info)
domain = sdb.lookup(name, True)
if domain == None:
domain = sdb.create_domain(name)
now = strftime("%Y-%m-%d %H:%M:%S", gmtime())
# add the snapshot for expiration
backup = domain.new_item(snapshot[0])
backup.add_value('snapshot', snapshot[0])
backup.add_value('created', now)
backup.add_value('expires', snapshot[1])
backup.save()
# add the latest (for automatic restores)
latest = domain.new_item('snapshot')
latest.add_value('snapshot', snapshot[0])
latest.save()
class statWriter(object):
def __init__(self,domain=None):
self.conn = SDBConnection(keys.aws.AWS_ACCESS_KEY_ID, keys.aws.AWS_SECRET_KEY)
self.domain_name = domain
def write(self,stat,key=None,domain=None):
if domain is None:
if self.domain_name is None:
return False
else:
domain = self.domain_name
if key is None:
key = uuid.uuid1()
if len(stat) > 0:
try:
domain = self.conn.get_domain(domain)
except SDBResponseError:
domain = self.conn.create_domain(domain)
domain.put_attributes(key, stat, replace=False)
return True
def add_snapshot(key, access, cluster, snapshot):
sdb = SDBConnection(key, access, region=region_info)
domain = sdb.create_domain(cluster)
now = strftime("%Y-%m-%d %H:%M:%S", gmtime())
# add the latest rdb (for automatic restores)
latest = domain.new_item('snapshot')
latest.add_value('snapshot', snapshot[0])
# get the expiration date from the object name (for comparison)
latest.add_value('created', now)
latest.save()
# add the snapshot for expiration
backup = domain.new_item(snapshot[0])
backup.add_value('snapshot', snapshot[0])
backup.add_value('created', now)
backup.add_value('expires', snapshot[1])
backup.save()
# add the latest (for automatic restores)
latest = domain.new_item('snapshot')
latest.add_value('snapshot', snapshot[0])
latest.save()
def get_cluster_metadata(key, access, cluster):
sdb = SDBConnection(key, access, region=region_info)
domain = sdb.create_domain(cluster)
return domain.get_item('metadata', True)
def test_1_basic(self):
print '--- running SDBConnection tests ---'
c = SDBConnection()
rs = c.get_all_domains()
num_domains = len(rs)
# try illegal name
try:
domain = c.create_domain('bad:domain:name')
except SDBResponseError:
pass
# now create one that should work and should be unique (i.e. a new one)
domain_name = 'test%d' % int(time.time())
domain = c.create_domain(domain_name)
rs = c.get_all_domains()
assert len(rs) == num_domains + 1
# now let's a couple of items and attributes
item_1 = 'item1'
same_value = 'same_value'
attrs_1 = {'name1' : same_value, 'name2' : 'diff_value_1'}
domain.put_attributes(item_1, attrs_1)
item_2 = 'item2'
attrs_2 = {'name1' : same_value, 'name2' : 'diff_value_2'}
domain.put_attributes(item_2, attrs_2)
# try to get the attributes and see if they match
item = domain.get_attributes(item_1, consistent_read=True)
assert len(item.keys()) == len(attrs_1.keys())
assert item['name1'] == attrs_1['name1']
assert item['name2'] == attrs_1['name2']
# try a search or two
query = 'select * from %s where name1="%s"' % (domain_name, same_value)
rs = domain.select(query, consistent_read=True)
n = 0
for item in rs:
n += 1
assert n == 2
query = 'select * from %s where name2="diff_value_2"' % domain_name
rs = domain.select(query, consistent_read=True)
n = 0
for item in rs:
n += 1
assert n == 1
# delete all attributes associated with item_1
stat = domain.delete_attributes(item_1)
assert stat
# now try a batch put operation on the domain
item3 = {'name3_1' : 'value3_1',
'name3_2' : 'value3_2',
'name3_3' : ['value3_3_1', 'value3_3_2']}
item4 = {'name4_1' : 'value4_1',
'name4_2' : ['value4_2_1', 'value4_2_2'],
'name4_3' : 'value4_3'}
items = {'item3' : item3, 'item4' : item4}
domain.batch_put_attributes(items)
item = domain.get_attributes('item3', consistent_read=True)
assert item['name3_2'] == 'value3_2'
# now try a batch delete operation (variation #1)
items = {'item3' : item3}
stat = domain.batch_delete_attributes(items)
item = domain.get_attributes('item3', consistent_read=True)
assert not item
# now try a batch delete operation (variation #2)
stat = domain.batch_delete_attributes({'item4' : None})
item = domain.get_attributes('item4', consistent_read=True)
assert not item
# now delete the domain
stat = c.delete_domain(domain)
assert stat
print '--- tests completed ---'
class SimpleDB(object):
def __init__(self, domain=None):
self.conn = SDBConnection(keys.aws.AWS_ACCESS_KEY_ID, keys.aws.AWS_SECRET_KEY)
self.domain_name = None
self.domains = {}
if domain is None and is_ec2():
stack = get_stack()
stack_name = str(stack['instance']['stack'])
self.domain_name = stack_name
elif domain is not None:
self.domain_name = domain
def addStat(self, stat):
if self.domain_name is None:
return
try:
# Only add specific parameters
data = {}
if 'user_id' in stat:
data['uid'] = stat['user_id']
if 'path' in stat:
data['uri'] = stat['path']
if 'method' in stat:
data['mtd'] = stat['method']
if 'form' in stat:
try:
for k, v in stat['form'].items():
try:
if not isinstance(v, basestring):
v = str(v)
if len(v.encode('utf-8')) > 1024:
v = '<INPUT TOO LONG>'
data['frm_%s' % k] = v
except Exception as e:
print e
except Exception as e:
print e
if 'result' in stat:
data['cde'] = str(stat['result'])
if 'begin' in stat:
data['bgn'] = stat['begin'].isoformat()
if 'finish' in stat:
data['end'] = stat['finish'].isoformat()
if 'node' in stat:
data['nde'] = stat['node']
if 'client_id' in stat:
data['cid'] = stat['client_id']
if 'duration' in stat:
data['dur'] = "{0:10d}".format(stat['duration'])
if len(data) > 0:
statId = str(ObjectId())
if data['uri'] != '/v1/ping.json' and data['uri'] != '/v1/temp/ping.json':
suffix = '0%s' % (sha1(statId).hexdigest()[0])
if suffix in self.domains:
domain = self.domains[suffix]
else:
try:
domain = self.conn.get_domain('%s_%s' % (self.domain_name, suffix))
except SDBResponseError:
domain = self.conn.create_domain('%s_%s' % (self.domain_name, suffix))
self.domains[suffix] = domain
domain.put_attributes(statId, data, replace=False)
except Exception as e:
print e
raise
def test_1_basic(self):
print '--- running SDBConnection tests ---'
c = SDBConnection()
rs = c.get_all_domains()
num_domains = len(rs)
# try illegal name
try:
domain = c.create_domain('bad:domain:name')
except SDBResponseError:
pass
# now create one that should work and should be unique (i.e. a new one)
domain_name = 'test%d' % int(time.time())
domain = c.create_domain(domain_name)
rs = c.get_all_domains()
assert len(rs) == num_domains+1
# now let's a couple of items and attributes
item_1 = 'item1'
same_value = 'same_value'
attrs_1 = {'name1' : same_value, 'name2' : 'diff_value_1'}
domain.put_attributes(item_1, attrs_1)
item_2 = 'item2'
attrs_2 = {'name1' : same_value, 'name2' : 'diff_value_2'}
domain.put_attributes(item_2, attrs_2)
time.sleep(10)
# try to get the attributes and see if they match
item = domain.get_attributes(item_1)
assert len(item.keys()) == len(attrs_1.keys())
assert item['name1'] == attrs_1['name1']
assert item['name2'] == attrs_1['name2']
# try a search or two
query = 'select * from %s where name1="%s"' % (domain_name, same_value)
rs = domain.select(query)
n = 0
for item in rs:
n += 1
assert n == 2
query = 'select * from %s where name2="diff_value_2"' % domain_name
rs = domain.select(query)
n = 0
for item in rs:
n += 1
assert n == 1
# delete all attributes associated with item_1
stat = domain.delete_attributes(item_1)
assert stat
# now try a batch put operation on the domain
item3 = {'name3_1' : 'value3_1',
'name3_2' : 'value3_2',
'name3_3' : ['value3_3_1', 'value3_3_2']}
item4 = {'name4_1' : 'value4_1',
'name4_2' : ['value4_2_1', 'value4_2_2'],
'name4_3' : 'value4_3'}
items = {'item3' : item3, 'item4' : item4}
domain.batch_put_attributes(items)
time.sleep(10)
item = domain.get_attributes('item3')
assert item['name3_2'] == 'value3_2'
# now delete the domain
stat = c.delete_domain(domain)
assert stat
print '--- tests completed ---'
def delete_snapshot(key, access, cluster, snapshot_id): sdb = SDBConnection(key, access, region=region_info) domain = sdb.create_domain(cluster) return domain.delete_item(domain.get_item(snapshot_id))
