def name_uni_delete_item(name, uni):
# name must be string
# item must be dict
try:
table = Table('AssignmentTwo', connection=client_dynamo)
table.delete_item(name=name, CUID=uni)
return True
except KeyboardInterrupt:
exit
except:
print "Error"
return False
def test_delete_item(self):
self.storage_mocker.StubOutWithMock(storage, 'delete_item')
storage.delete_item(
IgnoreArg(), IgnoreArg(),
expected_condition_map=IgnoreArg()).AndReturn(True)
self.storage_mocker.ReplayAll()
table = Table('test_table', connection=self.DYNAMODB_CON)
table.delete_item(hash_key=1, range_key="range")
self.storage_mocker.VerifyAll()
def test_delete_item(self):
self.storage_mocker.StubOutWithMock(storage, 'delete_item')
storage.delete_item(IgnoreArg(),
IgnoreArg(),
expected_condition_map=IgnoreArg()).AndReturn(True)
self.storage_mocker.ReplayAll()
table = Table('test_table', connection=self.DYNAMODB_CON)
table.delete_item(hash_key=1, range_key="range")
self.storage_mocker.VerifyAll()
def deleteStatus(self, reportname, datemodified):
""" Purpose: Construct the nzsql command
:param self: class object itself
:param reportname: id of the report for status insert
:param datemodified: exact range key value to delete particular record of the day...
"""
## Get report Id for report Name
report_id = self.getReportID(reportname.upper())
## Create table object for the dynamo DB table
tab = Table('TBL_AWS_REPORT_DTL', connection=self.conn)
##Check if element is there before delete to avoid exception.
tab.delete_item(REPORT_ID=report_id, Date_Modified=datemodified)
class KVStore(KVStoreBase):
def __init__(self):
super(KVStore, self).__init__()
region = settings.AWS_REGION_NAME
access_key = settings.AWS_ACCESS_KEY_ID
secret = settings.AWS_SECRET_ACCESS_KEY
conn = boto.dynamodb2.connect_to_region(region, aws_access_key_id=access_key, aws_secret_access_key=secret)
self.table = Table(settings.THUMBNAIL_DYNAMODB_NAME, connection=conn)
def _get_raw(self, key):
try:
return self.table.get_item(key=key)["value"]
except boto.dynamodb2.exceptions.ItemNotFound:
pass
def _set_raw(self, key, value):
try:
item = self.table.get_item(key=key)
except boto.dynamodb2.exceptions.ItemNotFound:
item = self.table.new_item()
item["key"] = key
item["value"] = value
item.save(overwrite=True)
def _delete_raw(self, *keys):
[self.table.delete_item(key=k) for k in keys]
def _find_keys_raw(self, prefix):
return [i["key"] for i in self.table.scan(key__beginswith=prefix)]
class KVStore(KVStoreBase):
def __init__(self):
super(KVStore, self).__init__()
region = settings.AWS_REGION_NAME
access_key = settings.AWS_ACCESS_KEY_ID
secret = settings.AWS_SECRET_ACCESS_KEY
conn = boto.dynamodb2.connect_to_region(region,
aws_access_key_id=access_key,
aws_secret_access_key=secret)
self.table = Table(settings.THUMBNAIL_DYNAMODB_NAME, connection=conn)
def _get_raw(self, key):
try:
return self.table.get_item(key=key)['value']
except boto.dynamodb2.exceptions.ItemNotFound:
pass
def _set_raw(self, key, value):
try:
item = self.table.get_item(key=key)
except boto.dynamodb2.exceptions.ItemNotFound:
item = self.table.new_item()
item['key'] = key
item['value'] = value
item.save(overwrite=True)
def _delete_raw(self, *keys):
[self.table.delete_item(key=k) for k in keys]
def _find_keys_raw(self, prefix):
return [i['key'] for i in self.table.scan(key__beginswith=prefix)]
def delete_minion(instanceid):
result = False
try:
minions = Table('minions')
result = minions.delete_item(instanceid=instanceid)
except Exception, e:
raise e
def delete_minion(instanceid):
result = False
try:
minions = Table("minions")
result = minions.delete_item(instanceid=instanceid)
except Exception, e:
raise e
class dynamoDB:
def __init__(self, db_name, partition_key_name):
self.table_dynamo = None
self.partition_key_name = partition_key_name
try:
self.table_dynamo = Table.create(db_name, schema=[HashKey(partition_key_name)], connection=client_dynamo)
print ("Wait 20 sec until the table is created")
time.sleep(20)
print ("New table created.")
except Exception as e:
self.table_dynamo = Table(db_name, connection=client_dynamo)
print ("Table already exists.")
def add(self, **kwargs):
try:
record = self.get(kwargs[self.partition_key_name])
for k,v in kwargs.items():
record[k] = v
record.save(overwrite=True)
#print("Record has been updated.\n")
except Exception as e:
self.table_dynamo.put_item(data=kwargs)
#print("New entry created.\n")
def delete(self, pk):
try:
record = self.table_dynamo.get_item(**{self.partition_key_name:pk})
self.table_dynamo.delete_item(**{self.partition_key_name:pk})
#print("The record has been deleted.")
return record
except Exception as e:
#print("Cannot delete the record, it does not exist.")
pass
return None
def get(self,pk):
try:
item = self.table_dynamo.get_item(**{self.partition_key_name:pk})
return item
except Exception as e:
#print("Cannot get the record, it does not exist.")
pass
return None
def scan(self,**filter_kwargs):
return self.table_dynamo.scan(**filter_kwargs)
def execute(self, args, config):
glacier_connection = boto.connect_glacier(aws_access_key_id=config.get('configuration', 'aws_key'), aws_secret_access_key=config.get('configuration', 'aws_secret'))
try:
vault = glacier_connection.get_vault(args[4])
except:
vault = None
if vault is None:
print "Vault named '{}' does not exist.".format(args[4])
else:
try:
vault.delete_archive(args[2])
dynamo_connection=DynamoDBConnection(aws_access_key_id=config.get('configuration', 'aws_key'), aws_secret_access_key=config.get('configuration', 'aws_secret'))
archive_table = Table(config.get('configuration', 'dynamodb_table'), connection=dynamo_connection)
archive_table.delete_item(archive_id=args[2])
print "Archive deleted: '{}'".format(args[2])
except UnexpectedHTTPResponseError as error:
print "Archive can not be deleted:\n\t {}".format(error)
FLAG_TABLE_NAME,
connection=conn
)
tokenTable = Table(
TOKEN_TABLE_NAME,
connection=conn
)
flags = flagTable.scan()
for flag in flags:
#Delete it
flagTable.delete_item(**{
'flag': flag['flag']
})
# now update all the users!
users = tokenTable.scan()
for user in users:
#Delete the user .. Eaakkk
tokenTable.delete_item(**{
'token': user['token']
})
if user['flags']:
i = 0
for userFlag in user['flags']:
target_table = command[3]
print "adding item"
add_item(new_item, target_table)
elif command[0] == "delete" and len(command) == 4:
# command[1].delete()
table = Table(command[3])
print "Sex:"
input_0 = raw_input('>>')
print "CUID:"
input_1 = raw_input('>>')
print "First_Name:"
input_2 = raw_input('>>')
print "Last_Name:"
input_3 = raw_input('>>')
table.delete_item(Sex=input_0,CUID=input_1,First_Name=input_2,Last_Nameinput_3)
print "run deleting function"
elif command[0] == "view" and len(command) == 2:
target_table = command[1]
view_table(target_table)
print "view data"
elif command[0] == 'exit':
print "exiting"
time.sleep(1)
sys.exit()
elif command[0] == 'search':
target_table = command[1]
target_index = command[3]
print "searching...."
else:
print("DELETE (Name) (CUID)")
print("SEARCH (Name or CUID)")
print("VIEW")
print("EXIT")
command_str = "NONE"
while command_str != "EXIT":
command_str = raw_input(">>>>")
cmd_keyword = command_str.partition(" ")[0]
if cmd_keyword == "ADD":
this_table.put_item(data={
'Name': command_str.split(" ")[1],
'CUID': command_str.split(" ")[2]
})
elif cmd_keyword == "DELETE":
this_table.delete_item(Name=command_str.split(" ")[1],
CUID=command_str.split(" ")[2])
elif cmd_keyword == "VIEW":
all_items = this_table.scan()
for item in all_items:
print(item['Name'] + " " + item['CUID'])
elif cmd_keyword == "SEARCH":
target_items_Name = this_table.query_2(
Name__eq=command_str.split(" ")[1])
target_items_CUID = []
all_items = this_table.scan()
for item in all_items:
if item["CUID"] == command_str.split(" ")[1]:
target_items_CUID.append(item)
if target_items_Name:
oldHashedFlag = flag['flag']
newSalt = "XXX";
#make a hash of it with the new salt
sha1 = hashlib.sha1()
sha1.update(oldHashedFlag + newSalt)
newHashedFlag = sha1.hexdigest()
print oldHashedFlag
print newHashedFlag
flagMap[oldHashedFlag] = newHashedFlag
#Delete it
flagTable.delete_item(**{
'flag': oldHashedFlag
})
#change it
flag['flag'] = newHashedFlag
#put it back
flagTable.put_item(flag)
print flagMap
# now update all the users!
users = tokenTable.scan()
for user in users:
class ClientRadiusSecretsDDB(RadiusSecrets):
"""
get secrets for a particular client
ddb=DynamoDBConnection(
aws_access_key_id=aws_access_key,
aws_secret_access_key=aws_secret_key
)
secrets = Table.create('qradius_secrets',schema=[\
HashKey('ip_address'),
RangeKey('not_before',data_type=NUMBER),
], indexes=[
AllIndex('IPNotAfter',parts=[
HashKey('ip_address'),
RangeKey('not_after',data_type=NUMBER),
])
],connection=ddb)
we will normally want secrets where
ip_address = client ip address
not_before < now
not_after >= now
likely we'll want to limit the # of secrets we look at, here i limit it to 3
For queries (these are the least impactful on dynamo:
res=secrets.query(ip_address__eq='10.25.95.158',not_before__lt=now,limit=3,consistent=False)
res=secrets.query(ip_address__eq='10.25.95.158',not_after__gt=now,limit=3,consistent=False,index='IPNotAfter')
if you really need everything, this is a full scan:
res=secrets.scan(ip_address__eq='10.25.95.158',not_before__lt=now,not_after__gt=now,limit=3)
"""
def __init__(self, encryption_key=None, aws_keys=None, table_name=None):
"""
@param encryption_key : string containing encryption key
@param aws_keys : object containing aws keys
@param table_name : dynamo table name
@type encryption_key string
@type aws_keys AWSKeys
@type table_name string
"""
if encryption_key is None:
raise ValueError("encryption_key must be specified")
if not isinstance(aws_keys,AWSKeys):
raise ValueError("aws_material must be specified and of type AWSKeys")
if table_name is None:
raise ValueError("dynamo table containing secrets must be specified")
self._encryption_key = encryption_key
self._encryptor = DataEncryptor(self._encryption_key)
self._ddb_connection = DynamoDBConnection(
aws_access_key_id=aws_keys.aws_access_key,
aws_secret_access_key=aws_keys.aws_secret_key
)
if self._ddb_connection is None:
raise ValueError("unable to obtain dynamo connection using %s" % aws_material)
self._secret_table = Table(table_name,connection=self._ddb_connection)
logging.debug('connectd to dynamo table %s' % table_name)
if self._secret_table is None:
raise ValueError("unable to connect to dynamo table %s" % table_name)
def encryptSecret(self, secret):
"""
call the included encryption module to encrypt/encode
secrets
@param secret
@type secret string
"""
if len(secret) > MAX_SECRET_LENGTH:
raise ValueError("secret may not be more than %d bytes" % MAX_SECRET_LENGTH)
encoded_secret = self._encryptor.encrypt(secret)
return(encoded_secret)
def decryptSecret(self, encoded_secret):
"""
call the included encryption module to decrypt/decode
secrets
@param encoded_secret
@type encoded_secret string
"""
plain_secret = self._encryptor.decrypt(encoded_secret)
if len(plain_secret) > MAX_SECRET_LENGTH:
raise ValueError("decryption resulted in a plain secret longer than the maximum length of %d bytes" % MAX_SECRET_LENGTH)
return(plain_secret)
def putSecret(self, clientIP, secret, not_before=None,not_after=None,tries=0):
"""
store a secret for clientIP
@param clientIP : client ip address
@param secret : radius secret for client
@param tries : internal parameter to constrain recursion depth for self-calls
@type clientIP string
@type secret
example usage:
from radiussecrets import *
rs=ClientRadiusSecrets(encryption_key='someencryptionkey',
aws_keys=AWSKeys('myaccesskey','mysecretkey'),table_name='qradius_secrets')
ValidationException
rs.putSecret('1.2.3.4','shhdonottellanyone')
"""
now = time.time()
if not_before is None:
not_before = now
if not_after is None:
not_after = now + DEFAULT_KEY_LIFE
if not isinstance(not_before,(int,float,long)) or not_before < 0:
raise ValueError("not_before must be a number representing seconds since epoch")
if not isinstance(not_after,(int,float,long)) or not_after < 0:
raise ValueError("not_before must be a number representing seconds since epoch")
if len(secret) > MAX_SECRET_LENGTH:
raise ValueError("length of secret may not exceed %d bytes" % MAX_SECRET_LENGTH)
result = None
try:
result = self._secret_table.put_item(data={
'ip_address': clientIP,
'not_before': not_before,
'not_after': not_after,
'secret': self.encryptSecret(secret)
})
except boto.dynamodb2.exceptions.ConditionalCheckFailedException as e:
tries += 1
if tries > 5:
logging.crit('pk violation for client %s not_before %d after %d tries at incrementing' % (clientIP,not_before,tries))
raise e
#increment not_before to avoid pk violation
not_before += 1
logging.warn('pk violation for client %s not_before %d; retrying with higher not_before ' % (clientIP,not_before))
result = self.putSecret(clientIP, secret, not_before=not_before,not_after=not_after,tries=tries)
return result
def deleteSecret(self, clientIP, not_before):
"""
delete a secret
this should be used carefully
@param clientIP
@param not_before
@type clientIP string
@type not_before number
"""
return self._secret_table.delete_item(ip_address=clientIP,not_before=not_before)
def getSecret(self, clientIP):
"""
return the secret associated with IP address
if multiple secrets are found, selection is by:
1) not_before < now
2) not_after >= now
3) the highest value of not before, if there are still multiple secrets
@param clientIP
@param not_before seconds since epoch that the secret becomes valid
@param not_after seconds since epoch after which the secret is not valid
@type clientIP string representing an ip address
"""
now = time.time()
#i wanted to limit to 3 (limit=3) but, boto kept barfing
results = self._secret_table.query(ip_address__eq=clientIP,not_before__lt=now,consistent=False)
client_secret = None
client_secret_not_before = 0
for result in results:
if result['not_after'] >= now:
if client_secret_not_before < result['not_before']:
client_secret_not_before = result['not_before']
client_secret = self.decryptSecret(result['secret'])
logging.debug('retrieved secret for %s' % (clientIP))
return client_secret
def purgeSecrets(self, clientIP):
"""
purge stale secrets associated with IP address
1) remove all keys for clientIP where not after is older than
current time - PURGE_TIME_BEFORE_NOW
2) scan remaining keys, keep
@param clientIP
@type string
@returns # of purged secrets
"""
now = time.time()
min_purge_time = now
nr_purged = 0
# first get rid of expired keys
results = self._secret_table.query(ip_address__eq=clientIP,
not_after__lt=min_purge_time,
consistent=False,
index='IPNotAfter')
for result in results:
logging.info('purging secret: %s %d' % (result['ip'],result['not_before']))
result.delete()
nr_purged += 1
# now the fun...
result_list = []
results = self._secret_table.query(ip_address__eq=clientIP,
not_before__lt=min_purge_time,
consistent=False)
for result in results:
result_list.append(result)
# delete results if there are more than PURGE_RETAIN_NR_ACTIVE results,
# we want the oldest not befores to be removed first
if len(result_list) > PURGE_RETAIN_NR_ACTIVE_KEYS:
for result in sorted(result_list, key=lambda result: result['not_before'])[:-PURGE_RETAIN_NR_ACTIVE_KEYS]:
logging.info('purging secret: %s %d' % (result['ip'],result['not_before']))
result.delete()
nr_purged += 1
return nr_purged
def delete_item(self, schema_name, _id):
table = Table(schema_name)
return table.delete_item(**self._kwargs_for_id(schema_name, _id))
class DynamoTable(object):
conn = None
table = None
table_name = 'test-table'
hash_key = 'hash_key'
range_key = 'range_key'
indexes = []
read_units = 10
write_units = 10
counters = {'reads':0,'writes':0,'delete':0,'batch_w':0}
def __init__(self, table_name, hash_key, range_key, indexes, read_units=10, write_units=10 ):
self.table_name = table_name
self.hash_key = hash_key
self.range_key = range_key
self.indexes = indexes
self.read_units = read_units
self.write_units = write_units
try:
self.connect()
self.setup()
except:
logger.warn('Unable to connect or handle DynamoDB Table')
traceback.print_exc()
def connect(self):
# create initial database tables
self.conn = boto.dynamodb2.connect_to_region( settings.AWS_DYNAMODB_REGION,
aws_access_key_id=settings.AWS_DYNAMODB_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_DYNAMODB_SECRET_ACCESS_KEY
)
def setup(self):
'''
Set's up the table schema if table does not exists yet
Return the Table
'''
try:
self.table = Table.create(self.table_name, connection=self.conn, schema=[
HashKey(self.hash_key),
RangeKey(self.range_key),
],
throughput={'read':self.read_units,'write':self.write_units})
logger.warning('Created new DynamoDB Table')
except:
self.table = Table(self.table_name, connection=self.conn, schema=[
HashKey(self.hash_key),
RangeKey(self.range_key),
],
throughput={'read':self.read_units,'write':self.write_units})
return self.table
def put(self, hash_key, range_key, data):
'''
puts the data to the table
if key/range_key exists
'''
if settings.DEBUG:
bench_start = time()
data[self.hash_key] = hash_key
data[self.range_key] = range_key
item = self.table.put_item( data=data, overwrite=True )
if settings.DEBUG:
if not hash_key in self.counters:
self.counters[hash_key] = {'reads':0,'writes':0}
self.counters[hash_key]['writes'] +=1
self.counters['writes'] +=1
elapsed_time = time() - bench_start
logger.info(data)
logger.info("R%sW%s - write %0.5f seconds" % (self.counters[hash_key]['reads'], self.counters[hash_key]['writes'], elapsed_time))
return item
def get_latest(self, hash_key ):
'''
retreive the last recorded data hash_key item for the hash key
'''
if settings.DEBUG:
bench_start = time()
kwargs = {}
kwargs[self.hash_key+'__eq'] = hash_key
kwargs['limit'] = 1
items = self.table.query( **kwargs )
if items:
data = {}
for item in items:
for key in item.keys():
if not key in (self.hash_key, self.range_key):
data[key] = item[key]
else:
return None
if not len(data):
return None
if settings.DEBUG:
if not hash_key in self.counters:
self.counters[hash_key] = {'reads':0,'writes':0}
self.counters[hash_key]['reads'] +=1
self.counters['reads'] +=1
elapsed_time = time() - bench_start
logger.info("R%sW%s - %s - read %0.5f seconds" % (self.counters[hash_key]['reads'], self.counters[hash_key]['writes'], hash_key, elapsed_time))
return data
def get_range_obj(self, hash_key):
if settings.DEBUG:
bench_start = time()
kwargs = {}
kwargs[self.hash_key+'__eq'] = hash_key
# TODO - use batch_get
items = self.table.query( **kwargs )
self.counters['reads'] +=1
data = {}
for item in items:
rkey_data = {}
rkey = item[self.range_key]
if rkey == 'index':
data = json.loads(item['value'])
break
else:
for key in item.keys():
if key != None and not key in (self.hash_key, self.range_key) and key != 'index':
if key == 'value':
value = item[key]
try:
rkey_data = json.loads(str(value))
except:
rkey_data = value
#else:
# rkey_data[key] = item[key]
data[rkey] = rkey_data
if settings.DEBUG:
if not hash_key in self.counters:
self.counters[hash_key] = {'reads':0,'writes':0}
self.counters[hash_key]['reads'] +=1
self.counters['reads'] +=1
elapsed_time = time() - bench_start
#logger.info(data)
logger.info("R%sW%s - %s - read %0.5f seconds" % (self.counters[hash_key]['reads'], self.counters[hash_key]['writes'], hash_key, elapsed_time))
return data
def set_range_obj(self, hash_key, data, range_keys=None):
# avoid crashing on attempt to write None data
if data == None:
return
if range_keys == None:
range_keys = data.keys()
# TODO
# add better size estimate
datablocks = 0
for range_key in data.keys():
try:
len_size = len( data[range_key] )
except:
len_size = 1
datablocks += len_size
# update date in msecs since epoch
update_date = time()
if datablocks > 1000:
#print hash_key,
#print datablocks
# split over multiple items by data dict key
with self.table.batch_write() as batch:
for range_key in range_keys:
value = json.dumps( data[range_key] )
batch_data = {}
batch_data[self.hash_key] = hash_key
batch_data[self.range_key] = range_key
batch_data['value'] = value
batch_data['update_date'] = update_date
batch.put_item(data=batch_data)
self.counters['batch_w'] +=1
# delete index if exists
self.remove_range_obj(hash_key, range_keys=['index'])
else:
value = json.dumps(data)
batch_data = {}
batch_data[self.hash_key] = hash_key
batch_data[self.range_key] = 'index'
batch_data['value'] = value
batch_data['update_date'] = update_date
self.table.put_item(data=batch_data, overwrite=True)
self.counters['writes'] +=1
return True
def remove_range_obj(self, hash_key, range_keys=None):
'''
deletes ranged object or specific range_keys
'''
# get range object
if range_keys == None:
data = self.get_range_obj(hash_key)
range_keys = data.keys()
# remove possible index
try:
kwargs = {}
kwargs[self.hash_key] = hash_key
kwargs[self.range_key] = 'index'
self.table.delete_item( **kwargs )
except:
pass
with self.table.batch_write() as batch:
for range_key in range_keys:
kwargs = {}
kwargs[self.hash_key] = hash_key
kwargs[self.range_key] = range_key
batch.delete_item( **kwargs )
self.counters['delete'] +=1
return True
class Index(Base):
'''An Index for docker-registry that uses Amazon AWS DynamoDB as the storage engine.
Boto is used to do all access to DynamoDB.
Configure the following dynamodb_config variables or environment variables:
dynamodb_index_database - optional, if not specified will default to 'docker-registry'
and the repository and version table names will be constructed using the
{dynamodb_index_database}-repository and {dynamodb_index_database}-version.
DynamoDB does not have a database concept, just tables in the data store.
dynamodb_index_repository_table - override the default table name (above) with a new name
dynamodb_index_version_table - override the default table name with a new name
dynamodb_region - the AWS region for the dynamodb. This will default to the s3_region and if
that is not defined, it defaults to 'us-east-1'.
dynamodb_access_key - the AWS access key to use
dynamodb_secret_access_key - the AWS secret part of the access key
'''
_initLock = Lock()
def __init__(self, database=None, dynamodb_access_key=None, dynamodb_secret_access_key=None):
'''
Constructor
'''
cfg = dynamodb_config.load()
if database is None:
database = cfg['extensions.dynamodb_index.database']
if dynamodb_access_key is None:
dynamodb_access_key = cfg['extensions.dynamodb_index.access_key']
if dynamodb_secret_access_key is None:
dynamodb_secret_access_key = cfg['extensions.dynamodb_index.secret_access_key']
self.repositoryTableName = cfg['extensions.dynamodb_index.repository_table']
self.versionTableName = cfg['extensions.dynamodb_index.version_table']
if dynamodb_access_key is None:
self._db = dynamodb2.connect_to_region(cfg['extensions.dynamodb_index.region'])
else:
self._db = dynamodb2.connect_to_region(cfg['extensions.dynamodb_index.region'],
aws_access_key_id=dynamodb_access_key,
aws_secret_access_key=dynamodb_secret_access_key)
self._repositoryTable = Table(self.repositoryTableName,
schema=[HashKey('name', data_type=STRING)],
global_indexes=[GlobalAllIndex('Repositories-By-Description-Index',
parts=[HashKey('description', data_type=STRING)])],
connection=self._db)
self._versionTable = Table(self.versionTableName,
schema=[HashKey('version', data_type=NUMBER)],
connection=self._db)
self.version = 1
Index._initLock.acquire()
try:
self._setup_database()
finally:
Index._initLock.release()
super(Index, self).__init__()
def _describe_or_create_tables(self):
dynamodb_util.create_table_if_not_exists(self._repositoryTable)
dynamodb_util.create_table_if_not_exists(self._versionTable)
def _wait_for_tables(self):
dynamodb_util.wait_for_table_active(self._repositoryTable)
dynamodb_util.wait_for_table_active(self._versionTable)
def _read_or_set_schema_version(self, default_version):
def read_schema_version():
v = 0
try:
results = self._versionTable.scan()
row = results.next()
v = row['version']
except:
v = -1
return v
# Read or insert the schema_version. Keep doing it until one
# of them works. This is in case another thread is attempting the same
# thing. Reading first will allow this thread to complete.
schemaVersion = read_schema_version()
while (schemaVersion <= 0):
try:
self._versionTable.put_item(data={'version': default_version})
schemaVersion = default_version
except:
sleep(0.5)
schemaVersion = read_schema_version()
return schemaVersion
def _setup_database(self):
needs_index = not dynamodb_util.table_exists(self._versionTable)
self._describe_or_create_tables()
self._wait_for_tables()
version = self._read_or_set_schema_version(self.version)
if (version != self.version):
raise NotImplementedError('unrecognized search index version {0}'.format(version))
if needs_index:
self._generate_index()
def _generate_index(self):
store = storage.load()
with self._repositoryTable.batch_write() as batch:
for repository in self._walk_storage(store=store):
logger.info('Populating repository: {0}'.format(repository['name']))
batch.put_item(data=repository)
def _handle_repository_created(
self, sender, namespace, repository, value):
name = '{0}/{1}'.format(namespace, repository)
description = '' # TODO(wking): store descriptions
logger.info('Creating new repository {0}'.format(name))
self._repositoryTable.put_item(data={'name': name, 'description': description})
def _handle_repository_updated(
self, sender, namespace, repository, value):
name = '{0}/{1}'.format(namespace, repository)
description = '' # TODO(wking): store descriptions
logger.info('Updating repository {0}'.format(name))
repo = self._repositoryTable.get_item(name=name)
repo['description'] = description
repo.save(overwrite=True)
def _handle_repository_deleted(self, sender, namespace, repository):
name = '{0}/{1}'.format(namespace, repository)
logger.info('Deleting repository {0}'.format(name))
self._repositoryTable.delete_item(name=name)
def results(self, search_term=None):
"""Return a list of results matching search_term
The list elements should be dictionaries:
{'name': name, 'description': description}
"""
if not search_term or len(search_term) == 0:
logger.info('Index query: full table scan')
repositories = self._repositoryTable.scan()
else:
logger.info('Index query: {0}'.format(search_term))
repositories = self._repositoryTable.scan(conditional_operator='OR',
name__contains=search_term,
description__contains=search_term)
if repositories:
return [{'name': repo['name'],
'description': repo['description'],
} for repo in repositories]
return []
connection=conn
)
tokenTable = Table(
TOKEN_TABLE_NAME,
connection=conn
)
# Everybody get's 0 score!
users = tokenTable.scan()
for user in users:
#Delete the user .. Eaakkk
tokenTable.delete_item(**{
'token': user['token']
})
if user['flags']:
i = 0
for userFlag in user['flags']:
del user['flags'][i]
user['score'] = 0
user['level'] = 0
#Put them back .. pheww
tokenTable.put_item(user)
users = tokenTable.scan()
tweets.get_item(id='2222').items()
###########################
## Deleting an item
###########################
# If you already have an Item instance, the easiest approach is just to call Item.delete
_tweet.delete()
tweets.get_item(id='1111').items() # return item not found
# If you don't have an Item instance
from boto.dynamodb2.table import Table
tweets = Table('tweets')
tweets.delete_item(id='2222')
###########################
## Batch writing
###########################
# If you’re loading a lot of data at a time, making use of batch writing can both speed up the process &
# reduce the number of write requests made to the service.
# Batch writing involves wrapping the calls you want batched in a context manager. The context manager
# imitates the Table.put_item & Table.delete_item APIs. Getting & using the context manager looks like:
import time
from boto.dynamodb2.table import Table
tweets = Table('tweets')
class Backend():
def __init__(self, region, link_prefix, bucket_name, table_name):
if link_prefix[-1] != u'/':
link_prefix += u'/'
self.link_prefix = link_prefix
conn = boto.s3.connect_to_region(region)
self.bucket = conn.get_bucket(bucket_name)
conn = boto.dynamodb2.connect_to_region(region)
self.table = Table(table_name, connection=conn)
def write_to_backend(self, station, title, timestamp, description, files_path, files):
if self._get_item(station, title):
msg = u'%s %s' % (station, title)
raise TitleExistError(msg)
if files_path and files:
if files_path[-1] != u'/':
files_path = files_path + u'/'
files_with_path = []
for f in files:
k = Key(self.bucket)
k.key = u''.join([station, u'_', title, u'_', f])
k.set_contents_from_filename(files_path+f)
k.set_acl(u'public-read')
files_with_path.append(self.link_prefix+k.key)
files_str = u' '.join(files_with_path)
else:
files_str = u'empty'
self.table.put_item(data={u'station':station, u'title':title, \
u'timestamp':timestamp, u'description': description, u'files':files_str})
return True
def update_description_and_append_files(self, station, title, timestamp, description, files_path, files):
if files_path and files:
if files_path[-1] != u'/':
files_path = files_path + u'/'
files_with_path = []
for f in files:
k = Key(self.bucket)
k.key = u''.join([station, u'_', title, u'_', f])
k.set_contents_from_filename(files_path+f)
k.set_acl(u'public-read')
files_with_path.append(self.link_prefix+k.key)
new_files_str = u' '.join(files_with_path)
else:
new_files_str = u'empty'
item = self._get_item(station, title)
if not item:
return False
old_files_str = unicode(item[u'files'])
if old_files_str == u'empty':
files_str = new_files_str
elif new_files_str == u'empty':
files_str = old_files_str
else:
files_str = u''.join([old_files_str, u' ', new_files_str])
item[u'files'] = files_str
item[u'description'] = description
item.partial_save()
return True
def delete_item(self, station, title):
item = self._get_item(station, title)
if not item:
return False
files_str = unicode(item[u'files'])
if files_str == u'empty':
files = []
elif not files_str:
files = []
else:
files = files_str.split(u' ')
self.table.delete_item(station=station, title=title)
for f in files:
k = Key(self.bucket)
k.key = f[len(self.link_prefix):]
k.delete()
return True
def delete_file(self, station, title, f):
item = self._get_item(station, title)
files = unicode(item[u'files']).split(u' ')
if f in files:
files.remove(f)
if files:
files_str = u' '.join(files)
else:
files_str = u'empty'
item[u'files'] = files_str
item.partial_save()
k =Key(self.bucket)
k.key = f[len(self.link_prefix):]
k.delete()
return True
return False
def get_page(self, station, direction, timestamp, page_size):
need_reverse = False
if timestamp == u'0':
items = self.table.query(station__eq=station, index=u'timestamp-index', reverse=False, limit=page_size)
elif direction == u'next':
items = self.table.query(station__eq=station, timestamp__lt=timestamp,
index=u'timestamp-index', reverse=False, limit=page_size)
elif direction == u'prev':
items = self.table.query(station__eq=station, timestamp__gt=timestamp,
index=u'timestamp-index', reverse=True, limit=page_size)
need_reverse = True
else:
return []
ret = []
for item in items:
r = {}
r[u'title'] = unicode(item[u'title'])
r[u'timestamp'] = unicode(item[u'timestamp'])
r[u'description'] = unicode(item[u'description'])
r[u'files'] = unicode(item[u'files']).split(u' ')
ret.append(r)
if need_reverse:
ret.reverse()
return ret
def get_item(self, station, title):
item = self._get_item(station, title)
if not item:
return None
r = {}
r[u'title'] = unicode(item[u'title'])
r[u'timestamp'] = unicode(item[u'timestamp'])
r[u'description'] = unicode(item[u'description'])
r[u'files'] = unicode(item[u'files']).split(u' ')
return r
def _get_item(self, station, title):
items = self.table.query(station__eq=station, title__eq=title)
for item in items:
return item
return None
class dynamoMethods:
def __init__(self, dbName):
self.table_dynamo = None
try:
#1. create new table
self.table_dynamo = Table.create(
dbName, schema=[HashKey('CUID')], connection=client_dynamo
) #HINT: Table.create; #HINT 2: Use CUID as your hashkey
#self.table_dynamo = client_dynamo.create_table(name=dbName, schema=s)
print "New Table Created"
except Exception as e:
#2.table already exists, so get the table
self.table_dynamo = Table(
dbName, connection=client_dynamo
) #HINT: Remember to use "connection=client.dynamo"
'''
self.table_dynamo.put_item(data={
'CUID': 'sh3658',
'name': 'scott',
'password': '******',
})
'''
print "Table Already Exists"
def dynamoAdd(self, cuid, name, password):
# Check if the cuid exists in the database.
user = None
try:
print 'try to get item'
user = self.table_dynamo.get_item(
CUID=cuid
) #, name=name, password=password)#, name=name, password=password)
print "Found user."
print user
except Exception as e:
print e
# If the CUID already exists inside db, perform a partial update on the entry.
if user != None and user['CUID'] == cuid:
user['name'] = name
user['password'] = password
try:
user.partial_save()
print "Updated entry: " + cuid
except Exception as e:
print e
# Otherwise, create a new entry
else:
try:
self.table_dynamo.put_item(data={
'CUID': cuid,
'name': name,
'password': password,
})
print "New entry created."
except Exception as e:
print e
def dynamoDelete(self, cuid):
####################################################################
# YOUR CODE HERE
#1. Check table for entries that have the same CUID, if so, DELETE
####################################################################
try:
user = self.table_dynamo.get_item(
CUID=cuid
) #, name=name, password=password)#, name=name, password=password)
self.table_dynamo.delete_item(CUID=cuid)
print 'Item deleted'
except Exception as e:
print e
def dynamoViewAll(self):
string_db = "CUID: NAME\n"
print string_db
entries = self.table_dynamo.scan()
for entry in entries:
print entry['CUID'], ':', entry['name']
def delete_item(target_table, feature1):
try:
table = Table(target_table)
table.delete_item(CUID = feature1)
time.sleep(0.5)
return
except IOError:
print "Error deleting item"
return
def view_table(target_table):
try:
table = Table(target_table)
result_set = table.scan()
for user in result_set:
print user['Sex'], ":", user
except IOError:
print "Error viewing items"
return
def func_search(target_table, target_index):
try:
table = Table(target_table)
_index = target_index
#TODO
print "result is: "
except IOError:
print "Error searching item!"
if __name__ == "__main__":
flag = True
print "please enter commands 'add<>to<>' or 'delete<>from<>' or 'view<>' or 'search<>by<>' or exit"
while (flag):
print "Commands:"
raw = raw_input('>>')
command = raw.split()
try:
if command[0] == "add" and len(command) == 4:
print "Sex:"
input_0 = raw_input('>>')
print "CUID:"
input_1 = raw_input('>>')
print "First_Name:"
input_2 = raw_input('>>')
print "Last_Name:"
input_3 = raw_input('>>')
new_item = {'Sex':input_0,'CUID':input_1,'First_Name':input_2,'Last_Name':input_3,}
target_table = command[3]
print "adding item"
add_item(new_item, target_table)
elif command[0] == "delete" and len(command) == 4:
item_to_delete = command[1]
target_table = command[3]
print "run deleting function"
delete_item(target_table, item_to_delete)
elif command[0] == "view" and len(command) == 2:
target_table = command[1]
view_table(target_table)
print "view data"
elif command[0] == 'exit':
print "exiting"
time.sleep(1)
sys.exit()
elif command[0] == 'search':
target_table = command[1]
target_index = command[3]
print "searching...."
else:
print "invalid command!"
except KeyboardInterrupt:
print "exiting"
sys.exit()
class ClientRadiusSecretsDDB(RadiusSecrets):
"""
get secrets for a particular client
ddb=DynamoDBConnection(
aws_access_key_id=aws_access_key,
aws_secret_access_key=aws_secret_key
)
secrets = Table.create('qradius_secrets',schema=[\
HashKey('ip_address'),
RangeKey('not_before',data_type=NUMBER),
], indexes=[
AllIndex('IPNotAfter',parts=[
HashKey('ip_address'),
RangeKey('not_after',data_type=NUMBER),
])
],connection=ddb)
we will normally want secrets where
ip_address = client ip address
not_before < now
not_after >= now
likely we'll want to limit the # of secrets we look at, here i limit it to 3
For queries (these are the least impactful on dynamo:
res=secrets.query(ip_address__eq='10.25.95.158',not_before__lt=now,limit=3,consistent=False)
res=secrets.query(ip_address__eq='10.25.95.158',not_after__gt=now,limit=3,consistent=False,index='IPNotAfter')
if you really need everything, this is a full scan:
res=secrets.scan(ip_address__eq='10.25.95.158',not_before__lt=now,not_after__gt=now,limit=3)
"""
def __init__(self, encryption_key=None, aws_keys=None, table_name=None):
"""
@param encryption_key : string containing encryption key
@param aws_keys : object containing aws keys
@param table_name : dynamo table name
@type encryption_key string
@type aws_keys AWSKeys
@type table_name string
"""
if encryption_key is None:
raise ValueError("encryption_key must be specified")
if not isinstance(aws_keys, AWSKeys):
raise ValueError(
"aws_material must be specified and of type AWSKeys")
if table_name is None:
raise ValueError(
"dynamo table containing secrets must be specified")
self._encryption_key = encryption_key
self._encryptor = DataEncryptor(self._encryption_key)
self._ddb_connection = DynamoDBConnection(
aws_access_key_id=aws_keys.aws_access_key,
aws_secret_access_key=aws_keys.aws_secret_key)
if self._ddb_connection is None:
raise ValueError("unable to obtain dynamo connection using %s" %
aws_material)
self._secret_table = Table(table_name, connection=self._ddb_connection)
logging.debug('connectd to dynamo table %s' % table_name)
if self._secret_table is None:
raise ValueError("unable to connect to dynamo table %s" %
table_name)
def encryptSecret(self, secret):
"""
call the included encryption module to encrypt/encode
secrets
@param secret
@type secret string
"""
if len(secret) > MAX_SECRET_LENGTH:
raise ValueError("secret may not be more than %d bytes" %
MAX_SECRET_LENGTH)
encoded_secret = self._encryptor.encrypt(secret)
return (encoded_secret)
def decryptSecret(self, encoded_secret):
"""
call the included encryption module to decrypt/decode
secrets
@param encoded_secret
@type encoded_secret string
"""
plain_secret = self._encryptor.decrypt(encoded_secret)
if len(plain_secret) > MAX_SECRET_LENGTH:
raise ValueError(
"decryption resulted in a plain secret longer than the maximum length of %d bytes"
% MAX_SECRET_LENGTH)
return (plain_secret)
def putSecret(self,
clientIP,
secret,
not_before=None,
not_after=None,
tries=0):
"""
store a secret for clientIP
@param clientIP : client ip address
@param secret : radius secret for client
@param tries : internal parameter to constrain recursion depth for self-calls
@type clientIP string
@type secret
example usage:
from radiussecrets import *
rs=ClientRadiusSecrets(encryption_key='someencryptionkey',
aws_keys=AWSKeys('myaccesskey','mysecretkey'),table_name='qradius_secrets')
ValidationException
rs.putSecret('1.2.3.4','shhdonottellanyone')
"""
now = time.time()
if not_before is None:
not_before = now
if not_after is None:
not_after = now + DEFAULT_KEY_LIFE
if not isinstance(not_before, (int, float, long)) or not_before < 0:
raise ValueError(
"not_before must be a number representing seconds since epoch")
if not isinstance(not_after, (int, float, long)) or not_after < 0:
raise ValueError(
"not_before must be a number representing seconds since epoch")
if len(secret) > MAX_SECRET_LENGTH:
raise ValueError("length of secret may not exceed %d bytes" %
MAX_SECRET_LENGTH)
result = None
try:
result = self._secret_table.put_item(
data={
'ip_address': clientIP,
'not_before': not_before,
'not_after': not_after,
'secret': self.encryptSecret(secret)
})
except boto.dynamodb2.exceptions.ConditionalCheckFailedException as e:
tries += 1
if tries > 5:
logging.crit(
'pk violation for client %s not_before %d after %d tries at incrementing'
% (clientIP, not_before, tries))
raise e
#increment not_before to avoid pk violation
not_before += 1
logging.warn(
'pk violation for client %s not_before %d; retrying with higher not_before '
% (clientIP, not_before))
result = self.putSecret(clientIP,
secret,
not_before=not_before,
not_after=not_after,
tries=tries)
return result
def deleteSecret(self, clientIP, not_before):
"""
delete a secret
this should be used carefully
@param clientIP
@param not_before
@type clientIP string
@type not_before number
"""
return self._secret_table.delete_item(ip_address=clientIP,
not_before=not_before)
def getSecret(self, clientIP):
"""
return the secret associated with IP address
if multiple secrets are found, selection is by:
1) not_before < now
2) not_after >= now
3) the highest value of not before, if there are still multiple secrets
@param clientIP
@param not_before seconds since epoch that the secret becomes valid
@param not_after seconds since epoch after which the secret is not valid
@type clientIP string representing an ip address
"""
now = time.time()
#i wanted to limit to 3 (limit=3) but, boto kept barfing
results = self._secret_table.query(ip_address__eq=clientIP,
not_before__lt=now,
consistent=False)
client_secret = None
client_secret_not_before = 0
for result in results:
if result['not_after'] >= now:
if client_secret_not_before < result['not_before']:
client_secret_not_before = result['not_before']
client_secret = self.decryptSecret(result['secret'])
logging.debug('retrieved secret for %s' % (clientIP))
return client_secret
def purgeSecrets(self, clientIP):
"""
purge stale secrets associated with IP address
1) remove all keys for clientIP where not after is older than
current time - PURGE_TIME_BEFORE_NOW
2) scan remaining keys, keep
@param clientIP
@type string
@returns # of purged secrets
"""
now = time.time()
min_purge_time = now
nr_purged = 0
# first get rid of expired keys
results = self._secret_table.query(ip_address__eq=clientIP,
not_after__lt=min_purge_time,
consistent=False,
index='IPNotAfter')
for result in results:
logging.info('purging secret: %s %d' %
(result['ip'], result['not_before']))
result.delete()
nr_purged += 1
# now the fun...
result_list = []
results = self._secret_table.query(ip_address__eq=clientIP,
not_before__lt=min_purge_time,
consistent=False)
for result in results:
result_list.append(result)
# delete results if there are more than PURGE_RETAIN_NR_ACTIVE results,
# we want the oldest not befores to be removed first
if len(result_list) > PURGE_RETAIN_NR_ACTIVE_KEYS:
for result in sorted(result_list,
key=lambda result: result['not_before']
)[:-PURGE_RETAIN_NR_ACTIVE_KEYS]:
logging.info('purging secret: %s %d' %
(result['ip'], result['not_before']))
result.delete()
nr_purged += 1
return nr_purged
class dynamoMethods:
def __init__(self, dbName):
self.table_dynamo = None
####################################################################
# YOUR CODE HERE
#1. create new table
try:
self.table_dynamo = Table.create(
dbName, schema=[HashKey('CUID')], connection=client_dynamo
) #HINT: Table.create; #HINT 2: Use CUID as your hashkey
print("Wait 20 sec until the table is created")
time.sleep(20)
print("New Table Created")
#2.table already exists, so get the table
except Exception as e:
self.table_dynamo = Table(
dbName, connection=client_dynamo
) #HINT: Remember to use "connection=client.dynamo"
print("Table Already Exists")
####################################################################
def dynamoAdd(self, cuid, name, password):
####################################################################
# YOUR CODE HERE
#1. Check table for entries that have the same CUID, if so, UPDATE (Don't delete)
try:
record = self.table_dynamo.get_item(CUID=cuid)
record['name'] = name
record['password'] = password
record.save(overwrite=True)
print("Record has been updated.\n")
#2. Otherwise, create a new entry
except Exception as e:
# if query occurs exception, add record
self.table_dynamo.put_item(data={
'CUID': cuid,
'name': name,
'password': password
})
print("New entry created.\n")
####################################################################
def dynamoDelete(self, cuid):
####################################################################
# YOUR CODE HERE
#1. Check table for entries that have the same CUID, if so, DELETE
try:
self.table_dynamo.delete_item(CUID=cuid)
print("The Record has benn deleted")
except Exception as e:
print("Cannot delete the record, it is not exists")
raise
####################################################################
def dynamoViewAll(self):
####################################################################
# YOUR CODE HERE
string_db = "CUID: NAME\n"
print(string_db)
#1. Get all entries in the table
#2. Print the CUID: NAME for each entry
items = self.table_dynamo.scan()
all_users = list(items)
for user in all_users:
print(user['CUID'] + ':' + user['name'])
print("\n")
