def name_uni_delete_item(name, uni): # name must be string # item must be dict try: table = Table('AssignmentTwo', connection=client_dynamo) table.delete_item(name=name, CUID=uni) return True except KeyboardInterrupt: exit except: print "Error" return False
def test_delete_item(self): self.storage_mocker.StubOutWithMock(storage, 'delete_item') storage.delete_item( IgnoreArg(), IgnoreArg(), expected_condition_map=IgnoreArg()).AndReturn(True) self.storage_mocker.ReplayAll() table = Table('test_table', connection=self.DYNAMODB_CON) table.delete_item(hash_key=1, range_key="range") self.storage_mocker.VerifyAll()
def test_delete_item(self): self.storage_mocker.StubOutWithMock(storage, 'delete_item') storage.delete_item(IgnoreArg(), IgnoreArg(), expected_condition_map=IgnoreArg()).AndReturn(True) self.storage_mocker.ReplayAll() table = Table('test_table', connection=self.DYNAMODB_CON) table.delete_item(hash_key=1, range_key="range") self.storage_mocker.VerifyAll()
def deleteStatus(self, reportname, datemodified): """ Purpose: Construct the nzsql command :param self: class object itself :param reportname: id of the report for status insert :param datemodified: exact range key value to delete particular record of the day... """ ## Get report Id for report Name report_id = self.getReportID(reportname.upper()) ## Create table object for the dynamo DB table tab = Table('TBL_AWS_REPORT_DTL', connection=self.conn) ##Check if element is there before delete to avoid exception. tab.delete_item(REPORT_ID=report_id, Date_Modified=datemodified)
class KVStore(KVStoreBase): def __init__(self): super(KVStore, self).__init__() region = settings.AWS_REGION_NAME access_key = settings.AWS_ACCESS_KEY_ID secret = settings.AWS_SECRET_ACCESS_KEY conn = boto.dynamodb2.connect_to_region(region, aws_access_key_id=access_key, aws_secret_access_key=secret) self.table = Table(settings.THUMBNAIL_DYNAMODB_NAME, connection=conn) def _get_raw(self, key): try: return self.table.get_item(key=key)["value"] except boto.dynamodb2.exceptions.ItemNotFound: pass def _set_raw(self, key, value): try: item = self.table.get_item(key=key) except boto.dynamodb2.exceptions.ItemNotFound: item = self.table.new_item() item["key"] = key item["value"] = value item.save(overwrite=True) def _delete_raw(self, *keys): [self.table.delete_item(key=k) for k in keys] def _find_keys_raw(self, prefix): return [i["key"] for i in self.table.scan(key__beginswith=prefix)]
class KVStore(KVStoreBase): def __init__(self): super(KVStore, self).__init__() region = settings.AWS_REGION_NAME access_key = settings.AWS_ACCESS_KEY_ID secret = settings.AWS_SECRET_ACCESS_KEY conn = boto.dynamodb2.connect_to_region(region, aws_access_key_id=access_key, aws_secret_access_key=secret) self.table = Table(settings.THUMBNAIL_DYNAMODB_NAME, connection=conn) def _get_raw(self, key): try: return self.table.get_item(key=key)['value'] except boto.dynamodb2.exceptions.ItemNotFound: pass def _set_raw(self, key, value): try: item = self.table.get_item(key=key) except boto.dynamodb2.exceptions.ItemNotFound: item = self.table.new_item() item['key'] = key item['value'] = value item.save(overwrite=True) def _delete_raw(self, *keys): [self.table.delete_item(key=k) for k in keys] def _find_keys_raw(self, prefix): return [i['key'] for i in self.table.scan(key__beginswith=prefix)]
def delete_minion(instanceid): result = False try: minions = Table('minions') result = minions.delete_item(instanceid=instanceid) except Exception, e: raise e
def delete_minion(instanceid): result = False try: minions = Table("minions") result = minions.delete_item(instanceid=instanceid) except Exception, e: raise e
class dynamoDB: def __init__(self, db_name, partition_key_name): self.table_dynamo = None self.partition_key_name = partition_key_name try: self.table_dynamo = Table.create(db_name, schema=[HashKey(partition_key_name)], connection=client_dynamo) print ("Wait 20 sec until the table is created") time.sleep(20) print ("New table created.") except Exception as e: self.table_dynamo = Table(db_name, connection=client_dynamo) print ("Table already exists.") def add(self, **kwargs): try: record = self.get(kwargs[self.partition_key_name]) for k,v in kwargs.items(): record[k] = v record.save(overwrite=True) #print("Record has been updated.\n") except Exception as e: self.table_dynamo.put_item(data=kwargs) #print("New entry created.\n") def delete(self, pk): try: record = self.table_dynamo.get_item(**{self.partition_key_name:pk}) self.table_dynamo.delete_item(**{self.partition_key_name:pk}) #print("The record has been deleted.") return record except Exception as e: #print("Cannot delete the record, it does not exist.") pass return None def get(self,pk): try: item = self.table_dynamo.get_item(**{self.partition_key_name:pk}) return item except Exception as e: #print("Cannot get the record, it does not exist.") pass return None def scan(self,**filter_kwargs): return self.table_dynamo.scan(**filter_kwargs)
def execute(self, args, config): glacier_connection = boto.connect_glacier(aws_access_key_id=config.get('configuration', 'aws_key'), aws_secret_access_key=config.get('configuration', 'aws_secret')) try: vault = glacier_connection.get_vault(args[4]) except: vault = None if vault is None: print "Vault named '{}' does not exist.".format(args[4]) else: try: vault.delete_archive(args[2]) dynamo_connection=DynamoDBConnection(aws_access_key_id=config.get('configuration', 'aws_key'), aws_secret_access_key=config.get('configuration', 'aws_secret')) archive_table = Table(config.get('configuration', 'dynamodb_table'), connection=dynamo_connection) archive_table.delete_item(archive_id=args[2]) print "Archive deleted: '{}'".format(args[2]) except UnexpectedHTTPResponseError as error: print "Archive can not be deleted:\n\t {}".format(error)
FLAG_TABLE_NAME, connection=conn ) tokenTable = Table( TOKEN_TABLE_NAME, connection=conn ) flags = flagTable.scan() for flag in flags: #Delete it flagTable.delete_item(**{ 'flag': flag['flag'] }) # now update all the users! users = tokenTable.scan() for user in users: #Delete the user .. Eaakkk tokenTable.delete_item(**{ 'token': user['token'] }) if user['flags']: i = 0 for userFlag in user['flags']:
target_table = command[3] print "adding item" add_item(new_item, target_table) elif command[0] == "delete" and len(command) == 4: # command[1].delete() table = Table(command[3]) print "Sex:" input_0 = raw_input('>>') print "CUID:" input_1 = raw_input('>>') print "First_Name:" input_2 = raw_input('>>') print "Last_Name:" input_3 = raw_input('>>') table.delete_item(Sex=input_0,CUID=input_1,First_Name=input_2,Last_Nameinput_3) print "run deleting function" elif command[0] == "view" and len(command) == 2: target_table = command[1] view_table(target_table) print "view data" elif command[0] == 'exit': print "exiting" time.sleep(1) sys.exit() elif command[0] == 'search': target_table = command[1] target_index = command[3] print "searching...." else:
print("DELETE (Name) (CUID)") print("SEARCH (Name or CUID)") print("VIEW") print("EXIT") command_str = "NONE" while command_str != "EXIT": command_str = raw_input(">>>>") cmd_keyword = command_str.partition(" ")[0] if cmd_keyword == "ADD": this_table.put_item(data={ 'Name': command_str.split(" ")[1], 'CUID': command_str.split(" ")[2] }) elif cmd_keyword == "DELETE": this_table.delete_item(Name=command_str.split(" ")[1], CUID=command_str.split(" ")[2]) elif cmd_keyword == "VIEW": all_items = this_table.scan() for item in all_items: print(item['Name'] + " " + item['CUID']) elif cmd_keyword == "SEARCH": target_items_Name = this_table.query_2( Name__eq=command_str.split(" ")[1]) target_items_CUID = [] all_items = this_table.scan() for item in all_items: if item["CUID"] == command_str.split(" ")[1]: target_items_CUID.append(item) if target_items_Name:
oldHashedFlag = flag['flag'] newSalt = "XXX"; #make a hash of it with the new salt sha1 = hashlib.sha1() sha1.update(oldHashedFlag + newSalt) newHashedFlag = sha1.hexdigest() print oldHashedFlag print newHashedFlag flagMap[oldHashedFlag] = newHashedFlag #Delete it flagTable.delete_item(**{ 'flag': oldHashedFlag }) #change it flag['flag'] = newHashedFlag #put it back flagTable.put_item(flag) print flagMap # now update all the users! users = tokenTable.scan() for user in users:
class ClientRadiusSecretsDDB(RadiusSecrets): """ get secrets for a particular client ddb=DynamoDBConnection( aws_access_key_id=aws_access_key, aws_secret_access_key=aws_secret_key ) secrets = Table.create('qradius_secrets',schema=[\ HashKey('ip_address'), RangeKey('not_before',data_type=NUMBER), ], indexes=[ AllIndex('IPNotAfter',parts=[ HashKey('ip_address'), RangeKey('not_after',data_type=NUMBER), ]) ],connection=ddb) we will normally want secrets where ip_address = client ip address not_before < now not_after >= now likely we'll want to limit the # of secrets we look at, here i limit it to 3 For queries (these are the least impactful on dynamo: res=secrets.query(ip_address__eq='10.25.95.158',not_before__lt=now,limit=3,consistent=False) res=secrets.query(ip_address__eq='10.25.95.158',not_after__gt=now,limit=3,consistent=False,index='IPNotAfter') if you really need everything, this is a full scan: res=secrets.scan(ip_address__eq='10.25.95.158',not_before__lt=now,not_after__gt=now,limit=3) """ def __init__(self, encryption_key=None, aws_keys=None, table_name=None): """ @param encryption_key : string containing encryption key @param aws_keys : object containing aws keys @param table_name : dynamo table name @type encryption_key string @type aws_keys AWSKeys @type table_name string """ if encryption_key is None: raise ValueError("encryption_key must be specified") if not isinstance(aws_keys,AWSKeys): raise ValueError("aws_material must be specified and of type AWSKeys") if table_name is None: raise ValueError("dynamo table containing secrets must be specified") self._encryption_key = encryption_key self._encryptor = DataEncryptor(self._encryption_key) self._ddb_connection = DynamoDBConnection( aws_access_key_id=aws_keys.aws_access_key, aws_secret_access_key=aws_keys.aws_secret_key ) if self._ddb_connection is None: raise ValueError("unable to obtain dynamo connection using %s" % aws_material) self._secret_table = Table(table_name,connection=self._ddb_connection) logging.debug('connectd to dynamo table %s' % table_name) if self._secret_table is None: raise ValueError("unable to connect to dynamo table %s" % table_name) def encryptSecret(self, secret): """ call the included encryption module to encrypt/encode secrets @param secret @type secret string """ if len(secret) > MAX_SECRET_LENGTH: raise ValueError("secret may not be more than %d bytes" % MAX_SECRET_LENGTH) encoded_secret = self._encryptor.encrypt(secret) return(encoded_secret) def decryptSecret(self, encoded_secret): """ call the included encryption module to decrypt/decode secrets @param encoded_secret @type encoded_secret string """ plain_secret = self._encryptor.decrypt(encoded_secret) if len(plain_secret) > MAX_SECRET_LENGTH: raise ValueError("decryption resulted in a plain secret longer than the maximum length of %d bytes" % MAX_SECRET_LENGTH) return(plain_secret) def putSecret(self, clientIP, secret, not_before=None,not_after=None,tries=0): """ store a secret for clientIP @param clientIP : client ip address @param secret : radius secret for client @param tries : internal parameter to constrain recursion depth for self-calls @type clientIP string @type secret example usage: from radiussecrets import * rs=ClientRadiusSecrets(encryption_key='someencryptionkey', aws_keys=AWSKeys('myaccesskey','mysecretkey'),table_name='qradius_secrets') ValidationException rs.putSecret('1.2.3.4','shhdonottellanyone') """ now = time.time() if not_before is None: not_before = now if not_after is None: not_after = now + DEFAULT_KEY_LIFE if not isinstance(not_before,(int,float,long)) or not_before < 0: raise ValueError("not_before must be a number representing seconds since epoch") if not isinstance(not_after,(int,float,long)) or not_after < 0: raise ValueError("not_before must be a number representing seconds since epoch") if len(secret) > MAX_SECRET_LENGTH: raise ValueError("length of secret may not exceed %d bytes" % MAX_SECRET_LENGTH) result = None try: result = self._secret_table.put_item(data={ 'ip_address': clientIP, 'not_before': not_before, 'not_after': not_after, 'secret': self.encryptSecret(secret) }) except boto.dynamodb2.exceptions.ConditionalCheckFailedException as e: tries += 1 if tries > 5: logging.crit('pk violation for client %s not_before %d after %d tries at incrementing' % (clientIP,not_before,tries)) raise e #increment not_before to avoid pk violation not_before += 1 logging.warn('pk violation for client %s not_before %d; retrying with higher not_before ' % (clientIP,not_before)) result = self.putSecret(clientIP, secret, not_before=not_before,not_after=not_after,tries=tries) return result def deleteSecret(self, clientIP, not_before): """ delete a secret this should be used carefully @param clientIP @param not_before @type clientIP string @type not_before number """ return self._secret_table.delete_item(ip_address=clientIP,not_before=not_before) def getSecret(self, clientIP): """ return the secret associated with IP address if multiple secrets are found, selection is by: 1) not_before < now 2) not_after >= now 3) the highest value of not before, if there are still multiple secrets @param clientIP @param not_before seconds since epoch that the secret becomes valid @param not_after seconds since epoch after which the secret is not valid @type clientIP string representing an ip address """ now = time.time() #i wanted to limit to 3 (limit=3) but, boto kept barfing results = self._secret_table.query(ip_address__eq=clientIP,not_before__lt=now,consistent=False) client_secret = None client_secret_not_before = 0 for result in results: if result['not_after'] >= now: if client_secret_not_before < result['not_before']: client_secret_not_before = result['not_before'] client_secret = self.decryptSecret(result['secret']) logging.debug('retrieved secret for %s' % (clientIP)) return client_secret def purgeSecrets(self, clientIP): """ purge stale secrets associated with IP address 1) remove all keys for clientIP where not after is older than current time - PURGE_TIME_BEFORE_NOW 2) scan remaining keys, keep @param clientIP @type string @returns # of purged secrets """ now = time.time() min_purge_time = now nr_purged = 0 # first get rid of expired keys results = self._secret_table.query(ip_address__eq=clientIP, not_after__lt=min_purge_time, consistent=False, index='IPNotAfter') for result in results: logging.info('purging secret: %s %d' % (result['ip'],result['not_before'])) result.delete() nr_purged += 1 # now the fun... result_list = [] results = self._secret_table.query(ip_address__eq=clientIP, not_before__lt=min_purge_time, consistent=False) for result in results: result_list.append(result) # delete results if there are more than PURGE_RETAIN_NR_ACTIVE results, # we want the oldest not befores to be removed first if len(result_list) > PURGE_RETAIN_NR_ACTIVE_KEYS: for result in sorted(result_list, key=lambda result: result['not_before'])[:-PURGE_RETAIN_NR_ACTIVE_KEYS]: logging.info('purging secret: %s %d' % (result['ip'],result['not_before'])) result.delete() nr_purged += 1 return nr_purged
def delete_item(self, schema_name, _id): table = Table(schema_name) return table.delete_item(**self._kwargs_for_id(schema_name, _id))
class DynamoTable(object): conn = None table = None table_name = 'test-table' hash_key = 'hash_key' range_key = 'range_key' indexes = [] read_units = 10 write_units = 10 counters = {'reads':0,'writes':0,'delete':0,'batch_w':0} def __init__(self, table_name, hash_key, range_key, indexes, read_units=10, write_units=10 ): self.table_name = table_name self.hash_key = hash_key self.range_key = range_key self.indexes = indexes self.read_units = read_units self.write_units = write_units try: self.connect() self.setup() except: logger.warn('Unable to connect or handle DynamoDB Table') traceback.print_exc() def connect(self): # create initial database tables self.conn = boto.dynamodb2.connect_to_region( settings.AWS_DYNAMODB_REGION, aws_access_key_id=settings.AWS_DYNAMODB_ACCESS_KEY_ID, aws_secret_access_key=settings.AWS_DYNAMODB_SECRET_ACCESS_KEY ) def setup(self): ''' Set's up the table schema if table does not exists yet Return the Table ''' try: self.table = Table.create(self.table_name, connection=self.conn, schema=[ HashKey(self.hash_key), RangeKey(self.range_key), ], throughput={'read':self.read_units,'write':self.write_units}) logger.warning('Created new DynamoDB Table') except: self.table = Table(self.table_name, connection=self.conn, schema=[ HashKey(self.hash_key), RangeKey(self.range_key), ], throughput={'read':self.read_units,'write':self.write_units}) return self.table def put(self, hash_key, range_key, data): ''' puts the data to the table if key/range_key exists ''' if settings.DEBUG: bench_start = time() data[self.hash_key] = hash_key data[self.range_key] = range_key item = self.table.put_item( data=data, overwrite=True ) if settings.DEBUG: if not hash_key in self.counters: self.counters[hash_key] = {'reads':0,'writes':0} self.counters[hash_key]['writes'] +=1 self.counters['writes'] +=1 elapsed_time = time() - bench_start logger.info(data) logger.info("R%sW%s - write %0.5f seconds" % (self.counters[hash_key]['reads'], self.counters[hash_key]['writes'], elapsed_time)) return item def get_latest(self, hash_key ): ''' retreive the last recorded data hash_key item for the hash key ''' if settings.DEBUG: bench_start = time() kwargs = {} kwargs[self.hash_key+'__eq'] = hash_key kwargs['limit'] = 1 items = self.table.query( **kwargs ) if items: data = {} for item in items: for key in item.keys(): if not key in (self.hash_key, self.range_key): data[key] = item[key] else: return None if not len(data): return None if settings.DEBUG: if not hash_key in self.counters: self.counters[hash_key] = {'reads':0,'writes':0} self.counters[hash_key]['reads'] +=1 self.counters['reads'] +=1 elapsed_time = time() - bench_start logger.info("R%sW%s - %s - read %0.5f seconds" % (self.counters[hash_key]['reads'], self.counters[hash_key]['writes'], hash_key, elapsed_time)) return data def get_range_obj(self, hash_key): if settings.DEBUG: bench_start = time() kwargs = {} kwargs[self.hash_key+'__eq'] = hash_key # TODO - use batch_get items = self.table.query( **kwargs ) self.counters['reads'] +=1 data = {} for item in items: rkey_data = {} rkey = item[self.range_key] if rkey == 'index': data = json.loads(item['value']) break else: for key in item.keys(): if key != None and not key in (self.hash_key, self.range_key) and key != 'index': if key == 'value': value = item[key] try: rkey_data = json.loads(str(value)) except: rkey_data = value #else: # rkey_data[key] = item[key] data[rkey] = rkey_data if settings.DEBUG: if not hash_key in self.counters: self.counters[hash_key] = {'reads':0,'writes':0} self.counters[hash_key]['reads'] +=1 self.counters['reads'] +=1 elapsed_time = time() - bench_start #logger.info(data) logger.info("R%sW%s - %s - read %0.5f seconds" % (self.counters[hash_key]['reads'], self.counters[hash_key]['writes'], hash_key, elapsed_time)) return data def set_range_obj(self, hash_key, data, range_keys=None): # avoid crashing on attempt to write None data if data == None: return if range_keys == None: range_keys = data.keys() # TODO # add better size estimate datablocks = 0 for range_key in data.keys(): try: len_size = len( data[range_key] ) except: len_size = 1 datablocks += len_size # update date in msecs since epoch update_date = time() if datablocks > 1000: #print hash_key, #print datablocks # split over multiple items by data dict key with self.table.batch_write() as batch: for range_key in range_keys: value = json.dumps( data[range_key] ) batch_data = {} batch_data[self.hash_key] = hash_key batch_data[self.range_key] = range_key batch_data['value'] = value batch_data['update_date'] = update_date batch.put_item(data=batch_data) self.counters['batch_w'] +=1 # delete index if exists self.remove_range_obj(hash_key, range_keys=['index']) else: value = json.dumps(data) batch_data = {} batch_data[self.hash_key] = hash_key batch_data[self.range_key] = 'index' batch_data['value'] = value batch_data['update_date'] = update_date self.table.put_item(data=batch_data, overwrite=True) self.counters['writes'] +=1 return True def remove_range_obj(self, hash_key, range_keys=None): ''' deletes ranged object or specific range_keys ''' # get range object if range_keys == None: data = self.get_range_obj(hash_key) range_keys = data.keys() # remove possible index try: kwargs = {} kwargs[self.hash_key] = hash_key kwargs[self.range_key] = 'index' self.table.delete_item( **kwargs ) except: pass with self.table.batch_write() as batch: for range_key in range_keys: kwargs = {} kwargs[self.hash_key] = hash_key kwargs[self.range_key] = range_key batch.delete_item( **kwargs ) self.counters['delete'] +=1 return True
class Index(Base): '''An Index for docker-registry that uses Amazon AWS DynamoDB as the storage engine. Boto is used to do all access to DynamoDB. Configure the following dynamodb_config variables or environment variables: dynamodb_index_database - optional, if not specified will default to 'docker-registry' and the repository and version table names will be constructed using the {dynamodb_index_database}-repository and {dynamodb_index_database}-version. DynamoDB does not have a database concept, just tables in the data store. dynamodb_index_repository_table - override the default table name (above) with a new name dynamodb_index_version_table - override the default table name with a new name dynamodb_region - the AWS region for the dynamodb. This will default to the s3_region and if that is not defined, it defaults to 'us-east-1'. dynamodb_access_key - the AWS access key to use dynamodb_secret_access_key - the AWS secret part of the access key ''' _initLock = Lock() def __init__(self, database=None, dynamodb_access_key=None, dynamodb_secret_access_key=None): ''' Constructor ''' cfg = dynamodb_config.load() if database is None: database = cfg['extensions.dynamodb_index.database'] if dynamodb_access_key is None: dynamodb_access_key = cfg['extensions.dynamodb_index.access_key'] if dynamodb_secret_access_key is None: dynamodb_secret_access_key = cfg['extensions.dynamodb_index.secret_access_key'] self.repositoryTableName = cfg['extensions.dynamodb_index.repository_table'] self.versionTableName = cfg['extensions.dynamodb_index.version_table'] if dynamodb_access_key is None: self._db = dynamodb2.connect_to_region(cfg['extensions.dynamodb_index.region']) else: self._db = dynamodb2.connect_to_region(cfg['extensions.dynamodb_index.region'], aws_access_key_id=dynamodb_access_key, aws_secret_access_key=dynamodb_secret_access_key) self._repositoryTable = Table(self.repositoryTableName, schema=[HashKey('name', data_type=STRING)], global_indexes=[GlobalAllIndex('Repositories-By-Description-Index', parts=[HashKey('description', data_type=STRING)])], connection=self._db) self._versionTable = Table(self.versionTableName, schema=[HashKey('version', data_type=NUMBER)], connection=self._db) self.version = 1 Index._initLock.acquire() try: self._setup_database() finally: Index._initLock.release() super(Index, self).__init__() def _describe_or_create_tables(self): dynamodb_util.create_table_if_not_exists(self._repositoryTable) dynamodb_util.create_table_if_not_exists(self._versionTable) def _wait_for_tables(self): dynamodb_util.wait_for_table_active(self._repositoryTable) dynamodb_util.wait_for_table_active(self._versionTable) def _read_or_set_schema_version(self, default_version): def read_schema_version(): v = 0 try: results = self._versionTable.scan() row = results.next() v = row['version'] except: v = -1 return v # Read or insert the schema_version. Keep doing it until one # of them works. This is in case another thread is attempting the same # thing. Reading first will allow this thread to complete. schemaVersion = read_schema_version() while (schemaVersion <= 0): try: self._versionTable.put_item(data={'version': default_version}) schemaVersion = default_version except: sleep(0.5) schemaVersion = read_schema_version() return schemaVersion def _setup_database(self): needs_index = not dynamodb_util.table_exists(self._versionTable) self._describe_or_create_tables() self._wait_for_tables() version = self._read_or_set_schema_version(self.version) if (version != self.version): raise NotImplementedError('unrecognized search index version {0}'.format(version)) if needs_index: self._generate_index() def _generate_index(self): store = storage.load() with self._repositoryTable.batch_write() as batch: for repository in self._walk_storage(store=store): logger.info('Populating repository: {0}'.format(repository['name'])) batch.put_item(data=repository) def _handle_repository_created( self, sender, namespace, repository, value): name = '{0}/{1}'.format(namespace, repository) description = '' # TODO(wking): store descriptions logger.info('Creating new repository {0}'.format(name)) self._repositoryTable.put_item(data={'name': name, 'description': description}) def _handle_repository_updated( self, sender, namespace, repository, value): name = '{0}/{1}'.format(namespace, repository) description = '' # TODO(wking): store descriptions logger.info('Updating repository {0}'.format(name)) repo = self._repositoryTable.get_item(name=name) repo['description'] = description repo.save(overwrite=True) def _handle_repository_deleted(self, sender, namespace, repository): name = '{0}/{1}'.format(namespace, repository) logger.info('Deleting repository {0}'.format(name)) self._repositoryTable.delete_item(name=name) def results(self, search_term=None): """Return a list of results matching search_term The list elements should be dictionaries: {'name': name, 'description': description} """ if not search_term or len(search_term) == 0: logger.info('Index query: full table scan') repositories = self._repositoryTable.scan() else: logger.info('Index query: {0}'.format(search_term)) repositories = self._repositoryTable.scan(conditional_operator='OR', name__contains=search_term, description__contains=search_term) if repositories: return [{'name': repo['name'], 'description': repo['description'], } for repo in repositories] return []
connection=conn ) tokenTable = Table( TOKEN_TABLE_NAME, connection=conn ) # Everybody get's 0 score! users = tokenTable.scan() for user in users: #Delete the user .. Eaakkk tokenTable.delete_item(**{ 'token': user['token'] }) if user['flags']: i = 0 for userFlag in user['flags']: del user['flags'][i] user['score'] = 0 user['level'] = 0 #Put them back .. pheww tokenTable.put_item(user) users = tokenTable.scan()
tweets.get_item(id='2222').items() ########################### ## Deleting an item ########################### # If you already have an Item instance, the easiest approach is just to call Item.delete _tweet.delete() tweets.get_item(id='1111').items() # return item not found # If you don't have an Item instance from boto.dynamodb2.table import Table tweets = Table('tweets') tweets.delete_item(id='2222') ########################### ## Batch writing ########################### # If you’re loading a lot of data at a time, making use of batch writing can both speed up the process & # reduce the number of write requests made to the service. # Batch writing involves wrapping the calls you want batched in a context manager. The context manager # imitates the Table.put_item & Table.delete_item APIs. Getting & using the context manager looks like: import time from boto.dynamodb2.table import Table tweets = Table('tweets')
class Backend(): def __init__(self, region, link_prefix, bucket_name, table_name): if link_prefix[-1] != u'/': link_prefix += u'/' self.link_prefix = link_prefix conn = boto.s3.connect_to_region(region) self.bucket = conn.get_bucket(bucket_name) conn = boto.dynamodb2.connect_to_region(region) self.table = Table(table_name, connection=conn) def write_to_backend(self, station, title, timestamp, description, files_path, files): if self._get_item(station, title): msg = u'%s %s' % (station, title) raise TitleExistError(msg) if files_path and files: if files_path[-1] != u'/': files_path = files_path + u'/' files_with_path = [] for f in files: k = Key(self.bucket) k.key = u''.join([station, u'_', title, u'_', f]) k.set_contents_from_filename(files_path+f) k.set_acl(u'public-read') files_with_path.append(self.link_prefix+k.key) files_str = u' '.join(files_with_path) else: files_str = u'empty' self.table.put_item(data={u'station':station, u'title':title, \ u'timestamp':timestamp, u'description': description, u'files':files_str}) return True def update_description_and_append_files(self, station, title, timestamp, description, files_path, files): if files_path and files: if files_path[-1] != u'/': files_path = files_path + u'/' files_with_path = [] for f in files: k = Key(self.bucket) k.key = u''.join([station, u'_', title, u'_', f]) k.set_contents_from_filename(files_path+f) k.set_acl(u'public-read') files_with_path.append(self.link_prefix+k.key) new_files_str = u' '.join(files_with_path) else: new_files_str = u'empty' item = self._get_item(station, title) if not item: return False old_files_str = unicode(item[u'files']) if old_files_str == u'empty': files_str = new_files_str elif new_files_str == u'empty': files_str = old_files_str else: files_str = u''.join([old_files_str, u' ', new_files_str]) item[u'files'] = files_str item[u'description'] = description item.partial_save() return True def delete_item(self, station, title): item = self._get_item(station, title) if not item: return False files_str = unicode(item[u'files']) if files_str == u'empty': files = [] elif not files_str: files = [] else: files = files_str.split(u' ') self.table.delete_item(station=station, title=title) for f in files: k = Key(self.bucket) k.key = f[len(self.link_prefix):] k.delete() return True def delete_file(self, station, title, f): item = self._get_item(station, title) files = unicode(item[u'files']).split(u' ') if f in files: files.remove(f) if files: files_str = u' '.join(files) else: files_str = u'empty' item[u'files'] = files_str item.partial_save() k =Key(self.bucket) k.key = f[len(self.link_prefix):] k.delete() return True return False def get_page(self, station, direction, timestamp, page_size): need_reverse = False if timestamp == u'0': items = self.table.query(station__eq=station, index=u'timestamp-index', reverse=False, limit=page_size) elif direction == u'next': items = self.table.query(station__eq=station, timestamp__lt=timestamp, index=u'timestamp-index', reverse=False, limit=page_size) elif direction == u'prev': items = self.table.query(station__eq=station, timestamp__gt=timestamp, index=u'timestamp-index', reverse=True, limit=page_size) need_reverse = True else: return [] ret = [] for item in items: r = {} r[u'title'] = unicode(item[u'title']) r[u'timestamp'] = unicode(item[u'timestamp']) r[u'description'] = unicode(item[u'description']) r[u'files'] = unicode(item[u'files']).split(u' ') ret.append(r) if need_reverse: ret.reverse() return ret def get_item(self, station, title): item = self._get_item(station, title) if not item: return None r = {} r[u'title'] = unicode(item[u'title']) r[u'timestamp'] = unicode(item[u'timestamp']) r[u'description'] = unicode(item[u'description']) r[u'files'] = unicode(item[u'files']).split(u' ') return r def _get_item(self, station, title): items = self.table.query(station__eq=station, title__eq=title) for item in items: return item return None
class dynamoMethods: def __init__(self, dbName): self.table_dynamo = None try: #1. create new table self.table_dynamo = Table.create( dbName, schema=[HashKey('CUID')], connection=client_dynamo ) #HINT: Table.create; #HINT 2: Use CUID as your hashkey #self.table_dynamo = client_dynamo.create_table(name=dbName, schema=s) print "New Table Created" except Exception as e: #2.table already exists, so get the table self.table_dynamo = Table( dbName, connection=client_dynamo ) #HINT: Remember to use "connection=client.dynamo" ''' self.table_dynamo.put_item(data={ 'CUID': 'sh3658', 'name': 'scott', 'password': '******', }) ''' print "Table Already Exists" def dynamoAdd(self, cuid, name, password): # Check if the cuid exists in the database. user = None try: print 'try to get item' user = self.table_dynamo.get_item( CUID=cuid ) #, name=name, password=password)#, name=name, password=password) print "Found user." print user except Exception as e: print e # If the CUID already exists inside db, perform a partial update on the entry. if user != None and user['CUID'] == cuid: user['name'] = name user['password'] = password try: user.partial_save() print "Updated entry: " + cuid except Exception as e: print e # Otherwise, create a new entry else: try: self.table_dynamo.put_item(data={ 'CUID': cuid, 'name': name, 'password': password, }) print "New entry created." except Exception as e: print e def dynamoDelete(self, cuid): #################################################################### # YOUR CODE HERE #1. Check table for entries that have the same CUID, if so, DELETE #################################################################### try: user = self.table_dynamo.get_item( CUID=cuid ) #, name=name, password=password)#, name=name, password=password) self.table_dynamo.delete_item(CUID=cuid) print 'Item deleted' except Exception as e: print e def dynamoViewAll(self): string_db = "CUID: NAME\n" print string_db entries = self.table_dynamo.scan() for entry in entries: print entry['CUID'], ':', entry['name']
def delete_item(target_table, feature1): try: table = Table(target_table) table.delete_item(CUID = feature1) time.sleep(0.5) return except IOError: print "Error deleting item" return def view_table(target_table): try: table = Table(target_table) result_set = table.scan() for user in result_set: print user['Sex'], ":", user except IOError: print "Error viewing items" return def func_search(target_table, target_index): try: table = Table(target_table) _index = target_index #TODO print "result is: " except IOError: print "Error searching item!" if __name__ == "__main__": flag = True print "please enter commands 'add<>to<>' or 'delete<>from<>' or 'view<>' or 'search<>by<>' or exit" while (flag): print "Commands:" raw = raw_input('>>') command = raw.split() try: if command[0] == "add" and len(command) == 4: print "Sex:" input_0 = raw_input('>>') print "CUID:" input_1 = raw_input('>>') print "First_Name:" input_2 = raw_input('>>') print "Last_Name:" input_3 = raw_input('>>') new_item = {'Sex':input_0,'CUID':input_1,'First_Name':input_2,'Last_Name':input_3,} target_table = command[3] print "adding item" add_item(new_item, target_table) elif command[0] == "delete" and len(command) == 4: item_to_delete = command[1] target_table = command[3] print "run deleting function" delete_item(target_table, item_to_delete) elif command[0] == "view" and len(command) == 2: target_table = command[1] view_table(target_table) print "view data" elif command[0] == 'exit': print "exiting" time.sleep(1) sys.exit() elif command[0] == 'search': target_table = command[1] target_index = command[3] print "searching...." else: print "invalid command!" except KeyboardInterrupt: print "exiting" sys.exit()
class ClientRadiusSecretsDDB(RadiusSecrets): """ get secrets for a particular client ddb=DynamoDBConnection( aws_access_key_id=aws_access_key, aws_secret_access_key=aws_secret_key ) secrets = Table.create('qradius_secrets',schema=[\ HashKey('ip_address'), RangeKey('not_before',data_type=NUMBER), ], indexes=[ AllIndex('IPNotAfter',parts=[ HashKey('ip_address'), RangeKey('not_after',data_type=NUMBER), ]) ],connection=ddb) we will normally want secrets where ip_address = client ip address not_before < now not_after >= now likely we'll want to limit the # of secrets we look at, here i limit it to 3 For queries (these are the least impactful on dynamo: res=secrets.query(ip_address__eq='10.25.95.158',not_before__lt=now,limit=3,consistent=False) res=secrets.query(ip_address__eq='10.25.95.158',not_after__gt=now,limit=3,consistent=False,index='IPNotAfter') if you really need everything, this is a full scan: res=secrets.scan(ip_address__eq='10.25.95.158',not_before__lt=now,not_after__gt=now,limit=3) """ def __init__(self, encryption_key=None, aws_keys=None, table_name=None): """ @param encryption_key : string containing encryption key @param aws_keys : object containing aws keys @param table_name : dynamo table name @type encryption_key string @type aws_keys AWSKeys @type table_name string """ if encryption_key is None: raise ValueError("encryption_key must be specified") if not isinstance(aws_keys, AWSKeys): raise ValueError( "aws_material must be specified and of type AWSKeys") if table_name is None: raise ValueError( "dynamo table containing secrets must be specified") self._encryption_key = encryption_key self._encryptor = DataEncryptor(self._encryption_key) self._ddb_connection = DynamoDBConnection( aws_access_key_id=aws_keys.aws_access_key, aws_secret_access_key=aws_keys.aws_secret_key) if self._ddb_connection is None: raise ValueError("unable to obtain dynamo connection using %s" % aws_material) self._secret_table = Table(table_name, connection=self._ddb_connection) logging.debug('connectd to dynamo table %s' % table_name) if self._secret_table is None: raise ValueError("unable to connect to dynamo table %s" % table_name) def encryptSecret(self, secret): """ call the included encryption module to encrypt/encode secrets @param secret @type secret string """ if len(secret) > MAX_SECRET_LENGTH: raise ValueError("secret may not be more than %d bytes" % MAX_SECRET_LENGTH) encoded_secret = self._encryptor.encrypt(secret) return (encoded_secret) def decryptSecret(self, encoded_secret): """ call the included encryption module to decrypt/decode secrets @param encoded_secret @type encoded_secret string """ plain_secret = self._encryptor.decrypt(encoded_secret) if len(plain_secret) > MAX_SECRET_LENGTH: raise ValueError( "decryption resulted in a plain secret longer than the maximum length of %d bytes" % MAX_SECRET_LENGTH) return (plain_secret) def putSecret(self, clientIP, secret, not_before=None, not_after=None, tries=0): """ store a secret for clientIP @param clientIP : client ip address @param secret : radius secret for client @param tries : internal parameter to constrain recursion depth for self-calls @type clientIP string @type secret example usage: from radiussecrets import * rs=ClientRadiusSecrets(encryption_key='someencryptionkey', aws_keys=AWSKeys('myaccesskey','mysecretkey'),table_name='qradius_secrets') ValidationException rs.putSecret('1.2.3.4','shhdonottellanyone') """ now = time.time() if not_before is None: not_before = now if not_after is None: not_after = now + DEFAULT_KEY_LIFE if not isinstance(not_before, (int, float, long)) or not_before < 0: raise ValueError( "not_before must be a number representing seconds since epoch") if not isinstance(not_after, (int, float, long)) or not_after < 0: raise ValueError( "not_before must be a number representing seconds since epoch") if len(secret) > MAX_SECRET_LENGTH: raise ValueError("length of secret may not exceed %d bytes" % MAX_SECRET_LENGTH) result = None try: result = self._secret_table.put_item( data={ 'ip_address': clientIP, 'not_before': not_before, 'not_after': not_after, 'secret': self.encryptSecret(secret) }) except boto.dynamodb2.exceptions.ConditionalCheckFailedException as e: tries += 1 if tries > 5: logging.crit( 'pk violation for client %s not_before %d after %d tries at incrementing' % (clientIP, not_before, tries)) raise e #increment not_before to avoid pk violation not_before += 1 logging.warn( 'pk violation for client %s not_before %d; retrying with higher not_before ' % (clientIP, not_before)) result = self.putSecret(clientIP, secret, not_before=not_before, not_after=not_after, tries=tries) return result def deleteSecret(self, clientIP, not_before): """ delete a secret this should be used carefully @param clientIP @param not_before @type clientIP string @type not_before number """ return self._secret_table.delete_item(ip_address=clientIP, not_before=not_before) def getSecret(self, clientIP): """ return the secret associated with IP address if multiple secrets are found, selection is by: 1) not_before < now 2) not_after >= now 3) the highest value of not before, if there are still multiple secrets @param clientIP @param not_before seconds since epoch that the secret becomes valid @param not_after seconds since epoch after which the secret is not valid @type clientIP string representing an ip address """ now = time.time() #i wanted to limit to 3 (limit=3) but, boto kept barfing results = self._secret_table.query(ip_address__eq=clientIP, not_before__lt=now, consistent=False) client_secret = None client_secret_not_before = 0 for result in results: if result['not_after'] >= now: if client_secret_not_before < result['not_before']: client_secret_not_before = result['not_before'] client_secret = self.decryptSecret(result['secret']) logging.debug('retrieved secret for %s' % (clientIP)) return client_secret def purgeSecrets(self, clientIP): """ purge stale secrets associated with IP address 1) remove all keys for clientIP where not after is older than current time - PURGE_TIME_BEFORE_NOW 2) scan remaining keys, keep @param clientIP @type string @returns # of purged secrets """ now = time.time() min_purge_time = now nr_purged = 0 # first get rid of expired keys results = self._secret_table.query(ip_address__eq=clientIP, not_after__lt=min_purge_time, consistent=False, index='IPNotAfter') for result in results: logging.info('purging secret: %s %d' % (result['ip'], result['not_before'])) result.delete() nr_purged += 1 # now the fun... result_list = [] results = self._secret_table.query(ip_address__eq=clientIP, not_before__lt=min_purge_time, consistent=False) for result in results: result_list.append(result) # delete results if there are more than PURGE_RETAIN_NR_ACTIVE results, # we want the oldest not befores to be removed first if len(result_list) > PURGE_RETAIN_NR_ACTIVE_KEYS: for result in sorted(result_list, key=lambda result: result['not_before'] )[:-PURGE_RETAIN_NR_ACTIVE_KEYS]: logging.info('purging secret: %s %d' % (result['ip'], result['not_before'])) result.delete() nr_purged += 1 return nr_purged
class dynamoMethods: def __init__(self, dbName): self.table_dynamo = None #################################################################### # YOUR CODE HERE #1. create new table try: self.table_dynamo = Table.create( dbName, schema=[HashKey('CUID')], connection=client_dynamo ) #HINT: Table.create; #HINT 2: Use CUID as your hashkey print("Wait 20 sec until the table is created") time.sleep(20) print("New Table Created") #2.table already exists, so get the table except Exception as e: self.table_dynamo = Table( dbName, connection=client_dynamo ) #HINT: Remember to use "connection=client.dynamo" print("Table Already Exists") #################################################################### def dynamoAdd(self, cuid, name, password): #################################################################### # YOUR CODE HERE #1. Check table for entries that have the same CUID, if so, UPDATE (Don't delete) try: record = self.table_dynamo.get_item(CUID=cuid) record['name'] = name record['password'] = password record.save(overwrite=True) print("Record has been updated.\n") #2. Otherwise, create a new entry except Exception as e: # if query occurs exception, add record self.table_dynamo.put_item(data={ 'CUID': cuid, 'name': name, 'password': password }) print("New entry created.\n") #################################################################### def dynamoDelete(self, cuid): #################################################################### # YOUR CODE HERE #1. Check table for entries that have the same CUID, if so, DELETE try: self.table_dynamo.delete_item(CUID=cuid) print("The Record has benn deleted") except Exception as e: print("Cannot delete the record, it is not exists") raise #################################################################### def dynamoViewAll(self): #################################################################### # YOUR CODE HERE string_db = "CUID: NAME\n" print(string_db) #1. Get all entries in the table #2. Print the CUID: NAME for each entry items = self.table_dynamo.scan() all_users = list(items) for user in all_users: print(user['CUID'] + ':' + user['name']) print("\n")