def writeDB(ngram, idx): tbl = Table('project6-20141589') ngram = ngram.split('\n') cnt = 0 for elem in ngram: elem = elem.split('\t') if len(elem) == 2: try: elem[1] = int(elem[1]) except ValueError: continue atom = {'words': elem[0], 'counts': elem[1]} try: tbl.put_item(data = atom) except ConditionalCheckFailedException: try: item = tbl.get_item(words = elem[0]) item['counts'] += elem[1] item.save() except ItemNotFound: print 'Unknown exception raised for (%s, %d)' % (elem[0], elem[1]) cnt += 1 if cnt % 10000 == 0: print 'Thread %d running (%d)' % (idx, cnt) print 'Thread %d finished with %d' % (idx, cnt)
def isValidUser(self, username, passwd): print('this is ' + _platform + ' system') users = {'*****@*****.**':'admin'} if _platform == "linux" or _platform == "linux2": try: print('Before accessing DynamoDB') users = Table('Users') validuser = users.get_item(EmailId=username,Password=passwd) print('Linux - dynamodb authorization successful') except: validuser = None print('Accessing DynamoDB failed') pass else: try: if (username in users and users[username] == passwd and _platform == "win32"): validuser = '******' print('Windows - authorization successful') else: validuser = None print('Windows authorization failed!') except: validuser = None print('local authentication failed') return validuser
def createDynamoObject(name): try: users = Table.create(name, schema=[HashKey('id')], throughput={'read': db_read_cap, 'write': db_write_cap}, global_indexes=[GlobalAllIndex('EverythingIndex' , parts=[HashKey('name')])], connection=boto.dynamodb2.connect_to_region(AWS_REGION)) except: users = Table(name, connection=boto.dynamodb2.connect_to_region('us-west-2' )) print "1) Table 'data' already created for table: " + name # On first Run this wont insert data because of delay to create table on aws server side. try: users.put_item(data={ 'id': '3', 'type': 'person', 'name': 'dummy', 'activities': ['activity one'], }) except: print '2) Dummy Data already added for tabe: ' + name return users
def get_all_minions(): results = False try: minions = Table("minions") results = minions.scan() except Exception, e: raise e
def retrieve_id(db_name, user_id): #Grabs the table from dynamodb users = Table(db_name, connection=boto.dynamodb2.connect_to_region(AWS_REGION)) #Attempt to grab the item from the table try: #Call for getting item from table user = users.get_item(id=user_id) #Gets the set of activites and changes it to json after converting to list activities = user['activities'] output = json.dumps(list(activities)) #Report success, and adds the relevant information into a tuple result = 200, { "data": { "type": user['type'], "id": str(user['id']), "name": user['name'], "activities": output } } #Catches the exception of when the item isn't found except boto.dynamodb2.exceptions.ItemNotFound: #Reports failure, telling the user that the item isn't found result = 404, { "errors": [{ "not_found": { "id": str(user_id) } }] } #Returns the result back to the main file return result
def test_delete_table(self): self.storage_mocker.StubOutWithMock(storage, 'delete_table') self.storage_mocker.StubOutWithMock(storage, 'describe_table') storage.delete_table(IgnoreArg(), 'test_table') storage.describe_table(IgnoreArg(), 'test_table').AndReturn( models.TableMeta( models.TableSchema( { 'city1': models.ATTRIBUTE_TYPE_STRING, 'id': models.ATTRIBUTE_TYPE_STRING, 'name': models.ATTRIBUTE_TYPE_STRING }, ['id', 'name'], {'index_name': models.IndexDefinition('city1')} ), models.TableMeta.TABLE_STATUS_ACTIVE ) ) self.storage_mocker.ReplayAll() table = Table('test_table', connection=self.DYNAMODB_CON) self.assertTrue(table.delete()) self.storage_mocker.VerifyAll()
def main(): """ """ logging.basicConfig( format='%(asctime)s [%(levelname)s] [%(module)s] [%(funcName)s] [%(message)s]', level=logging.INFO ) logger = logging.getLogger(__name__) try: logger.info( 'Starting' ) spot_request_table_name = 'spotbatch.spotrequest' spot_request_uuid = '90719024-e546-11e4-9020-101f74edff46' dynamodb_conn = boto.dynamodb2.connect_to_region( 'us-east-1', profile_name='ipc-training') ts_pending_termination_detected = '2015-01-05T18:02:00Z' spot_request_table = Table( spot_request_table_name, connection=dynamodb_conn ) spot_request_item = spot_request_table.get_item( spot_request_uuid=spot_request_uuid ) spot_request_item[TableSpotRequest.ts_pending_termination_detected] = ts_pending_termination_detected partial_save_result = spot_request_item.partial_save() logger.info(partial_save_result) logger.info( 'Completed Successfully' ) except StandardError as e: logger.error( e ) logger.error( traceback.format_exc() ) sys.exit(8)
def retrieve_name(db_name, username): #Get the table from dynamodb users = Table(db_name, connection=boto.dynamodb2.connect_to_region(AWS_REGION)) #Assuming a failure first, before assigning a success result = 404, { "errors": [{ "not_found": { "name": username } }] } #Scan database for user with specified username user_with_name = users.scan(name__eq=username) #Assuming there is only one item with this username for user in user_with_name: #Get the list of activities activities = user['activities'] #Changes the set into a json format after changing to list output = json.dumps(list(activities)) #Returns success and gets information from the item, overwriting the failure result = 200, { "data": { "type": user['type'], #Gets the type of the item "id": str(user['id']), #Gets the ID of the item, in string format "name": user['name'], #Gets the name of the item "activities": output #Assign the list of activites to the returning object } } #Returns the result of the retrieval to the main file return result
def test_create_table(self): self.storage_mocker.StubOutWithMock(storage, 'create_table') storage.create_table(IgnoreArg(), IgnoreArg()) self.storage_mocker.ReplayAll() Table.create( "test", schema=[ fields.HashKey('hash', data_type=schema_types.NUMBER), fields.RangeKey('range', data_type=schema_types.STRING) ], throughput={ 'read': 20, 'write': 10, }, indexes=[ fields.KeysOnlyIndex( 'index_name', parts=[ fields.RangeKey('indexed_field', data_type=schema_types.STRING) ] ) ], connection=self.DYNAMODB_CON ) self.storage_mocker.VerifyAll()
def search(request): keyword = request.GET.get('keyword', '') #connection= boto.dynamodb2.connect_to_region('us-east-1a', aws_access_key_id='', # aws_secret_access_key='') conn = boto.connect_dynamodb() tweetMap = Table('tweet') #result = tweetMap.batch_get(keys=[ # {'tweetid': '525403743609970688'}, #]) result = tweetMap.scan(text__contains=keyword) coordinates = {} i = 0 for item in result: i = i + 1 latitude = item['latitude'].encode('utf-8') longitude = item['longitude'].encode('utf-8') text = item['text'].encode('utf-8') element = (latitude, longitude, text) #element = keyword coordinates[i] = element jsondata = simplejson.dumps(coordinates) return HttpResponse(jsondata, content_type="application/json")
def putSecret(name, secret, version, kms_key="alias/credstash", region="us-east-1", table="credential-store", context=None): ''' put a secret called `name` into the secret-store, protected by the key kms_key ''' kms = boto.kms.connect_to_region(region) # generate a a 64 byte key. # Half will be for data encryption, the other half for HMAC try: kms_response = kms.generate_data_key(kms_key, context, 64) except: raise KmsError("Could not generate key using KMS key %s" % kms_key) data_key = kms_response['Plaintext'][:32] hmac_key = kms_response['Plaintext'][32:] wrapped_key = kms_response['CiphertextBlob'] enc_ctr = Counter.new(128) encryptor = AES.new(data_key, AES.MODE_CTR, counter=enc_ctr) c_text = encryptor.encrypt(secret) # compute an HMAC using the hmac key and the ciphertext hmac = HMAC(hmac_key, msg=c_text, digestmod=SHA256) b64hmac = hmac.hexdigest() secretStore = Table(table, connection=boto.dynamodb2.connect_to_region(region)) data = {} data['name'] = name data['version'] = version if version != "" else "1" data['key'] = b64encode(wrapped_key) data['contents'] = b64encode(c_text) data['hmac'] = b64hmac return secretStore.put_item(data=data)
def deleteSecrets(name, region="us-east-1", table="credential-store"): secretStore = Table(table, connection=boto.dynamodb2.connect_to_region(region)) rs = secretStore.scan(name__eq=name) for i in rs: print("Deleting %s -- version %s" % (i["name"], i["version"])) i.delete()
def getReportID(self, val_hashkey, tablename=""): """ Purpose: Used to query any Dynamo DB table having HASH_KEY as REPORT_NAME by providing the parameters. Method will return an resultset object of all the records matching for criteria. :param self: class object itself :param val_hashkey: Hash Key value for the Dynamo DB table. :param tablename: Dynamo DB table name. """ if not tablename: tablename = 'TBL_AWS_REPORT_HDR' ## Create table object for the dynamo DB table tab = Table(tablename, connection=self.conn) try: #get the record from table based on reportname item = tab.get_item(Report_Name=val_hashkey.upper()) except Exception, e: self.m_logger.info("Error: Error While running the getReportID method..") self.m_logger.info("Exception: "+ str(e) +"occured while getReportID method") self.m_logger.sendlog() sys.exit(1)
def get_clients(self, raw_idadmin): t1 = Table('client') query = t1.scan(idadmin__eq=raw_idadmin, limit=50) j = 0 arr1 = [] arr2 = [] datos = {0:'idclient', 1:'birthdate', 2:'loanamount', 3:'loanperiod', 4:'loanpurpose', 5:'status', 6:'risk', 7:'created', 8:'modified' } for q in query: for i in range(9): arr2.append(q[datos[i]]) print arr2 arr1.append(arr2) arr2 = [] j += 1 return arr1
class TestMisconfiguredSchemaException(unittest.TestCase): def setUp(self): # Configuration options self.table_name = 'conf' self.store_name = 'test' # Instanciate the store DynamoDBConfigStore(connection, self.table_name, self.store_name) # Get an Table instance for validation self.table = Table(self.table_name, connection=connection) def test_misconfigured_schema_store_key(self): """ Test that an exception is raised if the store key is not an hash """ with self.assertRaises(MisconfiguredSchemaException): DynamoDBConfigStore( connection, self.table_name, self.store_name, store_key='test') def test_misconfigured_schema_option_key(self): """ Test that an exception is raised if the option key isn't a range """ with self.assertRaises(MisconfiguredSchemaException): DynamoDBConfigStore( connection, self.table_name, self.store_name, option_key='test') def tearDown(self): """ Tear down the test case """ self.table.delete()
def run(self): """ """ region_name = 'us-east-1' profile_name = 'ipc-training' dummy_message = None dynamodb_conn = boto.dynamodb2.connect_to_region( region_name, profile_name=profile_name ) spot_request_table = Table( awsspotbatch.common.const.SPOT_REQUEST_TABLE_NAME, connection=dynamodb_conn ) spot_request_item = spot_request_table.get_item( spot_request_uuid=self.spot_request_uuid ) # spot_batch_job_parm_table = Table( awsspotbatch.common.const.SPOT_BATCH_JOB_PARM_TABLE_NAME, connection=dynamodb_conn ) # spot_batch_job_parm_item = spot_batch_job_parm_table.get_item( spot_master_uuid=spot_request_item[TableSpotRequest.spot_master_uuid] ) # batch_job_parm_item = BatchJobParmItem( stringParmFile=spot_batch_job_parm_item[TableSpotBatchJobParm.raw_batch_job_parm_item] ) client_bootstrap_service_cmds_results, client_bootstrap_user_cmds_results = launch_remote_client( spot_request_item ) logger.info( 'spot_request_uuid: ' + self.spot_request_uuid ) for cmd_result in client_bootstrap_service_cmds_results: logger.info( ' service cmd: ' + cmd_result['cmd']) logger.info( ' remote_exit_status: ' + str(cmd_result['remote_exit_status']) ) logger.info( ' buf_std_out: ' + cmd_result['buf_std_out'] ) logger.info( ' buf_std_err: ' + cmd_result['buf_std_err'] ) for cmd_result in client_bootstrap_user_cmds_results: logger.info( ' user cmd: ' + cmd_result['cmd']) logger.info( ' remote_exit_status: ' + str(cmd_result['remote_exit_status']) ) logger.info( ' buf_std_out: ' + cmd_result['buf_std_out'] ) logger.info( ' buf_std_err: ' + cmd_result['buf_std_err'] )
def setUp(self): logging.getLogger('boto').setLevel(logging.CRITICAL) err = 'Use a real DynamoDB %s. Add datastore/dynamo/test_settings.py.' assert aws_access_key != '<aws access key>', err % 'access key.' assert aws_secret_key != '<aws secret key>', err % 'secret key.' self.conn = boto.dynamodb2.connect_to_region(aws_region, aws_access_key_id=aws_access_key,aws_secret_access_key=aws_secret_key) # Create an indexed table table = Table(self.INDEXED_TABLE, connection=self.conn) try: status = table.describe() except: table = Table.create(self.INDEXED_TABLE, schema=[ HashKey('department', data_type=STRING), RangeKey('name', data_type=STRING) ], indexes=[ AllIndex('ScoreIndex', parts=[ HashKey('department'), RangeKey('score', data_type=NUMBER) ]) ], global_indexes=[ GlobalAllIndex('GroupIndex', parts=[ HashKey('group'), RangeKey('age', data_type=NUMBER) ]) ], connection=self.conn) # make sure we're clean :) self._delete_keys_from_table(self.SIMPLE_TABLE) self._delete_keys_from_table(self.INDEXED_TABLE) self._delete_keys_from_table(self.RANGEKEY_TABLE)
class TestCustomThroughput(unittest.TestCase): def setUp(self): # Configuration options self.table_name = 'conf' self.store_name = 'test' self.read_units = 10 self.write_units = 8 # Instanciate the store self.store = DynamoDBConfigStore( connection, self.table_name, self.store_name, read_units=self.read_units, write_units=self.write_units) # Get an Table instance for validation self.table = Table(self.table_name, connection=connection) def test_custom_throughput(self): """ Test that we can set custom thoughput for new tables """ throughput = self.table.describe()[u'Table'][u'ProvisionedThroughput'] self.assertEqual(throughput[u'ReadCapacityUnits'], self.read_units) self.assertEqual(throughput[u'WriteCapacityUnits'], self.write_units) def tearDown(self): """ Tear down the test case """ self.table.delete()
def tables_create(self): """ Creates a new table.. throws exception if tables already exist """ for table_name, schema in dynamo_schema.iteritems(): logging.info("Creating " + self.prefix + table_name) Table.create(self.prefix + table_name, **schema)
def refresh(self, items, consistent=False): """ Overwrite model data with freshest from database Parameters ---------- items : list or :class:`~flywheel.models.Model` Models to sync consistent : bool, optional If True, force a consistent read from the db. (default False) """ if isinstance(items, Model): items = [items] if not items: return tables = defaultdict(list) for item in items: tables[item.meta_.ddb_tablename].append(item) for tablename, items in tables.iteritems(): table = Table(tablename, connection=self.dynamo) keys = [item.pk_dict_ for item in items] results = table.batch_get(keys, consistent=consistent) for item, data in itertools.izip(items, results): with item.loading_(self): for key, val in data.items(): item.set_ddb_val_(key, val)
def create_table(region): connection = boto.dynamodb2.connect_to_region(region) Table.create( 'Word', schema=[ HashKey('user'), RangeKey('word') ], global_indexes = [ GlobalAllIndex('WordList-index', parts=[ HashKey('user'), RangeKey('wordList') ]) ], throughput={ 'read': 1, 'write': 1 }, connection=connection ) Table.create( 'WordList', schema=[ HashKey('user'), RangeKey('name') ], throughput={ 'read': 1, 'write': 1 }, connection=connection )
def getTenantIDs(environment, blacklist): """ Returns a list of Tenant IDs for the current environment :param environment: Current environment :return: [ (tid, [email1, ...], name), ... ] """ # hash=id (tenant ID) | contentMarketing | enabled | licenses | name tInfoTable = Table(environment + "_idp_tenant") # hash=tenant_id | range=id | email | enabled | password | user_profile | last_login_time | login_token tUserTable = Table(environment + "_idp_user") tenantResults = tInfoTable.scan() tenants = [] for res in tenantResults: if str(res['id']) in blacklist: continue tenant = [res["id"], [], res["name"]] for email in tUserTable.query(tenant_id__eq=tenant[0]): tenant[1].append(email["email"]) tenants.append(tenant) return tenants
def __init__(self, region, prefix = 'cicada_'): self.prefix = prefix self.connection= boto.dynamodb2.connect_to_region(region) self.project_job = Table(prefix + 'project_job', connection=self.connection, **dynamo_schema['project_job']) self.job_history = Table(prefix + 'job_history', connection=self.connection, **dynamo_schema['job_history']) self.worker = Table(prefix + 'worker', connection=self.connection, **dynamo_schema['workers']) self.poll = Table(prefix + 'poll', connection=self.connection, **dynamo_schema['poll'])
def table_exists(name): t = Table(name) try: t.describe() return True except: return False
def debug_delete_table(): try: users = Table('users', connection=boto.dynamodb2.connect_to_region('us-west-2')) Table.delete(users) print "Deleting users!" except Exception, c: print c
def getUsersWithTenantBlackList(environment, blacklist): """ Gets users for all tenants except blacklisted tenants :param environment: environment to use :param blacklist: [ tid1, tid2, tid3 ... ] all str' return [ {uid:str, email:str, last_login_time:str, tid:str}, ... ] """ # hash=id (tenant ID) | contentMarketing | enabled | licenses | name tInfoTable = Table(environment + "_idp_tenant") # hash=tenant_id | range=id | email | enabled | password | user_profile | last_login_time | login_token tUserTable = Table(environment + "_idp_user") tenantResults = tInfoTable.scan() users = [] for res in tenantResults: if str(res['id']) in blacklist: continue for userItem in tUserTable.query(tenant_id__eq=res['id']): user = {} user['tid'] = res['id'] user['uid'] = userItem['id'] user['email'] = userItem['email'] user['last_login_time'] = userItem['last_login_time'] users.append(user) return users
def get_minion(instanceid): result = False try: minions = Table("minions") result = minions.get_item(instanceid=instanceid) except Exception, e: raise e
def main(): print 'starting cleanDB.py...' items_table = Table('items') usr_itemQuery = items_table.scan() for usr_item in usr_itemQuery: endTime = usr_item['endTime'] if endTime is None: print 'nonetype listing! deleting...' usr_item.delete() continue now = datetime.utcnow() list_date, list_time = (endTime.encode('utf-8').decode('ascii', 'ignore')).split('T') list_time = list_time.replace('.000Z','') together = list_date+list_time c_end_time = datetime.strptime(together, '%Y-%m-%d%H:%M:%S') diff = c_end_time - now diff = diff.total_seconds() if diff < 0: print 'expired listing by time! deleting...' usr_item.delete() continue print 'finished'
def add_minion(instanceid): result = False try: minions = Table("minions") result = minions.put_item(data={"instanceid": instanceid}) except Exception, e: raise e
def display_keypair(): """ """ dynamodb_conn = boto.dynamodb2.connect_to_region( 'us-east-1', profile_name='ipc-training' ) spot_rsa_key_table = Table( awsspotbatch.common.const.SPOT_RSA_KEY_TABLE_NAME, connection=dynamodb_conn ) rsa_key_item = spot_rsa_key_table.get_item( spot_master_uuid='4d70b3da-f5af-11e4-b866-101f74edff46' ) kp_material_dec = decode( kp_enc_key, str( rsa_key_item[ TableSpotRSAKey.rsa_key_encoded ]) ) print kp_material_dec
def get_table(): table = Table('test_bot') return table
def test_get_item_with_undeclared_table(): table = Table('undeclared-table') table.get_item.when.called_with(test_hash=3241526475).should.throw(JSONResponseError)
def get_table(self, table_name=None): """ Convenience method for client who may wish to get a specific table from the DynamoDB connection """ table_name = table_name or self.table.table_name return Table(table_name, connection=self.connection)
def create_tables(): conn = init_connection() tables = conn.list_tables()["TableNames"] for name in tables: Table(name, connection=conn).delete() Table.create("DelegateIt_Customers", schema=[ HashKey("uuid"), ], global_indexes=[ GlobalAllIndex("phone_number-index", parts=[ HashKey("phone_number"), ]), GlobalAllIndex("fbuser_id-index", parts=[ HashKey("fbuser_id"), ]), ], connection=conn) Table.create("DelegateIt_Delegators", schema=[ HashKey("uuid"), ], global_indexes=[ GlobalAllIndex("phone_number-index", parts=[ HashKey("phone_number"), ]), GlobalAllIndex("email-index", parts=[ HashKey("email"), ]), GlobalAllIndex("fbuser_id-index", parts=[ HashKey("fbuser_id"), ]), ], connection=conn) Table.create("DelegateIt_Transactions_CD", schema=[HashKey("customer_uuid"), RangeKey("timestamp", "N")], global_indexes=[ GlobalAllIndex("status-index", parts=[ HashKey("status"), ]), GlobalAllIndex("delegator_uuid-index", parts=[ HashKey("delegator_uuid"), RangeKey("timestamp", "N") ]), ], connection=conn) Table.create("DelegateIt_Handlers", schema=[HashKey("ip_address")], connection=conn) Table.create("DelegateIt_PushEndpoints", schema=[HashKey("device_id")], global_indexes=[ GlobalAllIndex("customer_uuid-index", parts=[ HashKey("customer_uuid"), ]) ], connection=conn) tables = conn.list_tables()["TableNames"] for name in tables: print("Describing", name, "\n", Table(name, connection=conn).describe())
def tprint(string): now = time.ctime() print(now + '\t ' + str(string)) if __name__ == '__main__': ddb = boto.dynamodb2.connect_to_region(REGION) tprint(str(ddb)) conn = psycopg2.connect(CONN_STRING) while True: for z in ZONE_IDS: table_name = 'sg_%s_21' % z tprint("Processing table %s" % table_name) table = Table(table_name, connection=ddb) now = time.time() try: res = table.query_2(is_etl_processed__eq=0, etl_ts__lte=now, index='ETLIndex', limit=QLIMIT) except Exception, err: print Exception, err else: tprint('processing...') for r in res: print "DDB ok", if r['elaboration_ts'] is None:
from boto.exception import JSONResponseError from time import sleep import sys if len(sys.argv) != 3: print 'Usage: %s <source_table_name> <destination_table_name>' % sys.argv[0] sys.exit(1) src_table = sys.argv[1] dst_table = sys.argv[2] ddbc = DynamoDBConnection() # 1. Read and copy the target table to be copied table_struct = None try: logs = Table(src_table) table_struct = logs.describe() except JSONResponseError: print "%s not existing" % src_table sys.exit(1) print '*** Reading key schema from %s table' % src_table src = ddbc.describe_table(src_table)['Table'] hash_key = '' range_key = '' for schema in src['KeySchema']: attr_name = schema['AttributeName'] key_type = schema['KeyType'] if key_type == 'HASH': hash_key = attr_name elif key_type == 'RANGE':
def test_create_table_duplicate(self): self.storage_mocker.StubOutWithMock(storage, 'create_table') storage.create_table(IgnoreArg(), IgnoreArg(), IgnoreArg()).AndReturn( models.TableMeta( models.TableSchema( { 'hash': models.ATTRIBUTE_TYPE_NUMBER, 'range': models.ATTRIBUTE_TYPE_STRING, 'indexed_field': models.ATTRIBUTE_TYPE_STRING }, ['hash', 'range'], {"index_name": models.IndexDefinition('indexed_field')}), models.TableMeta.TABLE_STATUS_ACTIVE)) storage.create_table(IgnoreArg(), IgnoreArg(), IgnoreArg()).AndRaise(TableAlreadyExistsException) self.storage_mocker.ReplayAll() Table.create("test", schema=[ fields.HashKey('hash', data_type=schema_types.NUMBER), fields.RangeKey('range', data_type=schema_types.STRING) ], throughput={ 'read': 20, 'write': 10, }, indexes=[ fields.KeysOnlyIndex( 'index_name', parts=[ fields.RangeKey('indexed_field', data_type=schema_types.STRING) ]) ], connection=self.DYNAMODB_CON) try: Table.create("test", schema=[ fields.HashKey('hash', data_type=schema_types.NUMBER), fields.RangeKey('range', data_type=schema_types.STRING) ], throughput={ 'read': 20, 'write': 10, }, indexes=[ fields.KeysOnlyIndex( 'index_name', parts=[ fields.RangeKey( 'indexed_field', data_type=schema_types.STRING) ]) ], connection=self.DYNAMODB_CON) self.fail() except JSONResponseError as e: self.assertEqual('ResourceInUseException', e.error_code) self.storage_mocker.VerifyAll() except Exception as e: self.fail()
import boto.dynamodb2 import boto.dynamodb2.types import boto.dynamodb2.table from boto.dynamodb2.table import Table from boto.dynamodb2.items import Item dynamodb = boto3.resource('dynamodb') parser = argparse.ArgumentParser(description="MongoDB JSON to dynamoDB importer") parser.add_argument("-j", "--json", required=True, help="JSON file name") args = parser.parse_args(sys.argv[1:]) jsonIn = args.json test3 = Table('test3') with open(jsonIn, 'r') as f: lines = f.read().splitlines() f.close() for line in lines: with open(line, 'r') as l: data = json.load(l) dynamodb_json = json.dumps(data) test3.put_item(data=json.loads(dynamodb_json)) print line + " added to test2 dynamoDB" l.close()
def main(args): dynamo = dynaconnect(args) cfn = cfnconnect(args) s3 = s3connect(args) try: dynamo.describe_table(args.table) except boto.exception.JSONResponseError as details: error("Unable to access parameter table", details.message) else: table = Table(args.table, connection=dynamo) stackName = args.name try: stack = cfn.describe_stacks(stackName) except boto.exception.BotoServerError as details: if args.action == "delete": error("No such stack", "") else: print("Creating stack") action = cfn.create_stack else: if args.action == "delete": print("Deleting stack") else: print("Updating stack") action = cfn.update_stack if args.action == "delete": cfn.delete_stack(stackName) else: # Initial values from explicit parameters param_values_list = [ dict([x.strip() for x in k.strip().split("=")] for k in args.key) ] # Get DynamoDB parameters conf_item = table.get_item(name=stackName) type = conf_item['type'] param_values_list.append(conf_item['config']) while (conf_item['depends']): conf_item = table.get_item(name=conf_item['depends']) param_values_list.append(conf_item['outputs']) param_values_list.append(conf_item['config']) # We reverse the list of dictionnaries to prioritize explicit parameters over stack parameters (over dependant stack parameters) param_values = dict((k, v) for p in reversed(param_values_list) for (k, v) in p.items()) # Get template from bucket add parse param keys bucket = s3.get_bucket(args.bucket) template = bucket.get_key(type).get_contents_as_string( encoding='utf-8') param_list = json.loads(template)['Parameters'].keys() # We set all the parameters from the template that we found in ddb or on the command line params = [(p, param_values[p]) for p in param_list if p in param_values] url = "https://s3-%s.amazonaws.com/%s/%s" % (args.region, args.bucket, type) try: action(stackName, template_url=url, parameters=params, capabilities=['CAPABILITY_IAM'], tags={'env': args.tag}) except boto.exception.BotoServerError as details: if (details.error_message == "No updates are to be performed."): print("Stack already up to date") exit(0) else: error("Unable to Create/Update", details) while cfn.describe_stacks(stackName)[0].stack_status in [ "CREATE_IN_PROGRESS", "UPDATE_IN_PROGRESS", "UPDATE_COMPLETE_CLEANUP_IN_PROGRESS" ]: time.sleep(10) resources = cfn.list_stack_resources(stackName) r = sorted([(i.resource_status, i.logical_resource_id) for i in resources]) for t in r: print("%s\t%s" % t) print( "===============================================================" ) desc = cfn.describe_stacks(stackName)[0] if desc.stack_status != "CREATE_COMPLETE" and desc.stack_status != "UPDATE_COMPLETE": error("Stack creation failed", desc.stack_status_reason) print("Stack created/updated successfully, udpdating referential data") stack_params = table.get_item(name=stackName) stack_params['outputs'] = dict((o.key, o.value) for o in desc.outputs) stack_params.partial_save()
#!/usr/bin/python #Aplicacao demo integradno python e dynamoDB # import boto.dynamodb2 import exceptions from datetime import datetime from boto.dynamodb2.table import Table studentsTable = Table("Estudantes", connection=boto.dynamodb2.connect_to_region('us-west-2')) def existeRegistro(hash_key, value): try: item = studentsTable.get_item(identidade=value) print "Aluno: %s " % (item['nome']) except boto.dynamodb2.exceptions.ItemNotFound: print "Aluno nao encontrado " item = None except boto.dynamodb.exceptions.DynamoDBKeyNotFoundError: print "Aluno nao encontrado " item = None return item empresa = {'nome': "FIAP", 'empregados': 105, 'Faturamento': 10623.12} hora = datetime.now() print empresa['nome']
import os from pyspark import SparkConf, SparkContext from pyspark.streaming import StreamingContext from pyspark.streaming.kafka import KafkaUtils, OffsetRange, TopicAndPartition from boto import dynamodb2 from boto.dynamodb2.table import Table, Item #import boto3 import decimal import datetime from time import sleep dynamoDB = dynamodb2.connect_to_region('us-east-1') dyntable = Table('BestArrivalTimeFinalTask2', connection=dynamoDB) def updateFunction(newValues, minimum): if minimum is None: minimum = newValues[0] newValues.append(minimum) minimum = min(newValues, key=lambda x: x[1]) return minimum def printResult(rdd): result = rdd.take(10) #Ordered(10,key=lambda x:-x[1]) print("*******") for airport in result: print(airport) def saveToDynamodb(rdd):
def create_or_update_dynamo_table(connection, module, boto3_dynamodb=None, boto3_sts=None, region=None): table_name = module.params.get('name') hash_key_name = module.params.get('hash_key_name') hash_key_type = module.params.get('hash_key_type') range_key_name = module.params.get('range_key_name') range_key_type = module.params.get('range_key_type') read_capacity = module.params.get('read_capacity') write_capacity = module.params.get('write_capacity') all_indexes = module.params.get('indexes') tags = module.params.get('tags') wait_for_active_timeout = module.params.get('wait_for_active_timeout') for index in all_indexes: validate_index(index, module) schema = get_schema_param(hash_key_name, hash_key_type, range_key_name, range_key_type) throughput = { 'read': read_capacity, 'write': write_capacity } indexes, global_indexes = get_indexes(all_indexes) result = dict( region=region, table_name=table_name, hash_key_name=hash_key_name, hash_key_type=hash_key_type, range_key_name=range_key_name, range_key_type=range_key_type, read_capacity=read_capacity, write_capacity=write_capacity, indexes=all_indexes, ) try: table = Table(table_name, connection=connection) if dynamo_table_exists(table): result['changed'] = update_dynamo_table(table, throughput=throughput, check_mode=module.check_mode, global_indexes=global_indexes) else: if not module.check_mode: Table.create(table_name, connection=connection, schema=schema, throughput=throughput, indexes=indexes, global_indexes=global_indexes) result['changed'] = True if not module.check_mode: result['table_status'] = table.describe()['Table']['TableStatus'] if tags: # only tables which are active can be tagged wait_until_table_active(module, table, wait_for_active_timeout) account_id = get_account_id(boto3_sts) boto3_dynamodb.tag_resource( ResourceArn='arn:jctanner.cloud_amazon.aws:dynamodb:' + region + ':' + account_id + ':table/' + table_name, Tags=ansible_dict_to_boto3_tag_list(tags)) result['tags'] = tags except BotoServerError: result['msg'] = 'Failed to create/update dynamo table due to error: ' + traceback.format_exc() module.fail_json(**result) else: module.exit_json(**result)
import sys sys.path.append('/Users/pgregg/nosql-service') sys.path.append('/Users/pgregg/nosql-service/tests') sys.path.append('/Users/pgregg/nosql-service/tests/integration') sys.path.append('/Users/pgregg/nosql-service/tests/integration/dynamodb2') import unittest from boto.dynamodb2 import exceptions from boto.dynamodb2.fields import HashKey, RangeKey, KeysOnlyIndex from boto.dynamodb2.items import Item from boto.dynamodb2.table import Table from boto.dynamodb2.types import NUMBER dynamodb = True # Test creating a full table with all options specified. users = Table.create( 'users', schema=[HashKey('username'), RangeKey('friend_count', data_type=NUMBER)], throughput={ 'read': 5, 'write': 5, }, indexes=[ KeysOnlyIndex('LastNameIndex', parts=[HashKey('username'), RangeKey('last_name')]), ])
def create_table(table_name, region=None, key=None, keyid=None, profile=None, read_capacity_units=None, write_capacity_units=None, hash_key=None, hash_key_data_type=None, range_key=None, range_key_data_type=None, local_indexes=None, global_indexes=None): ''' Creates a DynamoDB table. CLI Example: .. code-block:: bash salt myminion boto_dynamodb.create_table table_name / region=us-east-1 / hash_key=id / hash_key_data_type=N / range_key=created_at / range_key_data_type=N / read_capacity_units=1 / write_capacity_units=1 ''' schema = [] primary_index_fields = [] primary_index_name = '' if hash_key: hash_key_obj = HashKey(hash_key, data_type=hash_key_data_type) schema.append(hash_key_obj) primary_index_fields.append(hash_key_obj) primary_index_name += hash_key if range_key: range_key_obj = RangeKey(range_key, data_type=range_key_data_type) schema.append(range_key_obj) primary_index_fields.append(range_key_obj) primary_index_name += '_' primary_index_name += range_key primary_index_name += '_index' throughput = {'read': read_capacity_units, 'write': write_capacity_units} local_table_indexes = [] if local_indexes: # Add the table's key local_table_indexes.append( AllIndex(primary_index_name, parts=primary_index_fields)) for index in local_indexes: local_table_indexes.append(_extract_index(index)) global_table_indexes = [] if global_indexes: for index in global_indexes: global_table_indexes.append( _extract_index(index, global_index=True)) conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) Table.create(table_name, schema=schema, throughput=throughput, indexes=local_table_indexes, global_indexes=global_table_indexes, connection=conn) # Table creation can take several seconds to propagate. # We will check MAX_ATTEMPTS times. MAX_ATTEMPTS = 30 for i in range(MAX_ATTEMPTS): if exists(table_name, region, key, keyid, profile): return True else: time.sleep(1) # sleep for one second and try again return False
import os from pyspark import SparkConf, SparkContext from pyspark.streaming import StreamingContext from pyspark.streaming.kafka import KafkaUtils,OffsetRange,TopicAndPartition #import boto3 from boto import dynamodb2 from boto.dynamodb2.table import Table,Item import decimal #dynamodb = boto3.resource('dynamodb', region_name='us-east-1') #table = dynamodb.Table('Top10Airports2') dynamoDB = dynamodb2.connect_to_region('us-east-1') dyntable = Table('Top10AirportsTask2', connection = dynamoDB) def updateFunction(newValues, runningCount): if runningCount is None: runningCount = (0, 0, 0) depDelaySum = sum(newValues, runningCount[0]) count = runningCount[1] + len(newValues) avgDepDelay = depDelaySum/float(count) return (depDelaySum,count,avgDepDelay) def printResult(rdd): result = rdd.take(10)#Ordered(10,key=lambda x:-x[1]) for airport in result: print(airport) def sortLocal(top10, newVal): top10.append(newVal) top10.sort(key=lambda element: element[1]) return top10[0:10]
item = {} payload_hash = item['payload_hash'] = m.message_attributes[ 'payload_hash']['string_value'] topic = item['topic'] = m.message_attributes['topic']['string_value'] item['receive_ts'] = float( m.message_attributes['receive_ts']['string_value']) zone_id, profile_id, sender_id, recipient_id, msg_class_id = topic.split( '/') item['sender_id'] = sender_id payload = base64.decodestring(m.get_body()) logs = Table('sg_msg_log', connection=ddb) log_items = logs.query_2(payload_hash__eq=payload_hash, limit=1) header = payload[:HEADER_LEN] header_fields = struct.unpack(ENDIANESS + HEADER_FMT, header) for i, field_name in enumerate(HEADER_FIELDS): item[field_name] = header_fields[i] try: msg_name, msg_fmt, msg_fields = MESSAGES[header_fields[3]] except KeyError: tprint('message read with an unknown MSG-ID (%d)' % header_fields[3]) for log_item in log_items: log_item['err'] = 'ERR_UNKNOWN_MSG_ID_%d' % header_fields[3]
def test_scan(self): t = Table('users', connection=self.conn.conn) x = t.scan() self.assertGreater(map(lambda x: x._data, x),1)
import time from boto import dynamodb2 from boto.dynamodb2.table import Table from boto.dynamodb2.fields import HashKey, RangeKey, GlobalAllIndex #from boto.dynamodb2.table import Table from boto.dynamodb2.types import NUMBER TABLE_NAME = "ParkingPass" REGION = "us-east-2" conn = dynamodb2.connect_to_region( REGION, aws_access_key_id='AKIAINKNLJLNPOG7BJAQ', aws_secret_access_key='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') table = Table(TABLE_NAME, connection=conn) def write(counter): attrib = {"Type": "A", "Available": "YES", "Spot": 21} with table.batch_write() as table_batch: #for example_counter in xrange(10): reqhashdata = {"Space": counter, "timestamp": time.time()} final_dynamo_data = dict(attrib.items() + reqhashdata.items()) table_batch.put_item(data=final_dynamo_data) def request(space): results = table.query(Space__eq=space) #results = table.query(timestamp__gte=0) print(results)
def connect_to_dynamo(name, amazon): # CONNECT TO AMAZON DYNAMODB TABLE table = Table(name, connection=amazon) return table
from boto.dynamodb2.fields import HashKey from boto.dynamodb2.table import Table from flask import Flask from flask.ext.dynamo import Dynamo import boto import boto3 app = Flask(__name__) app.config['DYNAMO_TABLES'] = [ Table('users', schema=[HashKey('username')]), Table('groups', schema=[HashKey('name')]), ] dynamo = Dynamo(app) # @app.route('/create_user') # def create_user(): # dynamo.users.put_item(data={ # 'username': '******', # 'first_name': 'Randall', # 'last_name': 'Degges', # 'email': '*****@*****.**', # }) # # # or ... # # dynamo.tables['users'].put_item(data={ # 'username': '******', # 'first_name': 'Randall', # 'last_name': 'Degges', # 'email': '*****@*****.**', # })
def testPathwayToDynamoDB(self): """ Test metric data pathway to dynamodb """ metricName = "TEST." + "".join(random.sample(string.ascii_letters, 16)) nativeMetric = { "modelParams": { "minResolution": 0.2, "min": 0.0, "max": 10000.0, }, "datasource": "custom", "metricSpec": { "metric": metricName, "resource": "Test", "userInfo": { "symbol": "TEST", "metricType": "TwitterVolume", "metricTypeName": "Twitter Volume", } } } metricName = nativeMetric["metricSpec"]["metric"] instanceName = nativeMetric["metricSpec"]["resource"] userInfo = nativeMetric["metricSpec"]["userInfo"] now = datetime.datetime.utcnow().replace(minute=0, second=0, microsecond=0) data = [ (5000.0, now - datetime.timedelta(minutes=10)), (6000.0, now - datetime.timedelta(minutes=5)), (7000.0, now), ] # We'll be explicitly deleting the metric below, but we need to add a # cleanup step that runs in case there is some other failure that prevents # that part of the test from being reached. def gracefulDelete(): try: self._deleteMetric(metricName) except ObjectNotFoundError: pass self.addCleanup(gracefulDelete) # Add custom metric data sock = socket.socket() sock.connect(("localhost", self.plaintextPort)) for metricValue, ts in data: sock.sendall( "%s %r %s\n" % (metricName, metricValue, epochFromNaiveUTCDatetime(ts))) self.gracefullyCloseSocket(sock) uid = self.checkMetricCreated(metricName) # Save the uid for later LOGGER.info("Metric %s has uid: %s", metricName, uid) # Send model creation request model = self._createModel(nativeMetric) parameters = json.loads(model.parameters) self.assertEqual(parameters["metricSpec"]["userInfo"], userInfo) for _ in xrange(60): with self.engine.begin() as conn: metric = repository.getMetric(conn, uid) if metric.status == MetricStatus.ACTIVE: break LOGGER.info("Model=%s not ready. Sleeping 1 second...", uid) time.sleep(1) else: self.fail("Model results not available within 5 minutes") # Check that the data all got processed self.checkModelResultsSize(uid, 3) # Now check that the data was published to dynamodb... dynamodb = DynamoDBService.connectDynamoDB() metricTable = Table(MetricDynamoDBDefinition().tableName, connection=dynamodb) metricItem = metricTable.lookup(uid) self.assertEqual(metricItem["uid"], uid) self.assertEqual(metricItem["name"], metricName) self.assertEqual(metricItem["metricType"], "TwitterVolume") self.assertEqual(metricItem["metricTypeName"], "Twitter Volume") self.assertEqual(metricItem["symbol"], "TEST") metricDataTable = Table(MetricDataDynamoDBDefinition().tableName, connection=dynamodb) instanceDataAnomalyScores = {} for metricValue, ts in data: metricDataItem = _RETRY_ON_ITEM_NOT_FOUND_DYNAMODB_ERROR( metricDataTable.lookup)(uid, ts.isoformat()) # There is no server-side cleanup for metric data, so remove it here for # now to avoid accumulating test data self.addCleanup(metricDataItem.delete) self.assertEqual(metricValue, metricDataItem["metric_value"]) dt = datetime.datetime.strptime(metricDataItem["timestamp"], "%Y-%m-%dT%H:%M:%S") self.assertEqual(ts, dt) ts = ts.replace(minute=0, second=0, microsecond=0) date = ts.strftime("%Y-%m-%d") hour = ts.strftime("%H") key = (date, hour) maxVal = instanceDataAnomalyScores.get(key, 0.0) instanceDataAnomalyScores[key] = max( maxVal, metricDataItem["anomaly_score"]) # And check that the aggregated instance data is updated instanceDataHourlyTable = Table( InstanceDataHourlyDynamoDBDefinition().tableName, connection=dynamodb) for key, anomalyScore in instanceDataAnomalyScores.iteritems(): date, hour = key instanceDataHourlyItem = _RETRY_ON_ITEM_NOT_FOUND_DYNAMODB_ERROR( instanceDataHourlyTable.lookup)(instanceName, "%sT%s" % (date, hour)) self.addCleanup(instanceDataHourlyItem.delete) self.assertAlmostEqual( anomalyScore, float( instanceDataHourlyItem["anomaly_score"]["TwitterVolume"])) self.assertEqual(date, instanceDataHourlyItem["date"]) self.assertEqual(hour, instanceDataHourlyItem["hour"]) # Now send some twitter data and validate that it made it to dynamodb twitterData = [{ "metric_name": metricName, "tweet_uid": uid, "created_at": "2015-02-19T19:43:24.870109", "agg_ts": "2015-02-19T19:43:24.870118", "text": "Tweet text", "userid": "10", "username": "******", "retweet_count": "0" }] with MessageBusConnector() as messageBus: messageBus.publishExg( exchange=self.config.get("non_metric_data", "exchange_name"), routingKey=( self.config.get("non_metric_data", "exchange_name") + ".twitter"), body=json.dumps(twitterData)) metricTweetsTable = Table(MetricTweetsDynamoDBDefinition().tableName, connection=dynamodb) metricTweetItem = metricTweetsTable.lookup( "-".join((metricName, uid)), "2015-02-19T19:43:24.870118") # There is no server-side cleanup for tweet data, so remove it here for # now to avoid accumulating test data self.addCleanup(metricTweetItem.delete) self.assertEqual(metricTweetItem["username"], twitterData[0]["username"]) self.assertEqual(metricTweetItem["tweet_uid"], twitterData[0]["tweet_uid"]) self.assertEqual(metricTweetItem["created_at"], twitterData[0]["created_at"]) self.assertEqual(metricTweetItem["agg_ts"], twitterData[0]["agg_ts"]) self.assertEqual(metricTweetItem["text"], twitterData[0]["text"]) self.assertEqual(metricTweetItem["userid"], twitterData[0]["userid"]) self.assertEqual(metricTweetItem["username"], twitterData[0]["username"]) self.assertEqual(metricTweetItem["retweet_count"], twitterData[0]["retweet_count"]) queryResult = metricTweetsTable.query_2( metric_name__eq=metricName, agg_ts__eq=twitterData[0]["agg_ts"], index="taurus.metric_data-metric_name_index") queriedMetricTweetItem = next(queryResult) self.assertEqual(queriedMetricTweetItem["username"], twitterData[0]["username"]) self.assertEqual(queriedMetricTweetItem["tweet_uid"], twitterData[0]["tweet_uid"]) self.assertEqual(queriedMetricTweetItem["created_at"], twitterData[0]["created_at"]) self.assertEqual(queriedMetricTweetItem["agg_ts"], twitterData[0]["agg_ts"]) self.assertEqual(queriedMetricTweetItem["text"], twitterData[0]["text"]) self.assertEqual(queriedMetricTweetItem["userid"], twitterData[0]["userid"]) self.assertEqual(queriedMetricTweetItem["username"], twitterData[0]["username"]) self.assertEqual(queriedMetricTweetItem["retweet_count"], twitterData[0]["retweet_count"]) # Delete metric and ensure metric is deleted from dynamodb, too self._deleteMetric(metricName) for _ in xrange(60): time.sleep(1) try: metricItem = metricTable.lookup(uid) except ItemNotFound as err: break else: self.fail("Metric not deleted from dynamodb")
def test_query_with_reverse(self): posts = Table.create('more-posts', schema=[ HashKey('thread'), RangeKey('posted_on') ], throughput={ 'read': 5, 'write': 5, }) self.addCleanup(posts.delete) # Wait for it. time.sleep(60) # Add some data. test_data_path = os.path.join( os.path.dirname(__file__), 'forum_test_data.json' ) with open(test_data_path, 'r') as test_data: data = json.load(test_data) with posts.batch_write() as batch: for post in data: batch.put_item(post) time.sleep(5) # Test the default order (ascending). results = posts.query_2( thread__eq='Favorite chiptune band?', posted_on__gte='2013-12-24T00:00:00' ) self.assertEqual( [post['posted_on'] for post in results], [ '2013-12-24T12:30:54', '2013-12-24T12:35:40', '2013-12-24T13:45:30', '2013-12-24T14:15:14', '2013-12-24T14:25:33', '2013-12-24T15:22:22', ] ) # Test the explicit ascending order. results = posts.query_2( thread__eq='Favorite chiptune band?', posted_on__gte='2013-12-24T00:00:00', reverse=False ) self.assertEqual( [post['posted_on'] for post in results], [ '2013-12-24T12:30:54', '2013-12-24T12:35:40', '2013-12-24T13:45:30', '2013-12-24T14:15:14', '2013-12-24T14:25:33', '2013-12-24T15:22:22', ] ) # Test the explicit descending order. results = posts.query_2( thread__eq='Favorite chiptune band?', posted_on__gte='2013-12-24T00:00:00', reverse=True ) self.assertEqual( [post['posted_on'] for post in results], [ '2013-12-24T15:22:22', '2013-12-24T14:25:33', '2013-12-24T14:15:14', '2013-12-24T13:45:30', '2013-12-24T12:35:40', '2013-12-24T12:30:54', ] ) # Test the old, broken style. results = posts.query( thread__eq='Favorite chiptune band?', posted_on__gte='2013-12-24T00:00:00' ) self.assertEqual( [post['posted_on'] for post in results], [ '2013-12-24T15:22:22', '2013-12-24T14:25:33', '2013-12-24T14:15:14', '2013-12-24T13:45:30', '2013-12-24T12:35:40', '2013-12-24T12:30:54', ] ) results = posts.query( thread__eq='Favorite chiptune band?', posted_on__gte='2013-12-24T00:00:00', reverse=True ) self.assertEqual( [post['posted_on'] for post in results], [ '2013-12-24T12:30:54', '2013-12-24T12:35:40', '2013-12-24T13:45:30', '2013-12-24T14:15:14', '2013-12-24T14:25:33', '2013-12-24T15:22:22', ] )
# check if the file is present """ c.execute('''CREATE TABLE users (email text, token text)''') c.execute('''CREATE TABLE result (token text, q1 text, q2 text, q3 text)''') """ conn = boto.dynamodb2.connect_to_region( 'us-west-1', aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY ) users = Table('survey2_users', connection=conn) unique_emails = set() with users.batch_write() as batch: with open(csvfile) as csvfile: csvreader = csv.reader(csvfile, delimiter=',', quotechar='"') for line in csvreader: if line[0] not in unique_emails: batch.put_item(data={'email':line[0], 'token': str(uuid.uuid4()), }) unique_emails.add(line[0]) else: print "DUPE: %s"%line[0]
class DBTable(object): item_class = None simple_schema = [HashKey('_id')] def __init__(self, table_name, schema, global_indexes, dbconn): self.table_name = table_name self._table = Table(table_name, connection=dbconn.conn) self.schema = schema self.global_indexes = global_indexes self.dbconn = dbconn.conn return self def get_table(self): return self._table def exists(self): return self._table.table_name in self.dbconn.list_tables( )['TableNames'] def create(self): try: # print self.schema # print self.table_name self._table = Table.create(self.table_name, schema=self.schema, global_indexes=self.global_indexes, connection=self.dbconn) except JSONResponseError as e: logging.warn("Error creating table " + self.table_name) log_boto_error(e) if 'message' in e.body and e.body['message'].startswith( 'Table already exists'): self._table = Table(self.table_name, connection=self.dbconn) elif 'Message' in e.body and e.body['Message'].startswith( "Cannot create preexisting"): self._table = Table(self.table_name, connection=self.dbconn) else: raise e return self def delete(self): try: self._table.delete() except JSONResponseError as e: logging.warn("Error deleting table " + self.table_name) log_boto_error(e) if e.body['Message'].startswith( 'Cannot do operations on a non-existent tabl'): return else: raise e return def insert(self, data): self._table.put_item(data=data, overwrite=True) return def get_item(self, **kwargs): item = self._table.get_item(**kwargs) # TODO return item or class?? # new_item = self.item_class(self) # new_item.__dict__ = item._data # return new_item return item def query_2(self, **kwargs): return self._table.query_2(**kwargs) def scan(self, **kwargs): return self._table.scan(**kwargs) def remove(self): return self._table.remove()
def upload(self, filename, file_hash): """ Upload filename and store the archive id for future retrieval """ rel_filename = os.path.relpath(filename, bagparent) archive_id = self.vault.concurrent_create_archive_from_file( filename, description=rel_filename) # Storing the archive_id, filename, file_hash, and bag_date relationships in dynamodb try: archives.put_item( data={ 'archive_id': archive_id, 'vault_name': vault_name, 'file_hash': file_hash, 'filename': rel_filename, }) ## If the database doesn't exist, create it except JSONResponseError as e: if e.status == 400 and e.message == 'Requested resource not found': print('freezerbag_archives table missing, creating now') Table.create('freezerbag_archives', schema=[ HashKey('archive_id'), RangeKey('vault_name', data_type='S') ]) time.sleep(30) ## Bail if we hit a JSON error we don't understand else: print(e.status) print(e.message) exit() ## Write out the hash too hashes.put_item( data={ 'file_hash': file_hash, 'archive_id': archive_id, 'vault_name': vault_name, }) try: names.put_item( data={ 'filename': rel_filename, 'bag_date': bag_date, 'bagname': bagname, 'archive_id': archive_id, 'file_hash': file_hash, 'vault_name': vault_name, }) ## If the database doesn't exist, create it except JSONResponseError as e: if e.status == 400 and e.message == 'Requested resource not found': print('freezerbag_names table missing, creating now') print('freezerbag_names table missing, creating now') Table.create('freezerbag_names', schema=[ HashKey('filename'), RangeKey('bag_date', data_type=NUMBER) ]) time.sleep(30) ## Bail if we hit a JSON error we don't understand else: print(e.status) print(e.message) exit()
import boto.dynamodb2 from boto.dynamodb2.table import Table from boto.dynamodb2.fields import HashKey from boto.regioninfo import RegionInfo from boto.dynamodb2.layer1 import DynamoDBConnection from faker import Factory import uuid import time try: sessions = Table( table_name='usertable', schema=[HashKey('id')], connection=DynamoDBConnection(region=RegionInfo( name='eu-west-1', endpoint='dynamodb.eu-west-1.amazonaws.com'))) except: print("Dynamo can't connect") def create_session(): id = str(uuid.uuid4()) timestamp = time.strftime("%Y%m%d%H%M%S") ipv4 = Factory.create().ipv4() users_id = Factory.create().slug() users_name = Factory.create().first_name() users_surname = Factory.create().last_name() res = sessions.put_item( data={ 'username': id, 'data': {
def test_integration(self): # Test creating a full table with all options specified. users = Table.create('users', schema=[ HashKey('username'), RangeKey('friend_count', data_type=NUMBER) ], throughput={ 'read': 5, 'write': 5, }, indexes=[ KeysOnlyIndex('LastNameIndex', parts=[ HashKey('username'), RangeKey('last_name') ]), ]) self.addCleanup(users.delete) self.assertEqual(len(users.schema), 2) self.assertEqual(users.throughput['read'], 5) # Wait for it. time.sleep(60) # Make sure things line up if we're introspecting the table. users_hit_api = Table('users') users_hit_api.describe() self.assertEqual(len(users.schema), len(users_hit_api.schema)) self.assertEqual(users.throughput, users_hit_api.throughput) self.assertEqual(len(users.indexes), len(users_hit_api.indexes)) # Test putting some items individually. users.put_item(data={ 'username': '******', 'first_name': 'John', 'last_name': 'Doe', 'friend_count': 4 }) users.put_item(data={ 'username': '******', 'first_name': 'Alice', 'last_name': 'Expert', 'friend_count': 2 }) time.sleep(5) # Test batch writing. with users.batch_write() as batch: batch.put_item({ 'username': '******', 'first_name': 'Jane', 'last_name': 'Doe', 'friend_count': 3 }) batch.delete_item(username='******', friend_count=2) batch.put_item({ 'username': '******', 'first_name': 'Bob', 'last_name': 'Smith', 'friend_count': 1 }) time.sleep(5) # Test getting an item & updating it. # This is the "safe" variant (only write if there have been no # changes). jane = users.get_item(username='******', friend_count=3) self.assertEqual(jane['first_name'], 'Jane') jane['last_name'] = 'Doh' self.assertTrue(jane.save()) # Test strongly consistent getting of an item. # Additionally, test the overwrite behavior. client_1_jane = users.get_item( username='******', friend_count=3, consistent=True ) self.assertEqual(jane['first_name'], 'Jane') client_2_jane = users.get_item( username='******', friend_count=3, consistent=True ) self.assertEqual(jane['first_name'], 'Jane') # Write & assert the ``first_name`` is gone, then... del client_1_jane['first_name'] self.assertTrue(client_1_jane.save()) check_name = users.get_item( username='******', friend_count=3, consistent=True ) self.assertEqual(check_name['first_name'], None) # ...overwrite the data with what's in memory. client_2_jane['first_name'] = 'Joan' # Now a write that fails due to default expectations... self.assertRaises(exceptions.JSONResponseError, client_2_jane.save) # ... so we force an overwrite. self.assertTrue(client_2_jane.save(overwrite=True)) check_name_again = users.get_item( username='******', friend_count=3, consistent=True ) self.assertEqual(check_name_again['first_name'], 'Joan') # Reset it. jane['username'] = '******' jane['first_name'] = 'Jane' jane['last_name'] = 'Doe' jane['friend_count'] = 3 self.assertTrue(jane.save(overwrite=True)) # Test the partial update behavior. client_3_jane = users.get_item( username='******', friend_count=3, consistent=True ) client_4_jane = users.get_item( username='******', friend_count=3, consistent=True ) client_3_jane['favorite_band'] = 'Feed Me' # No ``overwrite`` needed due to new data. self.assertTrue(client_3_jane.save()) # Expectations are only checked on the ``first_name``, so what wouldn't # have succeeded by default does succeed here. client_4_jane['first_name'] = 'Jacqueline' self.assertTrue(client_4_jane.partial_save()) partial_jane = users.get_item( username='******', friend_count=3, consistent=True ) self.assertEqual(partial_jane['favorite_band'], 'Feed Me') self.assertEqual(partial_jane['first_name'], 'Jacqueline') # Reset it. jane['username'] = '******' jane['first_name'] = 'Jane' jane['last_name'] = 'Doe' jane['friend_count'] = 3 self.assertTrue(jane.save(overwrite=True)) # Ensure that partial saves of a brand-new object work. sadie = Item(users, data={ 'username': '******', 'first_name': 'Sadie', 'favorite_band': 'Zedd', 'friend_count': 7 }) self.assertTrue(sadie.partial_save()) serverside_sadie = users.get_item( username='******', friend_count=7, consistent=True ) self.assertEqual(serverside_sadie['first_name'], 'Sadie') # Test the eventually consistent query. results = users.query( username__eq='johndoe', last_name__eq='Doe', index='LastNameIndex', attributes=('username',), reverse=True ) for res in results: self.assertTrue(res['username'] in ['johndoe',]) self.assertEqual(res.keys(), ['username']) # Test the strongly consistent query. c_results = users.query( username__eq='johndoe', last_name__eq='Doe', index='LastNameIndex', reverse=True, consistent=True ) for res in c_results: self.assertTrue(res['username'] in ['johndoe',]) # Test scans without filters. all_users = users.scan(limit=7) self.assertEqual(all_users.next()['username'], 'bob') self.assertEqual(all_users.next()['username'], 'jane') self.assertEqual(all_users.next()['username'], 'johndoe') # Test scans with a filter. filtered_users = users.scan(limit=2, username__beginswith='j') self.assertEqual(filtered_users.next()['username'], 'jane') self.assertEqual(filtered_users.next()['username'], 'johndoe') # Test deleting a single item. johndoe = users.get_item(username='******', friend_count=4) johndoe.delete() # Test the eventually consistent batch get. results = users.batch_get(keys=[ {'username': '******', 'friend_count': 1}, {'username': '******', 'friend_count': 3} ]) batch_users = [] for res in results: batch_users.append(res) self.assertTrue(res['first_name'] in ['Bob', 'Jane']) self.assertEqual(len(batch_users), 2) # Test the strongly consistent batch get. c_results = users.batch_get(keys=[ {'username': '******', 'friend_count': 1}, {'username': '******', 'friend_count': 3} ], consistent=True) c_batch_users = [] for res in c_results: c_batch_users.append(res) self.assertTrue(res['first_name'] in ['Bob', 'Jane']) self.assertEqual(len(c_batch_users), 2) # Test count, but in a weak fashion. Because lag time. self.assertTrue(users.count() > -1) # Test query count count = users.query_count( username__eq='bob', ) self.assertEqual(count, 1) # Test without LSIs (describe calls shouldn't fail). admins = Table.create('admins', schema=[ HashKey('username') ]) self.addCleanup(admins.delete) time.sleep(60) admins.describe() self.assertEqual(admins.throughput['read'], 5) self.assertEqual(admins.indexes, []) # A single query term should fail on a table with *ONLY* a HashKey. self.assertRaises( exceptions.QueryError, admins.query, username__eq='johndoe' ) # But it shouldn't break on more complex tables. res = users.query(username__eq='johndoe') # Test putting with/without sets. mau5_created = users.put_item(data={ 'username': '******', 'first_name': 'dead', 'last_name': 'mau5', 'friend_count': 2, 'friends': set(['skrill', 'penny']), }) self.assertTrue(mau5_created) penny_created = users.put_item(data={ 'username': '******', 'first_name': 'Penny', 'friend_count': 0, 'friends': set([]), }) self.assertTrue(penny_created)
def retrieve(self, archive_id, filename, bag_date, wait_mode=False): """ Initiate a Job, check its status, and download the archive when it's completed. """ filename = os.path.normpath(filename.translate(filetranstable)) bag_date = str(bag_date) #archive_id = self.get_archive_id(filename, bag_date) destdir = os.path.normpath( os.path.abspath(os.path.join(bagparent, bag_date))) destfile = os.path.normpath( os.path.abspath(os.path.join(destdir, filename))) archive_id = str(archive_id) if not archive_id: return # We'll use this to reference the retrieval job, whether it already exists or if we're firing it up. job = None # Is the job in the db? try: jobcheck = jobs.get_item(archive_id=archive_id, vault_name=vault_name) job_id = str(jobcheck['job_id']) ## If so, try to get the job from glacier try: job = self.vault.get_job(job_id) ## TODO we probably need to clean the job from the table in this case. except UnexpectedHTTPResponseError: # Return a 404 if the job is no longer available pass ## If the database doesn't exist, create it then initialize the job except JSONResponseError as e: if e.status == 400 and e.message == 'Requested resource not found': print('freezerbag_jobs table missing, creating now') Table.create('freezerbag_jobs', schema=[ HashKey('archive_id', data_type='S'), RangeKey('vault_name', data_type='S') ]) time.sleep(30) # Job initialization job = self.vault.retrieve_archive(archive_id) # Note it in the db job_id = job.id jobs.put_item( data={ 'archive_id': archive_id, 'job_id': job_id, 'vault_name': vault_name, 'filename': filename, }) ## Bail if we hit a JSON error we don't understand else: print(e.status) print(e.message) exit() ## If the db exists, but the job isn't in the db, initialize it except ItemNotFound: # Job initialization job = self.vault.retrieve_archive(archive_id) # Note it in the db job_id = job.id print(job_id) jobs.put_item( data={ 'archive_id': archive_id, 'job_id': job_id, 'vault_name': vault_name, 'filename': filename, }) print( ("Job {action}: {status_code} ({creation_date}/{completion_date})". format(**job.__dict__))) # checking manually if job is completed every 10 seconds instead of using Amazon SNS if wait_mode: while 1: job = self.vault.get_job(job_id) if not job.completed: time.sleep(10) else: break if job.completed: print("Downloading to %s" % (destfile)) if not os.path.exists(os.path.dirname(destfile)): os.makedirs(os.path.dirname(destfile)) ## verify_hashes=False is do to a boto bug in 2.36.0 + python3.x. # We're doing bag validation anyway, so this isn't a deal-breaker, though it hurts efficiency ## https://github.com/boto/boto/issues/3059 job.download_to_file(destfile, verify_hashes=False) else: print("destfile: %s" % (destfile))
#!/usr/bin/python #Criar uma tabela no DynamoDB com indice # import boto.dynamodb2 from boto.dynamodb2.fields import HashKey, RangeKey, KeysOnlyIndex, AllIndex from boto.dynamodb2.table import Table from boto.dynamodb2.types import NUMBER print("Criacao de Tabela Key/Value Pair no Dynamo") users = Table.create('users', schema=[ HashKey('account_type', data_type=NUMBER), RangeKey('last_name'), ], throughput={ 'read': 5, 'write': 15, }, indexes=[ AllIndex('EverythingIndex', parts=[ HashKey('account_type', data_type=NUMBER), RangeKey('last_name'), ]) ], # Definindo paramtro na regiao connection= boto.dynamodb2.connect_to_region('us-west-2'))
''') exit() ## Set those variables to something useful fulbagpath = os.path.normpath( os.path.abspath(os.path.join(args.path, args.bag))) vault_name = args.vaultid # We'll record stuff in amazon relative to the bag directory, rather than the # absolute path bagname = os.path.basename(fulbagpath) bagparent = os.path.dirname(fulbagpath) relbagpath = os.path.relpath(fulbagpath, bagparent) ## Our DynamoDB tables archives = Table('freezerbag_archives') hashes = Table('freezerbag_hashes') names = Table('freezerbag_names') jobs = Table('freezerbag_jobs') ## Set up translation table for character replacement in filenames filetranstable = dict.fromkeys(map(ord, '\\'), u'/') class GlacierVault: """ Wrapper for uploading/download archive to/from Amazon Glacier Vault Makes use of DynamoDB to store archive id corresponding to filename and waiting jobs. Backup: >>> GlacierVault("myvault")upload("myfile")