def test_integration(self): # Test creating a full table with all options specified. users = Table.create('users', schema=[ HashKey('username'), RangeKey('friend_count', data_type=NUMBER) ], throughput={ 'read': 5, 'write': 5, }, indexes=[ KeysOnlyIndex('LastNameIndex', parts=[ HashKey('username'), RangeKey('last_name') ]), ]) self.addCleanup(users.delete) self.assertEqual(len(users.schema), 2) self.assertEqual(users.throughput['read'], 5) # Wait for it. time.sleep(60) # Make sure things line up if we're introspecting the table. users_hit_api = Table('users') users_hit_api.describe() self.assertEqual(len(users.schema), len(users_hit_api.schema)) self.assertEqual(users.throughput, users_hit_api.throughput) self.assertEqual(len(users.indexes), len(users_hit_api.indexes)) # Test putting some items individually. users.put_item( data={ 'username': '******', 'first_name': 'John', 'last_name': 'Doe', 'friend_count': 4 }) users.put_item( data={ 'username': '******', 'first_name': 'Alice', 'last_name': 'Expert', 'friend_count': 2 }) time.sleep(5) # Test batch writing. with users.batch_write() as batch: batch.put_item({ 'username': '******', 'first_name': 'Jane', 'last_name': 'Doe', 'friend_count': 3 }) batch.delete_item(username='******', friend_count=2) batch.put_item({ 'username': '******', 'first_name': 'Bob', 'last_name': 'Smith', 'friend_count': 1 }) time.sleep(5) # Does it exist? It should? self.assertTrue(users.has_item(username='******', friend_count=3)) # But this shouldn't be there... self.assertFalse( users.has_item(username='******', friend_count=72948)) # Test getting an item & updating it. # This is the "safe" variant (only write if there have been no # changes). jane = users.get_item(username='******', friend_count=3) self.assertEqual(jane['first_name'], 'Jane') jane['last_name'] = 'Doh' self.assertTrue(jane.save()) # Test strongly consistent getting of an item. # Additionally, test the overwrite behavior. client_1_jane = users.get_item(username='******', friend_count=3, consistent=True) self.assertEqual(jane['first_name'], 'Jane') client_2_jane = users.get_item(username='******', friend_count=3, consistent=True) self.assertEqual(jane['first_name'], 'Jane') # Write & assert the ``first_name`` is gone, then... del client_1_jane['first_name'] self.assertTrue(client_1_jane.save()) check_name = users.get_item(username='******', friend_count=3, consistent=True) self.assertEqual(check_name['first_name'], None) # ...overwrite the data with what's in memory. client_2_jane['first_name'] = 'Joan' # Now a write that fails due to default expectations... self.assertRaises(exceptions.JSONResponseError, client_2_jane.save) # ... so we force an overwrite. self.assertTrue(client_2_jane.save(overwrite=True)) check_name_again = users.get_item(username='******', friend_count=3, consistent=True) self.assertEqual(check_name_again['first_name'], 'Joan') # Reset it. jane['username'] = '******' jane['first_name'] = 'Jane' jane['last_name'] = 'Doe' jane['friend_count'] = 3 self.assertTrue(jane.save(overwrite=True)) # Test the partial update behavior. client_3_jane = users.get_item(username='******', friend_count=3, consistent=True) client_4_jane = users.get_item(username='******', friend_count=3, consistent=True) client_3_jane['favorite_band'] = 'Feed Me' # No ``overwrite`` needed due to new data. self.assertTrue(client_3_jane.save()) # Expectations are only checked on the ``first_name``, so what wouldn't # have succeeded by default does succeed here. client_4_jane['first_name'] = 'Jacqueline' self.assertTrue(client_4_jane.partial_save()) partial_jane = users.get_item(username='******', friend_count=3, consistent=True) self.assertEqual(partial_jane['favorite_band'], 'Feed Me') self.assertEqual(partial_jane['first_name'], 'Jacqueline') # Reset it. jane['username'] = '******' jane['first_name'] = 'Jane' jane['last_name'] = 'Doe' jane['friend_count'] = 3 self.assertTrue(jane.save(overwrite=True)) # Ensure that partial saves of a brand-new object work. sadie = Item(users, data={ 'username': '******', 'first_name': 'Sadie', 'favorite_band': 'Zedd', 'friend_count': 7 }) self.assertTrue(sadie.partial_save()) serverside_sadie = users.get_item(username='******', friend_count=7, consistent=True) self.assertEqual(serverside_sadie['first_name'], 'Sadie') # Test the eventually consistent query. results = users.query_2(username__eq='johndoe', last_name__eq='Doe', index='LastNameIndex', attributes=('username', ), reverse=True) for res in results: self.assertTrue(res['username'] in [ 'johndoe', ]) self.assertEqual(list(res.keys()), ['username']) # Ensure that queries with attributes don't return the hash key. results = users.query_2(username__eq='johndoe', friend_count__eq=4, attributes=('first_name', )) for res in results: self.assertEqual(res['first_name'], 'John') self.assertEqual(list(res.keys()), ['first_name']) # Test the strongly consistent query. c_results = users.query_2(username__eq='johndoe', last_name__eq='Doe', index='LastNameIndex', reverse=True, consistent=True) for res in c_results: self.assertEqual(res['username'], 'johndoe') # Test a query with query filters results = users.query_2(username__eq='johndoe', query_filter={'first_name__beginswith': 'J'}, attributes=('first_name', )) for res in results: self.assertTrue(res['first_name'] in ['John']) # Test scans without filters. all_users = users.scan(limit=7) self.assertEqual(next(all_users)['username'], 'bob') self.assertEqual(next(all_users)['username'], 'jane') self.assertEqual(next(all_users)['username'], 'johndoe') # Test scans with a filter. filtered_users = users.scan(limit=2, username__beginswith='j') self.assertEqual(next(filtered_users)['username'], 'jane') self.assertEqual(next(filtered_users)['username'], 'johndoe') # Test deleting a single item. johndoe = users.get_item(username='******', friend_count=4) johndoe.delete() # Set batch get limit to ensure keys with no results are # handled correctly. users.max_batch_get = 2 # Test the eventually consistent batch get. results = users.batch_get(keys=[{ 'username': '******', 'friend_count': 4 }, { 'username': '******', 'friend_count': 10 }, { 'username': '******', 'friend_count': 1 }, { 'username': '******', 'friend_count': 3 }]) batch_users = [] for res in results: batch_users.append(res) self.assertIn(res['first_name'], ['Bob', 'Jane']) self.assertEqual(len(batch_users), 2) # Test the strongly consistent batch get. c_results = users.batch_get(keys=[{ 'username': '******', 'friend_count': 1 }, { 'username': '******', 'friend_count': 3 }], consistent=True) c_batch_users = [] for res in c_results: c_batch_users.append(res) self.assertTrue(res['first_name'] in ['Bob', 'Jane']) self.assertEqual(len(c_batch_users), 2) # Test count, but in a weak fashion. Because lag time. self.assertTrue(users.count() > -1) # Test query count count = users.query_count(username__eq='bob', ) self.assertEqual(count, 1) # Test without LSIs (describe calls shouldn't fail). admins = Table.create('admins', schema=[HashKey('username')]) self.addCleanup(admins.delete) time.sleep(60) admins.describe() self.assertEqual(admins.throughput['read'], 5) self.assertEqual(admins.indexes, []) # A single query term should fail on a table with *ONLY* a HashKey. self.assertRaises(exceptions.QueryError, admins.query, username__eq='johndoe') # But it shouldn't break on more complex tables. res = users.query_2(username__eq='johndoe') # Test putting with/without sets. mau5_created = users.put_item( data={ 'username': '******', 'first_name': 'dead', 'last_name': 'mau5', 'friend_count': 2, 'friends': set(['skrill', 'penny']), }) self.assertTrue(mau5_created) penny_created = users.put_item( data={ 'username': '******', 'first_name': 'Penny', 'friend_count': 0, 'friends': set([]), }) self.assertTrue(penny_created) # Test attributes. mau5 = users.get_item(username='******', friend_count=2, attributes=['username', 'first_name']) self.assertEqual(mau5['username'], 'mau5') self.assertEqual(mau5['first_name'], 'dead') self.assertTrue('last_name' not in mau5)
""" DynamoDB table configuration Author: Angad Gill """ from boto.dynamodb2.fields import HashKey from boto.dynamodb2.table import Table DYNAMO_TABLES = [ Table('pages', schema=[HashKey('pagename')]), ]
class JBoxUserV2(JBoxDB): """ - user_id (primary hash key) - create_month (global secondary hash key) - create_time (global secondary range index) - update_month (global secondary hash key) - update_time (global secondary index) - activation_code (global secondary hash key) - activation_status (global secondary range key) - image (optional: global secondary hash key) - resource_profile (optional: global secondary range key) - status - organization - role - gtok - courses_owned """ NAME = 'jbox_users_v2' SCHEMA = [HashKey('user_id', data_type=STRING)] INDEXES = [ GlobalKeysOnlyIndex('create_month-create_time-index', parts=[ HashKey('create_month', data_type=NUMBER), RangeKey('create_time', data_type=NUMBER) ]), GlobalKeysOnlyIndex('update_month-update_time-index', parts=[ HashKey('update_month', data_type=NUMBER), RangeKey('update_time', data_type=NUMBER) ]), GlobalKeysOnlyIndex('activation_code-activation_status-index', parts=[ HashKey('activation_code', data_type=STRING), RangeKey('activation_status', data_type=NUMBER) ]) ] TABLE = None KEYS = ['user_id'] ATTRIBUTES = [ 'create_month', 'create_time', 'update_month', 'update_time', 'status', 'activation_code', 'activation_status', 'resource_profile', 'role', 'gtok', 'courses_offered', 'balance', 'max_cluster_cores' ] STATUS_ACTIVE = 0 STATUS_INACTIVE = 1 ROLE_USER = 0 ROLE_ACCESS_STATS = 1 << 0 ROLE_MANAGE_INVITES = 1 << 1 ROLE_MANAGE_CONTAINERS = 1 << 2 ROLE_OFFER_COURSES = 1 << 3 ROLE_SUPER = (1 << 33) - 1 ACTIVATION_NONE = 0 ACTIVATION_GRANTED = 1 ACTIVATION_REQUESTED = 2 ACTIVATION_CODE_AUTO = 'AUTO' RES_PROF_BASIC = 0 RES_PROF_DISK_EBS_10G = 1 << 0 RES_PROF_JULIA_PKG_PRECOMP = 1 << 12 RES_PROF_CLUSTER = 1 << 13 RES_PROF_API_PUBLISHER = 1 << 14 STATS = None STAT_NAME = "stat_users" DEF_MAX_CLUSTER_CORES = 64 def __init__(self, user_id, create=False): try: self.item = self.fetch(user_id=user_id) self.is_new = False except JBoxDBItemNotFound: if create: data = { 'user_id': user_id, 'resource_profile': JBoxUserV2.RES_PROF_JULIA_PKG_PRECOMP } JBoxUserV2._set_time(data, "create") JBoxUserV2._set_activation_state(data, '-', JBoxUserV2.ACTIVATION_NONE) self.create(data) self.item = self.fetch(user_id=user_id) self.is_new = True else: raise def get_user_id(self): return self.get_attrib('user_id') def get_status(self): return self.get_attrib('status', JBoxUserV2.STATUS_ACTIVE) def get_role(self): return int(self.get_attrib('role', JBoxUserV2.ROLE_USER)) def set_role(self, role): r = int(self.get_attrib('role', JBoxUserV2.ROLE_USER)) self.set_attrib('role', r | role) def has_role(self, role): return self.get_role() & role == role def set_status(self, status): self.set_attrib('status', status) def set_time(self, prefix, dt=None): JBoxUserV2._set_time(self.item, prefix, dt) @staticmethod def _set_time(item, prefix, dt=None): if None == dt: dt = datetime.datetime.now(pytz.utc) if prefix not in ["create", "update"]: raise (Exception("invalid prefix for setting time")) item[prefix + "_month"] = JBoxUserV2.datetime_to_yyyymm(dt) item[prefix + "_time"] = JBoxUserV2.datetime_to_epoch_secs(dt) def get_time(self, prefix): if prefix not in ["create", "update"]: raise (Exception("invalid prefix for setting time")) return JBoxUserV2.epoch_secs_to_datetime(self.item[prefix + "_time"]) def save(self, set_time=True): self.set_time("update") super(JBoxUserV2, self).save() def set_activation_state(self, activation_code, activation_status): JBoxUserV2.log_debug("setting activation state of %s to %s, %d", self.get_user_id(), activation_code, activation_status) JBoxUserV2._set_activation_state(self.item, activation_code, activation_status) @staticmethod def _set_activation_state(item, activation_code, activation_status): item['activation_code'] = activation_code item['activation_status'] = activation_status def get_activation_state(self): return self.get_attrib('activation_code', '-'), self.get_attrib( 'activation_status', JBoxUserV2.ACTIVATION_NONE) def set_gtok(self, gtok): self.set_attrib('gtok', encrypt(gtok, self.enckey())) def get_gtok(self): gtok = self.get_attrib('gtok') return decrypt(gtok, self.enckey()) if (gtok is not None) else None def set_container_type(self, image, resource_profile): self.set_attrib('image', image) self.set_attrib('resource_profile', resource_profile) def get_container_type(self): return self.get_attrib('image'), int( self.get_attrib('resource_profile', JBoxUserV2.RES_PROF_BASIC)) def get_resource_profile(self): return int( self.get_attrib('resource_profile', JBoxUserV2.RES_PROF_BASIC)) def set_resource_profile(self, mask): resource_profile = self.get_resource_profile() new_resource_profile = resource_profile | mask if new_resource_profile != resource_profile: self.set_attrib('resource_profile', new_resource_profile) def unset_resource_profile(self, mask): resource_profile = self.get_resource_profile() new_resource_profile = resource_profile & (~mask) if new_resource_profile != resource_profile: self.set_attrib('resource_profile', new_resource_profile) def has_resource_profile(self, mask): resource_profile = self.get_resource_profile() if mask == 0: return resource_profile == 0 return (resource_profile & mask) == mask def get_courses_offered(self): return json.loads(self.get_attrib('courses_offered', '[]')) def set_courses_offered(self, courses_offered): self.set_attrib('courses_offered', json.dumps(courses_offered)) def set_balance(self, amt): self.set_attrib('balance', amt) def credit_balance(self, amt): self.set_attrib('balance', self.get_attrib('balance', 0.0) + amt) def debit_balance(self, amt): self.set_attrib('balance', self.get_attrib('balance', 0.0) - amt) def get_balance(self): return self.get_attrib('balance', 0.0) def set_max_cluster_cores(self, cores): self.set_attrib('max_cluster_cores', cores) def get_max_cluster_cores(self): return int( self.get_attrib('max_cluster_cores', JBoxUserV2.DEF_MAX_CLUSTER_CORES)) @staticmethod def get_pending_activations(max_count): records = JBoxUserV2.query( activation_code__eq=JBoxUserV2.ACTIVATION_CODE_AUTO, activation_status__eq=JBoxUserV2.ACTIVATION_REQUESTED, index='activation_code-activation_status-index', limit=max_count) user_ids = [] for rec in records: user_ids.append(rec['user_id']) return user_ids @staticmethod def count_pending_activations(): count = JBoxUserV2.query_count( activation_code__eq='AUTO', activation_status__eq=JBoxUserV2.ACTIVATION_REQUESTED, index='activation_code-activation_status-index') return count @staticmethod def count_created(hours_before, tilldate=None): if None == tilldate: tilldate = datetime.datetime.now(pytz.utc) fromdate = tilldate - datetime.timedelta(hours=hours_before) till_month = JBoxUserV2.datetime_to_yyyymm(tilldate) till_time = JBoxUserV2.datetime_to_epoch_secs(tilldate) from_month = JBoxUserV2.datetime_to_yyyymm(fromdate) from_time = JBoxUserV2.datetime_to_epoch_secs(fromdate) count = 0 mon = from_month while mon <= till_month: count += JBoxUserV2.query_count( create_month__eq=mon, create_time__between=(from_time, till_time), index='create_month-create_time-index') JBoxUserV2.log_debug( "adding accounts created in mon %d, from %d till %d. count %d", mon, from_time, till_time, count) if (mon % 100) == 12: mon = (mon / 100 + 1) * 100 + 1 else: mon += 1 return count @staticmethod def calc_stat(user, weeks, days): stats = JBoxUserV2.STATS stats['num_users'] += 1 gtok_val = user.get('gtok', None) if gtok_val is not None: stats['sync']['gdrive'] += 1 role = stats['role'] role_val = int(user['role']) if user.get( 'role', None) is not None else JBoxUserV2.ROLE_USER if role_val == JBoxUserV2.ROLE_USER: role['user'] += 1 else: if (role_val & JBoxUserV2.ROLE_SUPER) == JBoxUserV2.ROLE_SUPER: role['superuser'] += 1 if (role_val & JBoxUserV2.ROLE_ACCESS_STATS ) == JBoxUserV2.ROLE_ACCESS_STATS: role['access_stats'] += 1 act_status = stats['activation_status'] user_act_status = user.get('activation_status', None) act_status_val = int( user_act_status ) if user_act_status is not None else JBoxUserV2.ACTIVATION_NONE if act_status_val == JBoxUserV2.ACTIVATION_NONE: act_status['none'] += 1 elif act_status_val == JBoxUserV2.ACTIVATION_GRANTED: act_status['granted'] += 1 elif act_status_val == JBoxUserV2.ACTIVATION_REQUESTED: act_status['requested'] += 1 res_profile = stats['resource_profile'] user_res_profile = user['resource_profile'] res_profile_val = int( user_res_profile ) if user_res_profile is not None else JBoxUserV2.RES_PROF_BASIC if res_profile_val == JBoxUserV2.RES_PROF_BASIC: res_profile['basic'] += 1 else: if (res_profile_val & JBoxUserV2.RES_PROF_DISK_EBS_10G ) == JBoxUserV2.RES_PROF_DISK_EBS_10G: res_profile['disk_ebs_10G'] += 1 elif (res_profile_val & JBoxUserV2.RES_PROF_JULIA_PKG_PRECOMP ) == JBoxUserV2.RES_PROF_JULIA_PKG_PRECOMP: res_profile['julia_packages_precompiled'] += 1 elif (res_profile_val & JBoxUserV2.RES_PROF_CLUSTER) == JBoxUserV2.RES_PROF_CLUSTER: res_profile['julia_cluster'] += 1 elif (res_profile_val & JBoxUserV2.RES_PROF_API_PUBLISHER ) == JBoxUserV2.RES_PROF_API_PUBLISHER: res_profile['api_publisher'] += 1 create_month_val = int(user['create_month']) create_month = stats['created_time']['months'] if create_month_val not in create_month: create_month[create_month_val] = 1 else: create_month[create_month_val] += 1 create_time_val = int(user['create_time']) last_n_weeks = JBoxUserV2.STATS['created_time']['last_n_weeks'] last_n_days = JBoxUserV2.STATS['created_time']['last_n_days'] for week in range(0, len(weeks)): if create_time_val >= weeks[week]: last_n_weeks[week + 1] += 1 break for day in range(0, len(days)): if create_time_val >= days[day]: last_n_days[day + 1] += 1 break @staticmethod def calc_stats(): JBoxUserV2.STATS = { 'date': '', 'num_users': 0, 'sync': { 'gdrive': 0 }, 'role': { 'user': 0, 'superuser': 0, 'access_stats': 0 }, 'activation_status': { 'none': 0, 'granted': 0, 'requested': 0 }, 'resource_profile': { 'basic': 0, 'disk_ebs_10G': 0, 'julia_packages_precompiled': 0, 'julia_cluster': 0, 'api_publisher': 0 }, 'created_time': { 'months': {}, 'last_n_weeks': {}, 'last_n_days': {} } } secs_day = 24 * 60 * 60 secs_week = secs_day * 7 now = datetime.datetime.now(pytz.utc) secs_now = int(JBoxUserV2.datetime_to_epoch_secs(now)) weeks = [(secs_now - secs_week * week) for week in range(1, 5)] days = [(secs_now - secs_day * day) for day in range(1, 8)] last_n_weeks = JBoxUserV2.STATS['created_time']['last_n_weeks'] last_n_days = JBoxUserV2.STATS['created_time']['last_n_days'] for week in range(0, len(weeks)): last_n_weeks[week + 1] = 0 for day in range(0, len(days)): last_n_days[day + 1] = 0 result_set = JBoxUserV2.scan(attributes=('user_id', 'create_month', 'create_time', 'gtok', 'role', 'resource_profile', 'activation_status')) for user in result_set: JBoxUserV2.calc_stat(user, weeks, days) JBoxUserV2.STATS['date'] = now.isoformat()
hash_key = "" range_key = "" for schema in src_table["KeySchema"]: attr_name = schema["AttributeName"] key_type = schema["KeyType"] if key_type == "HASH": hash_key = attr_name elif key_type == "RANGE": range_key = attr_name # destnation table try: dst_logs = Table(dst_name, connection=ddbc, schema=[ HashKey(hash_key), RangeKey(range_key), ]) dst_logs.describe() except JSONResponseError: # create table schema = [] if hash_key != "": schema.append(HashKey(hash_key)) if range_key != "": schema.append(RangeKey(range_key)) print("# Create table [%s]." % dst_name) dst_logs = Table.create(
from boto.dynamodb2.table import Table from boto.dynamodb2.types import NUMBER # make a connection to dynamodb conn = boto.dynamodb2.connect_to_region(region_name='us-east-1', aws_access_key_id='1234567890', aws_secret_access_key='0987654321') ############################## # create a new tweets tables ############################## # create a new dynamo table - example 2 tweets = Table.create('tweets', schema=[ HashKey('id'), ], throughput={ 'read': 5, 'write': 10, }, connection=conn) ########################### ## Creating new item ########################### # use put_item method tweets = Table('tweets', connection=conn) tweets.put_item( data={
class JBoxInstanceProps(JBoxDB): NAME = 'jbox_instance' SCHEMA = [HashKey('instance_id', data_type=STRING)] INDEXES = None GLOBAL_INDEXES = None TABLE = None KEYS = ['instance_id'] ATTRIBUTES = ['load', 'accept', 'api_status', 'publish_time'] SQL_INDEXES = None KEYS_TYPES = [JBoxDB.VCHAR] TYPES = [JBoxDB.VCHAR, JBoxDB.INT, JBoxDB.TEXT, JBoxDB.INT] # maintenance runs are once in 5 minutes # TODO: make configurable SESS_UPDATE_INTERVAL = (5 * 1.5) * 60 def __init__(self, cluster, instance_id, create=False): qinstance_id = JBoxDB.qual(cluster, instance_id) try: self.item = self.fetch(instance_id=qinstance_id) self.is_new = False except JBoxDBItemNotFound: if create: data = {'instance_id': qinstance_id} self.create(data) self.item = self.fetch(instance_id=qinstance_id) self.is_new = True else: raise def get_load(self): return self.get_attrib('load', '0.0') def set_load(self, load): self.set_attrib('load', str(load)) def get_accept(self): return self.get_attrib('accept', 0) == 1 def set_accept(self, accept): self.set_attrib('accept', 1 if accept else 0) def get_api_status(self): try: return json.loads(self.get_attrib('api_status', '{}')) except: return dict() def set_api_status(self, api_status): self.set_attrib('api_status', json.dumps(api_status)) def set_publish_time(self): now = datetime.datetime.now(pytz.utc) self.set_attrib('publish_time', JBoxInstanceProps.datetime_to_epoch_secs(now)) def get_publish_time(self): now = datetime.datetime.now(pytz.utc) return int( self.get_attrib('publish_time', JBoxInstanceProps.datetime_to_epoch_secs(now))) @staticmethod def set_props(cluster, instance_id, load=None, accept=None, api_status=None): instance_props = JBoxInstanceProps(cluster, instance_id, create=True) if load is not None: instance_props.set_load(load) if accept is not None: instance_props.set_accept(accept) if api_status is not None: instance_props.set_api_status(api_status) instance_props.set_publish_time() instance_props.save() @staticmethod def purge_stale_instances(cluster): for iid in JBoxInstanceProps.get_stale_instances(cluster): instance = JBoxInstanceProps(cluster, iid) instance.delete() @staticmethod def get_stale_instances(cluster): now = datetime.datetime.now(pytz.utc) nowsecs = JBoxInstanceProps.datetime_to_epoch_secs(now) valid_time = nowsecs - JBoxInstanceProps.SESS_UPDATE_INTERVAL stale = [] for record in JBoxInstanceProps.scan( instance_id__beginswith=JBoxDB.qual(cluster, ''), publish_time__lt=valid_time): stale.append(record.get('instance_id').split('.', 1)[1]) return stale @staticmethod def get_instance_status(cluster): now = datetime.datetime.now(pytz.utc) nowsecs = JBoxInstanceProps.datetime_to_epoch_secs(now) valid_time = nowsecs - JBoxInstanceProps.SESS_UPDATE_INTERVAL result = dict() for record in JBoxInstanceProps.scan( instance_id__beginswith=JBoxDB.qual(cluster, ''), publish_time__gte=valid_time): iid = record.get('instance_id').split('.', 1)[1] props = { 'load': float(record.get('load', '0.0')), 'accept': bool(record.get('accept', 0)), 'api_status': json.loads(record.get('api_status', '{}')) } result[iid] = props return result @staticmethod def get_available_instances(cluster): now = datetime.datetime.now(pytz.utc) nowsecs = JBoxInstanceProps.datetime_to_epoch_secs(now) valid_time = nowsecs - JBoxInstanceProps.SESS_UPDATE_INTERVAL result = list() for record in JBoxInstanceProps.scan( instance_id__beginswith=JBoxDB.qual(cluster, ''), publish_time__gte=valid_time, accept__eq=1): result.append(record.get('instance_id').split('.', 1)[1]) return result
''' Create the table for the BasicDB exercise. You only execute this ONCE. ''' import boto.dynamodb2 from boto.dynamodb2.table import Table from boto.dynamodb2.fields import HashKey from boto.dynamodb2.types import NUMBER # Modify these as necessary TABLE_NAME = "activities" READ_CAPACITY = 1 WRITE_CAPACITY = 1 if __name__ == "__main__": acts = Table.create ( TABLE_NAME, schema=[ HashKey('id', data_type=NUMBER) ], throughput = { 'read': READ_CAPACITY, 'write': WRITE_CAPACITY }, connection=boto.dynamodb2.connect_to_region('us-west-2') )
#k=Key(bucket) DYNAMODB_TABLE_NAME = 'mtaData' # Prepare DynamoDB client client_dynamo = boto.dynamodb2.connect_to_region( 'us-east-1', aws_access_key_id=assumedRoleObject.credentials.access_key, aws_secret_access_key=assumedRoleObject.credentials.secret_key, security_token=assumedRoleObject.credentials.session_token) from boto.dynamodb2.table import Table table_dynamo = Table(DYNAMODB_TABLE_NAME, connection=client_dynamo) try: mtaData = Table.create('mtaData', schema=[HashKey('tripId')], connection=client_dynamo) mtaData = Table('mtaData', schema=[HashKey('tripId')], connection=client_dynamo) time.sleep(.12) except: mtaData = Table('mtaData', schema=[HashKey('tripId')], connection=client_dynamo) #########before this is all AWS requirements importmta = mtaUpdates.mtaUpdates('80e73d31e83802678e719e770763da90') #tu,timest = importmta.getTripUpdates() #print t[10].futureStops,v[0].currentStopNumber,timest
def create_table(self): super(DynamoFaceSignatureClient, self).create_table(schema=[HashKey('box_id', data_type=NUMBER)])
class JBoxDynConfig(JBoxDB): NAME = 'jbox_dynconfig' SCHEMA = [HashKey('name', data_type=STRING)] INDEXES = None TABLE = None DEFAULT_REGISTRATION_RATE = 60 def __init__(self, prop, create=False, value=None): if self.table() is None: return self.item = None try: self.item = self.table().get_item(name=prop) self.is_new = False except boto.dynamodb2.exceptions.ItemNotFound: if create: data = {'name': prop} if value is not None: data['value'] = value self.create(data) self.item = self.table().get_item(name=prop) self.is_new = True else: raise @staticmethod def _n(cluster, name): return '.'.join([cluster, name]) def set_value(self, value): self.set_attrib('value', value) def get_value(self): return self.get_attrib('value') @staticmethod def unset_cluster_leader(cluster): try: record = JBoxDynConfig(JBoxDynConfig._n(cluster, 'leader')) record.delete() except boto.dynamodb2.exceptions.ItemNotFound: return @staticmethod def set_cluster_leader(cluster, instance): record = JBoxDynConfig(JBoxDynConfig._n(cluster, 'leader'), create=True, value=instance) if not record.is_new: record.set_value(instance) record.save() @staticmethod def get_cluster_leader(cluster): try: return JBoxDynConfig(JBoxDynConfig._n(cluster, 'leader')).get_value() except boto.dynamodb2.exceptions.ItemNotFound: return None @staticmethod def set_allow_registration(cluster, allow): record = JBoxDynConfig(JBoxDynConfig._n(cluster, 'allow_registration'), create=True, value=str(allow)) if not record.is_new: record.set_value(str(allow)) record.save() @staticmethod def get_allow_registration(cluster): try: record = JBoxDynConfig( JBoxDynConfig._n(cluster, 'allow_registration')) except boto.dynamodb2.exceptions.ItemNotFound: return True return record.get_value() == 'True' @staticmethod def get_registration_hourly_rate(cluster): try: return int( JBoxDynConfig( JBoxDynConfig._n(cluster, 'registrations_hourly_rate')).get_value()) except boto.dynamodb2.exceptions.ItemNotFound: return JBoxDynConfig.DEFAULT_REGISTRATION_RATE @staticmethod def set_registration_hourly_rate(cluster, rate): record = JBoxDynConfig(JBoxDynConfig._n(cluster, 'registrations_hourly_rate'), create=True, value=str(rate)) if not record.is_new: record.set_value(str(rate)) record.save() @staticmethod def set_message(cluster, message, valid_delta): tnow = datetime.datetime.now(pytz.utc) tvalid = tnow + valid_delta msg = { 'msg': message, 'valid_till': isodate.datetime_isoformat(tvalid) } msg = json.dumps(msg) record = JBoxDynConfig(JBoxDynConfig._n(cluster, 'message'), create=True, value=msg) if not record.is_new: record.set_value(msg) record.save() @staticmethod def get_message(cluster, del_expired=True): try: record = JBoxDynConfig(JBoxDynConfig._n(cluster, 'message')) except boto.dynamodb2.exceptions.ItemNotFound: return None msg = record.get_value() if msg is None: return None msg = json.loads(msg) tnow = datetime.datetime.now(pytz.utc) tvalid = parse_iso_time(msg['valid_till']) #JBoxDynConfig.log_debug("tnow: %s, tvalid: %s", str(tnow), str(tvalid)) if tvalid >= tnow: return msg['msg'] if del_expired: JBoxDynConfig.table().delete_item( name='.'.join([cluster, 'message'])) return None @staticmethod def get_user_home_image(cluster): try: record = JBoxDynConfig(JBoxDynConfig._n(cluster, 'user_home_image')) except boto.dynamodb2.exceptions.ItemNotFound: return None, None img = json.loads(record.get_value()) return img['bucket'], img['filename'] @staticmethod def set_user_home_image(cluster, bucket, filename): img = {'bucket': bucket, 'filename': filename} img = json.dumps(img) record = JBoxDynConfig(JBoxDynConfig._n(cluster, 'user_home_image'), create=True, value=img) if not record.is_new: record.set_value(img) record.save() @staticmethod def set_stat_collected_date(cluster): dt = datetime.datetime.now(pytz.utc).isoformat() record = JBoxDynConfig(JBoxDynConfig._n(cluster, 'stat_date'), create=True, value=dt) if not record.is_new: record.set_value(dt) record.save() @staticmethod def get_stat_collected_date(cluster): try: record = JBoxDynConfig(JBoxDynConfig._n(cluster, 'stat_date')) except boto.dynamodb2.exceptions.ItemNotFound: return None return parse_iso_time(record.get_value()) @staticmethod def is_stat_collected_within(cluster, days): last_date = JBoxDynConfig.get_stat_collected_date(cluster) if last_date is None: return False dt = datetime.datetime.now(pytz.utc) - datetime.timedelta(days=days) return last_date > dt @staticmethod def set_stat(cluster, stat_name, stat): val = json.dumps(stat) record = JBoxDynConfig(JBoxDynConfig._n(cluster, stat_name), create=True, value=val) if not record.is_new: record.set_value(val) record.save() @staticmethod def get_stat(cluster, stat_name): try: record = JBoxDynConfig(JBoxDynConfig._n(cluster, stat_name)) except boto.dynamodb2.exceptions.ItemNotFound: return None return json.loads(record.get_value()) @staticmethod def get_course(cluster, course_id): try: course_key = '|'.join(['course', course_id]) record = JBoxDynConfig(JBoxDynConfig._n(cluster, course_key)) except boto.dynamodb2.exceptions.ItemNotFound: return None return json.loads(record.get_value()) @staticmethod def set_course(cluster, course_id, course_details): val = json.dumps(course_details) course_key = '|'.join(['course', course_id]) record = JBoxDynConfig(JBoxDynConfig._n(cluster, course_key), create=True, value=val) if not record.is_new: record.set_value(val) record.save()
import os # imports for flask-dynamo from boto.dynamodb2.fields import HashKey, RangeKey from boto.dynamodb2.table import Table from flask.ext.dynamo import Dynamo from dateutil.parser import parse from dateutil.tz import gettz print(os.environ["AWS_ACCESS_KEY_ID"]) print(os.environ["AWS_SECRET_ACCESS_KEY"]) app = Flask(__name__) # Schema definition for dynamoDB app.config['DYNAMO_TABLES'] = [ Table('orders', schema=[HashKey('id')]), Table('stocks', schema=[HashKey('id')]), Table('priceHistory', schema=[HashKey('id'), RangeKey('time')]), ] # init object for handling dynamoDB dynamo = Dynamo(app) idBoerse = 'datBoerse' @app.route('/') def hello(): return "<h1 style='color:blue'>Hello There!</h1>"
def test_update_table_online_indexing_support(self): # Create a table using gsi to test the DynamoDB online indexing support # https://github.com/boto/boto/pull/2925 users = Table.create('online_indexing_support_users', schema=[HashKey('user_id')], throughput={ 'read': 5, 'write': 5 }, global_indexes=[ GlobalAllIndex('EmailGSIIndex', parts=[HashKey('email')], throughput={ 'read': 2, 'write': 2 }) ]) # Add this function to be called after tearDown() self.addCleanup(users.delete) # Wait for it. time.sleep(60) # Fetch fresh table desc from DynamoDB users.describe() # Assert if everything is fine so far self.assertEqual(len(users.global_indexes), 1) self.assertEqual(users.global_indexes[0].throughput['read'], 2) self.assertEqual(users.global_indexes[0].throughput['write'], 2) # Update a GSI throughput. it should work. users.update_global_secondary_index( global_indexes={'EmailGSIIndex': { 'read': 2, 'write': 1, }}) # Wait for it. time.sleep(60) # Fetch fresh table desc from DynamoDB users.describe() # Assert if everything is fine so far self.assertEqual(len(users.global_indexes), 1) self.assertEqual(users.global_indexes[0].throughput['read'], 2) self.assertEqual(users.global_indexes[0].throughput['write'], 1) # Update a GSI throughput using the old fashion way for compatibility # purposes. it should work. users.update( global_indexes={'EmailGSIIndex': { 'read': 3, 'write': 2, }}) # Wait for it. time.sleep(60) # Fetch fresh table desc from DynamoDB users.describe() # Assert if everything is fine so far self.assertEqual(len(users.global_indexes), 1) self.assertEqual(users.global_indexes[0].throughput['read'], 3) self.assertEqual(users.global_indexes[0].throughput['write'], 2) # Delete a GSI. it should work. users.delete_global_secondary_index('EmailGSIIndex') # Wait for it. time.sleep(60) # Fetch fresh table desc from DynamoDB users.describe() # Assert if everything is fine so far self.assertEqual(len(users.global_indexes), 0) # Create a GSI. it should work. users.create_global_secondary_index(global_index=GlobalAllIndex( 'AddressGSIIndex', parts=[HashKey('address', data_type=STRING)], throughput={ 'read': 1, 'write': 1, })) # Wait for it. This operation usually takes much longer than the others time.sleep(60 * 10) # Fetch fresh table desc from DynamoDB users.describe() # Assert if everything is fine so far self.assertEqual(len(users.global_indexes), 1) self.assertEqual(users.global_indexes[0].throughput['read'], 1) self.assertEqual(users.global_indexes[0].throughput['write'], 1)
def test_query_after_describe_with_gsi(self): # Create a table to using gsi to reproduce the error mentioned on issue # https://github.com/boto/boto/issues/2828 users = Table.create('more_gsi_query_users', schema=[HashKey('user_id')], throughput={ 'read': 5, 'write': 5 }, global_indexes=[ GlobalAllIndex('EmailGSIIndex', parts=[HashKey('email')], throughput={ 'read': 1, 'write': 1 }) ]) # Add this function to be called after tearDown() self.addCleanup(users.delete) # Wait for it. time.sleep(60) # populate a couple of items in it users.put_item( data={ 'user_id': '7', 'username': '******', 'first_name': 'John', 'last_name': 'Doe', 'email': '*****@*****.**', }) users.put_item( data={ 'user_id': '24', 'username': '******', 'first_name': 'Alice', 'last_name': 'Expert', 'email': '*****@*****.**', }) users.put_item( data={ 'user_id': '35', 'username': '******', 'first_name': 'Jane', 'last_name': 'Doe', 'email': '*****@*****.**', }) # Try the GSI. it should work. rs = users.query_2(email__eq='*****@*****.**', index='EmailGSIIndex') for rs_item in rs: self.assertEqual(rs_item['username'], ['johndoe']) # The issue arises when we're introspecting the table and try to # query_2 after call describe method users_hit_api = Table('more_gsi_query_users') users_hit_api.describe() # Try the GSI. This is what os going wrong on #2828 issue. It should # work fine now. rs = users_hit_api.query_2(email__eq='*****@*****.**', index='EmailGSIIndex') for rs_item in rs: self.assertEqual(rs_item['username'], ['johndoe'])
def test_query_with_reverse(self): posts = Table.create('more-posts', schema=[HashKey('thread'), RangeKey('posted_on')], throughput={ 'read': 5, 'write': 5, }) self.addCleanup(posts.delete) # Wait for it. time.sleep(60) # Add some data. test_data_path = os.path.join(os.path.dirname(__file__), 'forum_test_data.json') with open(test_data_path, 'r') as test_data: data = json.load(test_data) with posts.batch_write() as batch: for post in data: batch.put_item(post) time.sleep(5) # Test the default order (ascending). results = posts.query_2(thread__eq='Favorite chiptune band?', posted_on__gte='2013-12-24T00:00:00') self.assertEqual([post['posted_on'] for post in results], [ '2013-12-24T12:30:54', '2013-12-24T12:35:40', '2013-12-24T13:45:30', '2013-12-24T14:15:14', '2013-12-24T14:25:33', '2013-12-24T15:22:22', ]) # Test the explicit ascending order. results = posts.query_2(thread__eq='Favorite chiptune band?', posted_on__gte='2013-12-24T00:00:00', reverse=False) self.assertEqual([post['posted_on'] for post in results], [ '2013-12-24T12:30:54', '2013-12-24T12:35:40', '2013-12-24T13:45:30', '2013-12-24T14:15:14', '2013-12-24T14:25:33', '2013-12-24T15:22:22', ]) # Test the explicit descending order. results = posts.query_2(thread__eq='Favorite chiptune band?', posted_on__gte='2013-12-24T00:00:00', reverse=True) self.assertEqual([post['posted_on'] for post in results], [ '2013-12-24T15:22:22', '2013-12-24T14:25:33', '2013-12-24T14:15:14', '2013-12-24T13:45:30', '2013-12-24T12:35:40', '2013-12-24T12:30:54', ]) # Test the old, broken style. results = posts.query(thread__eq='Favorite chiptune band?', posted_on__gte='2013-12-24T00:00:00') self.assertEqual([post['posted_on'] for post in results], [ '2013-12-24T15:22:22', '2013-12-24T14:25:33', '2013-12-24T14:15:14', '2013-12-24T13:45:30', '2013-12-24T12:35:40', '2013-12-24T12:30:54', ]) results = posts.query(thread__eq='Favorite chiptune band?', posted_on__gte='2013-12-24T00:00:00', reverse=True) self.assertEqual([post['posted_on'] for post in results], [ '2013-12-24T12:30:54', '2013-12-24T12:35:40', '2013-12-24T13:45:30', '2013-12-24T14:15:14', '2013-12-24T14:25:33', '2013-12-24T15:22:22', ])
def create_table(): table = Table.create( "messages", schema=[HashKey("forum_name")], throughput={"read": 10, "write": 10} ) return table
aws_access_key_id=assumedRoleObject.credentials.access_key, aws_secret_access_key=assumedRoleObject.credentials.secret_key, security_token=assumedRoleObject.credentials.session_token) from boto import kinesis #kinesis = kinesis.connect_to_region("us-east-1") #stream = kinesis.create_stream("edisonDemoKinesis", 1) #kinesis.describe_stream("edisonDemoKinesis") #kinesis.list_streams() #global count_dyn count_dyn = 0 from boto.dynamodb2.table import Table table_dynamo = Table(DYNAMODB_TABLE_NAME, connection=client_dynamo) try: pingdata = Table.create('pingdata', schema=[HashKey('timestamp')], connection=client_dynamo) count_dyn = 0 pingdata = Table('pingdata', schema=[HashKey('timestamp')], connection=client_dynamo) except: #count_dyn = tempdata.count() + 1 pingdata = Table('pingdata', schema=[HashKey('timestamp')], connection=client_dynamo) existingitems = pingdata.scan() for i in existingitems: count_dyn = count_dyn + 1 time.sleep(.15) #tempdata = Table('tempdata', schema = [HashKey('timestamp')], connection = client_dynamo)
class JBoxDiskState(JBPluginDB): provides = [JBPluginDB.JBP_TABLE_DYNAMODB] NAME = 'jbox_diskstate' SCHEMA = [ HashKey('disk_key', data_type=STRING) ] INDEXES = [ GlobalKeysOnlyIndex('state-index', parts=[ HashKey('state', data_type=NUMBER) ]) ] TABLE = None STATE_ATTACHED = 1 STATE_ATTACHING = 2 STATE_DETACHING = 3 STATE_DETACHED = 0 def __init__(self, disk_key=None, cluster_id=None, region_id=None, user_id=None, volume_id=None, attach_time=None, create=False): if create and ((cluster_id is None) or (region_id is None) or (user_id is None)): raise AssertionError if disk_key is None: disk_key = '_'.join([user_id, cluster_id, region_id]) try: self.item = self.fetch(disk_key=disk_key) self.is_new = False except JBoxDBItemNotFound: if create: data = { 'disk_key': disk_key, 'cluster_id': cluster_id, 'region_id': region_id, 'user_id': user_id } if volume_id is not None: data['volume_id'] = volume_id if attach_time is None: attach_time = datetime.datetime.now(pytz.utc) data['attach_time'] = JBoxDiskState.datetime_to_epoch_secs(attach_time) self.create(data) self.item = self.fetch(disk_key=disk_key) self.is_new = True else: raise def set_attach_time(self, attach_time=None): if attach_time is None: attach_time = datetime.datetime.now(pytz.utc) self.set_attrib('attach_time', JBoxDiskState.datetime_to_epoch_secs(attach_time)) def get_attach_time(self): return JBoxDiskState.epoch_secs_to_datetime(self.item['attach_time']) def set_detach_time(self, detach_time=None): if detach_time is None: detach_time = datetime.datetime.now(pytz.utc) self.set_attrib('detach_time', JBoxDiskState.datetime_to_epoch_secs(detach_time)) def get_detach_time(self): return JBoxDiskState.epoch_secs_to_datetime(int(self.item['detach_time'])) def get_state(self): state = self.get_attrib('state') return int(state) if state is not None else None def set_state(self, state, time=None): self.set_attrib('state', state) if state == JBoxDiskState.STATE_ATTACHING or state == JBoxDiskState.STATE_ATTACHED: self.set_attach_time(time) else: self.set_detach_time(time) def get_user_id(self): return self.get_attrib('user_id') def set_user_id(self, user_id): self.set_attrib('user_id', user_id) def get_region_id(self): return self.get_attrib('region_id') def set_region_id(self, region_id): self.set_attrib('region_id', region_id) def get_cluster_id(self): return self.get_attrib('cluster_id') def set_cluster_id(self, cluster_id): self.set_attrib('cluster_id', cluster_id) def get_volume_id(self): return self.get_attrib('volume_id') def set_volume_id(self, volume_id): self.set_attrib('volume_id', volume_id) def get_snapshot_ids(self): snapshots = self.get_attrib('snapshot_id') if (snapshots is not None) and (len(snapshots) > 0): return json.loads(snapshots) return [] def add_snapshot_id(self, snapshot_id): ids = self.get_snapshot_ids() ids.append(snapshot_id) self.set_snapshot_ids(ids) def set_snapshot_ids(self, snapshot_ids): self.set_attrib('snapshot_id', json.dumps(snapshot_ids)) @staticmethod def get_detached_disks(max_count=None): disk_keys = [] try: records = JBoxDiskState.query(state__eq=JBoxDiskState.STATE_DETACHED, index='state-index', limit=max_count) for rec in records: disk_keys.append(rec['disk_key']) except: # boto bug: https://github.com/boto/boto/issues/2708 JBoxDiskState.TABLE = None JBoxDiskState.log_warn("Exception in getting detached disks. Probably empty table.") return disk_keys
def dynamodb_users_table(dynamodb_table_maker): schema = [HashKey('name'), RangeKey('last_name')] return dynamodb_table_maker('users', schema)
oidc['Token']) # table DYNAMODB_TABLE_NAME = 'USER1' # DynamoDB client_dynamo = boto.dynamodb2.connect_to_region( 'us-east-1', aws_access_key_id=assumedRoleObject.credentials.access_key, aws_secret_access_key=assumedRoleObject.credentials.secret_key, security_token=assumedRoleObject.credentials.session_token) from boto.dynamodb2.table import Table table_dynamo = Table(DYNAMODB_TABLE_NAME, connection=client_dynamo) try: MTA = Table.create('USER1', schema=[HashKey('TIME')], connection=client_dynamo) time.sleep(30) except: MTA = Table('USER1', connection=client_dynamo) ### YOUR CODE HERE #### def putdata(dentduration, stage): USER1.put_item( data={ 'TIME': str(time.time()), 'DENTDURATION': str(dentduration), 'STAGE': str(stage), })
def dynamodb_records_table(dynamodb_table_maker): schema = [HashKey('time_index_key'), RangeKey('range_key')] return dynamodb_table_maker('records', schema)
class JBoxUserProfile(JBoxDB): NAME = 'jbox_user_profiles' SCHEMA = [HashKey('user_id', data_type=STRING)] INDEXES = None GLOBAL_INDEXES = [ GlobalKeysOnlyIndex('create_month-create_time-index', parts=[ HashKey('create_month', data_type=NUMBER), RangeKey('create_time', data_type=NUMBER) ]), GlobalKeysOnlyIndex('update_month-update_time-index', parts=[ HashKey('update_month', data_type=NUMBER), RangeKey('update_time', data_type=NUMBER) ]) ] TABLE = None ATTR_FIRST_NAME = 'first_name' ATTR_LAST_NAME = 'last_name' ATTR_COUNTRY = 'country' ATTR_CITY = 'city' ATTR_LOCATION = 'location' # a fuzzy location string, indicative of country and city ATTR_IP = 'ip' # ip from which last accessed ATTR_INDUSTRY = 'industry' ATTR_ORGANIZATION = 'org' # workplace ATTR_ORG_TITLE = 'org_title' # job title KEYS = ['user_id'] ATTRIBUTES = [ 'create_month', 'create_time', 'update_month', 'update_time', ATTR_FIRST_NAME, ATTR_LAST_NAME, ATTR_COUNTRY, ATTR_CITY, ATTR_LOCATION, ATTR_IP, ATTR_INDUSTRY, ATTR_ORGANIZATION, ATTR_ORG_TITLE, 'sources' # a JSON field that indicates where each profile attribute was filled from ] SQL_INDEXES = [ { 'name': 'create_month-create_time-index', 'cols': ['create_month', 'create_time'] }, { 'name': 'update_month-update_time-index', 'cols': ['update_month', 'update_time'] }, ] KEYS_TYPES = [JBoxDB.VCHAR] TYPES = [ JBoxDB.INT, JBoxDB.INT, JBoxDB.INT, JBoxDB.INT, JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.VCHAR ] SRC_USER = 1 # filled in by the user SRC_DERIVED = 2 # derived from other fields def __init__(self, user_id, create=False): try: self.item = self.fetch(user_id=user_id) self.is_new = False except JBoxDBItemNotFound: if create: data = {'user_id': user_id} JBoxUserProfile._set_time(data, "create") self.create(data) self.item = self.fetch(user_id=user_id) self.is_new = True else: raise def get_user_id(self): return self.get_attrib('user_id') def get_attrib_source(self, attrib_name): sources_str = self.get_attrib('sources', '{}') if len(sources_str) == 0: return None sources = json.loads(sources_str) return sources[attrib_name] if attrib_name in sources else None def set_attrib_source(self, attrib_name, source): sources_str = self.get_attrib('sources', '{}') if len(sources_str) == 0: sources_str = '{}' sources = json.loads(sources_str) sources[attrib_name] = source self.set_attrib('sources', json.dumps(sources)) def is_set_by_user(self, attrib_name): return self.get_attrib_source(attrib_name) == JBoxUserProfile.SRC_USER def set_profile(self, attrib_name, value, source): # do not overwrite attributes set by the user if source != JBoxUserProfile.SRC_USER and self.is_set_by_user( attrib_name): return False self.set_attrib(attrib_name, value) self.set_attrib_source(attrib_name, source) return True def can_set(self, attrib_name, value): if value is None or len(value) == 0: return False return value != self.get_attrib(attrib_name) def get_profile(self, attrib_name, default=''): return self.get_attrib(attrib_name, default) def set_time(self, prefix, dt=None): JBoxUserProfile._set_time(self.item, prefix, dt) @staticmethod def _set_time(item, prefix, dt=None): if dt is None: dt = datetime.datetime.now(pytz.utc) if prefix not in ["create", "update"]: raise (Exception("invalid prefix for setting time")) item[prefix + "_month"] = JBoxUserProfile.datetime_to_yyyymm(dt) item[prefix + "_time"] = JBoxUserProfile.datetime_to_epoch_secs(dt) def get_time(self, prefix): if prefix not in ["create", "update"]: raise (Exception("invalid prefix for setting time")) return JBoxUserProfile.epoch_secs_to_datetime(self.item[prefix + "_time"]) def save(self, set_time=True): self.set_time("update") super(JBoxUserProfile, self).save()
class JBoxInvite(JBoxDB): NAME = 'jbox_invites' SCHEMA = [HashKey('invite_code', data_type=STRING)] INDEXES = None TABLE = None def __init__(self, invite_code, invited=None, create=False): if self.table() is None: return self.item = None try: self.item = self.table().get_item(invite_code=invite_code) self.is_new = False except boto.dynamodb2.exceptions.ItemNotFound: if create: if len(invite_code) < 6: raise (Exception( "Invite code is too short. Must be at least 6 chars.")) now = datetime.datetime.now(pytz.utc) data = { 'invite_code': invite_code, 'time_created': now.isoformat(), 'expires_on': (now + datetime.datetime.timedelta(1)).isoformat(), # 1 day 'invited': invited } self.create(data) self.item = self.table().get_item(invite_code=invite_code) self.is_new = True else: raise def save(self): self.set_attrib('time_updated', datetime.datetime.now(pytz.utc).isoformat()) super(JBoxInvite, self).save() def is_invited(self, user_id): if (self.table() is None) or (self.item is None): return # is this handled well? if self.item.get('invited', None) is None: return False max_count = self.item.get('max_count', None) if max_count is not None and max_count <= self.item.get('count', 0): return False try: expires = isodate.parse_datetime(self.item['expires_on']) except: self.log_info("Error parsing invite code expiry date: " + str(self.item['invite_id']) + str(self.item['expires_on'])) return False if expires < datetime.datetime.now(pytz.utc): # This invite code has expired, and hence invalid return False if self.item['invited'] == '*': # Anyone is allowed return True ids = map(str.strip, self.item['invited'].split(",")) return user_id in ids def increment_count(self): if (self.table() is None) or (self.item is None): return # is this handled well? c = self.item.get('count', 0) self.item['count'] = c + 1
if key_type == 'HASH': hash_key = attr_name hash_key_type = [ x["AttributeType"] for x in src['AttributeDefinitions'] if x["AttributeName"] == hash_key ][0] elif key_type == 'RANGE': range_key = attr_name range_key_type = [ x["AttributeType"] for x in src['AttributeDefinitions'] if x["AttributeName"] == range_key ][0] # 2. Create the new table table_struct = None schema = [HashKey(hash_key, data_type=hash_key_type)] if range_key != '': schema.append(RangeKey(range_key, data_type=range_key_type)) try: new_logs = Table(dst_table, connection=ddbc, schema=schema) table_struct = new_logs.describe() print 'Table %s already exists' % dst_table sys.exit(0) except JSONResponseError: new_logs = Table.create( dst_table, connection=ddbc, schema=schema, )
def get_args(): argp = argparse.ArgumentParser( description="Create table for Assignment 3 of CMPT 474, Summer 2016") argp.add_argument("name", help="Name of table to create") argp.add_argument( "--reads", type=int, default=DEF_READ_CAPACITY, help="Read capacity (default {0})".format(DEF_READ_CAPACITY)) argp.add_argument( "--writes", type=int, default=DEF_WRITE_CAPACITY, help="write capacity (default {0})".format(DEF_WRITE_CAPACITY)) argp.add_argument("--region", default=DEF_REGION, help="Region (default {0})".format(DEF_REGION)) return argp.parse_args() if __name__ == "__main__": args = get_args() acts = Table.create(args.name, schema=[HashKey('id', data_type=NUMBER)], throughput={ 'read': args.reads, 'write': args.writes }, connection=boto.dynamodb2.connect_to_region( args.region))
hash_key = '' range_key = '' for schema in src['KeySchema']: attr_name = schema['AttributeName'] key_type = schema['KeyType'] if key_type == 'HASH': hash_key = attr_name elif key_type == 'RANGE': range_key = attr_name # 2. Create the new table table_struct = None try: new_logs = Table(dst_table, connection=ddbc, schema=[HashKey(hash_key), RangeKey(range_key), ] ) table_struct = new_logs.describe() print 'Table %s already exists' % dst_table except JSONResponseError: schema = [HashKey(hash_key)] if range_key != '': schema.append(RangeKey(range_key)) new_logs = Table.create(dst_table, connection=ddbc, schema=schema, ) print '*** Waiting for the new table %s to become active' % dst_table
def init(app): ec2 = boto.ec2.connect_to_region( app.config["identity"]['region'], aws_access_key_id=app.config['keys.key_id'], aws_secret_access_key=app.config['keys.key_secret'], security_token=app.config['keys.key_token']) #print "instance id : ", app.config["instance_id"] # Get meta tags reservation = ec2.get_all_instances(instance_ids=app.config["instance_id"]) meta_tags = {} if reservation: for tag in reservation[0].instances[0].tags: meta_tags[str(tag)] = str(reservation[0].instances[0].tags[tag]) #print str(tag), str(reservation[0].instances[0].tags[tag]) #meta_tags = {} #for tag in ec2.get_all_tags(): # meta_tags[str(tag.name)] = str(tag.value) # Log the metadata tags app.config["instance.tags"] = meta_tags for k in meta_tags: logging.debug("[TAGS] {0} : {1}".format(k, meta_tags[k])) sqs = boto.sqs.connect_to_region( app.config["identity"]['region'], aws_access_key_id=app.config['keys.key_id'], aws_secret_access_key=app.config['keys.key_secret'], security_token=app.config['keys.key_token']) sns = boto.sns.connect_to_region( app.config["identity"]['region'], aws_access_key_id=app.config['keys.key_id'], aws_secret_access_key=app.config['keys.key_secret'], security_token=app.config['keys.key_token']) ses = boto.ses.connect_to_region( app.config["identity"]['region'], aws_access_key_id=app.config['keys.key_id'], aws_secret_access_key=app.config['keys.key_secret'], security_token=app.config['keys.key_token']) scale = boto.ec2.autoscale.AutoScaleConnection( aws_access_key_id=app.config['keys.key_id'], aws_secret_access_key=app.config['keys.key_secret'], security_token=app.config['keys.key_token']) s3 = S3Connection(aws_access_key_id=app.config['keys.key_id'], aws_secret_access_key=app.config['keys.key_secret'], security_token=app.config['keys.key_token']) dyno = Table( app.config["instance.tags"] ["DynamoDBTableName"], #app.config['dynamodb.table_name'], schema=[HashKey("job_id")], connection=ddb.connect_to_region( app.config['dynamodb.region'], aws_access_key_id=app.config['keys.key_id'], aws_secret_access_key=app.config['keys.key_secret'], security_token=app.config['keys.key_token'])) app.config["ec2.conn"] = ec2 app.config["sns.conn"] = sns app.config["sqs.conn"] = sqs app.config["ses.conn"] = ses app.config["s3.conn"] = s3 app.config["scale.conn"] = scale app.config["dyno.conn"] = dyno app.config["doReload"] = True return app
def create_table(table_name, region=None, key=None, keyid=None, profile=None, read_capacity_units=None, write_capacity_units=None, hash_key=None, hash_key_data_type=None, range_key=None, range_key_data_type=None, local_indexes=None, global_indexes=None): ''' Creates a DynamoDB table. CLI Example: .. code-block:: bash salt myminion boto_dynamodb.create_table table_name / region=us-east-1 / hash_key=id / hash_key_data_type=N / range_key=created_at / range_key_data_type=N / read_capacity_units=1 / write_capacity_units=1 ''' schema = [] primary_index_fields = [] primary_index_name = '' if hash_key: hash_key_obj = HashKey(hash_key, data_type=hash_key_data_type) schema.append(hash_key_obj) primary_index_fields.append(hash_key_obj) primary_index_name += hash_key if range_key: range_key_obj = RangeKey(range_key, data_type=range_key_data_type) schema.append(range_key_obj) primary_index_fields.append(range_key_obj) primary_index_name += '_' primary_index_name += range_key primary_index_name += '_index' throughput = {'read': read_capacity_units, 'write': write_capacity_units} local_table_indexes = [] if local_indexes: # Add the table's key local_table_indexes.append( AllIndex(primary_index_name, parts=primary_index_fields)) for index in local_indexes: local_table_indexes.append(_extract_index(index)) global_table_indexes = [] if global_indexes: for index in global_indexes: global_table_indexes.append( _extract_index(index, global_index=True)) conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) Table.create(table_name, schema=schema, throughput=throughput, indexes=local_table_indexes, global_indexes=global_table_indexes, connection=conn) # Table creation can take several seconds to propagate. # We will check MAX_ATTEMPTS times. MAX_ATTEMPTS = 30 for i in range(MAX_ATTEMPTS): if exists(table_name, region, key, keyid, profile): return True else: time.sleep(1) # sleep for one second and try again return False
import logging import time from boto.dynamodb2.table import Table from boto.dynamodb2.fields import HashKey, RangeKey logger = logging.getLogger('flotilla') SCHEMAS = { 'assignments': [HashKey('instance_id')], 'locks': [HashKey('lock_name')], 'regions': [HashKey('region_name')], 'revisions': [HashKey('rev_hash')], 'services': [HashKey('service_name')], 'stacks': [HashKey('stack_arn')], 'status': [HashKey('service'), RangeKey('instance_id')], 'units': [HashKey('unit_hash')], 'users': [HashKey('username')] } class DynamoDbTables(object): def __init__(self, dynamo, environment=None, backoff=0.5): self._dynamo = dynamo if environment: self._prefix = 'flotilla-{0}-'.format(environment) else: self._prefix = 'flotilla-' self.assignments = None self.locks = None self.revisions = None self.services = None
def extract_index(index_data, global_index=False): """ Instantiates and returns an AllIndex object given a valid index configuration CLI Example: .. code-block:: bash salt myminion boto_dynamodb.extract_index index """ parsed_data = {} keys = [] for key, value in index_data.items(): for item in value: for field, data in item.items(): if field == "hash_key": parsed_data["hash_key"] = data elif field == "hash_key_data_type": parsed_data["hash_key_data_type"] = data elif field == "range_key": parsed_data["range_key"] = data elif field == "range_key_data_type": parsed_data["range_key_data_type"] = data elif field == "name": parsed_data["name"] = data elif field == "read_capacity_units": parsed_data["read_capacity_units"] = data elif field == "write_capacity_units": parsed_data["write_capacity_units"] = data elif field == "includes": parsed_data["includes"] = data elif field == "keys_only": parsed_data["keys_only"] = True if parsed_data["hash_key"]: keys.append( HashKey(parsed_data["hash_key"], data_type=parsed_data["hash_key_data_type"])) if parsed_data.get("range_key"): keys.append( RangeKey(parsed_data["range_key"], data_type=parsed_data["range_key_data_type"])) if (global_index and parsed_data["read_capacity_units"] and parsed_data["write_capacity_units"]): parsed_data["throughput"] = { "read": parsed_data["read_capacity_units"], "write": parsed_data["write_capacity_units"], } if parsed_data["name"] and keys: if global_index: if parsed_data.get("keys_only") and parsed_data.get("includes"): raise SaltInvocationError( "Only one type of GSI projection can be used.") if parsed_data.get("includes"): return GlobalIncludeIndex( parsed_data["name"], parts=keys, throughput=parsed_data["throughput"], includes=parsed_data["includes"], ) elif parsed_data.get("keys_only"): return GlobalKeysOnlyIndex( parsed_data["name"], parts=keys, throughput=parsed_data["throughput"], ) else: return GlobalAllIndex( parsed_data["name"], parts=keys, throughput=parsed_data["throughput"], ) else: return AllIndex(parsed_data["name"], parts=keys)
aws_access_key_id=assumedRoleObject.credentials.access_key, aws_secret_access_key=assumedRoleObject.credentials.secret_key, security_token=assumedRoleObject.credentials.session_token) from boto import kinesis #kinesis = kinesis.connect_to_region("us-east-1") #stream = kinesis.create_stream("edisonDemoKinesis", 1) #kinesis.describe_stream("edisonDemoKinesis") #kinesis.list_streams() #global count_dyn count_dyn = 0 from boto.dynamodb2.table import Table table_dynamo = Table(DYNAMODB_TABLE_NAME, connection=client_dynamo) try: pingdata = Table.create('androidUser', schema=[HashKey('userNo', data_type=NUMBER)], connection=client_dynamo) count_dyn = 0 pingdata = Table('androidUser', schema=[HashKey('userNo', data_type=NUMBER)], connection=client_dynamo) except: #count_dyn = tempdata.count() + 1 pingdata = Table('androidUser', schema=[HashKey('userNo', data_type=NUMBER)], connection=client_dynamo) existingitems = pingdata.scan() for i in existingitems: count_dyn = count_dyn + 1 time.sleep(.15) #tempdata = Table('tempdata', schema = [HashKey('timestamp')], connection = client_dynamo)