def create(self): Table.create(self.table_name, schema=[ HashKey(keys.entity_profile), RangeKey(time_keys.ts_add, data_type=NUMBER) ], throughput=standard_throughput, indexes={ AllIndex(self.index_delta, parts=[ HashKey(keys.entity_profile), RangeKey(time_keys.ts_delta, data_type=NUMBER) ]), AllIndex(self.index_cut, parts=[ HashKey(keys.entity_profile), RangeKey(time_keys.ts_cut, data_type=NUMBER) ]) }, global_indexes={ GlobalAllIndex(self.index_team, parts=[ HashKey(keys.entity_team), RangeKey(time_keys.ts_add, data_type=NUMBER) ], throughput=standard_throughput), GlobalAllIndex(self.index_league, parts=[ HashKey(keys.entity_league), RangeKey(time_keys.ts_add, data_type=NUMBER) ], throughput=standard_throughput), GlobalAllIndex(self.index_twitter, parts=[ HashKey(keys.entity_twitter), RangeKey(time_keys.ts_add, data_type=NUMBER) ], throughput=standard_throughput), GlobalAllIndex(self.index_site, parts=[ HashKey(keys.entity_site), RangeKey(time_keys.ts_add, data_type=NUMBER) ], throughput=standard_throughput) }) print 'creating entity history table'
def __init__(self, connection, debug=False): super(DatabaseManager, self).__init__() self.connection = connection tables_list = self.connection.list_tables() if TABLE_NAME in tables_list.get("TableNames"): self.table = Table(table_name=TABLE_NAME, connection=self.connection) else: self.table = Table.create(TABLE_NAME, [ HashKey("entry_type"), RangeKey("date_created", data_type=NUMBER) ], indexes=[ AllIndex("DateJoinedIndex", parts=[ HashKey("entry_type"), RangeKey( "date_created", data_type=NUMBER) ]) ], connection=self.connection)
def get_indexes(all_indexes): indexes = [] global_indexes = [] for index in all_indexes: name = index['name'] schema = get_schema_param(index.get('hash_key_name'), index.get('hash_key_type'), index.get('range_key_name'), index.get('range_key_type')) throughput = { 'read': index.get('read_capacity', 1), 'write': index.get('write_capacity', 1) } if index['type'] == 'all': indexes.append(AllIndex(name, parts=schema)) elif index['type'] == 'global_all': global_indexes.append(GlobalAllIndex(name, parts=schema, throughput=throughput)) elif index['type'] == 'global_include': global_indexes.append(GlobalIncludeIndex(name, parts=schema, throughput=throughput, includes=index['includes'])) elif index['type'] == 'global_keys_only': global_indexes.append(GlobalKeysOnlyIndex(name, parts=schema, throughput=throughput)) elif index['type'] == 'include': indexes.append(IncludeIndex(name, parts=schema, includes=index['includes'])) elif index['type'] == 'keys_only': indexes.append(KeysOnlyIndex(name, parts=schema)) return indexes, global_indexes
def test_sanity_test_table_task(self, mock_config): mock_config.get_config.return_value.get.return_value = AWS_ACCESS_KEY t = TestSanityTestDynamoDBTableTask() # mock s3 location for writing output token s3_client = S3Client(AWS_ACCESS_KEY, AWS_SECRET_KEY) s3_client.s3.create_bucket('mybucket') # create table table_name = 'dynamo_table1' schema = [HashKey('my_hash', data_type=STRING)] indexes = [ AllIndex('IndexName', parts=[ HashKey('my_hash', data_type=STRING), RangeKey('range_index', data_type=NUMBER) ]) ] throughput = {'read': 2, 'write': 4} client = DynamoDBClient(aws_access_key_id=AWS_ACCESS_KEY, aws_secret_access_key=AWS_SECRET_KEY) client.create_table(table_name, schema, throughput, indexes=indexes) self.assertRaises(DynamoDBTaskException, luigi.build([t], local_scheduler=True))
def _extract_index(index_data, global_index=False): ''' Instantiates and returns an AllIndex object given a valid index configuration ''' parsed_data = {} keys = [] for key, value in six.iteritems(index_data): for item in value: for field, data in six.iteritems(item): if field == 'hash_key': parsed_data['hash_key'] = data elif field == 'hash_key_data_type': parsed_data['hash_key_data_type'] = data elif field == 'range_key': parsed_data['range_key'] = data elif field == 'range_key_data_type': parsed_data['range_key_data_type'] = data elif field == 'name': parsed_data['name'] = data elif field == 'read_capacity_units': parsed_data['read_capacity_units'] = data elif field == 'write_capacity_units': parsed_data['write_capacity_units'] = data if parsed_data['hash_key']: keys.append( HashKey( parsed_data['hash_key'], data_type=parsed_data['hash_key_data_type'] ) ) if parsed_data['range_key']: keys.append( RangeKey( parsed_data['range_key'], data_type=parsed_data['range_key_data_type'] ) ) if ( global_index and parsed_data['read_capacity_units'] and parsed_data['write_capacity_units']): parsed_data['throughput'] = { 'read': parsed_data['read_capacity_units'], 'write': parsed_data['write_capacity_units'] } if parsed_data['name'] and len(keys) > 0: if global_index: return GlobalAllIndex( parsed_data['name'], parts=keys, throughput=parsed_data['throughput'] ) else: return AllIndex( parsed_data['name'], parts=keys )
def create(self): Table.create(self.table_name, schema=[ HashKey(Tweet.tweet_user_id), RangeKey(Tweet.tweet_id), ], throughput=standard_throughput, indexes={ AllIndex(self.index_timestamp, parts=[ HashKey(Tweet.tweet_user_id), RangeKey(Tweet.ts_ms) ]) }, global_indexes={ GlobalAllIndex(self.index_site, parts=[ HashKey(keys.entity_site), RangeKey(Tweet.tweet_id) ], throughput=standard_throughput), GlobalAllIndex(self.index_league, parts=[ HashKey(keys.entity_league), RangeKey(Tweet.tweet_id) ], throughput=standard_throughput), GlobalAllIndex(self.index_team, parts=[ HashKey(keys.entity_team), RangeKey(Tweet.tweet_id) ], throughput=standard_throughput) })
def Init(): """ Connect to DynamoDB Local. If you want to connect to the real DynamoDB set the 'local' variable below to Fals, but make sure either you have a .boto file or you pass both the aws_access_key_id and aws_secret_access_key parameters to the create (this code fetches them from settings.cfg). """ local = True if local: # Connect to local DynamoDB server. Make sure you have that running first. conn = DynamoDBConnection(host='localhost', port=8001, aws_secret_access_key='anything', is_secure=False) else: # Connect to the real DynamoDB. config = ConfigParser.RawConfigParser() config.read("settings.cfg") id = config.get('DynamoDB', 'aws_access_key_id') key = config.get('DynamoDB', 'aws_secret_access_key') conn = boto.dynamodb2.connect_to_region('us-west-2', aws_access_key_id=id, aws_secret_access_key=key) # Get a list of all tables from DynamoDB. tables = conn.list_tables() #print "Tables:", tables """ If there isn't an employees table then create it. The table has a primary key of the employee type and id, allowing you to query them. It has a secondary index on the employee type and title, allowing you to query them as well. """ if 'employees' not in tables['TableNames']: # Create table of employees print "Creating new table" employees = Table.create( 'employees', schema=[HashKey('etype'), RangeKey('id')], indexes=[ AllIndex('TitleIndex', parts=[HashKey('etype'), RangeKey('title')]) ], connection=conn) # Wait for table to be created (DynamoDB has eventual consistency) while True: time.sleep(5) try: conn.describe_table('employees') except Exception, e: print e else: break
def __init__(self, region): self.schema = [ HashKey(args.hashkey, data_type=STRING), RangeKey(args.indexkey, data_type=STRING), ] self.indexes = [ AllIndex('FirstName', parts=[ HashKey(args.hashkey, data_type=STRING), RangeKey('FirstName', data_type=STRING), ]), AllIndex('LastName', parts=[ HashKey(args.hashkey, data_type=STRING), RangeKey('LastName', data_type=STRING), ]) ] self.conn = boto.dynamodb2.connect_to_region( region, aws_access_key_id=os.environ['AWS_KEY'], aws_secret_access_key=os.environ['AWS_SECRET'] ) return
def create(self): Table.create( self.table_name, schema=[ HashKey(communication_keys.websocket_key), RangeKey(communication_keys.websocket_ip) ], throughput=standard_throughput, indexes={ AllIndex(self.index_start, parts=[ HashKey(communication_keys.websocket_key), RangeKey(communication_keys.websocket_ts_start, data_type=NUMBER) ]), AllIndex(self.index_end, parts=[ HashKey(communication_keys.websocket_key), RangeKey(communication_keys.websocket_ts_end, data_type=NUMBER) ]) }, global_indexes={}) print 'creating connection table'
def _generate_indexes(self): """ Create boto-friendly index data structure. """ all_index = [] for index in self.indexes: all_index.append( AllIndex(index['name'], parts=[ HashKey(self.hash_key, data_type=self.range_key_type), RangeKey(index['range_key'], data_type=index['data_type']) ])) return all_index
def setUp(self): # Connect to DynamoDB Local self.conn = DynamoDBConnection(host='localhost', port=8000, aws_secret_access_key='anything', is_secure=False) tables = self.conn.list_tables() if 'employees' not in tables['TableNames']: # Create table of employees self.employees = Table.create( 'employees', schema=[HashKey('etype'), RangeKey('id')], indexes=[ AllIndex('TitleIndex', parts=[HashKey('etype'), RangeKey('title')]) ], connection=self.conn) else: self.employees = Table('employees', connection=self.conn) self.employeeData = [{ 'etype': 'E', 'first_name': 'John', 'last_name': 'Doe', 'id': '123456789', 'title': 'Head Bottle Washer', 'hiredate': 'June 5 1986' }, { 'etype': 'E', 'first_name': 'Alice', 'last_name': 'Kramden', 'id': '007', 'title': 'Assistant Bottle Washer', 'hiredate': 'July 1 1950' }, { 'etype': 'E', 'first_name': 'Bob', 'last_name': 'Dylan', 'id': '42', 'title': 'Assistant Bottle Washer', 'hiredate': 'January 1 1970' }] for data in self.employeeData: self.employees.put_item(data=data, overwrite=True)
def test_create_table(self): table_name = 'dynamo_table' schema = [HashKey('my_hash', data_type=STRING)] indexes = [ AllIndex('IndexName', parts=[ HashKey('my_hash', data_type=STRING), RangeKey('range_index', data_type=NUMBER) ]) ] throughput = {'read': 2, 'write': 4} client = DynamoDBClient(aws_access_key_id=AWS_ACCESS_KEY, aws_secret_access_key=AWS_SECRET_KEY) client.create_table(table_name, schema, throughput, indexes=indexes) table = client.get_table(table_name) self.assertEquals(2, table.throughput['read']) self.assertEquals(4, table.throughput['write']) self.assertEquals('my_hash', table.schema[0].name)
def create_table_with_local_indexes(): table = Table.create('messages', schema=[ HashKey('forum_name'), RangeKey('subject'), ], throughput={ 'read': 10, 'write': 10, }, indexes=[ AllIndex('threads_index', parts=[ HashKey('forum_name', data_type=STRING), RangeKey('threads', data_type=NUMBER), ]) ]) return table
class JBoxAccountingV2(JBoxDB): NAME = 'jbox_accounting_v2' SCHEMA = [ HashKey('stop_date', data_type=NUMBER), RangeKey('stop_time', data_type=NUMBER) ] INDEXES = [ AllIndex('container_id-stop_time-index', parts=[ HashKey('container_id', data_type=STRING), RangeKey('stop_time', data_type=NUMBER) ]), IncludeIndex('image_id-stop_time-index', parts=[ HashKey('image_id', data_type=STRING), RangeKey('stop_time', data_type=NUMBER) ], includes=['container_id']) ] TABLE = None _stats_cache = {} def __init__(self, container_id, image_id, start_time, stop_time=None): if None == self.table(): return if None == stop_time: stop_datetime = datetime.datetime.now(pytz.utc) else: stop_datetime = stop_time stop_time = JBoxAccountingV2.datetime_to_epoch_secs( stop_datetime, allow_microsecs=True) stop_date = JBoxAccountingV2.datetime_to_yyyymmdd(stop_datetime) data = { 'stop_date': stop_date, 'stop_time': stop_time, 'image_id': image_id, 'container_id': container_id, 'start_time': JBoxAccountingV2.datetime_to_epoch_secs(start_time), 'start_date': JBoxAccountingV2.datetime_to_yyyymmdd(start_time) } self.create(data) self.item = self.table().get_item(stop_date=stop_date, stop_time=stop_time) self.is_new = True @staticmethod def query_stats_date(date): # TODO: caching items is not a good idea. Should cache computed data instead. if None == JBoxAccountingV2.table(): return [] today = datetime.datetime.now() date_day = JBoxAccountingV2.datetime_to_yyyymmdd(date) today_day = JBoxAccountingV2.datetime_to_yyyymmdd(today) istoday = date_day == today_day if date_day in JBoxAccountingV2._stats_cache: return JBoxAccountingV2._stats_cache[date_day] res = JBoxAccountingV2.table().query_2(stop_date__eq=date_day, stop_time__gte=0) items = [] for item in res: items.append(item) if not istoday: JBoxAccountingV2._stats_cache[date_day] = items return items @staticmethod def get_stats(dates=(datetime.datetime.now(), )): sum_time = 0 item_count = 0 image_count = {} container_freq = {} for date in dates: items = JBoxAccountingV2.query_stats_date(date) for x in items: item_count += 1 if 'start_time' in x: sum_time += x['stop_time'] - int(x['start_time']) try: image_ids = json.loads(x['image_id']) except: image_ids = [] for image_id in image_ids: if image_id.startswith("juliabox/") and ( not image_id.endswith(":latest")): image_count[image_id] = image_count.get(image_id, 0) + 1 cid = x['container_id'] container_freq[cid] = container_freq.get(cid, 0) + 1 def fmt(seconds): hrs = int(seconds / 3600) mins = int(seconds / 60) secs = int(seconds) return "%dh %dm %ds" % (hrs, mins % 60, secs % 60) active_users = 0 for container in container_freq: if container_freq[container] > 2: active_users += 1 return dict(session_count=item_count, avg_time=fmt(float(sum_time) / item_count) if item_count != 0 else 'NA', images_used=image_count, unique_users=len(container_freq), active_users=active_users)
def create_table(table_name, region=None, key=None, keyid=None, profile=None, read_capacity_units=None, write_capacity_units=None, hash_key=None, hash_key_data_type=None, range_key=None, range_key_data_type=None, local_indexes=None, global_indexes=None): ''' Creates a DynamoDB table. CLI example:: salt myminion boto_dynamodb.create_table table_name / region=us-east-1 / hash_key=id / hash_key_data_type=N / range_key=created_at / range_key_data_type=N / read_capacity_units=1 / write_capacity_units=1 ''' schema = [] primary_index_fields = [] primary_index_name = '' if hash_key: hash_key_obj = HashKey(hash_key, data_type=hash_key_data_type) schema.append(hash_key_obj) primary_index_fields.append(hash_key_obj) primary_index_name += hash_key if range_key: range_key_obj = RangeKey(range_key, data_type=range_key_data_type) schema.append(range_key_obj) primary_index_fields.append(range_key_obj) primary_index_name += '_' primary_index_name += range_key primary_index_name += '_index' throughput = {'read': read_capacity_units, 'write': write_capacity_units} local_table_indexes = [] # Add the table's key local_table_indexes.append( AllIndex(primary_index_name, parts=primary_index_fields)) if local_indexes: for index in local_indexes: local_table_indexes.append(_extract_index(index)) global_table_indexes = [] if global_indexes: for index in global_indexes: global_table_indexes.append( _extract_index(index, global_index=True)) conn = _create_connection(region, key, keyid, profile) Table.create(table_name, schema=schema, throughput=throughput, indexes=local_table_indexes, global_indexes=global_table_indexes, connection=conn) # Table creation can take several seconds to propagate. # We will check MAX_ATTEMPTS times. MAX_ATTEMPTS = 30 for i in range(MAX_ATTEMPTS): if exists(table_name, region, key, keyid, profile): return True else: time.sleep(1) # sleep for one second and try again return False
aws_secret_access_key=aws_secret_access_key) ############################################# ###########Try to connect in our table. Otherwise, it creates one####### tables = conn.list_tables() if 'twitter' not in tables['TableNames']: # Create table of employees twitter = Table.create( 'twitter', schema=[HashKey('post_time'), RangeKey('username')], #secondary global indexes=[ AllIndex('PostIndex', parts=[HashKey('post_time'), RangeKey('post')]) ], connection=conn) else: twitter = Table('twitter', connection=conn) ##############Parse input data##################### if (len(sys.argv) is not 3 or sys.argv[1][0] is not '@'): print("Usage:\n%s @<user> <msg>" % (sys.argv[0])) sys.exit(1) ########## Handling our post - IT DOESN'T SAVE ON Dynamo item = Item(twitter, data={ \ 'post_time': time.ctime(), \
# Connect to DynamoDB Local conn = DynamoDBConnection(host='localhost', port=8000, aws_secret_access_key='anything', is_secure=False) tables = conn.list_tables() print tables if 'employees' not in tables['TableNames']: # Create table of employees employees = Table.create( 'employees', schema=[HashKey('etype'), RangeKey('id')], indexes=[ AllIndex('TitleIndex', parts=[HashKey('etype'), RangeKey('title')]) ], connection=conn) else: employees = Table('employees', connection=conn) print employees for data in [{ 'etype': 'E', 'first_name': 'John', 'last_name': 'Doe', 'id': '123456789', 'title': 'Head Bottle Washer', 'hiredate': 'June 5 1986' }, {
def _extract_index(index_data, global_index=False): ''' Instantiates and returns an AllIndex object given a valid index configuration ''' parsed_data = {} keys = [] for key, value in six.iteritems(index_data): for item in value: for field, data in six.iteritems(item): if field == 'hash_key': parsed_data['hash_key'] = data elif field == 'hash_key_data_type': parsed_data['hash_key_data_type'] = data elif field == 'range_key': parsed_data['range_key'] = data elif field == 'range_key_data_type': parsed_data['range_key_data_type'] = data elif field == 'name': parsed_data['name'] = data elif field == 'read_capacity_units': parsed_data['read_capacity_units'] = data elif field == 'write_capacity_units': parsed_data['write_capacity_units'] = data elif field == 'includes': parsed_data['includes'] = data elif field == 'keys_only': parsed_data['keys_only'] = True if parsed_data['hash_key']: keys.append( HashKey(parsed_data['hash_key'], data_type=parsed_data['hash_key_data_type'])) if parsed_data.get('range_key'): keys.append( RangeKey(parsed_data['range_key'], data_type=parsed_data['range_key_data_type'])) if (global_index and parsed_data['read_capacity_units'] and parsed_data['write_capacity_units']): parsed_data['throughput'] = { 'read': parsed_data['read_capacity_units'], 'write': parsed_data['write_capacity_units'] } if parsed_data['name'] and len(keys) > 0: if global_index: if parsed_data.get('keys_only') and parsed_data.get('includes'): raise SaltInvocationError( 'Only one type of GSI projection can be used.') if parsed_data.get('includes'): return GlobalIncludeIndex(parsed_data['name'], parts=keys, throughput=parsed_data['throughput'], includes=parsed_data['includes']) elif parsed_data.get('keys_only'): return GlobalKeysOnlyIndex( parsed_data['name'], parts=keys, throughput=parsed_data['throughput'], ) else: return GlobalAllIndex(parsed_data['name'], parts=keys, throughput=parsed_data['throughput']) else: return AllIndex(parsed_data['name'], parts=keys)
def extract_index(index_data, global_index=False): """ Instantiates and returns an AllIndex object given a valid index configuration CLI Example: .. code-block:: bash salt myminion boto_dynamodb.extract_index index """ parsed_data = {} keys = [] for key, value in index_data.items(): for item in value: for field, data in item.items(): if field == "hash_key": parsed_data["hash_key"] = data elif field == "hash_key_data_type": parsed_data["hash_key_data_type"] = data elif field == "range_key": parsed_data["range_key"] = data elif field == "range_key_data_type": parsed_data["range_key_data_type"] = data elif field == "name": parsed_data["name"] = data elif field == "read_capacity_units": parsed_data["read_capacity_units"] = data elif field == "write_capacity_units": parsed_data["write_capacity_units"] = data elif field == "includes": parsed_data["includes"] = data elif field == "keys_only": parsed_data["keys_only"] = True if parsed_data["hash_key"]: keys.append( HashKey(parsed_data["hash_key"], data_type=parsed_data["hash_key_data_type"])) if parsed_data.get("range_key"): keys.append( RangeKey(parsed_data["range_key"], data_type=parsed_data["range_key_data_type"])) if global_index and parsed_data["read_capacity_units"] and parsed_data[ "write_capacity_units"]: parsed_data["throughput"] = { "read": parsed_data["read_capacity_units"], "write": parsed_data["write_capacity_units"], } if parsed_data["name"] and keys: if global_index: if parsed_data.get("keys_only") and parsed_data.get("includes"): raise SaltInvocationError( "Only one type of GSI projection can be used.") if parsed_data.get("includes"): return GlobalIncludeIndex( parsed_data["name"], parts=keys, throughput=parsed_data["throughput"], includes=parsed_data["includes"], ) elif parsed_data.get("keys_only"): return GlobalKeysOnlyIndex( parsed_data["name"], parts=keys, throughput=parsed_data["throughput"], ) else: return GlobalAllIndex( parsed_data["name"], parts=keys, throughput=parsed_data["throughput"], ) else: return AllIndex(parsed_data["name"], parts=keys)
#!/usr/bin/python #Criar uma tabela no DynamoDB com indice # import boto.dynamodb2 from boto.dynamodb2.fields import HashKey, RangeKey, KeysOnlyIndex, AllIndex from boto.dynamodb2.table import Table from boto.dynamodb2.types import NUMBER print("Criacao de Tabela Key/Value Pair no Dynamo") users = Table.create('users', schema=[ HashKey('account_type', data_type=NUMBER), RangeKey('last_name'), ], throughput={ 'read': 5, 'write': 15, }, indexes=[ AllIndex('EverythingIndex', parts=[ HashKey('account_type', data_type=NUMBER), RangeKey('last_name'), ]) ], # Definindo paramtro na regiao connection= boto.dynamodb2.connect_to_region('us-west-2'))
class JBoxAccountingV2(JBPluginDB): provides = [JBPluginDB.JBP_TABLE_DYNAMODB, JBPluginDB.JBP_USAGE_ACCOUNTING] NAME = 'jbox_accounting_v2' SCHEMA = [ HashKey('stop_date', data_type=NUMBER), RangeKey('stop_time', data_type=NUMBER) ] INDEXES = [ AllIndex('container_id-stop_time-index', parts=[ HashKey('container_id', data_type=STRING), RangeKey('stop_time', data_type=NUMBER) ]), IncludeIndex('image_id-stop_time-index', parts=[ HashKey('image_id', data_type=STRING), RangeKey('stop_time', data_type=NUMBER) ], includes=['container_id']) ] KEYS = ['stop_date', 'stop_time'] ATTRIBUTES = ['image_id', 'container_id', 'start_date', 'start_time'] SQL_INDEXES = [ { 'name': 'container_id-stop_time-index', 'cols': ['container_id', 'stop_time'] }, { 'name': 'image_id-stop_time-index', 'cols': ['image_id', 'stop_time'] }, ] KEYS_TYPES = [JBoxDB.INT, JBoxDB.INT] TYPES = [JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.INT, JBoxDB.INT] TABLE = None _stats_cache = {} def __init__(self, container_id, image_id, start_time, stop_time=None): if None == stop_time: stop_datetime = datetime.datetime.now(pytz.utc) else: stop_datetime = stop_time stop_time = JBoxAccountingV2.datetime_to_epoch_secs( stop_datetime, allow_microsecs=True) stop_date = JBoxAccountingV2.datetime_to_yyyymmdd(stop_datetime) data = { 'stop_date': stop_date, 'stop_time': stop_time, 'image_id': image_id, 'container_id': container_id, 'start_time': JBoxAccountingV2.datetime_to_epoch_secs(start_time), 'start_date': JBoxAccountingV2.datetime_to_yyyymmdd(start_time) } self.create(data) self.item = self.fetch(stop_date=stop_date, stop_time=stop_time) self.is_new = True @staticmethod def _query_stats_date(date): # TODO: caching items is not a good idea. Should cache computed data instead. today = datetime.datetime.now() date_day = JBoxAccountingV2.datetime_to_yyyymmdd(date) today_day = JBoxAccountingV2.datetime_to_yyyymmdd(today) istoday = date_day == today_day if date_day in JBoxAccountingV2._stats_cache: return JBoxAccountingV2._stats_cache[date_day] res = JBoxAccountingV2.query(stop_date__eq=date_day, stop_time__gte=0) items = [] for item in res: items.append(item) if not istoday: JBoxAccountingV2._stats_cache[date_day] = items return items @staticmethod def get_stats(dates=(datetime.datetime.now(), )): sum_time = 0 item_count = 0 image_count = {} container_freq = {} for date in dates: items = JBoxAccountingV2._query_stats_date(date) for x in items: item_count += 1 if 'start_time' in x: sum_time += x['stop_time'] - int(x['start_time']) try: image_ids = json.loads(x['image_id']) except: image_ids = [] for image_id in image_ids: if image_id.startswith("juliabox/") and ( not image_id.endswith(":latest")): image_count[image_id] = image_count.get(image_id, 0) + 1 cid = x['container_id'] container_freq[cid] = container_freq.get(cid, 0) + 1 def fmt(seconds): hrs = int(seconds / 3600) mins = int(seconds / 60) secs = int(seconds) return "%dh %dm %ds" % (hrs, mins % 60, secs % 60) active_users = 0 for container in container_freq: if container_freq[container] > 2: active_users += 1 return dict(session_count=item_count, avg_time=fmt(float(sum_time) / item_count) if item_count != 0 else 'NA', images_used=image_count, unique_users=len(container_freq), active_users=active_users) @staticmethod def record_session_time(container_name, images_used, time_created, time_finished): for retry in range(1, 10): try: start_time = time_created finish_time = time_finished if retry > 1: finish_time += datetime.timedelta( microseconds=random.randint(1, 100)) acct = JBoxAccountingV2(container_name, json.dumps(images_used), start_time, stop_time=finish_time) acct.save() break except: if retry == 10: JBoxAccountingV2.log_exception("error recording usage") else: JBoxAccountingV2.log_warn( "error recording usage, shall retry.")
def create(self): Table.create(self.table_name, schema=[ HashKey(keys.entity_league), RangeKey(keys.entity_profile) ], throughput=standard_throughput, indexes={ AllIndex(self.index_wikipedia, parts=[ HashKey(keys.entity_league), RangeKey(keys.entity_wikipedia) ]), AllIndex(self.index_twitter, parts=[ HashKey(keys.entity_league), RangeKey(keys.entity_twitter) ]), AllIndex(self.index_instagram, parts=[ HashKey(keys.entity_league), RangeKey(keys.entity_instagram) ]), AllIndex(self.index_facebook, parts=[ HashKey(keys.entity_league), RangeKey(keys.entity_facebook) ]) }, global_indexes={ GlobalAllIndex(self.index_twitter_league, parts=[ HashKey(keys.entity_twitter), RangeKey(keys.entity_league) ], throughput=standard_throughput), GlobalAllIndex(self.index_facebook_league, parts=[ HashKey(keys.entity_facebook), RangeKey(keys.entity_league) ], throughput=standard_throughput), GlobalAllIndex(self.index_instagram_league, parts=[ HashKey(keys.entity_instagram), RangeKey(keys.entity_league) ], throughput=standard_throughput), GlobalAllIndex(self.index_team_profile, parts=[ HashKey(keys.entity_team), RangeKey(keys.entity_profile) ], throughput=standard_throughput), GlobalAllIndex(self.index_site_profile, parts=[ HashKey(keys.entity_site), RangeKey(keys.entity_profile) ], throughput=standard_throughput) }) print 'creating entity next table'
def create_dynamo_tables(self): with open( '%s/sb_docstore/management/commands' '/dynamo_table.json' % settings.PROJECT_DIR, 'r') as data_file: data = loads(data_file.read()) conn = connect_to_dynamo() reads = 1 writes = 1 if isinstance(conn, Exception): print "Unable to connect to dynamo table, potential error" for item in data: table_name = get_table_name(item['table_name']) ''' # Don't think we want to automatically delete the tables every # deployment anymore. We probably want to be able to hit an # endpoint that triggers a rebuilding of the tables that we can # more closely monitor. try: table = Table(table_name=table_name, connection=conn) table.delete() while (table.describe()['Table']['TableStatus'] == "DELETING"): time.sleep(1) except JSONResponseError: print 'The table %s does not exist' % table_name ''' try: if 'range_key' and 'local_index' in item.keys(): Table.create( table_name, schema=[ HashKey(item['hash_key'], data_type=STRING), RangeKey(item['range_key']), ], indexes=[ AllIndex(item['local_index_name'], parts=[ HashKey(item['hash_key']), RangeKey(item['local_index'], data_type=item['type']), ]) ], throughput={ 'read': reads, 'write': writes }, connection=conn) elif 'range_key' in item.keys(): Table.create(table_name, schema=[ HashKey(item['hash_key'], data_type=STRING), RangeKey(item['range_key']), ], throughput={ 'read': reads, 'write': writes }, connection=conn) else: Table.create(table_name, schema=[ HashKey(item['hash_key'], data_type=STRING), ], throughput={ 'read': reads, 'write': writes }, connection=conn) except JSONResponseError: print 'Table %s already exists' % item['table_name'] '''