def create(self):
Table.create(self.table_name,
schema=[
HashKey(keys.entity_profile),
RangeKey(time_keys.ts_add, data_type=NUMBER)
],
throughput=standard_throughput,
indexes={
AllIndex(self.index_delta,
parts=[
HashKey(keys.entity_profile),
RangeKey(time_keys.ts_delta,
data_type=NUMBER)
]),
AllIndex(self.index_cut,
parts=[
HashKey(keys.entity_profile),
RangeKey(time_keys.ts_cut,
data_type=NUMBER)
])
},
global_indexes={
GlobalAllIndex(self.index_team,
parts=[
HashKey(keys.entity_team),
RangeKey(time_keys.ts_add,
data_type=NUMBER)
],
throughput=standard_throughput),
GlobalAllIndex(self.index_league,
parts=[
HashKey(keys.entity_league),
RangeKey(time_keys.ts_add,
data_type=NUMBER)
],
throughput=standard_throughput),
GlobalAllIndex(self.index_twitter,
parts=[
HashKey(keys.entity_twitter),
RangeKey(time_keys.ts_add,
data_type=NUMBER)
],
throughput=standard_throughput),
GlobalAllIndex(self.index_site,
parts=[
HashKey(keys.entity_site),
RangeKey(time_keys.ts_add,
data_type=NUMBER)
],
throughput=standard_throughput)
})
print 'creating entity history table'
def __init__(self, connection, debug=False):
super(DatabaseManager, self).__init__()
self.connection = connection
tables_list = self.connection.list_tables()
if TABLE_NAME in tables_list.get("TableNames"):
self.table = Table(table_name=TABLE_NAME,
connection=self.connection)
else:
self.table = Table.create(TABLE_NAME, [
HashKey("entry_type"),
RangeKey("date_created", data_type=NUMBER)
],
indexes=[
AllIndex("DateJoinedIndex",
parts=[
HashKey("entry_type"),
RangeKey(
"date_created",
data_type=NUMBER)
])
],
connection=self.connection)
def get_indexes(all_indexes):
indexes = []
global_indexes = []
for index in all_indexes:
name = index['name']
schema = get_schema_param(index.get('hash_key_name'), index.get('hash_key_type'), index.get('range_key_name'), index.get('range_key_type'))
throughput = {
'read': index.get('read_capacity', 1),
'write': index.get('write_capacity', 1)
}
if index['type'] == 'all':
indexes.append(AllIndex(name, parts=schema))
elif index['type'] == 'global_all':
global_indexes.append(GlobalAllIndex(name, parts=schema, throughput=throughput))
elif index['type'] == 'global_include':
global_indexes.append(GlobalIncludeIndex(name, parts=schema, throughput=throughput, includes=index['includes']))
elif index['type'] == 'global_keys_only':
global_indexes.append(GlobalKeysOnlyIndex(name, parts=schema, throughput=throughput))
elif index['type'] == 'include':
indexes.append(IncludeIndex(name, parts=schema, includes=index['includes']))
elif index['type'] == 'keys_only':
indexes.append(KeysOnlyIndex(name, parts=schema))
return indexes, global_indexes
def test_sanity_test_table_task(self, mock_config):
mock_config.get_config.return_value.get.return_value = AWS_ACCESS_KEY
t = TestSanityTestDynamoDBTableTask()
# mock s3 location for writing output token
s3_client = S3Client(AWS_ACCESS_KEY, AWS_SECRET_KEY)
s3_client.s3.create_bucket('mybucket')
# create table
table_name = 'dynamo_table1'
schema = [HashKey('my_hash', data_type=STRING)]
indexes = [
AllIndex('IndexName',
parts=[
HashKey('my_hash', data_type=STRING),
RangeKey('range_index', data_type=NUMBER)
])
]
throughput = {'read': 2, 'write': 4}
client = DynamoDBClient(aws_access_key_id=AWS_ACCESS_KEY,
aws_secret_access_key=AWS_SECRET_KEY)
client.create_table(table_name, schema, throughput, indexes=indexes)
self.assertRaises(DynamoDBTaskException,
luigi.build([t], local_scheduler=True))
def _extract_index(index_data, global_index=False):
'''
Instantiates and returns an AllIndex object given a valid index
configuration
'''
parsed_data = {}
keys = []
for key, value in six.iteritems(index_data):
for item in value:
for field, data in six.iteritems(item):
if field == 'hash_key':
parsed_data['hash_key'] = data
elif field == 'hash_key_data_type':
parsed_data['hash_key_data_type'] = data
elif field == 'range_key':
parsed_data['range_key'] = data
elif field == 'range_key_data_type':
parsed_data['range_key_data_type'] = data
elif field == 'name':
parsed_data['name'] = data
elif field == 'read_capacity_units':
parsed_data['read_capacity_units'] = data
elif field == 'write_capacity_units':
parsed_data['write_capacity_units'] = data
if parsed_data['hash_key']:
keys.append(
HashKey(
parsed_data['hash_key'],
data_type=parsed_data['hash_key_data_type']
)
)
if parsed_data['range_key']:
keys.append(
RangeKey(
parsed_data['range_key'],
data_type=parsed_data['range_key_data_type']
)
)
if (
global_index and
parsed_data['read_capacity_units'] and
parsed_data['write_capacity_units']):
parsed_data['throughput'] = {
'read': parsed_data['read_capacity_units'],
'write': parsed_data['write_capacity_units']
}
if parsed_data['name'] and len(keys) > 0:
if global_index:
return GlobalAllIndex(
parsed_data['name'],
parts=keys,
throughput=parsed_data['throughput']
)
else:
return AllIndex(
parsed_data['name'],
parts=keys
)
def create(self):
Table.create(self.table_name,
schema=[
HashKey(Tweet.tweet_user_id),
RangeKey(Tweet.tweet_id),
],
throughput=standard_throughput,
indexes={
AllIndex(self.index_timestamp,
parts=[
HashKey(Tweet.tweet_user_id),
RangeKey(Tweet.ts_ms)
])
},
global_indexes={
GlobalAllIndex(self.index_site,
parts=[
HashKey(keys.entity_site),
RangeKey(Tweet.tweet_id)
],
throughput=standard_throughput),
GlobalAllIndex(self.index_league,
parts=[
HashKey(keys.entity_league),
RangeKey(Tweet.tweet_id)
],
throughput=standard_throughput),
GlobalAllIndex(self.index_team,
parts=[
HashKey(keys.entity_team),
RangeKey(Tweet.tweet_id)
],
throughput=standard_throughput)
})
def Init():
"""
Connect to DynamoDB Local. If you want to connect to the real DynamoDB set the 'local'
variable below to Fals, but make sure either you have a .boto file or you
pass both the aws_access_key_id and aws_secret_access_key parameters to the create
(this code fetches them from settings.cfg).
"""
local = True
if local:
# Connect to local DynamoDB server. Make sure you have that running first.
conn = DynamoDBConnection(host='localhost',
port=8001,
aws_secret_access_key='anything',
is_secure=False)
else:
# Connect to the real DynamoDB.
config = ConfigParser.RawConfigParser()
config.read("settings.cfg")
id = config.get('DynamoDB', 'aws_access_key_id')
key = config.get('DynamoDB', 'aws_secret_access_key')
conn = boto.dynamodb2.connect_to_region('us-west-2',
aws_access_key_id=id,
aws_secret_access_key=key)
# Get a list of all tables from DynamoDB.
tables = conn.list_tables()
#print "Tables:", tables
"""
If there isn't an employees table then create it. The table has a primary key of the
employee type and id, allowing you to query them. It has a secondary index on the
employee type and title, allowing you to query them as well.
"""
if 'employees' not in tables['TableNames']:
# Create table of employees
print "Creating new table"
employees = Table.create(
'employees',
schema=[HashKey('etype'), RangeKey('id')],
indexes=[
AllIndex('TitleIndex',
parts=[HashKey('etype'),
RangeKey('title')])
],
connection=conn)
# Wait for table to be created (DynamoDB has eventual consistency)
while True:
time.sleep(5)
try:
conn.describe_table('employees')
except Exception, e:
print e
else:
break
def __init__(self, region):
self.schema = [
HashKey(args.hashkey, data_type=STRING),
RangeKey(args.indexkey, data_type=STRING),
]
self.indexes = [
AllIndex('FirstName', parts=[
HashKey(args.hashkey, data_type=STRING),
RangeKey('FirstName', data_type=STRING),
]),
AllIndex('LastName', parts=[
HashKey(args.hashkey, data_type=STRING),
RangeKey('LastName', data_type=STRING),
])
]
self.conn = boto.dynamodb2.connect_to_region(
region,
aws_access_key_id=os.environ['AWS_KEY'],
aws_secret_access_key=os.environ['AWS_SECRET']
)
return
def create(self):
Table.create(
self.table_name,
schema=[
HashKey(communication_keys.websocket_key),
RangeKey(communication_keys.websocket_ip)
],
throughput=standard_throughput,
indexes={
AllIndex(self.index_start,
parts=[
HashKey(communication_keys.websocket_key),
RangeKey(communication_keys.websocket_ts_start,
data_type=NUMBER)
]),
AllIndex(self.index_end,
parts=[
HashKey(communication_keys.websocket_key),
RangeKey(communication_keys.websocket_ts_end,
data_type=NUMBER)
])
},
global_indexes={})
print 'creating connection table'
def _generate_indexes(self):
"""
Create boto-friendly index data structure.
"""
all_index = []
for index in self.indexes:
all_index.append(
AllIndex(index['name'],
parts=[
HashKey(self.hash_key,
data_type=self.range_key_type),
RangeKey(index['range_key'],
data_type=index['data_type'])
]))
return all_index
def setUp(self):
# Connect to DynamoDB Local
self.conn = DynamoDBConnection(host='localhost',
port=8000,
aws_secret_access_key='anything',
is_secure=False)
tables = self.conn.list_tables()
if 'employees' not in tables['TableNames']:
# Create table of employees
self.employees = Table.create(
'employees',
schema=[HashKey('etype'), RangeKey('id')],
indexes=[
AllIndex('TitleIndex',
parts=[HashKey('etype'),
RangeKey('title')])
],
connection=self.conn)
else:
self.employees = Table('employees', connection=self.conn)
self.employeeData = [{
'etype': 'E',
'first_name': 'John',
'last_name': 'Doe',
'id': '123456789',
'title': 'Head Bottle Washer',
'hiredate': 'June 5 1986'
}, {
'etype': 'E',
'first_name': 'Alice',
'last_name': 'Kramden',
'id': '007',
'title': 'Assistant Bottle Washer',
'hiredate': 'July 1 1950'
}, {
'etype': 'E',
'first_name': 'Bob',
'last_name': 'Dylan',
'id': '42',
'title': 'Assistant Bottle Washer',
'hiredate': 'January 1 1970'
}]
for data in self.employeeData:
self.employees.put_item(data=data, overwrite=True)
def test_create_table(self):
table_name = 'dynamo_table'
schema = [HashKey('my_hash', data_type=STRING)]
indexes = [
AllIndex('IndexName',
parts=[
HashKey('my_hash', data_type=STRING),
RangeKey('range_index', data_type=NUMBER)
])
]
throughput = {'read': 2, 'write': 4}
client = DynamoDBClient(aws_access_key_id=AWS_ACCESS_KEY,
aws_secret_access_key=AWS_SECRET_KEY)
client.create_table(table_name, schema, throughput, indexes=indexes)
table = client.get_table(table_name)
self.assertEquals(2, table.throughput['read'])
self.assertEquals(4, table.throughput['write'])
self.assertEquals('my_hash', table.schema[0].name)
def create_table_with_local_indexes():
table = Table.create('messages',
schema=[
HashKey('forum_name'),
RangeKey('subject'),
],
throughput={
'read': 10,
'write': 10,
},
indexes=[
AllIndex('threads_index',
parts=[
HashKey('forum_name',
data_type=STRING),
RangeKey('threads',
data_type=NUMBER),
])
])
return table
class JBoxAccountingV2(JBoxDB):
NAME = 'jbox_accounting_v2'
SCHEMA = [
HashKey('stop_date', data_type=NUMBER),
RangeKey('stop_time', data_type=NUMBER)
]
INDEXES = [
AllIndex('container_id-stop_time-index',
parts=[
HashKey('container_id', data_type=STRING),
RangeKey('stop_time', data_type=NUMBER)
]),
IncludeIndex('image_id-stop_time-index',
parts=[
HashKey('image_id', data_type=STRING),
RangeKey('stop_time', data_type=NUMBER)
],
includes=['container_id'])
]
TABLE = None
_stats_cache = {}
def __init__(self, container_id, image_id, start_time, stop_time=None):
if None == self.table():
return
if None == stop_time:
stop_datetime = datetime.datetime.now(pytz.utc)
else:
stop_datetime = stop_time
stop_time = JBoxAccountingV2.datetime_to_epoch_secs(
stop_datetime, allow_microsecs=True)
stop_date = JBoxAccountingV2.datetime_to_yyyymmdd(stop_datetime)
data = {
'stop_date': stop_date,
'stop_time': stop_time,
'image_id': image_id,
'container_id': container_id,
'start_time': JBoxAccountingV2.datetime_to_epoch_secs(start_time),
'start_date': JBoxAccountingV2.datetime_to_yyyymmdd(start_time)
}
self.create(data)
self.item = self.table().get_item(stop_date=stop_date,
stop_time=stop_time)
self.is_new = True
@staticmethod
def query_stats_date(date):
# TODO: caching items is not a good idea. Should cache computed data instead.
if None == JBoxAccountingV2.table():
return []
today = datetime.datetime.now()
date_day = JBoxAccountingV2.datetime_to_yyyymmdd(date)
today_day = JBoxAccountingV2.datetime_to_yyyymmdd(today)
istoday = date_day == today_day
if date_day in JBoxAccountingV2._stats_cache:
return JBoxAccountingV2._stats_cache[date_day]
res = JBoxAccountingV2.table().query_2(stop_date__eq=date_day,
stop_time__gte=0)
items = []
for item in res:
items.append(item)
if not istoday:
JBoxAccountingV2._stats_cache[date_day] = items
return items
@staticmethod
def get_stats(dates=(datetime.datetime.now(), )):
sum_time = 0
item_count = 0
image_count = {}
container_freq = {}
for date in dates:
items = JBoxAccountingV2.query_stats_date(date)
for x in items:
item_count += 1
if 'start_time' in x:
sum_time += x['stop_time'] - int(x['start_time'])
try:
image_ids = json.loads(x['image_id'])
except:
image_ids = []
for image_id in image_ids:
if image_id.startswith("juliabox/") and (
not image_id.endswith(":latest")):
image_count[image_id] = image_count.get(image_id,
0) + 1
cid = x['container_id']
container_freq[cid] = container_freq.get(cid, 0) + 1
def fmt(seconds):
hrs = int(seconds / 3600)
mins = int(seconds / 60)
secs = int(seconds)
return "%dh %dm %ds" % (hrs, mins % 60, secs % 60)
active_users = 0
for container in container_freq:
if container_freq[container] > 2:
active_users += 1
return dict(session_count=item_count,
avg_time=fmt(float(sum_time) /
item_count) if item_count != 0 else 'NA',
images_used=image_count,
unique_users=len(container_freq),
active_users=active_users)
def create_table(table_name,
region=None,
key=None,
keyid=None,
profile=None,
read_capacity_units=None,
write_capacity_units=None,
hash_key=None,
hash_key_data_type=None,
range_key=None,
range_key_data_type=None,
local_indexes=None,
global_indexes=None):
'''
Creates a DynamoDB table.
CLI example::
salt myminion boto_dynamodb.create_table table_name /
region=us-east-1 /
hash_key=id /
hash_key_data_type=N /
range_key=created_at /
range_key_data_type=N /
read_capacity_units=1 /
write_capacity_units=1
'''
schema = []
primary_index_fields = []
primary_index_name = ''
if hash_key:
hash_key_obj = HashKey(hash_key, data_type=hash_key_data_type)
schema.append(hash_key_obj)
primary_index_fields.append(hash_key_obj)
primary_index_name += hash_key
if range_key:
range_key_obj = RangeKey(range_key, data_type=range_key_data_type)
schema.append(range_key_obj)
primary_index_fields.append(range_key_obj)
primary_index_name += '_'
primary_index_name += range_key
primary_index_name += '_index'
throughput = {'read': read_capacity_units, 'write': write_capacity_units}
local_table_indexes = []
# Add the table's key
local_table_indexes.append(
AllIndex(primary_index_name, parts=primary_index_fields))
if local_indexes:
for index in local_indexes:
local_table_indexes.append(_extract_index(index))
global_table_indexes = []
if global_indexes:
for index in global_indexes:
global_table_indexes.append(
_extract_index(index, global_index=True))
conn = _create_connection(region, key, keyid, profile)
Table.create(table_name,
schema=schema,
throughput=throughput,
indexes=local_table_indexes,
global_indexes=global_table_indexes,
connection=conn)
# Table creation can take several seconds to propagate.
# We will check MAX_ATTEMPTS times.
MAX_ATTEMPTS = 30
for i in range(MAX_ATTEMPTS):
if exists(table_name, region, key, keyid, profile):
return True
else:
time.sleep(1) # sleep for one second and try again
return False
aws_secret_access_key=aws_secret_access_key)
#############################################
###########Try to connect in our table. Otherwise, it creates one#######
tables = conn.list_tables()
if 'twitter' not in tables['TableNames']:
# Create table of employees
twitter = Table.create(
'twitter',
schema=[HashKey('post_time'),
RangeKey('username')],
#secondary global
indexes=[
AllIndex('PostIndex',
parts=[HashKey('post_time'),
RangeKey('post')])
],
connection=conn)
else:
twitter = Table('twitter', connection=conn)
##############Parse input data#####################
if (len(sys.argv) is not 3 or sys.argv[1][0] is not '@'):
print("Usage:\n%s @<user> <msg>" % (sys.argv[0]))
sys.exit(1)
########## Handling our post - IT DOESN'T SAVE ON Dynamo
item = Item(twitter, data={ \
'post_time': time.ctime(), \
# Connect to DynamoDB Local
conn = DynamoDBConnection(host='localhost',
port=8000,
aws_secret_access_key='anything',
is_secure=False)
tables = conn.list_tables()
print tables
if 'employees' not in tables['TableNames']:
# Create table of employees
employees = Table.create(
'employees',
schema=[HashKey('etype'), RangeKey('id')],
indexes=[
AllIndex('TitleIndex', parts=[HashKey('etype'),
RangeKey('title')])
],
connection=conn)
else:
employees = Table('employees', connection=conn)
print employees
for data in [{
'etype': 'E',
'first_name': 'John',
'last_name': 'Doe',
'id': '123456789',
'title': 'Head Bottle Washer',
'hiredate': 'June 5 1986'
}, {
def _extract_index(index_data, global_index=False):
'''
Instantiates and returns an AllIndex object given a valid index
configuration
'''
parsed_data = {}
keys = []
for key, value in six.iteritems(index_data):
for item in value:
for field, data in six.iteritems(item):
if field == 'hash_key':
parsed_data['hash_key'] = data
elif field == 'hash_key_data_type':
parsed_data['hash_key_data_type'] = data
elif field == 'range_key':
parsed_data['range_key'] = data
elif field == 'range_key_data_type':
parsed_data['range_key_data_type'] = data
elif field == 'name':
parsed_data['name'] = data
elif field == 'read_capacity_units':
parsed_data['read_capacity_units'] = data
elif field == 'write_capacity_units':
parsed_data['write_capacity_units'] = data
elif field == 'includes':
parsed_data['includes'] = data
elif field == 'keys_only':
parsed_data['keys_only'] = True
if parsed_data['hash_key']:
keys.append(
HashKey(parsed_data['hash_key'],
data_type=parsed_data['hash_key_data_type']))
if parsed_data.get('range_key'):
keys.append(
RangeKey(parsed_data['range_key'],
data_type=parsed_data['range_key_data_type']))
if (global_index and parsed_data['read_capacity_units']
and parsed_data['write_capacity_units']):
parsed_data['throughput'] = {
'read': parsed_data['read_capacity_units'],
'write': parsed_data['write_capacity_units']
}
if parsed_data['name'] and len(keys) > 0:
if global_index:
if parsed_data.get('keys_only') and parsed_data.get('includes'):
raise SaltInvocationError(
'Only one type of GSI projection can be used.')
if parsed_data.get('includes'):
return GlobalIncludeIndex(parsed_data['name'],
parts=keys,
throughput=parsed_data['throughput'],
includes=parsed_data['includes'])
elif parsed_data.get('keys_only'):
return GlobalKeysOnlyIndex(
parsed_data['name'],
parts=keys,
throughput=parsed_data['throughput'],
)
else:
return GlobalAllIndex(parsed_data['name'],
parts=keys,
throughput=parsed_data['throughput'])
else:
return AllIndex(parsed_data['name'], parts=keys)
def extract_index(index_data, global_index=False):
"""
Instantiates and returns an AllIndex object given a valid index
configuration
CLI Example:
.. code-block:: bash
salt myminion boto_dynamodb.extract_index index
"""
parsed_data = {}
keys = []
for key, value in index_data.items():
for item in value:
for field, data in item.items():
if field == "hash_key":
parsed_data["hash_key"] = data
elif field == "hash_key_data_type":
parsed_data["hash_key_data_type"] = data
elif field == "range_key":
parsed_data["range_key"] = data
elif field == "range_key_data_type":
parsed_data["range_key_data_type"] = data
elif field == "name":
parsed_data["name"] = data
elif field == "read_capacity_units":
parsed_data["read_capacity_units"] = data
elif field == "write_capacity_units":
parsed_data["write_capacity_units"] = data
elif field == "includes":
parsed_data["includes"] = data
elif field == "keys_only":
parsed_data["keys_only"] = True
if parsed_data["hash_key"]:
keys.append(
HashKey(parsed_data["hash_key"],
data_type=parsed_data["hash_key_data_type"]))
if parsed_data.get("range_key"):
keys.append(
RangeKey(parsed_data["range_key"],
data_type=parsed_data["range_key_data_type"]))
if global_index and parsed_data["read_capacity_units"] and parsed_data[
"write_capacity_units"]:
parsed_data["throughput"] = {
"read": parsed_data["read_capacity_units"],
"write": parsed_data["write_capacity_units"],
}
if parsed_data["name"] and keys:
if global_index:
if parsed_data.get("keys_only") and parsed_data.get("includes"):
raise SaltInvocationError(
"Only one type of GSI projection can be used.")
if parsed_data.get("includes"):
return GlobalIncludeIndex(
parsed_data["name"],
parts=keys,
throughput=parsed_data["throughput"],
includes=parsed_data["includes"],
)
elif parsed_data.get("keys_only"):
return GlobalKeysOnlyIndex(
parsed_data["name"],
parts=keys,
throughput=parsed_data["throughput"],
)
else:
return GlobalAllIndex(
parsed_data["name"],
parts=keys,
throughput=parsed_data["throughput"],
)
else:
return AllIndex(parsed_data["name"], parts=keys)
#!/usr/bin/python
#Criar uma tabela no DynamoDB com indice
#
import boto.dynamodb2
from boto.dynamodb2.fields import HashKey, RangeKey, KeysOnlyIndex, AllIndex
from boto.dynamodb2.table import Table
from boto.dynamodb2.types import NUMBER
print("Criacao de Tabela Key/Value Pair no Dynamo")
users = Table.create('users', schema=[
HashKey('account_type', data_type=NUMBER),
RangeKey('last_name'),
], throughput={
'read': 5,
'write': 15,
}, indexes=[
AllIndex('EverythingIndex', parts=[
HashKey('account_type', data_type=NUMBER),
RangeKey('last_name'),
])
],
# Definindo paramtro na regiao
connection= boto.dynamodb2.connect_to_region('us-west-2'))
class JBoxAccountingV2(JBPluginDB):
provides = [JBPluginDB.JBP_TABLE_DYNAMODB, JBPluginDB.JBP_USAGE_ACCOUNTING]
NAME = 'jbox_accounting_v2'
SCHEMA = [
HashKey('stop_date', data_type=NUMBER),
RangeKey('stop_time', data_type=NUMBER)
]
INDEXES = [
AllIndex('container_id-stop_time-index',
parts=[
HashKey('container_id', data_type=STRING),
RangeKey('stop_time', data_type=NUMBER)
]),
IncludeIndex('image_id-stop_time-index',
parts=[
HashKey('image_id', data_type=STRING),
RangeKey('stop_time', data_type=NUMBER)
],
includes=['container_id'])
]
KEYS = ['stop_date', 'stop_time']
ATTRIBUTES = ['image_id', 'container_id', 'start_date', 'start_time']
SQL_INDEXES = [
{
'name': 'container_id-stop_time-index',
'cols': ['container_id', 'stop_time']
},
{
'name': 'image_id-stop_time-index',
'cols': ['image_id', 'stop_time']
},
]
KEYS_TYPES = [JBoxDB.INT, JBoxDB.INT]
TYPES = [JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.INT, JBoxDB.INT]
TABLE = None
_stats_cache = {}
def __init__(self, container_id, image_id, start_time, stop_time=None):
if None == stop_time:
stop_datetime = datetime.datetime.now(pytz.utc)
else:
stop_datetime = stop_time
stop_time = JBoxAccountingV2.datetime_to_epoch_secs(
stop_datetime, allow_microsecs=True)
stop_date = JBoxAccountingV2.datetime_to_yyyymmdd(stop_datetime)
data = {
'stop_date': stop_date,
'stop_time': stop_time,
'image_id': image_id,
'container_id': container_id,
'start_time': JBoxAccountingV2.datetime_to_epoch_secs(start_time),
'start_date': JBoxAccountingV2.datetime_to_yyyymmdd(start_time)
}
self.create(data)
self.item = self.fetch(stop_date=stop_date, stop_time=stop_time)
self.is_new = True
@staticmethod
def _query_stats_date(date):
# TODO: caching items is not a good idea. Should cache computed data instead.
today = datetime.datetime.now()
date_day = JBoxAccountingV2.datetime_to_yyyymmdd(date)
today_day = JBoxAccountingV2.datetime_to_yyyymmdd(today)
istoday = date_day == today_day
if date_day in JBoxAccountingV2._stats_cache:
return JBoxAccountingV2._stats_cache[date_day]
res = JBoxAccountingV2.query(stop_date__eq=date_day, stop_time__gte=0)
items = []
for item in res:
items.append(item)
if not istoday:
JBoxAccountingV2._stats_cache[date_day] = items
return items
@staticmethod
def get_stats(dates=(datetime.datetime.now(), )):
sum_time = 0
item_count = 0
image_count = {}
container_freq = {}
for date in dates:
items = JBoxAccountingV2._query_stats_date(date)
for x in items:
item_count += 1
if 'start_time' in x:
sum_time += x['stop_time'] - int(x['start_time'])
try:
image_ids = json.loads(x['image_id'])
except:
image_ids = []
for image_id in image_ids:
if image_id.startswith("juliabox/") and (
not image_id.endswith(":latest")):
image_count[image_id] = image_count.get(image_id,
0) + 1
cid = x['container_id']
container_freq[cid] = container_freq.get(cid, 0) + 1
def fmt(seconds):
hrs = int(seconds / 3600)
mins = int(seconds / 60)
secs = int(seconds)
return "%dh %dm %ds" % (hrs, mins % 60, secs % 60)
active_users = 0
for container in container_freq:
if container_freq[container] > 2:
active_users += 1
return dict(session_count=item_count,
avg_time=fmt(float(sum_time) /
item_count) if item_count != 0 else 'NA',
images_used=image_count,
unique_users=len(container_freq),
active_users=active_users)
@staticmethod
def record_session_time(container_name, images_used, time_created,
time_finished):
for retry in range(1, 10):
try:
start_time = time_created
finish_time = time_finished
if retry > 1:
finish_time += datetime.timedelta(
microseconds=random.randint(1, 100))
acct = JBoxAccountingV2(container_name,
json.dumps(images_used),
start_time,
stop_time=finish_time)
acct.save()
break
except:
if retry == 10:
JBoxAccountingV2.log_exception("error recording usage")
else:
JBoxAccountingV2.log_warn(
"error recording usage, shall retry.")
def create(self):
Table.create(self.table_name,
schema=[
HashKey(keys.entity_league),
RangeKey(keys.entity_profile)
],
throughput=standard_throughput,
indexes={
AllIndex(self.index_wikipedia,
parts=[
HashKey(keys.entity_league),
RangeKey(keys.entity_wikipedia)
]),
AllIndex(self.index_twitter,
parts=[
HashKey(keys.entity_league),
RangeKey(keys.entity_twitter)
]),
AllIndex(self.index_instagram,
parts=[
HashKey(keys.entity_league),
RangeKey(keys.entity_instagram)
]),
AllIndex(self.index_facebook,
parts=[
HashKey(keys.entity_league),
RangeKey(keys.entity_facebook)
])
},
global_indexes={
GlobalAllIndex(self.index_twitter_league,
parts=[
HashKey(keys.entity_twitter),
RangeKey(keys.entity_league)
],
throughput=standard_throughput),
GlobalAllIndex(self.index_facebook_league,
parts=[
HashKey(keys.entity_facebook),
RangeKey(keys.entity_league)
],
throughput=standard_throughput),
GlobalAllIndex(self.index_instagram_league,
parts=[
HashKey(keys.entity_instagram),
RangeKey(keys.entity_league)
],
throughput=standard_throughput),
GlobalAllIndex(self.index_team_profile,
parts=[
HashKey(keys.entity_team),
RangeKey(keys.entity_profile)
],
throughput=standard_throughput),
GlobalAllIndex(self.index_site_profile,
parts=[
HashKey(keys.entity_site),
RangeKey(keys.entity_profile)
],
throughput=standard_throughput)
})
print 'creating entity next table'
def create_dynamo_tables(self):
with open(
'%s/sb_docstore/management/commands'
'/dynamo_table.json' % settings.PROJECT_DIR, 'r') as data_file:
data = loads(data_file.read())
conn = connect_to_dynamo()
reads = 1
writes = 1
if isinstance(conn, Exception):
print "Unable to connect to dynamo table, potential error"
for item in data:
table_name = get_table_name(item['table_name'])
'''
# Don't think we want to automatically delete the tables every
# deployment anymore. We probably want to be able to hit an
# endpoint that triggers a rebuilding of the tables that we can
# more closely monitor.
try:
table = Table(table_name=table_name,
connection=conn)
table.delete()
while (table.describe()['Table']['TableStatus'] ==
"DELETING"):
time.sleep(1)
except JSONResponseError:
print 'The table %s does not exist' % table_name
'''
try:
if 'range_key' and 'local_index' in item.keys():
Table.create(
table_name,
schema=[
HashKey(item['hash_key'], data_type=STRING),
RangeKey(item['range_key']),
],
indexes=[
AllIndex(item['local_index_name'],
parts=[
HashKey(item['hash_key']),
RangeKey(item['local_index'],
data_type=item['type']),
])
],
throughput={
'read': reads,
'write': writes
},
connection=conn)
elif 'range_key' in item.keys():
Table.create(table_name,
schema=[
HashKey(item['hash_key'],
data_type=STRING),
RangeKey(item['range_key']),
],
throughput={
'read': reads,
'write': writes
},
connection=conn)
else:
Table.create(table_name,
schema=[
HashKey(item['hash_key'],
data_type=STRING),
],
throughput={
'read': reads,
'write': writes
},
connection=conn)
except JSONResponseError:
print 'Table %s already exists' % item['table_name']
'''