def get_indexes(all_indexes):
indexes = []
global_indexes = []
for index in all_indexes:
name = index['name']
schema = get_schema_param(index.get('hash_key_name'), index.get('hash_key_type'), index.get('range_key_name'), index.get('range_key_type'))
throughput = {
'read': index.get('read_capacity', 1),
'write': index.get('write_capacity', 1)
}
if index['type'] == 'all':
indexes.append(AllIndex(name, parts=schema))
elif index['type'] == 'global_all':
global_indexes.append(GlobalAllIndex(name, parts=schema, throughput=throughput))
elif index['type'] == 'global_include':
global_indexes.append(GlobalIncludeIndex(name, parts=schema, throughput=throughput, includes=index['includes']))
elif index['type'] == 'global_keys_only':
global_indexes.append(GlobalKeysOnlyIndex(name, parts=schema, throughput=throughput))
elif index['type'] == 'include':
indexes.append(IncludeIndex(name, parts=schema, includes=index['includes']))
elif index['type'] == 'keys_only':
indexes.append(KeysOnlyIndex(name, parts=schema))
return indexes, global_indexes
class JBoxAccountingV2(JBPluginDB):
provides = [JBPluginDB.JBP_TABLE_DYNAMODB, JBPluginDB.JBP_USAGE_ACCOUNTING]
NAME = 'jbox_accounting_v2'
SCHEMA = [
HashKey('stop_date', data_type=NUMBER),
RangeKey('stop_time', data_type=NUMBER)
]
INDEXES = [
AllIndex('container_id-stop_time-index',
parts=[
HashKey('container_id', data_type=STRING),
RangeKey('stop_time', data_type=NUMBER)
]),
IncludeIndex('image_id-stop_time-index',
parts=[
HashKey('image_id', data_type=STRING),
RangeKey('stop_time', data_type=NUMBER)
],
includes=['container_id'])
]
KEYS = ['stop_date', 'stop_time']
ATTRIBUTES = ['image_id', 'container_id', 'start_date', 'start_time']
SQL_INDEXES = [
{
'name': 'container_id-stop_time-index',
'cols': ['container_id', 'stop_time']
},
{
'name': 'image_id-stop_time-index',
'cols': ['image_id', 'stop_time']
},
]
KEYS_TYPES = [JBoxDB.INT, JBoxDB.INT]
TYPES = [JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.INT, JBoxDB.INT]
TABLE = None
_stats_cache = {}
def __init__(self, container_id, image_id, start_time, stop_time=None):
if None == stop_time:
stop_datetime = datetime.datetime.now(pytz.utc)
else:
stop_datetime = stop_time
stop_time = JBoxAccountingV2.datetime_to_epoch_secs(
stop_datetime, allow_microsecs=True)
stop_date = JBoxAccountingV2.datetime_to_yyyymmdd(stop_datetime)
data = {
'stop_date': stop_date,
'stop_time': stop_time,
'image_id': image_id,
'container_id': container_id,
'start_time': JBoxAccountingV2.datetime_to_epoch_secs(start_time),
'start_date': JBoxAccountingV2.datetime_to_yyyymmdd(start_time)
}
self.create(data)
self.item = self.fetch(stop_date=stop_date, stop_time=stop_time)
self.is_new = True
@staticmethod
def _query_stats_date(date):
# TODO: caching items is not a good idea. Should cache computed data instead.
today = datetime.datetime.now()
date_day = JBoxAccountingV2.datetime_to_yyyymmdd(date)
today_day = JBoxAccountingV2.datetime_to_yyyymmdd(today)
istoday = date_day == today_day
if date_day in JBoxAccountingV2._stats_cache:
return JBoxAccountingV2._stats_cache[date_day]
res = JBoxAccountingV2.query(stop_date__eq=date_day, stop_time__gte=0)
items = []
for item in res:
items.append(item)
if not istoday:
JBoxAccountingV2._stats_cache[date_day] = items
return items
@staticmethod
def get_stats(dates=(datetime.datetime.now(), )):
sum_time = 0
item_count = 0
image_count = {}
container_freq = {}
for date in dates:
items = JBoxAccountingV2._query_stats_date(date)
for x in items:
item_count += 1
if 'start_time' in x:
sum_time += x['stop_time'] - int(x['start_time'])
try:
image_ids = json.loads(x['image_id'])
except:
image_ids = []
for image_id in image_ids:
if image_id.startswith("juliabox/") and (
not image_id.endswith(":latest")):
image_count[image_id] = image_count.get(image_id,
0) + 1
cid = x['container_id']
container_freq[cid] = container_freq.get(cid, 0) + 1
def fmt(seconds):
hrs = int(seconds / 3600)
mins = int(seconds / 60)
secs = int(seconds)
return "%dh %dm %ds" % (hrs, mins % 60, secs % 60)
active_users = 0
for container in container_freq:
if container_freq[container] > 2:
active_users += 1
return dict(session_count=item_count,
avg_time=fmt(float(sum_time) /
item_count) if item_count != 0 else 'NA',
images_used=image_count,
unique_users=len(container_freq),
active_users=active_users)
@staticmethod
def record_session_time(container_name, images_used, time_created,
time_finished):
for retry in range(1, 10):
try:
start_time = time_created
finish_time = time_finished
if retry > 1:
finish_time += datetime.timedelta(
microseconds=random.randint(1, 100))
acct = JBoxAccountingV2(container_name,
json.dumps(images_used),
start_time,
stop_time=finish_time)
acct.save()
break
except:
if retry == 10:
JBoxAccountingV2.log_exception("error recording usage")
else:
JBoxAccountingV2.log_warn(
"error recording usage, shall retry.")
class JBoxAccountingV2(JBoxDB):
NAME = 'jbox_accounting_v2'
SCHEMA = [
HashKey('stop_date', data_type=NUMBER),
RangeKey('stop_time', data_type=NUMBER)
]
INDEXES = [
AllIndex('container_id-stop_time-index',
parts=[
HashKey('container_id', data_type=STRING),
RangeKey('stop_time', data_type=NUMBER)
]),
IncludeIndex('image_id-stop_time-index',
parts=[
HashKey('image_id', data_type=STRING),
RangeKey('stop_time', data_type=NUMBER)
],
includes=['container_id'])
]
TABLE = None
_stats_cache = {}
def __init__(self, container_id, image_id, start_time, stop_time=None):
if None == self.table():
return
if None == stop_time:
stop_datetime = datetime.datetime.now(pytz.utc)
else:
stop_datetime = stop_time
stop_time = JBoxAccountingV2.datetime_to_epoch_secs(
stop_datetime, allow_microsecs=True)
stop_date = JBoxAccountingV2.datetime_to_yyyymmdd(stop_datetime)
data = {
'stop_date': stop_date,
'stop_time': stop_time,
'image_id': image_id,
'container_id': container_id,
'start_time': JBoxAccountingV2.datetime_to_epoch_secs(start_time),
'start_date': JBoxAccountingV2.datetime_to_yyyymmdd(start_time)
}
self.create(data)
self.item = self.table().get_item(stop_date=stop_date,
stop_time=stop_time)
self.is_new = True
@staticmethod
def query_stats_date(date):
# TODO: caching items is not a good idea. Should cache computed data instead.
if None == JBoxAccountingV2.table():
return []
today = datetime.datetime.now()
date_day = JBoxAccountingV2.datetime_to_yyyymmdd(date)
today_day = JBoxAccountingV2.datetime_to_yyyymmdd(today)
istoday = date_day == today_day
if date_day in JBoxAccountingV2._stats_cache:
return JBoxAccountingV2._stats_cache[date_day]
res = JBoxAccountingV2.table().query_2(stop_date__eq=date_day,
stop_time__gte=0)
items = []
for item in res:
items.append(item)
if not istoday:
JBoxAccountingV2._stats_cache[date_day] = items
return items
@staticmethod
def get_stats(dates=(datetime.datetime.now(), )):
sum_time = 0
item_count = 0
image_count = {}
container_freq = {}
for date in dates:
items = JBoxAccountingV2.query_stats_date(date)
for x in items:
item_count += 1
if 'start_time' in x:
sum_time += x['stop_time'] - int(x['start_time'])
try:
image_ids = json.loads(x['image_id'])
except:
image_ids = []
for image_id in image_ids:
if image_id.startswith("juliabox/") and (
not image_id.endswith(":latest")):
image_count[image_id] = image_count.get(image_id,
0) + 1
cid = x['container_id']
container_freq[cid] = container_freq.get(cid, 0) + 1
def fmt(seconds):
hrs = int(seconds / 3600)
mins = int(seconds / 60)
secs = int(seconds)
return "%dh %dm %ds" % (hrs, mins % 60, secs % 60)
active_users = 0
for container in container_freq:
if container_freq[container] > 2:
active_users += 1
return dict(session_count=item_count,
avg_time=fmt(float(sum_time) /
item_count) if item_count != 0 else 'NA',
images_used=image_count,
unique_users=len(container_freq),
active_users=active_users)