def create_router_table(tablename="router",
read_throughput=5,
write_throughput=5):
# type: (str, int, int) -> Table
"""Create a new router table
The last_connect index is a value used to determine the last month a user
was seen in. To prevent hot-keys on this table during month switchovers the
key is determined based on the following scheme:
(YEAR)(MONTH)(DAY)(HOUR)(0001-0010)
Note that the random key is only between 1-10 at the moment, if the key is
still too hot during production the random range can be increased at the
cost of additional queries during GC to locate expired users.
"""
return Table.create(
tablename,
schema=[HashKey("uaid")],
throughput=dict(read=read_throughput, write=write_throughput),
global_indexes=[
GlobalKeysOnlyIndex(
'AccessIndex',
parts=[HashKey('last_connect', data_type=NUMBER)],
throughput=dict(read=5, write=5))
],
)
def get_indexes(all_indexes):
indexes = []
global_indexes = []
for index in all_indexes:
name = index['name']
schema = get_schema_param(index.get('hash_key_name'), index.get('hash_key_type'), index.get('range_key_name'), index.get('range_key_type'))
throughput = {
'read': index.get('read_capacity', 1),
'write': index.get('write_capacity', 1)
}
if index['type'] == 'all':
indexes.append(AllIndex(name, parts=schema))
elif index['type'] == 'global_all':
global_indexes.append(GlobalAllIndex(name, parts=schema, throughput=throughput))
elif index['type'] == 'global_include':
global_indexes.append(GlobalIncludeIndex(name, parts=schema, throughput=throughput, includes=index['includes']))
elif index['type'] == 'global_keys_only':
global_indexes.append(GlobalKeysOnlyIndex(name, parts=schema, throughput=throughput))
elif index['type'] == 'include':
indexes.append(IncludeIndex(name, parts=schema, includes=index['includes']))
elif index['type'] == 'keys_only':
indexes.append(KeysOnlyIndex(name, parts=schema))
return indexes, global_indexes
def test_gsi(self):
users = Table.create('gsi_users',
schema=[
HashKey('user_id'),
],
throughput={
'read': 5,
'write': 3,
},
global_indexes=[
GlobalKeysOnlyIndex(
'StuffIndex',
parts=[HashKey('user_id')],
throughput={
'read': 2,
'write': 1,
}),
])
self.addCleanup(users.delete)
# Wait for it.
time.sleep(60)
users.update(throughput={
'read': 3,
'write': 4
},
global_indexes={'StuffIndex': {
'read': 1,
'write': 2
}})
# Wait again for the changes to finish propagating.
time.sleep(150)
def create_router_table(tablename="router",
read_throughput=5,
write_throughput=5):
"""Create a new router table"""
return Table.create(
tablename,
schema=[HashKey("uaid")],
throughput=dict(read=read_throughput, write=write_throughput),
global_indexes=[
GlobalKeysOnlyIndex(
'AccessIndex',
parts=[HashKey('last_connect', data_type=NUMBER)],
throughput=dict(read=5, write=5))
],
)
def _extract_index(index_data, global_index=False):
'''
Instantiates and returns an AllIndex object given a valid index
configuration
'''
parsed_data = {}
keys = []
for key, value in six.iteritems(index_data):
for item in value:
for field, data in six.iteritems(item):
if field == 'hash_key':
parsed_data['hash_key'] = data
elif field == 'hash_key_data_type':
parsed_data['hash_key_data_type'] = data
elif field == 'range_key':
parsed_data['range_key'] = data
elif field == 'range_key_data_type':
parsed_data['range_key_data_type'] = data
elif field == 'name':
parsed_data['name'] = data
elif field == 'read_capacity_units':
parsed_data['read_capacity_units'] = data
elif field == 'write_capacity_units':
parsed_data['write_capacity_units'] = data
elif field == 'includes':
parsed_data['includes'] = data
elif field == 'keys_only':
parsed_data['keys_only'] = True
if parsed_data['hash_key']:
keys.append(
HashKey(parsed_data['hash_key'],
data_type=parsed_data['hash_key_data_type']))
if parsed_data.get('range_key'):
keys.append(
RangeKey(parsed_data['range_key'],
data_type=parsed_data['range_key_data_type']))
if (global_index and parsed_data['read_capacity_units']
and parsed_data['write_capacity_units']):
parsed_data['throughput'] = {
'read': parsed_data['read_capacity_units'],
'write': parsed_data['write_capacity_units']
}
if parsed_data['name'] and len(keys) > 0:
if global_index:
if parsed_data.get('keys_only') and parsed_data.get('includes'):
raise SaltInvocationError(
'Only one type of GSI projection can be used.')
if parsed_data.get('includes'):
return GlobalIncludeIndex(parsed_data['name'],
parts=keys,
throughput=parsed_data['throughput'],
includes=parsed_data['includes'])
elif parsed_data.get('keys_only'):
return GlobalKeysOnlyIndex(
parsed_data['name'],
parts=keys,
throughput=parsed_data['throughput'],
)
else:
return GlobalAllIndex(parsed_data['name'],
parts=keys,
throughput=parsed_data['throughput'])
else:
return AllIndex(parsed_data['name'], parts=keys)
def extract_index(index_data, global_index=False):
"""
Instantiates and returns an AllIndex object given a valid index
configuration
CLI Example:
.. code-block:: bash
salt myminion boto_dynamodb.extract_index index
"""
parsed_data = {}
keys = []
for key, value in index_data.items():
for item in value:
for field, data in item.items():
if field == "hash_key":
parsed_data["hash_key"] = data
elif field == "hash_key_data_type":
parsed_data["hash_key_data_type"] = data
elif field == "range_key":
parsed_data["range_key"] = data
elif field == "range_key_data_type":
parsed_data["range_key_data_type"] = data
elif field == "name":
parsed_data["name"] = data
elif field == "read_capacity_units":
parsed_data["read_capacity_units"] = data
elif field == "write_capacity_units":
parsed_data["write_capacity_units"] = data
elif field == "includes":
parsed_data["includes"] = data
elif field == "keys_only":
parsed_data["keys_only"] = True
if parsed_data["hash_key"]:
keys.append(
HashKey(parsed_data["hash_key"],
data_type=parsed_data["hash_key_data_type"]))
if parsed_data.get("range_key"):
keys.append(
RangeKey(parsed_data["range_key"],
data_type=parsed_data["range_key_data_type"]))
if global_index and parsed_data["read_capacity_units"] and parsed_data[
"write_capacity_units"]:
parsed_data["throughput"] = {
"read": parsed_data["read_capacity_units"],
"write": parsed_data["write_capacity_units"],
}
if parsed_data["name"] and keys:
if global_index:
if parsed_data.get("keys_only") and parsed_data.get("includes"):
raise SaltInvocationError(
"Only one type of GSI projection can be used.")
if parsed_data.get("includes"):
return GlobalIncludeIndex(
parsed_data["name"],
parts=keys,
throughput=parsed_data["throughput"],
includes=parsed_data["includes"],
)
elif parsed_data.get("keys_only"):
return GlobalKeysOnlyIndex(
parsed_data["name"],
parts=keys,
throughput=parsed_data["throughput"],
)
else:
return GlobalAllIndex(
parsed_data["name"],
parts=keys,
throughput=parsed_data["throughput"],
)
else:
return AllIndex(parsed_data["name"], parts=keys)
class JBoxAPISpec(JBoxDB):
NAME = 'jbox_apispec'
SCHEMA = [HashKey('api_name', data_type=STRING)]
INDEXES = [
GlobalKeysOnlyIndex('publisher-api_name-index',
parts=[
HashKey('publisher', data_type=STRING),
RangeKey('api_name', data_type=STRING)
])
]
TABLE = None
KEYS = ['api_name']
ATTRIBUTES = [
'publisher', 'cmd', 'image_name', 'description', 'timeout_secs',
'create_time'
]
def __init__(self,
api_name,
cmd=None,
image_name=None,
description=None,
publisher=None,
timeout_secs=None,
create=False):
try:
self.item = self.fetch(api_name=api_name)
self.is_new = False
except JBoxDBItemNotFound:
if create:
dt = datetime.datetime.now(pytz.utc)
data = {
'api_name': api_name,
'cmd': cmd,
'description': description,
'publisher': publisher,
'create_time': JBoxAPISpec.datetime_to_epoch_secs(dt)
}
if image_name is not None:
data['image_name'] = image_name
if timeout_secs is not None:
data['timeout_secs'] = timeout_secs
self.create(data)
self.item = self.fetch(api_name=api_name)
self.is_new = True
else:
raise
def get_api_name(self):
return self.get_attrib('api_name', None)
def get_timeout_secs(self):
return int(self.get_attrib('timeout_secs', 30))
def get_description(self):
return self.get_attrib('description', None)
def get_publisher(self):
return self.get_attrib('publisher', None)
def get_image_name(self):
return self.get_attrib('image_name', 'juliabox/juliaboxapi:latest')
def get_cmd(self):
return self.get_attrib('cmd', None)
def get_create_time(self):
return int(self.get_attrib('create_time', None))
def set_cmd(self, cmd):
self.set_attrib('cmd', cmd)
def set_description(self, description):
self.set_attrib('description', description)
def set_timeout_secs(self, timeout_secs):
self.set_attrib('timeout_secs', timeout_secs)
def set_publisher(self, publisher):
self.set_attrib('publisher', publisher)
def set_image_name(self, image_name):
self.set_attrib('image_name', image_name)
def as_json(self):
def _add_not_none(d, n, v):
if v is not None:
d[n] = v
jsonval = dict()
_add_not_none(jsonval, 'api_name', self.get_api_name())
_add_not_none(jsonval, 'cmd', self.get_cmd())
_add_not_none(jsonval, 'image_name', self.get_image_name())
_add_not_none(jsonval, 'description', self.get_description())
_add_not_none(jsonval, 'publisher', self.get_publisher())
_add_not_none(jsonval, 'timeout_secs', self.get_timeout_secs())
_add_not_none(jsonval, 'create_time', self.get_create_time())
return jsonval
@staticmethod
def get_api_info(publisher, api_name):
if publisher is None and api_name is None:
raise
ret = []
if publisher is None:
ret.append(JBoxAPISpec(api_name).as_json())
else:
if api_name is None:
api_name = ' '
records = JBoxAPISpec.query(publisher__eq=publisher,
api_name__gte=api_name,
index='publisher-api_name-index')
for api in records:
ret.append(JBoxAPISpec(api['api_name']).as_json())
return ret
@staticmethod
def set_api_info(api_name,
cmd=None,
image_name=None,
description=None,
publisher=None,
timeout_secs=None):
try:
api = JBoxAPISpec(api_name)
if cmd is not None:
api.set_cmd(cmd)
if image_name is not None:
api.set_image_name(image_name)
if description is not None:
api.set_description(description)
if publisher is not None:
api.set_publisher(publisher)
if timeout_secs is not None:
api.set_timeout_secs(timeout_secs)
api.save()
except JBoxDBItemNotFound:
JBoxAPISpec(api_name,
cmd=cmd,
image_name=image_name,
description=description,
publisher=publisher,
timeout_secs=timeout_secs,
create=True)
class JBoxDiskState(JBoxDB):
NAME = 'jbox_diskstate'
SCHEMA = [HashKey('disk_key', data_type=STRING)]
INDEXES = [
GlobalKeysOnlyIndex('state-index',
parts=[HashKey('state', data_type=NUMBER)])
]
TABLE = None
STATE_ATTACHED = 1
STATE_DETACHED = 0
def __init__(self,
disk_key=None,
cluster_id=None,
region_id=None,
user_id=None,
volume_id=None,
attach_time=None,
create=False):
if self.table() is None:
return
self.item = None
if create and ((cluster_id is None) or (region_id is None) or
(user_id is None)):
raise AssertionError
if disk_key is None:
disk_key = '_'.join([user_id, cluster_id, region_id])
try:
self.item = self.table().get_item(disk_key=disk_key)
self.is_new = False
except boto.dynamodb2.exceptions.ItemNotFound:
if create:
data = {
'disk_key': disk_key,
'cluster_id': cluster_id,
'region_id': region_id,
'user_id': user_id
}
if volume_id is not None:
data['volume_id'] = volume_id
if attach_time is None:
attach_time = datetime.datetime.now(pytz.utc)
data['attach_time'] = JBoxDiskState.datetime_to_epoch_secs(
attach_time)
self.create(data)
self.item = self.table().get_item(disk_key=disk_key)
self.is_new = True
else:
raise
def set_attach_time(self, attach_time=None):
if attach_time is None:
attach_time = datetime.datetime.now(pytz.utc)
self.set_attrib('attach_time',
JBoxDiskState.datetime_to_epoch_secs(attach_time))
def get_attach_time(self):
return JBoxDiskState.epoch_secs_to_datetime(self.item['attach_time'])
def set_detach_time(self, detach_time=None):
if detach_time is None:
detach_time = datetime.datetime.now(pytz.utc)
self.set_attrib('detach_time',
JBoxDiskState.datetime_to_epoch_secs(detach_time))
def get_detach_time(self):
return JBoxDiskState.epoch_secs_to_datetime(
int(self.item['detach_time']))
def get_state(self):
return self.get_attrib('state')
def set_state(self, state):
self.set_attrib('state', state)
def get_user_id(self):
return self.get_attrib('user_id')
def set_user_id(self, user_id):
self.set_attrib('user_id', user_id)
def get_region_id(self):
return self.get_attrib('region_id')
def set_region_id(self, region_id):
self.set_attrib('region_id', region_id)
def get_cluster_id(self):
return self.get_attrib('cluster_id')
def set_cluster_id(self, cluster_id):
self.set_attrib('cluster_id', cluster_id)
def get_volume_id(self):
return self.get_attrib('volume_id')
def set_volume_id(self, volume_id):
self.set_attrib('volume_id', volume_id)
def get_snapshot_ids(self):
snapshots = self.get_attrib('snapshot_id')
if (snapshots is not None) and (len(snapshots) > 0):
return json.loads(snapshots)
return []
def add_snapshot_id(self, snapshot_id):
ids = self.get_snapshot_ids()
ids.append(snapshot_id)
self.set_snapshot_ids(ids)
def set_snapshot_ids(self, snapshot_ids):
self.set_attrib('snapshot_id', json.dumps(snapshot_ids))
@staticmethod
def get_detached_disks(max_count=None):
disk_keys = []
try:
records = JBoxDiskState.table().query_2(
state__eq=JBoxDiskState.STATE_DETACHED,
index='state-index',
limit=max_count)
for rec in records:
disk_keys.append(rec['disk_key'])
except:
# boto bug: https://github.com/boto/boto/issues/2708
JBoxDiskState.TABLE = None
JBoxDiskState.log_warn(
"Exception in getting detached disks. Probably empty table.")
return disk_keys
class JBoxUserV2(JBoxDB):
"""
- user_id (primary hash key)
- create_month (global secondary hash key)
- create_time (global secondary range index)
- update_month (global secondary hash key)
- update_time (global secondary index)
- activation_code (global secondary hash key)
- activation_status (global secondary range key)
- image (optional: global secondary hash key)
- resource_profile (optional: global secondary range key)
- status
- organization
- role
- gtok
"""
NAME = 'jbox_users_v2'
SCHEMA = [HashKey('user_id', data_type=STRING)]
INDEXES = [
GlobalKeysOnlyIndex('create_month-create_time-index',
parts=[
HashKey('create_month', data_type=NUMBER),
RangeKey('create_time', data_type=NUMBER)
]),
GlobalKeysOnlyIndex('update_month-update_time-index',
parts=[
HashKey('update_month', data_type=NUMBER),
RangeKey('update_time', data_type=NUMBER)
]),
GlobalKeysOnlyIndex('activation_code-activation_status-index',
parts=[
HashKey('activation_code', data_type=STRING),
RangeKey('activation_status', data_type=NUMBER)
])
]
TABLE = None
STATUS_ACTIVE = 0
STATUS_INACTIVE = 1
ROLE_USER = 0
ROLE_ACCESS_STATS = 1 << 0
ROLE_MANAGE_INVITES = 1 << 1
ROLE_MANAGE_CONTAINERS = 1 << 2
ROLE_SUPER = (1 << 33) - 1
ACTIVATION_NONE = 0
ACTIVATION_GRANTED = 1
ACTIVATION_REQUESTED = 2
ACTIVATION_CODE_AUTO = 'AUTO'
RES_PROF_BASIC = 0
RES_PROF_DISK_EBS_1G = 1 << 0
RES_PROF_JULIA_PKG_PRECOMP = 1 << 12
STATS = None
STAT_NAME = "stat_users"
def __init__(self, user_id, create=False):
if self.table() is None:
self.is_new = False
self.item = None
return
try:
self.item = self.table().get_item(user_id=user_id)
self.is_new = False
except boto.dynamodb2.exceptions.ItemNotFound:
if create:
data = {
'user_id': user_id,
'resource_profile': JBoxUserV2.RES_PROF_JULIA_PKG_PRECOMP
}
JBoxUserV2._set_time(data, "create")
JBoxUserV2._set_activation_state(data, '-',
JBoxUserV2.ACTIVATION_NONE)
self.create(data)
self.item = self.table().get_item(user_id=user_id)
self.is_new = True
else:
raise
def get_user_id(self):
return self.get_attrib('user_id', None)
def get_status(self):
if self.item is not None:
return self.item.get('status', JBoxUserV2.STATUS_ACTIVE)
else:
return None
def get_role(self):
return int(self.get_attrib('role', JBoxUserV2.ROLE_USER))
def set_role(self, role):
if self.item is not None:
r = self.item.get('role', JBoxUserV2.ROLE_USER)
self.item['role'] = r | role
def has_role(self, role):
return self.get_role() & role == role
def set_status(self, status):
self.set_attrib('status', status)
def set_time(self, prefix, dt=None):
if self.item is None:
return
JBoxUserV2._set_time(self.item, prefix, dt)
@staticmethod
def _set_time(item, prefix, dt=None):
if None == dt:
dt = datetime.datetime.now(pytz.utc)
if prefix not in ["create", "update"]:
raise (Exception("invalid prefix for setting time"))
item[prefix + "_month"] = JBoxUserV2.datetime_to_yyyymm(dt)
item[prefix + "_time"] = JBoxUserV2.datetime_to_epoch_secs(dt)
def get_time(self, prefix):
if self.item is None:
return None
if prefix not in ["create", "update"]:
raise (Exception("invalid prefix for setting time"))
return JBoxUserV2.epoch_secs_to_datetime(self.item[prefix + "_time"])
def save(self, set_time=True):
if self.item is not None:
self.set_time("update")
super(JBoxUserV2, self).save()
def set_activation_state(self, activation_code, activation_status):
if self.item is not None:
JBoxUserV2.log_debug("setting activation state of %s to %s, %d",
self.get_user_id(), activation_code,
activation_status)
JBoxUserV2._set_activation_state(self.item, activation_code,
activation_status)
@staticmethod
def _set_activation_state(item, activation_code, activation_status):
item['activation_code'] = activation_code
item['activation_status'] = activation_status
def get_activation_state(self):
if self.item is None:
return None, None
return self.item.get('activation_code',
'-'), self.item.get('activation_status',
JBoxUserV2.ACTIVATION_NONE)
def set_gtok(self, gtok):
if self.item is not None:
self.item['gtok'] = encrypt(gtok, self.enckey())
def get_gtok(self):
if self.item is None:
return None
gtok = self.item.get('gtok', None)
return decrypt(gtok, self.enckey()) if (gtok is not None) else None
def set_container_type(self, image, resource_profile):
if self.item is not None:
self.item['image'] = image
self.item['resource_profile'] = resource_profile
def get_container_type(self):
if self.item is None:
return None, None
return self.item.get('image', None), int(
self.item.get('resource_profile', JBoxUserV2.RES_PROF_BASIC))
def get_resource_profile(self):
if self.item is None:
return JBoxUserV2.RES_PROF_BASIC
return int(self.item.get('resource_profile',
JBoxUserV2.RES_PROF_BASIC))
def set_resource_profile(self, mask):
if self.item is not None:
resource_profile = self.get_resource_profile()
new_resource_profile = resource_profile | mask
if new_resource_profile != resource_profile:
self.item['resource_profile'] = new_resource_profile
def unset_resource_profile(self, mask):
if self.item is not None:
resource_profile = self.get_resource_profile()
new_resource_profile = resource_profile & (~mask)
if new_resource_profile != resource_profile:
self.item['resource_profile'] = new_resource_profile
def has_resource_profile(self, mask):
resource_profile = self.get_resource_profile()
if mask == 0:
return resource_profile == 0
return (resource_profile & mask) == mask
@staticmethod
def get_pending_activations(max_count):
records = JBoxUserV2.table().query_2(
activation_code__eq=JBoxUserV2.ACTIVATION_CODE_AUTO,
activation_status__eq=JBoxUserV2.ACTIVATION_REQUESTED,
index='activation_code-activation_status-index',
limit=max_count)
user_ids = []
for rec in records:
user_ids.append(rec['user_id'])
return user_ids
@staticmethod
def count_pending_activations():
count = JBoxUserV2.table().query_count(
activation_code__eq='AUTO',
activation_status__eq=JBoxUserV2.ACTIVATION_REQUESTED,
index='activation_code-activation_status-index')
return count
@staticmethod
def count_created(hours_before, tilldate=None):
if None == tilldate:
tilldate = datetime.datetime.now(pytz.utc)
fromdate = tilldate - datetime.timedelta(hours=hours_before)
till_month = JBoxUserV2.datetime_to_yyyymm(tilldate)
till_time = JBoxUserV2.datetime_to_epoch_secs(tilldate)
from_month = JBoxUserV2.datetime_to_yyyymm(fromdate)
from_time = JBoxUserV2.datetime_to_epoch_secs(fromdate)
count = 0
mon = from_month
while mon <= till_month:
count += JBoxUserV2.table().query_count(
create_month__eq=mon,
create_time__between=(from_time, till_time),
index='create_month-create_time-index')
JBoxUserV2.log_debug(
"adding accounts created in mon %d, from %d till %d. count %d",
mon, from_time, till_time, count)
if (mon % 100) == 12:
mon = (mon / 100 + 1) * 100 + 1
else:
mon += 1
return count
@staticmethod
def calc_stat(user, weeks, days):
stats = JBoxUserV2.STATS
stats['num_users'] += 1
if 'gtok' in user:
stats['sync']['gdrive'] += 1
role = stats['role']
role_val = int(
user['role']) if 'role' in user else JBoxUserV2.ROLE_USER
if role_val == JBoxUserV2.ROLE_USER:
role['user'] += 1
else:
if (role_val & JBoxUserV2.ROLE_SUPER) == JBoxUserV2.ROLE_SUPER:
role['superuser'] += 1
if (role_val & JBoxUserV2.ROLE_ACCESS_STATS
) == JBoxUserV2.ROLE_ACCESS_STATS:
role['access_stats'] += 1
act_status = stats['activation_status']
act_status_val = int(
user['activation_status']
) if 'activation_status' in user else JBoxUserV2.ACTIVATION_NONE
if act_status_val == JBoxUserV2.ACTIVATION_NONE:
act_status['none'] += 1
elif act_status_val == JBoxUserV2.ACTIVATION_GRANTED:
act_status['granted'] += 1
elif act_status_val == JBoxUserV2.ACTIVATION_REQUESTED:
act_status['requested'] += 1
res_profile = stats['resource_profile']
res_profile_val = int(user['resource_profile']) if 'resource_profile' in user \
else JBoxUserV2.RES_PROF_BASIC
if res_profile_val == JBoxUserV2.RES_PROF_BASIC:
res_profile['basic'] += 1
else:
if (res_profile_val & JBoxUserV2.RES_PROF_DISK_EBS_1G
) == JBoxUserV2.RES_PROF_DISK_EBS_1G:
res_profile['disk_ebs_1G'] += 1
elif (res_profile_val & JBoxUserV2.RES_PROF_JULIA_PKG_PRECOMP
) == JBoxUserV2.RES_PROF_JULIA_PKG_PRECOMP:
res_profile['julia_packages_precompiled'] += 1
create_month_val = int(user['create_month'])
create_month = stats['created_time']['months']
if create_month_val not in create_month:
create_month[create_month_val] = 1
else:
create_month[create_month_val] += 1
create_time_val = int(user['create_time'])
last_n_weeks = JBoxUserV2.STATS['created_time']['last_n_weeks']
last_n_days = JBoxUserV2.STATS['created_time']['last_n_days']
for week in range(0, len(weeks)):
if create_time_val >= weeks[week]:
last_n_weeks[week + 1] += 1
break
for day in range(0, len(days)):
if create_time_val >= days[day]:
last_n_days[day + 1] += 1
break
@staticmethod
def calc_stats():
JBoxUserV2.STATS = {
'date': '',
'num_users': 0,
'sync': {
'gdrive': 0
},
'role': {
'user': 0,
'superuser': 0,
'access_stats': 0
},
'activation_status': {
'none': 0,
'granted': 0,
'requested': 0
},
'resource_profile': {
'basic': 0,
'disk_ebs_1G': 0,
'julia_packages_precompiled': 0
},
'created_time': {
'months': {},
'last_n_weeks': {},
'last_n_days': {}
}
}
secs_day = 24 * 60 * 60
secs_week = secs_day * 7
now = datetime.datetime.now(pytz.utc)
secs_now = int(JBoxUserV2.datetime_to_epoch_secs(now))
weeks = [(secs_now - secs_week * week) for week in range(1, 5)]
days = [(secs_now - secs_day * day) for day in range(1, 8)]
last_n_weeks = JBoxUserV2.STATS['created_time']['last_n_weeks']
last_n_days = JBoxUserV2.STATS['created_time']['last_n_days']
for week in range(0, len(weeks)):
last_n_weeks[week + 1] = 0
for day in range(0, len(days)):
last_n_days[day + 1] = 0
result_set = JBoxUserV2.table().scan(
attributes=('user_id', 'create_month', 'create_time', 'gtok',
'role', 'resource_profile', 'activation_status'))
for user in result_set:
JBoxUserV2.calc_stat(user, weeks, days)
JBoxUserV2.STATS['date'] = now.isoformat()
class JBoxUserProfile(JBoxDB):
NAME = 'jbox_user_profiles'
SCHEMA = [HashKey('user_id', data_type=STRING)]
INDEXES = None
GLOBAL_INDEXES = [
GlobalKeysOnlyIndex('create_month-create_time-index',
parts=[
HashKey('create_month', data_type=NUMBER),
RangeKey('create_time', data_type=NUMBER)
]),
GlobalKeysOnlyIndex('update_month-update_time-index',
parts=[
HashKey('update_month', data_type=NUMBER),
RangeKey('update_time', data_type=NUMBER)
])
]
TABLE = None
ATTR_FIRST_NAME = 'first_name'
ATTR_LAST_NAME = 'last_name'
ATTR_COUNTRY = 'country'
ATTR_CITY = 'city'
ATTR_LOCATION = 'location' # a fuzzy location string, indicative of country and city
ATTR_IP = 'ip' # ip from which last accessed
ATTR_INDUSTRY = 'industry'
ATTR_ORGANIZATION = 'org' # workplace
ATTR_ORG_TITLE = 'org_title' # job title
KEYS = ['user_id']
ATTRIBUTES = [
'create_month',
'create_time',
'update_month',
'update_time',
ATTR_FIRST_NAME,
ATTR_LAST_NAME,
ATTR_COUNTRY,
ATTR_CITY,
ATTR_LOCATION,
ATTR_IP,
ATTR_INDUSTRY,
ATTR_ORGANIZATION,
ATTR_ORG_TITLE,
'sources' # a JSON field that indicates where each profile attribute was filled from
]
SQL_INDEXES = [
{
'name': 'create_month-create_time-index',
'cols': ['create_month', 'create_time']
},
{
'name': 'update_month-update_time-index',
'cols': ['update_month', 'update_time']
},
]
KEYS_TYPES = [JBoxDB.VCHAR]
TYPES = [
JBoxDB.INT, JBoxDB.INT, JBoxDB.INT, JBoxDB.INT, JBoxDB.VCHAR,
JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.VCHAR,
JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.VCHAR
]
SRC_USER = 1 # filled in by the user
SRC_DERIVED = 2 # derived from other fields
def __init__(self, user_id, create=False):
try:
self.item = self.fetch(user_id=user_id)
self.is_new = False
except JBoxDBItemNotFound:
if create:
data = {'user_id': user_id}
JBoxUserProfile._set_time(data, "create")
self.create(data)
self.item = self.fetch(user_id=user_id)
self.is_new = True
else:
raise
def get_user_id(self):
return self.get_attrib('user_id')
def get_attrib_source(self, attrib_name):
sources_str = self.get_attrib('sources', '{}')
if len(sources_str) == 0:
return None
sources = json.loads(sources_str)
return sources[attrib_name] if attrib_name in sources else None
def set_attrib_source(self, attrib_name, source):
sources_str = self.get_attrib('sources', '{}')
if len(sources_str) == 0:
sources_str = '{}'
sources = json.loads(sources_str)
sources[attrib_name] = source
self.set_attrib('sources', json.dumps(sources))
def is_set_by_user(self, attrib_name):
return self.get_attrib_source(attrib_name) == JBoxUserProfile.SRC_USER
def set_profile(self, attrib_name, value, source):
# do not overwrite attributes set by the user
if source != JBoxUserProfile.SRC_USER and self.is_set_by_user(
attrib_name):
return False
self.set_attrib(attrib_name, value)
self.set_attrib_source(attrib_name, source)
return True
def can_set(self, attrib_name, value):
if value is None or len(value) == 0:
return False
return value != self.get_attrib(attrib_name)
def get_profile(self, attrib_name, default=''):
return self.get_attrib(attrib_name, default)
def set_time(self, prefix, dt=None):
JBoxUserProfile._set_time(self.item, prefix, dt)
@staticmethod
def _set_time(item, prefix, dt=None):
if dt is None:
dt = datetime.datetime.now(pytz.utc)
if prefix not in ["create", "update"]:
raise (Exception("invalid prefix for setting time"))
item[prefix + "_month"] = JBoxUserProfile.datetime_to_yyyymm(dt)
item[prefix + "_time"] = JBoxUserProfile.datetime_to_epoch_secs(dt)
def get_time(self, prefix):
if prefix not in ["create", "update"]:
raise (Exception("invalid prefix for setting time"))
return JBoxUserProfile.epoch_secs_to_datetime(self.item[prefix +
"_time"])
def save(self, set_time=True):
self.set_time("update")
super(JBoxUserProfile, self).save()
