def create_router_table(tablename="router", read_throughput=5, write_throughput=5): # type: (str, int, int) -> Table """Create a new router table The last_connect index is a value used to determine the last month a user was seen in. To prevent hot-keys on this table during month switchovers the key is determined based on the following scheme: (YEAR)(MONTH)(DAY)(HOUR)(0001-0010) Note that the random key is only between 1-10 at the moment, if the key is still too hot during production the random range can be increased at the cost of additional queries during GC to locate expired users. """ return Table.create( tablename, schema=[HashKey("uaid")], throughput=dict(read=read_throughput, write=write_throughput), global_indexes=[ GlobalKeysOnlyIndex( 'AccessIndex', parts=[HashKey('last_connect', data_type=NUMBER)], throughput=dict(read=5, write=5)) ], )
def get_indexes(all_indexes): indexes = [] global_indexes = [] for index in all_indexes: name = index['name'] schema = get_schema_param(index.get('hash_key_name'), index.get('hash_key_type'), index.get('range_key_name'), index.get('range_key_type')) throughput = { 'read': index.get('read_capacity', 1), 'write': index.get('write_capacity', 1) } if index['type'] == 'all': indexes.append(AllIndex(name, parts=schema)) elif index['type'] == 'global_all': global_indexes.append(GlobalAllIndex(name, parts=schema, throughput=throughput)) elif index['type'] == 'global_include': global_indexes.append(GlobalIncludeIndex(name, parts=schema, throughput=throughput, includes=index['includes'])) elif index['type'] == 'global_keys_only': global_indexes.append(GlobalKeysOnlyIndex(name, parts=schema, throughput=throughput)) elif index['type'] == 'include': indexes.append(IncludeIndex(name, parts=schema, includes=index['includes'])) elif index['type'] == 'keys_only': indexes.append(KeysOnlyIndex(name, parts=schema)) return indexes, global_indexes
def test_gsi(self): users = Table.create('gsi_users', schema=[ HashKey('user_id'), ], throughput={ 'read': 5, 'write': 3, }, global_indexes=[ GlobalKeysOnlyIndex( 'StuffIndex', parts=[HashKey('user_id')], throughput={ 'read': 2, 'write': 1, }), ]) self.addCleanup(users.delete) # Wait for it. time.sleep(60) users.update(throughput={ 'read': 3, 'write': 4 }, global_indexes={'StuffIndex': { 'read': 1, 'write': 2 }}) # Wait again for the changes to finish propagating. time.sleep(150)
def create_router_table(tablename="router", read_throughput=5, write_throughput=5): """Create a new router table""" return Table.create( tablename, schema=[HashKey("uaid")], throughput=dict(read=read_throughput, write=write_throughput), global_indexes=[ GlobalKeysOnlyIndex( 'AccessIndex', parts=[HashKey('last_connect', data_type=NUMBER)], throughput=dict(read=5, write=5)) ], )
def _extract_index(index_data, global_index=False): ''' Instantiates and returns an AllIndex object given a valid index configuration ''' parsed_data = {} keys = [] for key, value in six.iteritems(index_data): for item in value: for field, data in six.iteritems(item): if field == 'hash_key': parsed_data['hash_key'] = data elif field == 'hash_key_data_type': parsed_data['hash_key_data_type'] = data elif field == 'range_key': parsed_data['range_key'] = data elif field == 'range_key_data_type': parsed_data['range_key_data_type'] = data elif field == 'name': parsed_data['name'] = data elif field == 'read_capacity_units': parsed_data['read_capacity_units'] = data elif field == 'write_capacity_units': parsed_data['write_capacity_units'] = data elif field == 'includes': parsed_data['includes'] = data elif field == 'keys_only': parsed_data['keys_only'] = True if parsed_data['hash_key']: keys.append( HashKey(parsed_data['hash_key'], data_type=parsed_data['hash_key_data_type'])) if parsed_data.get('range_key'): keys.append( RangeKey(parsed_data['range_key'], data_type=parsed_data['range_key_data_type'])) if (global_index and parsed_data['read_capacity_units'] and parsed_data['write_capacity_units']): parsed_data['throughput'] = { 'read': parsed_data['read_capacity_units'], 'write': parsed_data['write_capacity_units'] } if parsed_data['name'] and len(keys) > 0: if global_index: if parsed_data.get('keys_only') and parsed_data.get('includes'): raise SaltInvocationError( 'Only one type of GSI projection can be used.') if parsed_data.get('includes'): return GlobalIncludeIndex(parsed_data['name'], parts=keys, throughput=parsed_data['throughput'], includes=parsed_data['includes']) elif parsed_data.get('keys_only'): return GlobalKeysOnlyIndex( parsed_data['name'], parts=keys, throughput=parsed_data['throughput'], ) else: return GlobalAllIndex(parsed_data['name'], parts=keys, throughput=parsed_data['throughput']) else: return AllIndex(parsed_data['name'], parts=keys)
def extract_index(index_data, global_index=False): """ Instantiates and returns an AllIndex object given a valid index configuration CLI Example: .. code-block:: bash salt myminion boto_dynamodb.extract_index index """ parsed_data = {} keys = [] for key, value in index_data.items(): for item in value: for field, data in item.items(): if field == "hash_key": parsed_data["hash_key"] = data elif field == "hash_key_data_type": parsed_data["hash_key_data_type"] = data elif field == "range_key": parsed_data["range_key"] = data elif field == "range_key_data_type": parsed_data["range_key_data_type"] = data elif field == "name": parsed_data["name"] = data elif field == "read_capacity_units": parsed_data["read_capacity_units"] = data elif field == "write_capacity_units": parsed_data["write_capacity_units"] = data elif field == "includes": parsed_data["includes"] = data elif field == "keys_only": parsed_data["keys_only"] = True if parsed_data["hash_key"]: keys.append( HashKey(parsed_data["hash_key"], data_type=parsed_data["hash_key_data_type"])) if parsed_data.get("range_key"): keys.append( RangeKey(parsed_data["range_key"], data_type=parsed_data["range_key_data_type"])) if global_index and parsed_data["read_capacity_units"] and parsed_data[ "write_capacity_units"]: parsed_data["throughput"] = { "read": parsed_data["read_capacity_units"], "write": parsed_data["write_capacity_units"], } if parsed_data["name"] and keys: if global_index: if parsed_data.get("keys_only") and parsed_data.get("includes"): raise SaltInvocationError( "Only one type of GSI projection can be used.") if parsed_data.get("includes"): return GlobalIncludeIndex( parsed_data["name"], parts=keys, throughput=parsed_data["throughput"], includes=parsed_data["includes"], ) elif parsed_data.get("keys_only"): return GlobalKeysOnlyIndex( parsed_data["name"], parts=keys, throughput=parsed_data["throughput"], ) else: return GlobalAllIndex( parsed_data["name"], parts=keys, throughput=parsed_data["throughput"], ) else: return AllIndex(parsed_data["name"], parts=keys)
class JBoxAPISpec(JBoxDB): NAME = 'jbox_apispec' SCHEMA = [HashKey('api_name', data_type=STRING)] INDEXES = [ GlobalKeysOnlyIndex('publisher-api_name-index', parts=[ HashKey('publisher', data_type=STRING), RangeKey('api_name', data_type=STRING) ]) ] TABLE = None KEYS = ['api_name'] ATTRIBUTES = [ 'publisher', 'cmd', 'image_name', 'description', 'timeout_secs', 'create_time' ] def __init__(self, api_name, cmd=None, image_name=None, description=None, publisher=None, timeout_secs=None, create=False): try: self.item = self.fetch(api_name=api_name) self.is_new = False except JBoxDBItemNotFound: if create: dt = datetime.datetime.now(pytz.utc) data = { 'api_name': api_name, 'cmd': cmd, 'description': description, 'publisher': publisher, 'create_time': JBoxAPISpec.datetime_to_epoch_secs(dt) } if image_name is not None: data['image_name'] = image_name if timeout_secs is not None: data['timeout_secs'] = timeout_secs self.create(data) self.item = self.fetch(api_name=api_name) self.is_new = True else: raise def get_api_name(self): return self.get_attrib('api_name', None) def get_timeout_secs(self): return int(self.get_attrib('timeout_secs', 30)) def get_description(self): return self.get_attrib('description', None) def get_publisher(self): return self.get_attrib('publisher', None) def get_image_name(self): return self.get_attrib('image_name', 'juliabox/juliaboxapi:latest') def get_cmd(self): return self.get_attrib('cmd', None) def get_create_time(self): return int(self.get_attrib('create_time', None)) def set_cmd(self, cmd): self.set_attrib('cmd', cmd) def set_description(self, description): self.set_attrib('description', description) def set_timeout_secs(self, timeout_secs): self.set_attrib('timeout_secs', timeout_secs) def set_publisher(self, publisher): self.set_attrib('publisher', publisher) def set_image_name(self, image_name): self.set_attrib('image_name', image_name) def as_json(self): def _add_not_none(d, n, v): if v is not None: d[n] = v jsonval = dict() _add_not_none(jsonval, 'api_name', self.get_api_name()) _add_not_none(jsonval, 'cmd', self.get_cmd()) _add_not_none(jsonval, 'image_name', self.get_image_name()) _add_not_none(jsonval, 'description', self.get_description()) _add_not_none(jsonval, 'publisher', self.get_publisher()) _add_not_none(jsonval, 'timeout_secs', self.get_timeout_secs()) _add_not_none(jsonval, 'create_time', self.get_create_time()) return jsonval @staticmethod def get_api_info(publisher, api_name): if publisher is None and api_name is None: raise ret = [] if publisher is None: ret.append(JBoxAPISpec(api_name).as_json()) else: if api_name is None: api_name = ' ' records = JBoxAPISpec.query(publisher__eq=publisher, api_name__gte=api_name, index='publisher-api_name-index') for api in records: ret.append(JBoxAPISpec(api['api_name']).as_json()) return ret @staticmethod def set_api_info(api_name, cmd=None, image_name=None, description=None, publisher=None, timeout_secs=None): try: api = JBoxAPISpec(api_name) if cmd is not None: api.set_cmd(cmd) if image_name is not None: api.set_image_name(image_name) if description is not None: api.set_description(description) if publisher is not None: api.set_publisher(publisher) if timeout_secs is not None: api.set_timeout_secs(timeout_secs) api.save() except JBoxDBItemNotFound: JBoxAPISpec(api_name, cmd=cmd, image_name=image_name, description=description, publisher=publisher, timeout_secs=timeout_secs, create=True)
class JBoxDiskState(JBoxDB): NAME = 'jbox_diskstate' SCHEMA = [HashKey('disk_key', data_type=STRING)] INDEXES = [ GlobalKeysOnlyIndex('state-index', parts=[HashKey('state', data_type=NUMBER)]) ] TABLE = None STATE_ATTACHED = 1 STATE_DETACHED = 0 def __init__(self, disk_key=None, cluster_id=None, region_id=None, user_id=None, volume_id=None, attach_time=None, create=False): if self.table() is None: return self.item = None if create and ((cluster_id is None) or (region_id is None) or (user_id is None)): raise AssertionError if disk_key is None: disk_key = '_'.join([user_id, cluster_id, region_id]) try: self.item = self.table().get_item(disk_key=disk_key) self.is_new = False except boto.dynamodb2.exceptions.ItemNotFound: if create: data = { 'disk_key': disk_key, 'cluster_id': cluster_id, 'region_id': region_id, 'user_id': user_id } if volume_id is not None: data['volume_id'] = volume_id if attach_time is None: attach_time = datetime.datetime.now(pytz.utc) data['attach_time'] = JBoxDiskState.datetime_to_epoch_secs( attach_time) self.create(data) self.item = self.table().get_item(disk_key=disk_key) self.is_new = True else: raise def set_attach_time(self, attach_time=None): if attach_time is None: attach_time = datetime.datetime.now(pytz.utc) self.set_attrib('attach_time', JBoxDiskState.datetime_to_epoch_secs(attach_time)) def get_attach_time(self): return JBoxDiskState.epoch_secs_to_datetime(self.item['attach_time']) def set_detach_time(self, detach_time=None): if detach_time is None: detach_time = datetime.datetime.now(pytz.utc) self.set_attrib('detach_time', JBoxDiskState.datetime_to_epoch_secs(detach_time)) def get_detach_time(self): return JBoxDiskState.epoch_secs_to_datetime( int(self.item['detach_time'])) def get_state(self): return self.get_attrib('state') def set_state(self, state): self.set_attrib('state', state) def get_user_id(self): return self.get_attrib('user_id') def set_user_id(self, user_id): self.set_attrib('user_id', user_id) def get_region_id(self): return self.get_attrib('region_id') def set_region_id(self, region_id): self.set_attrib('region_id', region_id) def get_cluster_id(self): return self.get_attrib('cluster_id') def set_cluster_id(self, cluster_id): self.set_attrib('cluster_id', cluster_id) def get_volume_id(self): return self.get_attrib('volume_id') def set_volume_id(self, volume_id): self.set_attrib('volume_id', volume_id) def get_snapshot_ids(self): snapshots = self.get_attrib('snapshot_id') if (snapshots is not None) and (len(snapshots) > 0): return json.loads(snapshots) return [] def add_snapshot_id(self, snapshot_id): ids = self.get_snapshot_ids() ids.append(snapshot_id) self.set_snapshot_ids(ids) def set_snapshot_ids(self, snapshot_ids): self.set_attrib('snapshot_id', json.dumps(snapshot_ids)) @staticmethod def get_detached_disks(max_count=None): disk_keys = [] try: records = JBoxDiskState.table().query_2( state__eq=JBoxDiskState.STATE_DETACHED, index='state-index', limit=max_count) for rec in records: disk_keys.append(rec['disk_key']) except: # boto bug: https://github.com/boto/boto/issues/2708 JBoxDiskState.TABLE = None JBoxDiskState.log_warn( "Exception in getting detached disks. Probably empty table.") return disk_keys
class JBoxUserV2(JBoxDB): """ - user_id (primary hash key) - create_month (global secondary hash key) - create_time (global secondary range index) - update_month (global secondary hash key) - update_time (global secondary index) - activation_code (global secondary hash key) - activation_status (global secondary range key) - image (optional: global secondary hash key) - resource_profile (optional: global secondary range key) - status - organization - role - gtok """ NAME = 'jbox_users_v2' SCHEMA = [HashKey('user_id', data_type=STRING)] INDEXES = [ GlobalKeysOnlyIndex('create_month-create_time-index', parts=[ HashKey('create_month', data_type=NUMBER), RangeKey('create_time', data_type=NUMBER) ]), GlobalKeysOnlyIndex('update_month-update_time-index', parts=[ HashKey('update_month', data_type=NUMBER), RangeKey('update_time', data_type=NUMBER) ]), GlobalKeysOnlyIndex('activation_code-activation_status-index', parts=[ HashKey('activation_code', data_type=STRING), RangeKey('activation_status', data_type=NUMBER) ]) ] TABLE = None STATUS_ACTIVE = 0 STATUS_INACTIVE = 1 ROLE_USER = 0 ROLE_ACCESS_STATS = 1 << 0 ROLE_MANAGE_INVITES = 1 << 1 ROLE_MANAGE_CONTAINERS = 1 << 2 ROLE_SUPER = (1 << 33) - 1 ACTIVATION_NONE = 0 ACTIVATION_GRANTED = 1 ACTIVATION_REQUESTED = 2 ACTIVATION_CODE_AUTO = 'AUTO' RES_PROF_BASIC = 0 RES_PROF_DISK_EBS_1G = 1 << 0 RES_PROF_JULIA_PKG_PRECOMP = 1 << 12 STATS = None STAT_NAME = "stat_users" def __init__(self, user_id, create=False): if self.table() is None: self.is_new = False self.item = None return try: self.item = self.table().get_item(user_id=user_id) self.is_new = False except boto.dynamodb2.exceptions.ItemNotFound: if create: data = { 'user_id': user_id, 'resource_profile': JBoxUserV2.RES_PROF_JULIA_PKG_PRECOMP } JBoxUserV2._set_time(data, "create") JBoxUserV2._set_activation_state(data, '-', JBoxUserV2.ACTIVATION_NONE) self.create(data) self.item = self.table().get_item(user_id=user_id) self.is_new = True else: raise def get_user_id(self): return self.get_attrib('user_id', None) def get_status(self): if self.item is not None: return self.item.get('status', JBoxUserV2.STATUS_ACTIVE) else: return None def get_role(self): return int(self.get_attrib('role', JBoxUserV2.ROLE_USER)) def set_role(self, role): if self.item is not None: r = self.item.get('role', JBoxUserV2.ROLE_USER) self.item['role'] = r | role def has_role(self, role): return self.get_role() & role == role def set_status(self, status): self.set_attrib('status', status) def set_time(self, prefix, dt=None): if self.item is None: return JBoxUserV2._set_time(self.item, prefix, dt) @staticmethod def _set_time(item, prefix, dt=None): if None == dt: dt = datetime.datetime.now(pytz.utc) if prefix not in ["create", "update"]: raise (Exception("invalid prefix for setting time")) item[prefix + "_month"] = JBoxUserV2.datetime_to_yyyymm(dt) item[prefix + "_time"] = JBoxUserV2.datetime_to_epoch_secs(dt) def get_time(self, prefix): if self.item is None: return None if prefix not in ["create", "update"]: raise (Exception("invalid prefix for setting time")) return JBoxUserV2.epoch_secs_to_datetime(self.item[prefix + "_time"]) def save(self, set_time=True): if self.item is not None: self.set_time("update") super(JBoxUserV2, self).save() def set_activation_state(self, activation_code, activation_status): if self.item is not None: JBoxUserV2.log_debug("setting activation state of %s to %s, %d", self.get_user_id(), activation_code, activation_status) JBoxUserV2._set_activation_state(self.item, activation_code, activation_status) @staticmethod def _set_activation_state(item, activation_code, activation_status): item['activation_code'] = activation_code item['activation_status'] = activation_status def get_activation_state(self): if self.item is None: return None, None return self.item.get('activation_code', '-'), self.item.get('activation_status', JBoxUserV2.ACTIVATION_NONE) def set_gtok(self, gtok): if self.item is not None: self.item['gtok'] = encrypt(gtok, self.enckey()) def get_gtok(self): if self.item is None: return None gtok = self.item.get('gtok', None) return decrypt(gtok, self.enckey()) if (gtok is not None) else None def set_container_type(self, image, resource_profile): if self.item is not None: self.item['image'] = image self.item['resource_profile'] = resource_profile def get_container_type(self): if self.item is None: return None, None return self.item.get('image', None), int( self.item.get('resource_profile', JBoxUserV2.RES_PROF_BASIC)) def get_resource_profile(self): if self.item is None: return JBoxUserV2.RES_PROF_BASIC return int(self.item.get('resource_profile', JBoxUserV2.RES_PROF_BASIC)) def set_resource_profile(self, mask): if self.item is not None: resource_profile = self.get_resource_profile() new_resource_profile = resource_profile | mask if new_resource_profile != resource_profile: self.item['resource_profile'] = new_resource_profile def unset_resource_profile(self, mask): if self.item is not None: resource_profile = self.get_resource_profile() new_resource_profile = resource_profile & (~mask) if new_resource_profile != resource_profile: self.item['resource_profile'] = new_resource_profile def has_resource_profile(self, mask): resource_profile = self.get_resource_profile() if mask == 0: return resource_profile == 0 return (resource_profile & mask) == mask @staticmethod def get_pending_activations(max_count): records = JBoxUserV2.table().query_2( activation_code__eq=JBoxUserV2.ACTIVATION_CODE_AUTO, activation_status__eq=JBoxUserV2.ACTIVATION_REQUESTED, index='activation_code-activation_status-index', limit=max_count) user_ids = [] for rec in records: user_ids.append(rec['user_id']) return user_ids @staticmethod def count_pending_activations(): count = JBoxUserV2.table().query_count( activation_code__eq='AUTO', activation_status__eq=JBoxUserV2.ACTIVATION_REQUESTED, index='activation_code-activation_status-index') return count @staticmethod def count_created(hours_before, tilldate=None): if None == tilldate: tilldate = datetime.datetime.now(pytz.utc) fromdate = tilldate - datetime.timedelta(hours=hours_before) till_month = JBoxUserV2.datetime_to_yyyymm(tilldate) till_time = JBoxUserV2.datetime_to_epoch_secs(tilldate) from_month = JBoxUserV2.datetime_to_yyyymm(fromdate) from_time = JBoxUserV2.datetime_to_epoch_secs(fromdate) count = 0 mon = from_month while mon <= till_month: count += JBoxUserV2.table().query_count( create_month__eq=mon, create_time__between=(from_time, till_time), index='create_month-create_time-index') JBoxUserV2.log_debug( "adding accounts created in mon %d, from %d till %d. count %d", mon, from_time, till_time, count) if (mon % 100) == 12: mon = (mon / 100 + 1) * 100 + 1 else: mon += 1 return count @staticmethod def calc_stat(user, weeks, days): stats = JBoxUserV2.STATS stats['num_users'] += 1 if 'gtok' in user: stats['sync']['gdrive'] += 1 role = stats['role'] role_val = int( user['role']) if 'role' in user else JBoxUserV2.ROLE_USER if role_val == JBoxUserV2.ROLE_USER: role['user'] += 1 else: if (role_val & JBoxUserV2.ROLE_SUPER) == JBoxUserV2.ROLE_SUPER: role['superuser'] += 1 if (role_val & JBoxUserV2.ROLE_ACCESS_STATS ) == JBoxUserV2.ROLE_ACCESS_STATS: role['access_stats'] += 1 act_status = stats['activation_status'] act_status_val = int( user['activation_status'] ) if 'activation_status' in user else JBoxUserV2.ACTIVATION_NONE if act_status_val == JBoxUserV2.ACTIVATION_NONE: act_status['none'] += 1 elif act_status_val == JBoxUserV2.ACTIVATION_GRANTED: act_status['granted'] += 1 elif act_status_val == JBoxUserV2.ACTIVATION_REQUESTED: act_status['requested'] += 1 res_profile = stats['resource_profile'] res_profile_val = int(user['resource_profile']) if 'resource_profile' in user \ else JBoxUserV2.RES_PROF_BASIC if res_profile_val == JBoxUserV2.RES_PROF_BASIC: res_profile['basic'] += 1 else: if (res_profile_val & JBoxUserV2.RES_PROF_DISK_EBS_1G ) == JBoxUserV2.RES_PROF_DISK_EBS_1G: res_profile['disk_ebs_1G'] += 1 elif (res_profile_val & JBoxUserV2.RES_PROF_JULIA_PKG_PRECOMP ) == JBoxUserV2.RES_PROF_JULIA_PKG_PRECOMP: res_profile['julia_packages_precompiled'] += 1 create_month_val = int(user['create_month']) create_month = stats['created_time']['months'] if create_month_val not in create_month: create_month[create_month_val] = 1 else: create_month[create_month_val] += 1 create_time_val = int(user['create_time']) last_n_weeks = JBoxUserV2.STATS['created_time']['last_n_weeks'] last_n_days = JBoxUserV2.STATS['created_time']['last_n_days'] for week in range(0, len(weeks)): if create_time_val >= weeks[week]: last_n_weeks[week + 1] += 1 break for day in range(0, len(days)): if create_time_val >= days[day]: last_n_days[day + 1] += 1 break @staticmethod def calc_stats(): JBoxUserV2.STATS = { 'date': '', 'num_users': 0, 'sync': { 'gdrive': 0 }, 'role': { 'user': 0, 'superuser': 0, 'access_stats': 0 }, 'activation_status': { 'none': 0, 'granted': 0, 'requested': 0 }, 'resource_profile': { 'basic': 0, 'disk_ebs_1G': 0, 'julia_packages_precompiled': 0 }, 'created_time': { 'months': {}, 'last_n_weeks': {}, 'last_n_days': {} } } secs_day = 24 * 60 * 60 secs_week = secs_day * 7 now = datetime.datetime.now(pytz.utc) secs_now = int(JBoxUserV2.datetime_to_epoch_secs(now)) weeks = [(secs_now - secs_week * week) for week in range(1, 5)] days = [(secs_now - secs_day * day) for day in range(1, 8)] last_n_weeks = JBoxUserV2.STATS['created_time']['last_n_weeks'] last_n_days = JBoxUserV2.STATS['created_time']['last_n_days'] for week in range(0, len(weeks)): last_n_weeks[week + 1] = 0 for day in range(0, len(days)): last_n_days[day + 1] = 0 result_set = JBoxUserV2.table().scan( attributes=('user_id', 'create_month', 'create_time', 'gtok', 'role', 'resource_profile', 'activation_status')) for user in result_set: JBoxUserV2.calc_stat(user, weeks, days) JBoxUserV2.STATS['date'] = now.isoformat()
class JBoxUserProfile(JBoxDB): NAME = 'jbox_user_profiles' SCHEMA = [HashKey('user_id', data_type=STRING)] INDEXES = None GLOBAL_INDEXES = [ GlobalKeysOnlyIndex('create_month-create_time-index', parts=[ HashKey('create_month', data_type=NUMBER), RangeKey('create_time', data_type=NUMBER) ]), GlobalKeysOnlyIndex('update_month-update_time-index', parts=[ HashKey('update_month', data_type=NUMBER), RangeKey('update_time', data_type=NUMBER) ]) ] TABLE = None ATTR_FIRST_NAME = 'first_name' ATTR_LAST_NAME = 'last_name' ATTR_COUNTRY = 'country' ATTR_CITY = 'city' ATTR_LOCATION = 'location' # a fuzzy location string, indicative of country and city ATTR_IP = 'ip' # ip from which last accessed ATTR_INDUSTRY = 'industry' ATTR_ORGANIZATION = 'org' # workplace ATTR_ORG_TITLE = 'org_title' # job title KEYS = ['user_id'] ATTRIBUTES = [ 'create_month', 'create_time', 'update_month', 'update_time', ATTR_FIRST_NAME, ATTR_LAST_NAME, ATTR_COUNTRY, ATTR_CITY, ATTR_LOCATION, ATTR_IP, ATTR_INDUSTRY, ATTR_ORGANIZATION, ATTR_ORG_TITLE, 'sources' # a JSON field that indicates where each profile attribute was filled from ] SQL_INDEXES = [ { 'name': 'create_month-create_time-index', 'cols': ['create_month', 'create_time'] }, { 'name': 'update_month-update_time-index', 'cols': ['update_month', 'update_time'] }, ] KEYS_TYPES = [JBoxDB.VCHAR] TYPES = [ JBoxDB.INT, JBoxDB.INT, JBoxDB.INT, JBoxDB.INT, JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.VCHAR ] SRC_USER = 1 # filled in by the user SRC_DERIVED = 2 # derived from other fields def __init__(self, user_id, create=False): try: self.item = self.fetch(user_id=user_id) self.is_new = False except JBoxDBItemNotFound: if create: data = {'user_id': user_id} JBoxUserProfile._set_time(data, "create") self.create(data) self.item = self.fetch(user_id=user_id) self.is_new = True else: raise def get_user_id(self): return self.get_attrib('user_id') def get_attrib_source(self, attrib_name): sources_str = self.get_attrib('sources', '{}') if len(sources_str) == 0: return None sources = json.loads(sources_str) return sources[attrib_name] if attrib_name in sources else None def set_attrib_source(self, attrib_name, source): sources_str = self.get_attrib('sources', '{}') if len(sources_str) == 0: sources_str = '{}' sources = json.loads(sources_str) sources[attrib_name] = source self.set_attrib('sources', json.dumps(sources)) def is_set_by_user(self, attrib_name): return self.get_attrib_source(attrib_name) == JBoxUserProfile.SRC_USER def set_profile(self, attrib_name, value, source): # do not overwrite attributes set by the user if source != JBoxUserProfile.SRC_USER and self.is_set_by_user( attrib_name): return False self.set_attrib(attrib_name, value) self.set_attrib_source(attrib_name, source) return True def can_set(self, attrib_name, value): if value is None or len(value) == 0: return False return value != self.get_attrib(attrib_name) def get_profile(self, attrib_name, default=''): return self.get_attrib(attrib_name, default) def set_time(self, prefix, dt=None): JBoxUserProfile._set_time(self.item, prefix, dt) @staticmethod def _set_time(item, prefix, dt=None): if dt is None: dt = datetime.datetime.now(pytz.utc) if prefix not in ["create", "update"]: raise (Exception("invalid prefix for setting time")) item[prefix + "_month"] = JBoxUserProfile.datetime_to_yyyymm(dt) item[prefix + "_time"] = JBoxUserProfile.datetime_to_epoch_secs(dt) def get_time(self, prefix): if prefix not in ["create", "update"]: raise (Exception("invalid prefix for setting time")) return JBoxUserProfile.epoch_secs_to_datetime(self.item[prefix + "_time"]) def save(self, set_time=True): self.set_time("update") super(JBoxUserProfile, self).save()