Beispiel #1
0
def create_router_table(tablename="router",
                        read_throughput=5,
                        write_throughput=5):
    # type: (str, int, int) -> Table
    """Create a new router table

    The last_connect index is a value used to determine the last month a user
    was seen in. To prevent hot-keys on this table during month switchovers the
    key is determined based on the following scheme:

        (YEAR)(MONTH)(DAY)(HOUR)(0001-0010)

    Note that the random key is only between 1-10 at the moment, if the key is
    still too hot during production the random range can be increased at the
    cost of additional queries during GC to locate expired users.

    """
    return Table.create(
        tablename,
        schema=[HashKey("uaid")],
        throughput=dict(read=read_throughput, write=write_throughput),
        global_indexes=[
            GlobalKeysOnlyIndex(
                'AccessIndex',
                parts=[HashKey('last_connect', data_type=NUMBER)],
                throughput=dict(read=5, write=5))
        ],
    )
def get_indexes(all_indexes):
    indexes = []
    global_indexes = []
    for index in all_indexes:
        name = index['name']
        schema = get_schema_param(index.get('hash_key_name'), index.get('hash_key_type'), index.get('range_key_name'), index.get('range_key_type'))
        throughput = {
            'read': index.get('read_capacity', 1),
            'write': index.get('write_capacity', 1)
        }

        if index['type'] == 'all':
            indexes.append(AllIndex(name, parts=schema))

        elif index['type'] == 'global_all':
            global_indexes.append(GlobalAllIndex(name, parts=schema, throughput=throughput))

        elif index['type'] == 'global_include':
            global_indexes.append(GlobalIncludeIndex(name, parts=schema, throughput=throughput, includes=index['includes']))

        elif index['type'] == 'global_keys_only':
            global_indexes.append(GlobalKeysOnlyIndex(name, parts=schema, throughput=throughput))

        elif index['type'] == 'include':
            indexes.append(IncludeIndex(name, parts=schema, includes=index['includes']))

        elif index['type'] == 'keys_only':
            indexes.append(KeysOnlyIndex(name, parts=schema))

    return indexes, global_indexes
Beispiel #3
0
    def test_gsi(self):
        users = Table.create('gsi_users',
                             schema=[
                                 HashKey('user_id'),
                             ],
                             throughput={
                                 'read': 5,
                                 'write': 3,
                             },
                             global_indexes=[
                                 GlobalKeysOnlyIndex(
                                     'StuffIndex',
                                     parts=[HashKey('user_id')],
                                     throughput={
                                         'read': 2,
                                         'write': 1,
                                     }),
                             ])
        self.addCleanup(users.delete)

        # Wait for it.
        time.sleep(60)

        users.update(throughput={
            'read': 3,
            'write': 4
        },
                     global_indexes={'StuffIndex': {
                         'read': 1,
                         'write': 2
                     }})

        # Wait again for the changes to finish propagating.
        time.sleep(150)
Beispiel #4
0
def create_router_table(tablename="router",
                        read_throughput=5,
                        write_throughput=5):
    """Create a new router table"""
    return Table.create(
        tablename,
        schema=[HashKey("uaid")],
        throughput=dict(read=read_throughput, write=write_throughput),
        global_indexes=[
            GlobalKeysOnlyIndex(
                'AccessIndex',
                parts=[HashKey('last_connect', data_type=NUMBER)],
                throughput=dict(read=5, write=5))
        ],
    )
Beispiel #5
0
def _extract_index(index_data, global_index=False):
    '''
    Instantiates and returns an AllIndex object given a valid index
    configuration
    '''
    parsed_data = {}
    keys = []

    for key, value in six.iteritems(index_data):
        for item in value:
            for field, data in six.iteritems(item):
                if field == 'hash_key':
                    parsed_data['hash_key'] = data
                elif field == 'hash_key_data_type':
                    parsed_data['hash_key_data_type'] = data
                elif field == 'range_key':
                    parsed_data['range_key'] = data
                elif field == 'range_key_data_type':
                    parsed_data['range_key_data_type'] = data
                elif field == 'name':
                    parsed_data['name'] = data
                elif field == 'read_capacity_units':
                    parsed_data['read_capacity_units'] = data
                elif field == 'write_capacity_units':
                    parsed_data['write_capacity_units'] = data
                elif field == 'includes':
                    parsed_data['includes'] = data
                elif field == 'keys_only':
                    parsed_data['keys_only'] = True

    if parsed_data['hash_key']:
        keys.append(
            HashKey(parsed_data['hash_key'],
                    data_type=parsed_data['hash_key_data_type']))
    if parsed_data.get('range_key'):
        keys.append(
            RangeKey(parsed_data['range_key'],
                     data_type=parsed_data['range_key_data_type']))
    if (global_index and parsed_data['read_capacity_units']
            and parsed_data['write_capacity_units']):
        parsed_data['throughput'] = {
            'read': parsed_data['read_capacity_units'],
            'write': parsed_data['write_capacity_units']
        }
    if parsed_data['name'] and len(keys) > 0:
        if global_index:
            if parsed_data.get('keys_only') and parsed_data.get('includes'):
                raise SaltInvocationError(
                    'Only one type of GSI projection can be used.')

            if parsed_data.get('includes'):
                return GlobalIncludeIndex(parsed_data['name'],
                                          parts=keys,
                                          throughput=parsed_data['throughput'],
                                          includes=parsed_data['includes'])
            elif parsed_data.get('keys_only'):
                return GlobalKeysOnlyIndex(
                    parsed_data['name'],
                    parts=keys,
                    throughput=parsed_data['throughput'],
                )
            else:
                return GlobalAllIndex(parsed_data['name'],
                                      parts=keys,
                                      throughput=parsed_data['throughput'])
        else:
            return AllIndex(parsed_data['name'], parts=keys)
def extract_index(index_data, global_index=False):
    """
    Instantiates and returns an AllIndex object given a valid index
    configuration

    CLI Example:

    .. code-block:: bash

        salt myminion boto_dynamodb.extract_index index
    """
    parsed_data = {}
    keys = []

    for key, value in index_data.items():
        for item in value:
            for field, data in item.items():
                if field == "hash_key":
                    parsed_data["hash_key"] = data
                elif field == "hash_key_data_type":
                    parsed_data["hash_key_data_type"] = data
                elif field == "range_key":
                    parsed_data["range_key"] = data
                elif field == "range_key_data_type":
                    parsed_data["range_key_data_type"] = data
                elif field == "name":
                    parsed_data["name"] = data
                elif field == "read_capacity_units":
                    parsed_data["read_capacity_units"] = data
                elif field == "write_capacity_units":
                    parsed_data["write_capacity_units"] = data
                elif field == "includes":
                    parsed_data["includes"] = data
                elif field == "keys_only":
                    parsed_data["keys_only"] = True

    if parsed_data["hash_key"]:
        keys.append(
            HashKey(parsed_data["hash_key"],
                    data_type=parsed_data["hash_key_data_type"]))
    if parsed_data.get("range_key"):
        keys.append(
            RangeKey(parsed_data["range_key"],
                     data_type=parsed_data["range_key_data_type"]))
    if global_index and parsed_data["read_capacity_units"] and parsed_data[
            "write_capacity_units"]:
        parsed_data["throughput"] = {
            "read": parsed_data["read_capacity_units"],
            "write": parsed_data["write_capacity_units"],
        }
    if parsed_data["name"] and keys:
        if global_index:
            if parsed_data.get("keys_only") and parsed_data.get("includes"):
                raise SaltInvocationError(
                    "Only one type of GSI projection can be used.")

            if parsed_data.get("includes"):
                return GlobalIncludeIndex(
                    parsed_data["name"],
                    parts=keys,
                    throughput=parsed_data["throughput"],
                    includes=parsed_data["includes"],
                )
            elif parsed_data.get("keys_only"):
                return GlobalKeysOnlyIndex(
                    parsed_data["name"],
                    parts=keys,
                    throughput=parsed_data["throughput"],
                )
            else:
                return GlobalAllIndex(
                    parsed_data["name"],
                    parts=keys,
                    throughput=parsed_data["throughput"],
                )
        else:
            return AllIndex(parsed_data["name"], parts=keys)
Beispiel #7
0
class JBoxAPISpec(JBoxDB):
    NAME = 'jbox_apispec'

    SCHEMA = [HashKey('api_name', data_type=STRING)]

    INDEXES = [
        GlobalKeysOnlyIndex('publisher-api_name-index',
                            parts=[
                                HashKey('publisher', data_type=STRING),
                                RangeKey('api_name', data_type=STRING)
                            ])
    ]

    TABLE = None

    KEYS = ['api_name']
    ATTRIBUTES = [
        'publisher', 'cmd', 'image_name', 'description', 'timeout_secs',
        'create_time'
    ]

    def __init__(self,
                 api_name,
                 cmd=None,
                 image_name=None,
                 description=None,
                 publisher=None,
                 timeout_secs=None,
                 create=False):
        try:
            self.item = self.fetch(api_name=api_name)
            self.is_new = False
        except JBoxDBItemNotFound:
            if create:
                dt = datetime.datetime.now(pytz.utc)
                data = {
                    'api_name': api_name,
                    'cmd': cmd,
                    'description': description,
                    'publisher': publisher,
                    'create_time': JBoxAPISpec.datetime_to_epoch_secs(dt)
                }
                if image_name is not None:
                    data['image_name'] = image_name
                if timeout_secs is not None:
                    data['timeout_secs'] = timeout_secs

                self.create(data)
                self.item = self.fetch(api_name=api_name)
                self.is_new = True
            else:
                raise

    def get_api_name(self):
        return self.get_attrib('api_name', None)

    def get_timeout_secs(self):
        return int(self.get_attrib('timeout_secs', 30))

    def get_description(self):
        return self.get_attrib('description', None)

    def get_publisher(self):
        return self.get_attrib('publisher', None)

    def get_image_name(self):
        return self.get_attrib('image_name', 'juliabox/juliaboxapi:latest')

    def get_cmd(self):
        return self.get_attrib('cmd', None)

    def get_create_time(self):
        return int(self.get_attrib('create_time', None))

    def set_cmd(self, cmd):
        self.set_attrib('cmd', cmd)

    def set_description(self, description):
        self.set_attrib('description', description)

    def set_timeout_secs(self, timeout_secs):
        self.set_attrib('timeout_secs', timeout_secs)

    def set_publisher(self, publisher):
        self.set_attrib('publisher', publisher)

    def set_image_name(self, image_name):
        self.set_attrib('image_name', image_name)

    def as_json(self):
        def _add_not_none(d, n, v):
            if v is not None:
                d[n] = v

        jsonval = dict()
        _add_not_none(jsonval, 'api_name', self.get_api_name())
        _add_not_none(jsonval, 'cmd', self.get_cmd())
        _add_not_none(jsonval, 'image_name', self.get_image_name())
        _add_not_none(jsonval, 'description', self.get_description())
        _add_not_none(jsonval, 'publisher', self.get_publisher())
        _add_not_none(jsonval, 'timeout_secs', self.get_timeout_secs())
        _add_not_none(jsonval, 'create_time', self.get_create_time())
        return jsonval

    @staticmethod
    def get_api_info(publisher, api_name):
        if publisher is None and api_name is None:
            raise
        ret = []
        if publisher is None:
            ret.append(JBoxAPISpec(api_name).as_json())
        else:
            if api_name is None:
                api_name = ' '
            records = JBoxAPISpec.query(publisher__eq=publisher,
                                        api_name__gte=api_name,
                                        index='publisher-api_name-index')
            for api in records:
                ret.append(JBoxAPISpec(api['api_name']).as_json())
        return ret

    @staticmethod
    def set_api_info(api_name,
                     cmd=None,
                     image_name=None,
                     description=None,
                     publisher=None,
                     timeout_secs=None):
        try:
            api = JBoxAPISpec(api_name)
            if cmd is not None:
                api.set_cmd(cmd)
            if image_name is not None:
                api.set_image_name(image_name)
            if description is not None:
                api.set_description(description)
            if publisher is not None:
                api.set_publisher(publisher)
            if timeout_secs is not None:
                api.set_timeout_secs(timeout_secs)
            api.save()
        except JBoxDBItemNotFound:
            JBoxAPISpec(api_name,
                        cmd=cmd,
                        image_name=image_name,
                        description=description,
                        publisher=publisher,
                        timeout_secs=timeout_secs,
                        create=True)
Beispiel #8
0
class JBoxDiskState(JBoxDB):
    NAME = 'jbox_diskstate'

    SCHEMA = [HashKey('disk_key', data_type=STRING)]

    INDEXES = [
        GlobalKeysOnlyIndex('state-index',
                            parts=[HashKey('state', data_type=NUMBER)])
    ]

    TABLE = None

    STATE_ATTACHED = 1
    STATE_DETACHED = 0

    def __init__(self,
                 disk_key=None,
                 cluster_id=None,
                 region_id=None,
                 user_id=None,
                 volume_id=None,
                 attach_time=None,
                 create=False):
        if self.table() is None:
            return

        self.item = None
        if create and ((cluster_id is None) or (region_id is None) or
                       (user_id is None)):
            raise AssertionError
        if disk_key is None:
            disk_key = '_'.join([user_id, cluster_id, region_id])
        try:
            self.item = self.table().get_item(disk_key=disk_key)
            self.is_new = False
        except boto.dynamodb2.exceptions.ItemNotFound:
            if create:
                data = {
                    'disk_key': disk_key,
                    'cluster_id': cluster_id,
                    'region_id': region_id,
                    'user_id': user_id
                }

                if volume_id is not None:
                    data['volume_id'] = volume_id
                    if attach_time is None:
                        attach_time = datetime.datetime.now(pytz.utc)
                    data['attach_time'] = JBoxDiskState.datetime_to_epoch_secs(
                        attach_time)

                self.create(data)
                self.item = self.table().get_item(disk_key=disk_key)
                self.is_new = True
            else:
                raise

    def set_attach_time(self, attach_time=None):
        if attach_time is None:
            attach_time = datetime.datetime.now(pytz.utc)
        self.set_attrib('attach_time',
                        JBoxDiskState.datetime_to_epoch_secs(attach_time))

    def get_attach_time(self):
        return JBoxDiskState.epoch_secs_to_datetime(self.item['attach_time'])

    def set_detach_time(self, detach_time=None):
        if detach_time is None:
            detach_time = datetime.datetime.now(pytz.utc)
        self.set_attrib('detach_time',
                        JBoxDiskState.datetime_to_epoch_secs(detach_time))

    def get_detach_time(self):
        return JBoxDiskState.epoch_secs_to_datetime(
            int(self.item['detach_time']))

    def get_state(self):
        return self.get_attrib('state')

    def set_state(self, state):
        self.set_attrib('state', state)

    def get_user_id(self):
        return self.get_attrib('user_id')

    def set_user_id(self, user_id):
        self.set_attrib('user_id', user_id)

    def get_region_id(self):
        return self.get_attrib('region_id')

    def set_region_id(self, region_id):
        self.set_attrib('region_id', region_id)

    def get_cluster_id(self):
        return self.get_attrib('cluster_id')

    def set_cluster_id(self, cluster_id):
        self.set_attrib('cluster_id', cluster_id)

    def get_volume_id(self):
        return self.get_attrib('volume_id')

    def set_volume_id(self, volume_id):
        self.set_attrib('volume_id', volume_id)

    def get_snapshot_ids(self):
        snapshots = self.get_attrib('snapshot_id')
        if (snapshots is not None) and (len(snapshots) > 0):
            return json.loads(snapshots)
        return []

    def add_snapshot_id(self, snapshot_id):
        ids = self.get_snapshot_ids()
        ids.append(snapshot_id)
        self.set_snapshot_ids(ids)

    def set_snapshot_ids(self, snapshot_ids):
        self.set_attrib('snapshot_id', json.dumps(snapshot_ids))

    @staticmethod
    def get_detached_disks(max_count=None):
        disk_keys = []
        try:
            records = JBoxDiskState.table().query_2(
                state__eq=JBoxDiskState.STATE_DETACHED,
                index='state-index',
                limit=max_count)
            for rec in records:
                disk_keys.append(rec['disk_key'])
        except:
            # boto bug: https://github.com/boto/boto/issues/2708
            JBoxDiskState.TABLE = None
            JBoxDiskState.log_warn(
                "Exception in getting detached disks. Probably empty table.")
        return disk_keys
Beispiel #9
0
class JBoxUserV2(JBoxDB):
    """
        - user_id (primary hash key)
        
        - create_month (global secondary hash key)
        - create_time (global secondary range index)
        
        - update_month  (global secondary hash key)
        - update_time (global secondary index)
        
        - activation_code (global secondary hash key)
        - activation_status (global secondary range key)
        
        - image (optional: global secondary hash key)
        - resource_profile (optional: global secondary range key)
        
        - status
        - organization
        - role
        - gtok
    """
    NAME = 'jbox_users_v2'

    SCHEMA = [HashKey('user_id', data_type=STRING)]

    INDEXES = [
        GlobalKeysOnlyIndex('create_month-create_time-index',
                            parts=[
                                HashKey('create_month', data_type=NUMBER),
                                RangeKey('create_time', data_type=NUMBER)
                            ]),
        GlobalKeysOnlyIndex('update_month-update_time-index',
                            parts=[
                                HashKey('update_month', data_type=NUMBER),
                                RangeKey('update_time', data_type=NUMBER)
                            ]),
        GlobalKeysOnlyIndex('activation_code-activation_status-index',
                            parts=[
                                HashKey('activation_code', data_type=STRING),
                                RangeKey('activation_status', data_type=NUMBER)
                            ])
    ]

    TABLE = None

    STATUS_ACTIVE = 0
    STATUS_INACTIVE = 1

    ROLE_USER = 0
    ROLE_ACCESS_STATS = 1 << 0
    ROLE_MANAGE_INVITES = 1 << 1
    ROLE_MANAGE_CONTAINERS = 1 << 2

    ROLE_SUPER = (1 << 33) - 1

    ACTIVATION_NONE = 0
    ACTIVATION_GRANTED = 1
    ACTIVATION_REQUESTED = 2

    ACTIVATION_CODE_AUTO = 'AUTO'

    RES_PROF_BASIC = 0
    RES_PROF_DISK_EBS_1G = 1 << 0

    RES_PROF_JULIA_PKG_PRECOMP = 1 << 12

    STATS = None
    STAT_NAME = "stat_users"

    def __init__(self, user_id, create=False):
        if self.table() is None:
            self.is_new = False
            self.item = None
            return

        try:
            self.item = self.table().get_item(user_id=user_id)
            self.is_new = False
        except boto.dynamodb2.exceptions.ItemNotFound:
            if create:
                data = {
                    'user_id': user_id,
                    'resource_profile': JBoxUserV2.RES_PROF_JULIA_PKG_PRECOMP
                }
                JBoxUserV2._set_time(data, "create")
                JBoxUserV2._set_activation_state(data, '-',
                                                 JBoxUserV2.ACTIVATION_NONE)
                self.create(data)
                self.item = self.table().get_item(user_id=user_id)
                self.is_new = True
            else:
                raise

    def get_user_id(self):
        return self.get_attrib('user_id', None)

    def get_status(self):
        if self.item is not None:
            return self.item.get('status', JBoxUserV2.STATUS_ACTIVE)
        else:
            return None

    def get_role(self):
        return int(self.get_attrib('role', JBoxUserV2.ROLE_USER))

    def set_role(self, role):
        if self.item is not None:
            r = self.item.get('role', JBoxUserV2.ROLE_USER)
            self.item['role'] = r | role

    def has_role(self, role):
        return self.get_role() & role == role

    def set_status(self, status):
        self.set_attrib('status', status)

    def set_time(self, prefix, dt=None):
        if self.item is None:
            return
        JBoxUserV2._set_time(self.item, prefix, dt)

    @staticmethod
    def _set_time(item, prefix, dt=None):
        if None == dt:
            dt = datetime.datetime.now(pytz.utc)

        if prefix not in ["create", "update"]:
            raise (Exception("invalid prefix for setting time"))

        item[prefix + "_month"] = JBoxUserV2.datetime_to_yyyymm(dt)
        item[prefix + "_time"] = JBoxUserV2.datetime_to_epoch_secs(dt)

    def get_time(self, prefix):
        if self.item is None:
            return None
        if prefix not in ["create", "update"]:
            raise (Exception("invalid prefix for setting time"))
        return JBoxUserV2.epoch_secs_to_datetime(self.item[prefix + "_time"])

    def save(self, set_time=True):
        if self.item is not None:
            self.set_time("update")
        super(JBoxUserV2, self).save()

    def set_activation_state(self, activation_code, activation_status):
        if self.item is not None:
            JBoxUserV2.log_debug("setting activation state of %s to %s, %d",
                                 self.get_user_id(), activation_code,
                                 activation_status)
            JBoxUserV2._set_activation_state(self.item, activation_code,
                                             activation_status)

    @staticmethod
    def _set_activation_state(item, activation_code, activation_status):
        item['activation_code'] = activation_code
        item['activation_status'] = activation_status

    def get_activation_state(self):
        if self.item is None:
            return None, None
        return self.item.get('activation_code',
                             '-'), self.item.get('activation_status',
                                                 JBoxUserV2.ACTIVATION_NONE)

    def set_gtok(self, gtok):
        if self.item is not None:
            self.item['gtok'] = encrypt(gtok, self.enckey())

    def get_gtok(self):
        if self.item is None:
            return None
        gtok = self.item.get('gtok', None)
        return decrypt(gtok, self.enckey()) if (gtok is not None) else None

    def set_container_type(self, image, resource_profile):
        if self.item is not None:
            self.item['image'] = image
            self.item['resource_profile'] = resource_profile

    def get_container_type(self):
        if self.item is None:
            return None, None
        return self.item.get('image', None), int(
            self.item.get('resource_profile', JBoxUserV2.RES_PROF_BASIC))

    def get_resource_profile(self):
        if self.item is None:
            return JBoxUserV2.RES_PROF_BASIC
        return int(self.item.get('resource_profile',
                                 JBoxUserV2.RES_PROF_BASIC))

    def set_resource_profile(self, mask):
        if self.item is not None:
            resource_profile = self.get_resource_profile()
            new_resource_profile = resource_profile | mask
            if new_resource_profile != resource_profile:
                self.item['resource_profile'] = new_resource_profile

    def unset_resource_profile(self, mask):
        if self.item is not None:
            resource_profile = self.get_resource_profile()
            new_resource_profile = resource_profile & (~mask)
            if new_resource_profile != resource_profile:
                self.item['resource_profile'] = new_resource_profile

    def has_resource_profile(self, mask):
        resource_profile = self.get_resource_profile()
        if mask == 0:
            return resource_profile == 0
        return (resource_profile & mask) == mask

    @staticmethod
    def get_pending_activations(max_count):
        records = JBoxUserV2.table().query_2(
            activation_code__eq=JBoxUserV2.ACTIVATION_CODE_AUTO,
            activation_status__eq=JBoxUserV2.ACTIVATION_REQUESTED,
            index='activation_code-activation_status-index',
            limit=max_count)
        user_ids = []
        for rec in records:
            user_ids.append(rec['user_id'])
        return user_ids

    @staticmethod
    def count_pending_activations():
        count = JBoxUserV2.table().query_count(
            activation_code__eq='AUTO',
            activation_status__eq=JBoxUserV2.ACTIVATION_REQUESTED,
            index='activation_code-activation_status-index')
        return count

    @staticmethod
    def count_created(hours_before, tilldate=None):
        if None == tilldate:
            tilldate = datetime.datetime.now(pytz.utc)

        fromdate = tilldate - datetime.timedelta(hours=hours_before)

        till_month = JBoxUserV2.datetime_to_yyyymm(tilldate)
        till_time = JBoxUserV2.datetime_to_epoch_secs(tilldate)

        from_month = JBoxUserV2.datetime_to_yyyymm(fromdate)
        from_time = JBoxUserV2.datetime_to_epoch_secs(fromdate)

        count = 0
        mon = from_month
        while mon <= till_month:
            count += JBoxUserV2.table().query_count(
                create_month__eq=mon,
                create_time__between=(from_time, till_time),
                index='create_month-create_time-index')

            JBoxUserV2.log_debug(
                "adding accounts created in mon %d, from %d till %d. count %d",
                mon, from_time, till_time, count)

            if (mon % 100) == 12:
                mon = (mon / 100 + 1) * 100 + 1
            else:
                mon += 1

        return count

    @staticmethod
    def calc_stat(user, weeks, days):
        stats = JBoxUserV2.STATS
        stats['num_users'] += 1

        if 'gtok' in user:
            stats['sync']['gdrive'] += 1

        role = stats['role']
        role_val = int(
            user['role']) if 'role' in user else JBoxUserV2.ROLE_USER
        if role_val == JBoxUserV2.ROLE_USER:
            role['user'] += 1
        else:
            if (role_val & JBoxUserV2.ROLE_SUPER) == JBoxUserV2.ROLE_SUPER:
                role['superuser'] += 1
            if (role_val & JBoxUserV2.ROLE_ACCESS_STATS
                ) == JBoxUserV2.ROLE_ACCESS_STATS:
                role['access_stats'] += 1

        act_status = stats['activation_status']
        act_status_val = int(
            user['activation_status']
        ) if 'activation_status' in user else JBoxUserV2.ACTIVATION_NONE
        if act_status_val == JBoxUserV2.ACTIVATION_NONE:
            act_status['none'] += 1
        elif act_status_val == JBoxUserV2.ACTIVATION_GRANTED:
            act_status['granted'] += 1
        elif act_status_val == JBoxUserV2.ACTIVATION_REQUESTED:
            act_status['requested'] += 1

        res_profile = stats['resource_profile']
        res_profile_val = int(user['resource_profile']) if 'resource_profile' in user \
            else JBoxUserV2.RES_PROF_BASIC
        if res_profile_val == JBoxUserV2.RES_PROF_BASIC:
            res_profile['basic'] += 1
        else:
            if (res_profile_val & JBoxUserV2.RES_PROF_DISK_EBS_1G
                ) == JBoxUserV2.RES_PROF_DISK_EBS_1G:
                res_profile['disk_ebs_1G'] += 1
            elif (res_profile_val & JBoxUserV2.RES_PROF_JULIA_PKG_PRECOMP
                  ) == JBoxUserV2.RES_PROF_JULIA_PKG_PRECOMP:
                res_profile['julia_packages_precompiled'] += 1

        create_month_val = int(user['create_month'])
        create_month = stats['created_time']['months']
        if create_month_val not in create_month:
            create_month[create_month_val] = 1
        else:
            create_month[create_month_val] += 1

        create_time_val = int(user['create_time'])
        last_n_weeks = JBoxUserV2.STATS['created_time']['last_n_weeks']
        last_n_days = JBoxUserV2.STATS['created_time']['last_n_days']
        for week in range(0, len(weeks)):
            if create_time_val >= weeks[week]:
                last_n_weeks[week + 1] += 1
                break
        for day in range(0, len(days)):
            if create_time_val >= days[day]:
                last_n_days[day + 1] += 1
                break

    @staticmethod
    def calc_stats():
        JBoxUserV2.STATS = {
            'date': '',
            'num_users': 0,
            'sync': {
                'gdrive': 0
            },
            'role': {
                'user': 0,
                'superuser': 0,
                'access_stats': 0
            },
            'activation_status': {
                'none': 0,
                'granted': 0,
                'requested': 0
            },
            'resource_profile': {
                'basic': 0,
                'disk_ebs_1G': 0,
                'julia_packages_precompiled': 0
            },
            'created_time': {
                'months': {},
                'last_n_weeks': {},
                'last_n_days': {}
            }
        }

        secs_day = 24 * 60 * 60
        secs_week = secs_day * 7
        now = datetime.datetime.now(pytz.utc)
        secs_now = int(JBoxUserV2.datetime_to_epoch_secs(now))

        weeks = [(secs_now - secs_week * week) for week in range(1, 5)]
        days = [(secs_now - secs_day * day) for day in range(1, 8)]

        last_n_weeks = JBoxUserV2.STATS['created_time']['last_n_weeks']
        last_n_days = JBoxUserV2.STATS['created_time']['last_n_days']
        for week in range(0, len(weeks)):
            last_n_weeks[week + 1] = 0
        for day in range(0, len(days)):
            last_n_days[day + 1] = 0

        result_set = JBoxUserV2.table().scan(
            attributes=('user_id', 'create_month', 'create_time', 'gtok',
                        'role', 'resource_profile', 'activation_status'))
        for user in result_set:
            JBoxUserV2.calc_stat(user, weeks, days)

        JBoxUserV2.STATS['date'] = now.isoformat()
Beispiel #10
0
class JBoxUserProfile(JBoxDB):
    NAME = 'jbox_user_profiles'

    SCHEMA = [HashKey('user_id', data_type=STRING)]

    INDEXES = None
    GLOBAL_INDEXES = [
        GlobalKeysOnlyIndex('create_month-create_time-index',
                            parts=[
                                HashKey('create_month', data_type=NUMBER),
                                RangeKey('create_time', data_type=NUMBER)
                            ]),
        GlobalKeysOnlyIndex('update_month-update_time-index',
                            parts=[
                                HashKey('update_month', data_type=NUMBER),
                                RangeKey('update_time', data_type=NUMBER)
                            ])
    ]

    TABLE = None

    ATTR_FIRST_NAME = 'first_name'
    ATTR_LAST_NAME = 'last_name'
    ATTR_COUNTRY = 'country'
    ATTR_CITY = 'city'
    ATTR_LOCATION = 'location'  # a fuzzy location string, indicative of country and city
    ATTR_IP = 'ip'  # ip from which last accessed
    ATTR_INDUSTRY = 'industry'
    ATTR_ORGANIZATION = 'org'  # workplace
    ATTR_ORG_TITLE = 'org_title'  # job title

    KEYS = ['user_id']
    ATTRIBUTES = [
        'create_month',
        'create_time',
        'update_month',
        'update_time',
        ATTR_FIRST_NAME,
        ATTR_LAST_NAME,
        ATTR_COUNTRY,
        ATTR_CITY,
        ATTR_LOCATION,
        ATTR_IP,
        ATTR_INDUSTRY,
        ATTR_ORGANIZATION,
        ATTR_ORG_TITLE,
        'sources'  # a JSON field that indicates where each profile attribute was filled from
    ]
    SQL_INDEXES = [
        {
            'name': 'create_month-create_time-index',
            'cols': ['create_month', 'create_time']
        },
        {
            'name': 'update_month-update_time-index',
            'cols': ['update_month', 'update_time']
        },
    ]
    KEYS_TYPES = [JBoxDB.VCHAR]
    TYPES = [
        JBoxDB.INT, JBoxDB.INT, JBoxDB.INT, JBoxDB.INT, JBoxDB.VCHAR,
        JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.VCHAR,
        JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.VCHAR, JBoxDB.VCHAR
    ]

    SRC_USER = 1  # filled in by the user
    SRC_DERIVED = 2  # derived from other fields

    def __init__(self, user_id, create=False):
        try:
            self.item = self.fetch(user_id=user_id)
            self.is_new = False
        except JBoxDBItemNotFound:
            if create:
                data = {'user_id': user_id}
                JBoxUserProfile._set_time(data, "create")
                self.create(data)
                self.item = self.fetch(user_id=user_id)
                self.is_new = True
            else:
                raise

    def get_user_id(self):
        return self.get_attrib('user_id')

    def get_attrib_source(self, attrib_name):
        sources_str = self.get_attrib('sources', '{}')
        if len(sources_str) == 0:
            return None
        sources = json.loads(sources_str)
        return sources[attrib_name] if attrib_name in sources else None

    def set_attrib_source(self, attrib_name, source):
        sources_str = self.get_attrib('sources', '{}')
        if len(sources_str) == 0:
            sources_str = '{}'
        sources = json.loads(sources_str)
        sources[attrib_name] = source
        self.set_attrib('sources', json.dumps(sources))

    def is_set_by_user(self, attrib_name):
        return self.get_attrib_source(attrib_name) == JBoxUserProfile.SRC_USER

    def set_profile(self, attrib_name, value, source):
        # do not overwrite attributes set by the user
        if source != JBoxUserProfile.SRC_USER and self.is_set_by_user(
                attrib_name):
            return False
        self.set_attrib(attrib_name, value)
        self.set_attrib_source(attrib_name, source)
        return True

    def can_set(self, attrib_name, value):
        if value is None or len(value) == 0:
            return False
        return value != self.get_attrib(attrib_name)

    def get_profile(self, attrib_name, default=''):
        return self.get_attrib(attrib_name, default)

    def set_time(self, prefix, dt=None):
        JBoxUserProfile._set_time(self.item, prefix, dt)

    @staticmethod
    def _set_time(item, prefix, dt=None):
        if dt is None:
            dt = datetime.datetime.now(pytz.utc)

        if prefix not in ["create", "update"]:
            raise (Exception("invalid prefix for setting time"))

        item[prefix + "_month"] = JBoxUserProfile.datetime_to_yyyymm(dt)
        item[prefix + "_time"] = JBoxUserProfile.datetime_to_epoch_secs(dt)

    def get_time(self, prefix):
        if prefix not in ["create", "update"]:
            raise (Exception("invalid prefix for setting time"))
        return JBoxUserProfile.epoch_secs_to_datetime(self.item[prefix +
                                                                "_time"])

    def save(self, set_time=True):
        self.set_time("update")
        super(JBoxUserProfile, self).save()