Пример #1
0
def test_is_true():
    from metrique.utils import is_true
    assert is_true(True) is True
    assert is_true(None, except_=False) is False
    try:
        is_true('', except_=True)
    except RuntimeError:
        pass
    else:
        assert False
Пример #2
0
def test_is_true():
    from metrique.utils import is_true
    assert is_true(True) is True
    assert is_true(None, except_=False) is False
    try:
        is_true('', except_=True)
    except RuntimeError:
        pass
    else:
        assert False
Пример #3
0
 def proxy_init(self, **kwargs):
     config = self.proxy_config
     config.update(kwargs)
     if self._proxy is None:
         self._proxy = self._proxy_cls
     msg = "Invalid proxy: %s" % self._proxy
     if isclass(self._proxy):
         self._proxy = self._proxy(**config)
     is_true(isinstance(self._proxy, self._proxy_cls), msg)
     self._proxy.initialize()
     return self._proxy
Пример #4
0
 def container_init(self, value=None, **kwargs):
     config = self.container_config
     # don't pass 'proxy' config section as kwarg, but rather as
     # proxy_config kwarg
     config['proxy_config'] = config.get(self.proxy_config_key)
     config[self.proxy_config_key] = None
     config.update(kwargs)
     if self._container is None:
         self._container = self._container_cls
     msg = "Invalid container: %s" % self._container
     if isclass(self._container):
         self._container = self._container(objects=value, **config)
     is_true(isinstance(self._container, self._container_cls), msg)
     return self._container
Пример #5
0
    def share(self, with_user, roles=None, table=None):
        '''
        Give cube access rights to another user

        Not, this method is NOT supported by SQLite3!
        '''
        table = self.get_table(table)
        is_true(table is not None, 'invalid table: %s' % table)
        with_user = validate_username(with_user)
        roles = roles or ['SELECT']
        roles = validate_roles(roles, self.VALID_SHARE_ROLES)
        roles = list2str(roles)
        logger.info('Sharing cube %s with %s (%s)' % (table, with_user, roles))
        sql = 'GRANT %s ON %s TO %s' % (roles, table, with_user)
        return self.session_auto.execute(sql)
Пример #6
0
    def share(self, with_user, roles=None, table=None):
        '''
        Give cube access rights to another user

        Not, this method is NOT supported by SQLite3!
        '''
        table = self.get_table(table)
        is_true(table is not None, 'invalid table: %s' % table)
        with_user = validate_username(with_user)
        roles = roles or ['SELECT']
        roles = validate_roles(roles, self.VALID_SHARE_ROLES)
        roles = list2str(roles)
        logger.info('Sharing cube %s with %s (%s)' % (table, with_user, roles))
        sql = 'GRANT %s ON %s TO %s' % (roles, table, with_user)
        return self.session_auto.execute(sql)
Пример #7
0
    def get_table(self, table=None, except_=True, as_cls=False,
                  reflect=True, schema=None):
        schema = schema or self.config.get('schema')
        if table is None:
            table = self.config.get('table')

        if isinstance(table, Table):
            # this is already the table we're looking for...
            _table = table
        else:
            is_defined(table, 'table must be defined!')
            _table = self.meta_tables.get(table)
            if reflect:
                if _table is None and schema:
                    # if we have a schema, try to load the full
                    # Table class definition
                    _table = self.autotable(schema=schema)
                if _table is None:
                    # this provides us ONLY with the SQL definied Table,
                    # which might not include custom Types, etc (JSONType,
                    # UTCEpoch, etc) but rather only the underlying SQL
                    # types (Text, Float, etc) would need to call autotable...
                    self.meta_reflect()
                    _table = self.meta_tables.get(table)
        except_ and is_true(isinstance(_table, Table),
                            'table (%s) not found! Got: %s' % (table, _table))
        if isinstance(_table, Table) and as_cls:
            defaults = dict(__tablename__=table, autoload=True)
            _table = type(str(table), (self.Base,), defaults)
        elif _table is not None:
            pass
        else:
            _table = None
        return _table
Пример #8
0
def get_engine_uri(db,
                   host='127.0.0.1',
                   port=5432,
                   dialect='sqlite',
                   driver=None,
                   username=None,
                   password=None,
                   connect_args=None,
                   cache_dir=None):
    cache_dir = cache_dir or CACHE_DIR
    is_defined(db, 'db can not be null')
    is_true(bool(dialect in [None, 'postgresql', 'sqlite', 'teiid']),
            'invalid dialect: %s' % dialect)
    if dialect and driver:
        dialect = '%s+%s' % (dialect, driver)
    elif dialect:
        pass
    else:
        dialect = 'sqlite'
        dialect = dialect.replace('://', '')

    if dialect == 'sqlite':
        # db is expected to be an absolute path to where
        # sqlite db will be saved
        db = os.path.join(cache_dir, '%s.sqlite' % db)
        uri = '%s:///%s' % (dialect, db)
    else:
        if username and password:
            u_p = '%s:%s@' % (username, password)
        elif username:
            u_p = '%s@' % username
        else:
            u_p = ''

        uri = '%s://%s%s:%s/%s' % (dialect, u_p, host, port, db)
        if connect_args:
            args = ['%s=%s' % (k, v) for k, v in connect_args.iteritems()]
            args = '?%s' % '&'.join(args)
            uri += args
    _uri = re.sub(':[^:]+@', ':***@', uri)
    logger.info("Engine URI: %s" % _uri)
    return uri
Пример #9
0
 def execute(self, query, cursor=False, retries=1):
     retries = int(retries or self.config.get('retries') or 1)
     is_true(retries >= 1, 'retries value must be >= 1')
     OK, i = False, 1
     while retries > 0:
         try:
             rows = self.session_auto.execute(query)
         except Exception as e:
             logger.error('[%s of %s] SQL Load Error: %s' % (i, retries, e))
             i += 1
             retries -= 1
         else:
             OK = True
             break
     if not OK:
         raise
     elif cursor:
         return rows
     else:
         return list(dict(row) for row in rows)
Пример #10
0
 def execute(self, query, cursor=False, retries=1):
     retries = int(retries or self.config.get('retries') or 1)
     is_true(retries >= 1, 'retries value must be >= 1')
     OK, i = False, 1
     while retries > 0:
         try:
             rows = self.session_auto.execute(query)
         except Exception as e:
             logger.error('[%s of %s] SQL Load Error: %s' % (i, retries, e))
             i += 1
             retries -= 1
         else:
             OK = True
             break
     if not OK:
         raise
     elif cursor:
         return rows
     else:
         return list(dict(row) for row in rows)
Пример #11
0
 def user_register(self, username, password):
     # FIXME: enable setting roles at creation time...
     is_true(bool(username and password), 'username and password required!')
     u = validate_username(username, self.RESERVED_USERNAMES)
     p = validate_password(password)
     logger.info('Registering new user %s' % u)
     # FIXME: make a generic method which runs list of sql statements
     s_u = "CREATE USER %s WITH PASSWORD '%s';" % (u, p)
     s_db = "CREATE DATABASE %s WITH OWNER %s;" % (u, u)
     cnx = self.engine.connect()
     # can't run in a transaction...
     cnx.connection.set_isolation_level(0)
     if not self.user_exists(username):
         cnx.execute(s_u)
         logger.info('User created: %s' % u)
     else:
         logger.info('User exists: %s' % u)
     if not self.db_exists(u):
         cnx.execute(s_db)
         logger.info('DB created: %s' % u)
     else:
         logger.info('DB exists: %s' % u)
Пример #12
0
 def user_register(self, username, password):
     # FIXME: enable setting roles at creation time...
     is_true(bool(username and password), 'username and password required!')
     u = validate_username(username, self.RESERVED_USERNAMES)
     p = validate_password(password)
     logger.info('Registering new user %s' % u)
     # FIXME: make a generic method which runs list of sql statements
     s_u = "CREATE USER %s WITH PASSWORD '%s';" % (u, p)
     s_db = "CREATE DATABASE %s WITH OWNER %s;" % (u, u)
     cnx = self.engine.connect()
     # can't run in a transaction...
     cnx.connection.set_isolation_level(0)
     if not self.user_exists(username):
         cnx.execute(s_u)
         logger.info('User created: %s' % u)
     else:
         logger.info('User exists: %s' % u)
     if not self.db_exists(u):
         cnx.execute(s_db)
         logger.info('DB created: %s' % u)
     else:
         logger.info('DB exists: %s' % u)
Пример #13
0
def get_engine_uri(db, host='127.0.0.1', port=5432, dialect='sqlite',
                   driver=None, username=None, password=None,
                   connect_args=None, cache_dir=None):
    cache_dir = cache_dir or CACHE_DIR
    is_defined(db, 'db can not be null')
    is_true(bool(dialect in [None, 'postgresql', 'sqlite', 'teiid']),
            'invalid dialect: %s' % dialect)
    if dialect and driver:
        dialect = '%s+%s' % (dialect, driver)
    elif dialect:
        pass
    else:
        dialect = 'sqlite'
        dialect = dialect.replace('://', '')

    if dialect == 'sqlite':
        # db is expected to be an absolute path to where
        # sqlite db will be saved
        db = os.path.join(cache_dir, '%s.sqlite' % db)
        uri = '%s:///%s' % (dialect, db)
    else:
        if username and password:
            u_p = '%s:%s@' % (username, password)
        elif username:
            u_p = '%s@' % username
        else:
            u_p = ''

        uri = '%s://%s%s:%s/%s' % (dialect, u_p, host, port, db)
        if connect_args:
            args = ['%s=%s' % (k, v) for k, v in connect_args.iteritems()]
            args = '?%s' % '&'.join(args)
            uri += args
    _uri = re.sub(':[^:]+@', ':***@', uri)
    logger.info("Engine URI: %s" % _uri)
    return uri
Пример #14
0
    def get_table(self,
                  table=None,
                  except_=True,
                  as_cls=False,
                  reflect=True,
                  schema=None):
        schema = schema or self.config.get('schema')
        if table is None:
            table = self.config.get('table')

        if isinstance(table, Table):
            # this is already the table we're looking for...
            _table = table
        else:
            is_defined(table, 'table must be defined!')
            _table = self.meta_tables.get(table)
            if reflect:
                if _table is None and schema:
                    # if we have a schema, try to load the full
                    # Table class definition
                    _table = self.autotable(schema=schema)
                if _table is None:
                    # this provides us ONLY with the SQL definied Table,
                    # which might not include custom Types, etc (JSONType,
                    # UTCEpoch, etc) but rather only the underlying SQL
                    # types (Text, Float, etc) would need to call autotable...
                    self.meta_reflect()
                    _table = self.meta_tables.get(table)
        except_ and is_true(isinstance(_table, Table),
                            'table (%s) not found! Got: %s' % (table, _table))
        if isinstance(_table, Table) and as_cls:
            defaults = dict(__tablename__=table, autoload=True)
            _table = type(str(table), (self.Base, ), defaults)
        elif _table is not None:
            pass
        else:
            _table = None
        return _table
Пример #15
0
    def __init__(self,
                 db=None,
                 table=None,
                 debug=None,
                 config=None,
                 dialect=None,
                 driver=None,
                 host=None,
                 port=None,
                 username=None,
                 password=None,
                 connect_args=None,
                 batch_size=None,
                 cache_dir=None,
                 db_schema=None,
                 log_file=None,
                 log_dir=None,
                 log2file=None,
                 log2stdout=None,
                 log_format=None,
                 schema=None,
                 retries=None,
                 **kwargs):
        '''
        Accept additional kwargs, but ignore them.
        '''
        is_true(HAS_SQLALCHEMY, '`pip install sqlalchemy` required')
        # use copy of class default value
        self.RESERVED_USERNAMES = copy(SQLAlchemyProxy.RESERVED_USERNAMES)
        self.type_map = copy(SQLAlchemyProxy.type_map)
        # default _start, _end is epoch timestamp

        options = dict(batch_size=batch_size,
                       cache_dir=cache_dir,
                       connect_args=connect_args,
                       db=db,
                       db_schema=db_schema,
                       default_fields=None,
                       debug=debug,
                       dialect=dialect,
                       driver=driver,
                       host=host,
                       log_dir=log_dir,
                       log_file=log_file,
                       log_format=log_format,
                       log2file=log2file,
                       log2stdout=log2stdout,
                       password=password,
                       port=None,
                       retries=retries,
                       schema=schema,
                       table=table,
                       username=username)
        defaults = dict(batch_size=999,
                        cache_dir=CACHE_DIR,
                        connect_args=None,
                        db=None,
                        db_schema=None,
                        default_fields={
                            '_start': 1,
                            '_end': 1,
                            '_oid': 1
                        },
                        debug=logging.INFO,
                        dialect='sqlite',
                        driver=None,
                        host='127.0.0.1',
                        log_file='metrique.log',
                        log_dir=LOG_DIR,
                        log_format=None,
                        log2file=True,
                        log2stdout=False,
                        password=None,
                        port=5432,
                        retries=1,
                        schema=None,
                        table=None,
                        username=getuser())
        self.config = copy(config or self.config or {})
        # FIXME: config expected to come from caller as kwarg or defaults
        # will be used. This is because loading from file causes problems
        # at the moment such as when container is loaded, it tries to
        # load top-level 'proxy' key from config_file, which is incorrect,
        # since that config key is meant for the data source proxy rather
        # than container proxy.
        self.config = configure(options,
                                defaults,
                                section_only=True,
                                update=self.config)
        # db is required; default db is db username else local username
        self.config['db'] = self.config['db'] or self.config['username']
        is_defined(self.config.get('db'), 'db can not be null')
        # setup sqlalchemy logging; redirect to metrique logger
        self._debug_setup_sqlalchemy_logging()

        if not self._object_cls:
            from metrique.core_api import metrique_object
            self._object_cls = metrique_object
Пример #16
0
    def __init__(self, db=None, table=None, debug=None, config=None,
                 dialect=None, driver=None, host=None,
                 port=None, username=None, password=None,
                 connect_args=None, batch_size=None,
                 cache_dir=None, db_schema=None,
                 log_file=None, log_dir=None, log2file=None,
                 log2stdout=None, log_format=None, schema=None,
                 retries=None, **kwargs):
        '''
        Accept additional kwargs, but ignore them.
        '''
        is_true(HAS_SQLALCHEMY, '`pip install sqlalchemy` required')
        # use copy of class default value
        self.RESERVED_USERNAMES = copy(SQLAlchemyProxy.RESERVED_USERNAMES)
        self.type_map = copy(SQLAlchemyProxy.type_map)
        # default _start, _end is epoch timestamp

        options = dict(
            batch_size=batch_size,
            cache_dir=cache_dir,
            connect_args=connect_args,
            db=db,
            db_schema=db_schema,
            default_fields=None,
            debug=debug,
            dialect=dialect,
            driver=driver,
            host=host,
            log_dir=log_dir,
            log_file=log_file,
            log_format=log_format,
            log2file=log2file,
            log2stdout=log2stdout,
            password=password,
            port=None,
            retries=retries,
            schema=schema,
            table=table,
            username=username)
        defaults = dict(
            batch_size=999,
            cache_dir=CACHE_DIR,
            connect_args=None,
            db=None,
            db_schema=None,
            default_fields={'_start': 1, '_end': 1, '_oid': 1},
            debug=logging.INFO,
            dialect='sqlite',
            driver=None,
            host='127.0.0.1',
            log_file='metrique.log',
            log_dir=LOG_DIR,
            log_format=None,
            log2file=True,
            log2stdout=False,
            password=None,
            port=5432,
            retries=1,
            schema=None,
            table=None,
            username=getuser())
        self.config = copy(config or self.config or {})
        # FIXME: config expected to come from caller as kwarg or defaults
        # will be used. This is because loading from file causes problems
        # at the moment such as when container is loaded, it tries to
        # load top-level 'proxy' key from config_file, which is incorrect,
        # since that config key is meant for the data source proxy rather
        # than container proxy. 
        self.config = configure(options, defaults,
                                section_only=True,
                                update=self.config)
        # db is required; default db is db username else local username
        self.config['db'] = self.config['db'] or self.config['username']
        is_defined(self.config.get('db'), 'db can not be null')
        # setup sqlalchemy logging; redirect to metrique logger
        self._debug_setup_sqlalchemy_logging()

        if not self._object_cls:
            from metrique.core_api import metrique_object
            self._object_cls = metrique_object