def test_is_true():
from metrique.utils import is_true
assert is_true(True) is True
assert is_true(None, except_=False) is False
try:
is_true('', except_=True)
except RuntimeError:
pass
else:
assert False
def test_is_true():
from metrique.utils import is_true
assert is_true(True) is True
assert is_true(None, except_=False) is False
try:
is_true('', except_=True)
except RuntimeError:
pass
else:
assert False
def proxy_init(self, **kwargs):
config = self.proxy_config
config.update(kwargs)
if self._proxy is None:
self._proxy = self._proxy_cls
msg = "Invalid proxy: %s" % self._proxy
if isclass(self._proxy):
self._proxy = self._proxy(**config)
is_true(isinstance(self._proxy, self._proxy_cls), msg)
self._proxy.initialize()
return self._proxy
def container_init(self, value=None, **kwargs):
config = self.container_config
# don't pass 'proxy' config section as kwarg, but rather as
# proxy_config kwarg
config['proxy_config'] = config.get(self.proxy_config_key)
config[self.proxy_config_key] = None
config.update(kwargs)
if self._container is None:
self._container = self._container_cls
msg = "Invalid container: %s" % self._container
if isclass(self._container):
self._container = self._container(objects=value, **config)
is_true(isinstance(self._container, self._container_cls), msg)
return self._container
def share(self, with_user, roles=None, table=None):
'''
Give cube access rights to another user
Not, this method is NOT supported by SQLite3!
'''
table = self.get_table(table)
is_true(table is not None, 'invalid table: %s' % table)
with_user = validate_username(with_user)
roles = roles or ['SELECT']
roles = validate_roles(roles, self.VALID_SHARE_ROLES)
roles = list2str(roles)
logger.info('Sharing cube %s with %s (%s)' % (table, with_user, roles))
sql = 'GRANT %s ON %s TO %s' % (roles, table, with_user)
return self.session_auto.execute(sql)
def share(self, with_user, roles=None, table=None):
'''
Give cube access rights to another user
Not, this method is NOT supported by SQLite3!
'''
table = self.get_table(table)
is_true(table is not None, 'invalid table: %s' % table)
with_user = validate_username(with_user)
roles = roles or ['SELECT']
roles = validate_roles(roles, self.VALID_SHARE_ROLES)
roles = list2str(roles)
logger.info('Sharing cube %s with %s (%s)' % (table, with_user, roles))
sql = 'GRANT %s ON %s TO %s' % (roles, table, with_user)
return self.session_auto.execute(sql)
def get_table(self, table=None, except_=True, as_cls=False,
reflect=True, schema=None):
schema = schema or self.config.get('schema')
if table is None:
table = self.config.get('table')
if isinstance(table, Table):
# this is already the table we're looking for...
_table = table
else:
is_defined(table, 'table must be defined!')
_table = self.meta_tables.get(table)
if reflect:
if _table is None and schema:
# if we have a schema, try to load the full
# Table class definition
_table = self.autotable(schema=schema)
if _table is None:
# this provides us ONLY with the SQL definied Table,
# which might not include custom Types, etc (JSONType,
# UTCEpoch, etc) but rather only the underlying SQL
# types (Text, Float, etc) would need to call autotable...
self.meta_reflect()
_table = self.meta_tables.get(table)
except_ and is_true(isinstance(_table, Table),
'table (%s) not found! Got: %s' % (table, _table))
if isinstance(_table, Table) and as_cls:
defaults = dict(__tablename__=table, autoload=True)
_table = type(str(table), (self.Base,), defaults)
elif _table is not None:
pass
else:
_table = None
return _table
def get_engine_uri(db,
host='127.0.0.1',
port=5432,
dialect='sqlite',
driver=None,
username=None,
password=None,
connect_args=None,
cache_dir=None):
cache_dir = cache_dir or CACHE_DIR
is_defined(db, 'db can not be null')
is_true(bool(dialect in [None, 'postgresql', 'sqlite', 'teiid']),
'invalid dialect: %s' % dialect)
if dialect and driver:
dialect = '%s+%s' % (dialect, driver)
elif dialect:
pass
else:
dialect = 'sqlite'
dialect = dialect.replace('://', '')
if dialect == 'sqlite':
# db is expected to be an absolute path to where
# sqlite db will be saved
db = os.path.join(cache_dir, '%s.sqlite' % db)
uri = '%s:///%s' % (dialect, db)
else:
if username and password:
u_p = '%s:%s@' % (username, password)
elif username:
u_p = '%s@' % username
else:
u_p = ''
uri = '%s://%s%s:%s/%s' % (dialect, u_p, host, port, db)
if connect_args:
args = ['%s=%s' % (k, v) for k, v in connect_args.iteritems()]
args = '?%s' % '&'.join(args)
uri += args
_uri = re.sub(':[^:]+@', ':***@', uri)
logger.info("Engine URI: %s" % _uri)
return uri
def execute(self, query, cursor=False, retries=1):
retries = int(retries or self.config.get('retries') or 1)
is_true(retries >= 1, 'retries value must be >= 1')
OK, i = False, 1
while retries > 0:
try:
rows = self.session_auto.execute(query)
except Exception as e:
logger.error('[%s of %s] SQL Load Error: %s' % (i, retries, e))
i += 1
retries -= 1
else:
OK = True
break
if not OK:
raise
elif cursor:
return rows
else:
return list(dict(row) for row in rows)
def execute(self, query, cursor=False, retries=1):
retries = int(retries or self.config.get('retries') or 1)
is_true(retries >= 1, 'retries value must be >= 1')
OK, i = False, 1
while retries > 0:
try:
rows = self.session_auto.execute(query)
except Exception as e:
logger.error('[%s of %s] SQL Load Error: %s' % (i, retries, e))
i += 1
retries -= 1
else:
OK = True
break
if not OK:
raise
elif cursor:
return rows
else:
return list(dict(row) for row in rows)
def user_register(self, username, password):
# FIXME: enable setting roles at creation time...
is_true(bool(username and password), 'username and password required!')
u = validate_username(username, self.RESERVED_USERNAMES)
p = validate_password(password)
logger.info('Registering new user %s' % u)
# FIXME: make a generic method which runs list of sql statements
s_u = "CREATE USER %s WITH PASSWORD '%s';" % (u, p)
s_db = "CREATE DATABASE %s WITH OWNER %s;" % (u, u)
cnx = self.engine.connect()
# can't run in a transaction...
cnx.connection.set_isolation_level(0)
if not self.user_exists(username):
cnx.execute(s_u)
logger.info('User created: %s' % u)
else:
logger.info('User exists: %s' % u)
if not self.db_exists(u):
cnx.execute(s_db)
logger.info('DB created: %s' % u)
else:
logger.info('DB exists: %s' % u)
def user_register(self, username, password):
# FIXME: enable setting roles at creation time...
is_true(bool(username and password), 'username and password required!')
u = validate_username(username, self.RESERVED_USERNAMES)
p = validate_password(password)
logger.info('Registering new user %s' % u)
# FIXME: make a generic method which runs list of sql statements
s_u = "CREATE USER %s WITH PASSWORD '%s';" % (u, p)
s_db = "CREATE DATABASE %s WITH OWNER %s;" % (u, u)
cnx = self.engine.connect()
# can't run in a transaction...
cnx.connection.set_isolation_level(0)
if not self.user_exists(username):
cnx.execute(s_u)
logger.info('User created: %s' % u)
else:
logger.info('User exists: %s' % u)
if not self.db_exists(u):
cnx.execute(s_db)
logger.info('DB created: %s' % u)
else:
logger.info('DB exists: %s' % u)
def get_engine_uri(db, host='127.0.0.1', port=5432, dialect='sqlite',
driver=None, username=None, password=None,
connect_args=None, cache_dir=None):
cache_dir = cache_dir or CACHE_DIR
is_defined(db, 'db can not be null')
is_true(bool(dialect in [None, 'postgresql', 'sqlite', 'teiid']),
'invalid dialect: %s' % dialect)
if dialect and driver:
dialect = '%s+%s' % (dialect, driver)
elif dialect:
pass
else:
dialect = 'sqlite'
dialect = dialect.replace('://', '')
if dialect == 'sqlite':
# db is expected to be an absolute path to where
# sqlite db will be saved
db = os.path.join(cache_dir, '%s.sqlite' % db)
uri = '%s:///%s' % (dialect, db)
else:
if username and password:
u_p = '%s:%s@' % (username, password)
elif username:
u_p = '%s@' % username
else:
u_p = ''
uri = '%s://%s%s:%s/%s' % (dialect, u_p, host, port, db)
if connect_args:
args = ['%s=%s' % (k, v) for k, v in connect_args.iteritems()]
args = '?%s' % '&'.join(args)
uri += args
_uri = re.sub(':[^:]+@', ':***@', uri)
logger.info("Engine URI: %s" % _uri)
return uri
def get_table(self,
table=None,
except_=True,
as_cls=False,
reflect=True,
schema=None):
schema = schema or self.config.get('schema')
if table is None:
table = self.config.get('table')
if isinstance(table, Table):
# this is already the table we're looking for...
_table = table
else:
is_defined(table, 'table must be defined!')
_table = self.meta_tables.get(table)
if reflect:
if _table is None and schema:
# if we have a schema, try to load the full
# Table class definition
_table = self.autotable(schema=schema)
if _table is None:
# this provides us ONLY with the SQL definied Table,
# which might not include custom Types, etc (JSONType,
# UTCEpoch, etc) but rather only the underlying SQL
# types (Text, Float, etc) would need to call autotable...
self.meta_reflect()
_table = self.meta_tables.get(table)
except_ and is_true(isinstance(_table, Table),
'table (%s) not found! Got: %s' % (table, _table))
if isinstance(_table, Table) and as_cls:
defaults = dict(__tablename__=table, autoload=True)
_table = type(str(table), (self.Base, ), defaults)
elif _table is not None:
pass
else:
_table = None
return _table
def __init__(self,
db=None,
table=None,
debug=None,
config=None,
dialect=None,
driver=None,
host=None,
port=None,
username=None,
password=None,
connect_args=None,
batch_size=None,
cache_dir=None,
db_schema=None,
log_file=None,
log_dir=None,
log2file=None,
log2stdout=None,
log_format=None,
schema=None,
retries=None,
**kwargs):
'''
Accept additional kwargs, but ignore them.
'''
is_true(HAS_SQLALCHEMY, '`pip install sqlalchemy` required')
# use copy of class default value
self.RESERVED_USERNAMES = copy(SQLAlchemyProxy.RESERVED_USERNAMES)
self.type_map = copy(SQLAlchemyProxy.type_map)
# default _start, _end is epoch timestamp
options = dict(batch_size=batch_size,
cache_dir=cache_dir,
connect_args=connect_args,
db=db,
db_schema=db_schema,
default_fields=None,
debug=debug,
dialect=dialect,
driver=driver,
host=host,
log_dir=log_dir,
log_file=log_file,
log_format=log_format,
log2file=log2file,
log2stdout=log2stdout,
password=password,
port=None,
retries=retries,
schema=schema,
table=table,
username=username)
defaults = dict(batch_size=999,
cache_dir=CACHE_DIR,
connect_args=None,
db=None,
db_schema=None,
default_fields={
'_start': 1,
'_end': 1,
'_oid': 1
},
debug=logging.INFO,
dialect='sqlite',
driver=None,
host='127.0.0.1',
log_file='metrique.log',
log_dir=LOG_DIR,
log_format=None,
log2file=True,
log2stdout=False,
password=None,
port=5432,
retries=1,
schema=None,
table=None,
username=getuser())
self.config = copy(config or self.config or {})
# FIXME: config expected to come from caller as kwarg or defaults
# will be used. This is because loading from file causes problems
# at the moment such as when container is loaded, it tries to
# load top-level 'proxy' key from config_file, which is incorrect,
# since that config key is meant for the data source proxy rather
# than container proxy.
self.config = configure(options,
defaults,
section_only=True,
update=self.config)
# db is required; default db is db username else local username
self.config['db'] = self.config['db'] or self.config['username']
is_defined(self.config.get('db'), 'db can not be null')
# setup sqlalchemy logging; redirect to metrique logger
self._debug_setup_sqlalchemy_logging()
if not self._object_cls:
from metrique.core_api import metrique_object
self._object_cls = metrique_object
def __init__(self, db=None, table=None, debug=None, config=None,
dialect=None, driver=None, host=None,
port=None, username=None, password=None,
connect_args=None, batch_size=None,
cache_dir=None, db_schema=None,
log_file=None, log_dir=None, log2file=None,
log2stdout=None, log_format=None, schema=None,
retries=None, **kwargs):
'''
Accept additional kwargs, but ignore them.
'''
is_true(HAS_SQLALCHEMY, '`pip install sqlalchemy` required')
# use copy of class default value
self.RESERVED_USERNAMES = copy(SQLAlchemyProxy.RESERVED_USERNAMES)
self.type_map = copy(SQLAlchemyProxy.type_map)
# default _start, _end is epoch timestamp
options = dict(
batch_size=batch_size,
cache_dir=cache_dir,
connect_args=connect_args,
db=db,
db_schema=db_schema,
default_fields=None,
debug=debug,
dialect=dialect,
driver=driver,
host=host,
log_dir=log_dir,
log_file=log_file,
log_format=log_format,
log2file=log2file,
log2stdout=log2stdout,
password=password,
port=None,
retries=retries,
schema=schema,
table=table,
username=username)
defaults = dict(
batch_size=999,
cache_dir=CACHE_DIR,
connect_args=None,
db=None,
db_schema=None,
default_fields={'_start': 1, '_end': 1, '_oid': 1},
debug=logging.INFO,
dialect='sqlite',
driver=None,
host='127.0.0.1',
log_file='metrique.log',
log_dir=LOG_DIR,
log_format=None,
log2file=True,
log2stdout=False,
password=None,
port=5432,
retries=1,
schema=None,
table=None,
username=getuser())
self.config = copy(config or self.config or {})
# FIXME: config expected to come from caller as kwarg or defaults
# will be used. This is because loading from file causes problems
# at the moment such as when container is loaded, it tries to
# load top-level 'proxy' key from config_file, which is incorrect,
# since that config key is meant for the data source proxy rather
# than container proxy.
self.config = configure(options, defaults,
section_only=True,
update=self.config)
# db is required; default db is db username else local username
self.config['db'] = self.config['db'] or self.config['username']
is_defined(self.config.get('db'), 'db can not be null')
# setup sqlalchemy logging; redirect to metrique logger
self._debug_setup_sqlalchemy_logging()
if not self._object_cls:
from metrique.core_api import metrique_object
self._object_cls = metrique_object