def __init__(self, name=None, db=None, schema=None, version=None,
objects=None, proxy=None, proxy_config=None,
batch_size=None, config=None, config_file=None,
config_key=None, cache_dir=None, autotable=None,
**kwargs):
# null name -> anonymous table; no native ability to persist
options = dict(autotable=autotable,
cache_dir=cache_dir,
batch_size=batch_size,
name=None,
schema=schema,
version=int(version or 0))
defaults = dict(autotable=True,
cache_dir=CACHE_DIR,
batch_size=999,
name=name,
schema={},
version=0)
# if config is passed in, set it, otherwise start
# with class assigned default or empty dict
self.config = copy(config or MetriqueContainer.config or {})
self.config_file = config_file or MetriqueContainer.config_file
self.config_key = config_key or MetriqueContainer.config_key
# load defaults + set args passed in
self.config = configure(options, defaults,
config_file=self.config_file,
section_key=self.config_key,
section_only=True,
update=self.config)
self.name = self.config.get('name') or MetriqueContainer.name
self.version = (self.config.get('version') or
MetriqueContainer.version)
proxy_config = dict(proxy_config or {})
proxy_config.setdefault('db', db)
proxy_config.setdefault('table', self.name)
proxy_config.setdefault('config_file', self.config_file)
self.config.setdefault(self.proxy_config_key, {}).update(proxy_config)
if self._object_cls is None:
self._object_cls = metrique_object
if self._proxy_cls is None:
from metrique.sqlalchemy import SQLAlchemyProxy
self._proxy_cls = SQLAlchemyProxy
self._proxy = proxy
# init and update internal store with passed in objects, if any
self.store = copy(MetriqueContainer.store or {})
self._update(objects)
def __init__(self, retries=None, batch_size=None, worker_batch_size=None,
**kwargs):
self.fields = self.fields or {}
super(Generic, self).__init__(**kwargs)
options = dict(retries=retries,
batch_size=batch_size,
worker_batch_size=worker_batch_size)
defaults = dict(retries=3,
batch_size=999,
worker_batch_size=5000)
self.config = configure(options, defaults,
config_file=self.config_file,
section_key=self.config_key,
update=self.config)
self.retry_on_error = (Exception, )
self._setup_inconsistency_log()
def __init__(self,
retries=None,
batch_size=None,
worker_batch_size=None,
**kwargs):
self.fields = self.fields or {}
super(Generic, self).__init__(**kwargs)
options = dict(retries=retries,
batch_size=batch_size,
worker_batch_size=worker_batch_size)
defaults = dict(retries=3, batch_size=999, worker_batch_size=5000)
self.config = configure(options,
defaults,
config_file=self.config_file,
section_key=self.config_key,
update=self.config)
self.retry_on_error = (Exception, )
self._setup_inconsistency_log()
def test_postgresql():
from metrique.sqlalchemy import SQLAlchemyProxy
from metrique.utils import rand_chars, configure
config = configure(config_file=default_config, section_key='proxy',
section_only=True)
_db = config['db'] = 'test'
_table = config['table'] = 'bla'
config['dialect'] = 'postgresql'
p = SQLAlchemyProxy(**config)
_u = p.config.get('username')
_p = p.config.get('password')
_po = p.config.get('port')
_expected_engine = 'postgresql://%s:%[email protected]:%s/%s' % (_u, _p,
_po, _db)
p.initialize()
assert p._engine_uri == _expected_engine
# FIXME: DROP ALL
db_tester(p)
schema = {
'col_1': {'type': int},
'col_3': {'type': datetime},
}
# FIXME: remove user! before test completes
# new user
new_u = rand_chars(chars='asdfghjkl')
new_p = rand_chars(8)
p.user_register(username=new_u, password=new_p)
assert p.user_exists(new_u) is True
# Sharing
p.share(new_u)
_new_table = 'blabla'
# switch to the new users db
p.config['username'] = new_u
p.config['password'] = new_p
p.config['db'] = new_u
p.config['table'] = _new_table
p.initialize()
assert p.ls() == []
q = 'SELECT current_user;'
assert p.execute(q)[0] == {'current_user': new_u}
p.autotable(name=_new_table, schema=schema, create=True)
assert p.ls() == [_new_table]
# switch to admin's db
p.config['username'] = _u
p.config['password'] = _p
p.config['db'] = _u
p.config['table'] = _table
p.initialize()
p.autotable(schema=schema)
assert p.ls() == [_table]
p.drop()
try:
assert p.count()
except RuntimeError:
pass
else:
assert False
assert p.ls() == []
def __init__(self,
db=None,
table=None,
debug=None,
config=None,
dialect=None,
driver=None,
host=None,
port=None,
username=None,
password=None,
connect_args=None,
batch_size=None,
cache_dir=None,
db_schema=None,
log_file=None,
log_dir=None,
log2file=None,
log2stdout=None,
log_format=None,
schema=None,
retries=None,
**kwargs):
'''
Accept additional kwargs, but ignore them.
'''
is_true(HAS_SQLALCHEMY, '`pip install sqlalchemy` required')
# use copy of class default value
self.RESERVED_USERNAMES = copy(SQLAlchemyProxy.RESERVED_USERNAMES)
self.type_map = copy(SQLAlchemyProxy.type_map)
# default _start, _end is epoch timestamp
options = dict(batch_size=batch_size,
cache_dir=cache_dir,
connect_args=connect_args,
db=db,
db_schema=db_schema,
default_fields=None,
debug=debug,
dialect=dialect,
driver=driver,
host=host,
log_dir=log_dir,
log_file=log_file,
log_format=log_format,
log2file=log2file,
log2stdout=log2stdout,
password=password,
port=None,
retries=retries,
schema=schema,
table=table,
username=username)
defaults = dict(batch_size=999,
cache_dir=CACHE_DIR,
connect_args=None,
db=None,
db_schema=None,
default_fields={
'_start': 1,
'_end': 1,
'_oid': 1
},
debug=logging.INFO,
dialect='sqlite',
driver=None,
host='127.0.0.1',
log_file='metrique.log',
log_dir=LOG_DIR,
log_format=None,
log2file=True,
log2stdout=False,
password=None,
port=5432,
retries=1,
schema=None,
table=None,
username=getuser())
self.config = copy(config or self.config or {})
# FIXME: config expected to come from caller as kwarg or defaults
# will be used. This is because loading from file causes problems
# at the moment such as when container is loaded, it tries to
# load top-level 'proxy' key from config_file, which is incorrect,
# since that config key is meant for the data source proxy rather
# than container proxy.
self.config = configure(options,
defaults,
section_only=True,
update=self.config)
# db is required; default db is db username else local username
self.config['db'] = self.config['db'] or self.config['username']
is_defined(self.config.get('db'), 'db can not be null')
# setup sqlalchemy logging; redirect to metrique logger
self._debug_setup_sqlalchemy_logging()
if not self._object_cls:
from metrique.core_api import metrique_object
self._object_cls = metrique_object
def __init__(self, name=None, db=None, config_file=None,
config=None, config_key=None, cube_pkgs=None,
cube_paths=None, debug=None, log_file=None,
log2file=None, log2stdout=None, log_format=None,
workers=None, log_dir=None, cache_dir=None,
etc_dir=None, tmp_dir=None, container=None,
container_config=None, container_config_key=None,
proxy=None, proxy_config=None, proxy_config_key=None,
version=None, schema=None):
super(Metrique, self).__init__()
self.name = name or self.name or Metrique.name
# cube class defined name
# FIXME: this is ugly... and not obvious...
# only used currently in sqldata.Generic
self._cube = type(self).name
options = dict(cache_dir=cache_dir,
cube_pkgs=cube_pkgs,
cube_paths=cube_paths,
db=db,
debug=debug,
etc_dir=etc_dir,
log_dir=log_dir,
log_file=log_file,
log_format=log_format,
log2file=log2file,
log2stdout=log2stdout,
name=self.name,
schema=schema,
tmp_dir=tmp_dir,
version=version,
workers=workers)
defaults = dict(cache_dir=CACHE_DIR,
cube_pkgs=['cubes'],
cube_paths=[],
db=getuser(),
debug=None,
etc_dir=ETC_DIR,
log_file='metrique.log',
log_dir=LOG_DIR,
log_format=None,
log2file=True,
log2stdout=False,
name=None,
schema={},
tmp_dir=TMP_DIR,
version=0,
workers=2)
if not self.config:
self.config = {}
if not self.config.get(self.config_key):
self.config[self.config_key] = {}
# FIXME: update os.environ LOG_DIR, ETC_DIR, etc to config'd value
# if config is passed in, set it, otherwise start
# with class assigned default or empty dict
self.config.update(copy(config or Metrique.config or {}))
self.config_file = config_file or Metrique.config_file
self.config_key = config_key or Metrique.config_key
# load defaults + set args passed in
self.config = configure(options, defaults,
config_file=self.config_file,
section_key=self.config_key,
update=self.config)
level = self.lconfig.get('debug')
log2stdout = self.lconfig.get('log2stdout')
log_format = self.lconfig.get('log_format')
log2file = self.lconfig.get('log2file')
log_dir = self.lconfig.get('log_dir')
log_file = self.lconfig.get('log_file')
if self.name:
log_file = filename_append(log_file, '.%s' % self.name)
self.lconfig['log_file'] = log_file
debug_setup(logger='metrique', level=level, log2stdout=log2stdout,
log_format=log_format, log2file=log2file,
log_dir=log_dir, log_file=log_file)
if not schema:
# schema (and more) might be defined within self.fields attr
schema = getattr(self, 'fields') or {}
# filter out invalid schema keys (eg, if derived from .fields)
schema = self._schema_filter(schema)
self.container_config_key = (container_config_key or
Metrique.container_config_key)
container_config = dict(container_config or {})
container_config.setdefault('name', self.name)
container_config.setdefault('config_file', self.config_file)
container_config.setdefault('schema', schema)
self.config[self.container_config_key].update(container_config)
self.proxy_config_key = proxy_config_key or Metrique.proxy_config_key
proxy_config = dict(proxy_config or {})
proxy_config.setdefault('table', self.name)
proxy_config.setdefault('config_file', self.config_file)
self.config.setdefault(self.proxy_config_key, {}).update(proxy_config)
self._proxy = proxy
self._container = container
if self._container_cls is None:
from metrique.core_api import MetriqueContainer
self._container_cls = MetriqueContainer
if self._proxy_cls is None:
from metrique.sqlalchemy import SQLAlchemyProxy
self._proxy_cls = SQLAlchemyProxy
def __init__(self,
name=None,
db=None,
schema=None,
version=None,
objects=None,
proxy=None,
proxy_config=None,
batch_size=None,
config=None,
config_file=None,
config_key=None,
cache_dir=None,
autotable=None,
**kwargs):
# null name -> anonymous table; no native ability to persist
options = dict(autotable=autotable,
cache_dir=cache_dir,
batch_size=batch_size,
name=None,
schema=schema,
version=int(version or 0))
defaults = dict(autotable=True,
cache_dir=CACHE_DIR,
batch_size=999,
name=name,
schema={},
version=0)
# if config is passed in, set it, otherwise start
# with class assigned default or empty dict
self.config = copy(config or MetriqueContainer.config or {})
self.config_file = config_file or MetriqueContainer.config_file
self.config_key = config_key or MetriqueContainer.config_key
# load defaults + set args passed in
self.config = configure(options,
defaults,
config_file=self.config_file,
section_key=self.config_key,
section_only=True,
update=self.config)
self.name = self.config.get('name') or MetriqueContainer.name
self.version = (self.config.get('version')
or MetriqueContainer.version)
proxy_config = dict(proxy_config or {})
proxy_config.setdefault('db', db)
proxy_config.setdefault('table', self.name)
proxy_config.setdefault('config_file', self.config_file)
self.config.setdefault(self.proxy_config_key, {}).update(proxy_config)
if self._object_cls is None:
self._object_cls = metrique_object
if self._proxy_cls is None:
from metrique.sqlalchemy import SQLAlchemyProxy
self._proxy_cls = SQLAlchemyProxy
self._proxy = proxy
# init and update internal store with passed in objects, if any
self.store = copy(MetriqueContainer.store or {})
self._update(objects)
def test_configure():
from metrique.utils import configure
assert configure() == {}
config = dict(
debug=100,
OK='OK')
defaults = dict(
debug=False,
log2file=False)
options = dict(
debug=20,
log2file=None) # when None, should be ignored
config_file = os.path.join(etc, 'test_conf.json')
# contents:
#{ "file": true
# "debug": true,
# "log2file": true }
# first, only defaults
x = configure(defaults=defaults)
assert is_in(x, 'debug', False)
assert is_in(x, 'log2file', False)
# then, where opt is not None, override
x = configure(defaults=defaults, options=options)
assert is_in(x, 'debug', 20)
assert is_in(x, 'log2file', False) # ignored options:None value
# update acts as 'template config' in place of {}
# but options will override values set already...
# so, except that we have a new key, this should
# be same as the one above
x = configure(update=config, defaults=defaults,
options=options)
assert is_in(x, 'debug', 20)
assert is_in(x, 'log2file', False) # ignored options:None value
assert is_in(x, 'OK', 'OK') # only in the template config
# first thing loaded is values from disk, then updated
# with 'update' config template
# since log2file is set in config_file to True, it will
# take that value
x = configure(config_file=config_file, update=config,
defaults=defaults, options=options)
assert is_in(x, 'debug', 20)
assert is_in(x, 'log2file', True) # ignored options:None value
assert is_in(x, 'OK', 'OK') # only in the template config
assert is_in(x, 'file', True) # only in the config_file config
# cf is loaded first and update config template applied on top
x = configure(config_file=config_file, update=config)
assert is_in(x, 'debug', 100)
assert is_in(x, 'log2file', True) # ignored options:None value
assert is_in(x, 'OK', 'OK') # only in the template config
assert is_in(x, 'file', True) # only in the config_file config
# cf is loaded first and update config template applied on top
x = configure(config_file=config_file, options=options)
assert is_in(x, 'debug', 20)
assert is_in(x, 'log2file', True) # ignored options:None value
assert is_in(x, 'file', True) # only in the config_file config
# cf is loaded first and where key:values aren't set or set to
# None defaults will be applied
x = configure(config_file=config_file, defaults=defaults)
assert is_in(x, 'debug', True)
assert is_in(x, 'log2file', True) # ignored options:None value
assert is_in(x, 'file', True) # only in the config_file config
config_file = os.path.join(etc, 'test_conf_nested.json')
# Contents are same, but one level nested under key 'metrique'
x = configure(config_file=config_file, defaults=defaults,
section_key='metrique', section_only=True)
assert is_in(x, 'debug', True)
assert is_in(x, 'log2file', True) # ignored options:None value
assert is_in(x, 'file', True) # only in the config_file config
_x = x.copy()
config_file = os.path.join(etc, 'test_conf_nested.json')
# Contents are same, but one level nested under key 'metrique'
x = configure(config_file=config_file, defaults=defaults,
section_key='metrique')
assert is_in(x, 'metrique', _x)
try: # should fail
x = configure(config_file='I_DO_NOT_EXIST')
except IOError:
pass
else:
assert False
# shouldn't fail, but instead, returns empty dict
x = configure(config_file=config_file, section_key='I_DO_NOT_EXIST',
section_only=True)
assert x == {}
for arg in ('update', 'options', 'defaults'):
try:
x = configure(**{arg: 'I_SHOULD_BE_A_DICT'})
except AttributeError:
pass
else:
assert False
def __init__(self, db=None, table=None, debug=None, config=None,
dialect=None, driver=None, host=None,
port=None, username=None, password=None,
connect_args=None, batch_size=None,
cache_dir=None, db_schema=None,
log_file=None, log_dir=None, log2file=None,
log2stdout=None, log_format=None, schema=None,
retries=None, **kwargs):
'''
Accept additional kwargs, but ignore them.
'''
is_true(HAS_SQLALCHEMY, '`pip install sqlalchemy` required')
# use copy of class default value
self.RESERVED_USERNAMES = copy(SQLAlchemyProxy.RESERVED_USERNAMES)
self.type_map = copy(SQLAlchemyProxy.type_map)
# default _start, _end is epoch timestamp
options = dict(
batch_size=batch_size,
cache_dir=cache_dir,
connect_args=connect_args,
db=db,
db_schema=db_schema,
default_fields=None,
debug=debug,
dialect=dialect,
driver=driver,
host=host,
log_dir=log_dir,
log_file=log_file,
log_format=log_format,
log2file=log2file,
log2stdout=log2stdout,
password=password,
port=None,
retries=retries,
schema=schema,
table=table,
username=username)
defaults = dict(
batch_size=999,
cache_dir=CACHE_DIR,
connect_args=None,
db=None,
db_schema=None,
default_fields={'_start': 1, '_end': 1, '_oid': 1},
debug=logging.INFO,
dialect='sqlite',
driver=None,
host='127.0.0.1',
log_file='metrique.log',
log_dir=LOG_DIR,
log_format=None,
log2file=True,
log2stdout=False,
password=None,
port=5432,
retries=1,
schema=None,
table=None,
username=getuser())
self.config = copy(config or self.config or {})
# FIXME: config expected to come from caller as kwarg or defaults
# will be used. This is because loading from file causes problems
# at the moment such as when container is loaded, it tries to
# load top-level 'proxy' key from config_file, which is incorrect,
# since that config key is meant for the data source proxy rather
# than container proxy.
self.config = configure(options, defaults,
section_only=True,
update=self.config)
# db is required; default db is db username else local username
self.config['db'] = self.config['db'] or self.config['username']
is_defined(self.config.get('db'), 'db can not be null')
# setup sqlalchemy logging; redirect to metrique logger
self._debug_setup_sqlalchemy_logging()
if not self._object_cls:
from metrique.core_api import metrique_object
self._object_cls = metrique_object
def test_configure():
from metrique.utils import configure
assert configure() == {}
config = dict(debug=100, OK='OK')
defaults = dict(debug=False, log2file=False)
options = dict(debug=20, log2file=None) # when None, should be ignored
config_file = os.path.join(etc, 'test_conf.json')
# contents:
#{ "file": true
# "debug": true,
# "log2file": true }
# first, only defaults
x = configure(defaults=defaults)
assert is_in(x, 'debug', False)
assert is_in(x, 'log2file', False)
# then, where opt is not None, override
x = configure(defaults=defaults, options=options)
assert is_in(x, 'debug', 20)
assert is_in(x, 'log2file', False) # ignored options:None value
# update acts as 'template config' in place of {}
# but options will override values set already...
# so, except that we have a new key, this should
# be same as the one above
x = configure(update=config, defaults=defaults, options=options)
assert is_in(x, 'debug', 20)
assert is_in(x, 'log2file', False) # ignored options:None value
assert is_in(x, 'OK', 'OK') # only in the template config
# first thing loaded is values from disk, then updated
# with 'update' config template
# since log2file is set in config_file to True, it will
# take that value
x = configure(config_file=config_file,
update=config,
defaults=defaults,
options=options)
assert is_in(x, 'debug', 20)
assert is_in(x, 'log2file', True) # ignored options:None value
assert is_in(x, 'OK', 'OK') # only in the template config
assert is_in(x, 'file', True) # only in the config_file config
# cf is loaded first and update config template applied on top
x = configure(config_file=config_file, update=config)
assert is_in(x, 'debug', 100)
assert is_in(x, 'log2file', True) # ignored options:None value
assert is_in(x, 'OK', 'OK') # only in the template config
assert is_in(x, 'file', True) # only in the config_file config
# cf is loaded first and update config template applied on top
x = configure(config_file=config_file, options=options)
assert is_in(x, 'debug', 20)
assert is_in(x, 'log2file', True) # ignored options:None value
assert is_in(x, 'file', True) # only in the config_file config
# cf is loaded first and where key:values aren't set or set to
# None defaults will be applied
x = configure(config_file=config_file, defaults=defaults)
assert is_in(x, 'debug', True)
assert is_in(x, 'log2file', True) # ignored options:None value
assert is_in(x, 'file', True) # only in the config_file config
config_file = os.path.join(etc, 'test_conf_nested.json')
# Contents are same, but one level nested under key 'metrique'
x = configure(config_file=config_file,
defaults=defaults,
section_key='metrique',
section_only=True)
assert is_in(x, 'debug', True)
assert is_in(x, 'log2file', True) # ignored options:None value
assert is_in(x, 'file', True) # only in the config_file config
_x = x.copy()
config_file = os.path.join(etc, 'test_conf_nested.json')
# Contents are same, but one level nested under key 'metrique'
x = configure(config_file=config_file,
defaults=defaults,
section_key='metrique')
assert is_in(x, 'metrique', _x)
try: # should fail
x = configure(config_file='I_DO_NOT_EXIST')
except IOError:
pass
else:
assert False
# shouldn't fail, but instead, returns empty dict
x = configure(config_file=config_file,
section_key='I_DO_NOT_EXIST',
section_only=True)
assert x == {}
for arg in ('update', 'options', 'defaults'):
try:
x = configure(**{arg: 'I_SHOULD_BE_A_DICT'})
except AttributeError:
pass
else:
assert False
def test_postgresql():
from metrique.sqlalchemy import SQLAlchemyProxy
from metrique.utils import rand_chars, configure
config = configure(config_file=default_config,
section_key='proxy',
section_only=True)
_db = config['db'] = 'test'
_table = config['table'] = 'bla'
config['dialect'] = 'postgresql'
p = SQLAlchemyProxy(**config)
_u = p.config.get('username')
_p = p.config.get('password')
_po = p.config.get('port')
_expected_engine = 'postgresql://%s:%[email protected]:%s/%s' % (_u, _p, _po,
_db)
p.initialize()
assert p._engine_uri == _expected_engine
# FIXME: DROP ALL
db_tester(p)
schema = {
'col_1': {
'type': int
},
'col_3': {
'type': datetime
},
}
# FIXME: remove user! before test completes
# new user
new_u = rand_chars(chars='asdfghjkl')
new_p = rand_chars(8)
p.user_register(username=new_u, password=new_p)
assert p.user_exists(new_u) is True
# Sharing
p.share(new_u)
_new_table = 'blabla'
# switch to the new users db
p.config['username'] = new_u
p.config['password'] = new_p
p.config['db'] = new_u
p.config['table'] = _new_table
p.initialize()
assert p.ls() == []
q = 'SELECT current_user;'
assert p.execute(q)[0] == {'current_user': new_u}
p.autotable(name=_new_table, schema=schema, create=True)
assert p.ls() == [_new_table]
# switch to admin's db
p.config['username'] = _u
p.config['password'] = _p
p.config['db'] = _u
p.config['table'] = _table
p.initialize()
p.autotable(schema=schema)
assert p.ls() == [_table]
p.drop()
try:
assert p.count()
except RuntimeError:
pass
else:
assert False
assert p.ls() == []
