def __init__(self, name=None, db=None, schema=None, version=None, objects=None, proxy=None, proxy_config=None, batch_size=None, config=None, config_file=None, config_key=None, cache_dir=None, autotable=None, **kwargs): # null name -> anonymous table; no native ability to persist options = dict(autotable=autotable, cache_dir=cache_dir, batch_size=batch_size, name=None, schema=schema, version=int(version or 0)) defaults = dict(autotable=True, cache_dir=CACHE_DIR, batch_size=999, name=name, schema={}, version=0) # if config is passed in, set it, otherwise start # with class assigned default or empty dict self.config = copy(config or MetriqueContainer.config or {}) self.config_file = config_file or MetriqueContainer.config_file self.config_key = config_key or MetriqueContainer.config_key # load defaults + set args passed in self.config = configure(options, defaults, config_file=self.config_file, section_key=self.config_key, section_only=True, update=self.config) self.name = self.config.get('name') or MetriqueContainer.name self.version = (self.config.get('version') or MetriqueContainer.version) proxy_config = dict(proxy_config or {}) proxy_config.setdefault('db', db) proxy_config.setdefault('table', self.name) proxy_config.setdefault('config_file', self.config_file) self.config.setdefault(self.proxy_config_key, {}).update(proxy_config) if self._object_cls is None: self._object_cls = metrique_object if self._proxy_cls is None: from metrique.sqlalchemy import SQLAlchemyProxy self._proxy_cls = SQLAlchemyProxy self._proxy = proxy # init and update internal store with passed in objects, if any self.store = copy(MetriqueContainer.store or {}) self._update(objects)
def __init__(self, retries=None, batch_size=None, worker_batch_size=None, **kwargs): self.fields = self.fields or {} super(Generic, self).__init__(**kwargs) options = dict(retries=retries, batch_size=batch_size, worker_batch_size=worker_batch_size) defaults = dict(retries=3, batch_size=999, worker_batch_size=5000) self.config = configure(options, defaults, config_file=self.config_file, section_key=self.config_key, update=self.config) self.retry_on_error = (Exception, ) self._setup_inconsistency_log()
def test_postgresql(): from metrique.sqlalchemy import SQLAlchemyProxy from metrique.utils import rand_chars, configure config = configure(config_file=default_config, section_key='proxy', section_only=True) _db = config['db'] = 'test' _table = config['table'] = 'bla' config['dialect'] = 'postgresql' p = SQLAlchemyProxy(**config) _u = p.config.get('username') _p = p.config.get('password') _po = p.config.get('port') _expected_engine = 'postgresql://%s:%[email protected]:%s/%s' % (_u, _p, _po, _db) p.initialize() assert p._engine_uri == _expected_engine # FIXME: DROP ALL db_tester(p) schema = { 'col_1': {'type': int}, 'col_3': {'type': datetime}, } # FIXME: remove user! before test completes # new user new_u = rand_chars(chars='asdfghjkl') new_p = rand_chars(8) p.user_register(username=new_u, password=new_p) assert p.user_exists(new_u) is True # Sharing p.share(new_u) _new_table = 'blabla' # switch to the new users db p.config['username'] = new_u p.config['password'] = new_p p.config['db'] = new_u p.config['table'] = _new_table p.initialize() assert p.ls() == [] q = 'SELECT current_user;' assert p.execute(q)[0] == {'current_user': new_u} p.autotable(name=_new_table, schema=schema, create=True) assert p.ls() == [_new_table] # switch to admin's db p.config['username'] = _u p.config['password'] = _p p.config['db'] = _u p.config['table'] = _table p.initialize() p.autotable(schema=schema) assert p.ls() == [_table] p.drop() try: assert p.count() except RuntimeError: pass else: assert False assert p.ls() == []
def __init__(self, db=None, table=None, debug=None, config=None, dialect=None, driver=None, host=None, port=None, username=None, password=None, connect_args=None, batch_size=None, cache_dir=None, db_schema=None, log_file=None, log_dir=None, log2file=None, log2stdout=None, log_format=None, schema=None, retries=None, **kwargs): ''' Accept additional kwargs, but ignore them. ''' is_true(HAS_SQLALCHEMY, '`pip install sqlalchemy` required') # use copy of class default value self.RESERVED_USERNAMES = copy(SQLAlchemyProxy.RESERVED_USERNAMES) self.type_map = copy(SQLAlchemyProxy.type_map) # default _start, _end is epoch timestamp options = dict(batch_size=batch_size, cache_dir=cache_dir, connect_args=connect_args, db=db, db_schema=db_schema, default_fields=None, debug=debug, dialect=dialect, driver=driver, host=host, log_dir=log_dir, log_file=log_file, log_format=log_format, log2file=log2file, log2stdout=log2stdout, password=password, port=None, retries=retries, schema=schema, table=table, username=username) defaults = dict(batch_size=999, cache_dir=CACHE_DIR, connect_args=None, db=None, db_schema=None, default_fields={ '_start': 1, '_end': 1, '_oid': 1 }, debug=logging.INFO, dialect='sqlite', driver=None, host='127.0.0.1', log_file='metrique.log', log_dir=LOG_DIR, log_format=None, log2file=True, log2stdout=False, password=None, port=5432, retries=1, schema=None, table=None, username=getuser()) self.config = copy(config or self.config or {}) # FIXME: config expected to come from caller as kwarg or defaults # will be used. This is because loading from file causes problems # at the moment such as when container is loaded, it tries to # load top-level 'proxy' key from config_file, which is incorrect, # since that config key is meant for the data source proxy rather # than container proxy. self.config = configure(options, defaults, section_only=True, update=self.config) # db is required; default db is db username else local username self.config['db'] = self.config['db'] or self.config['username'] is_defined(self.config.get('db'), 'db can not be null') # setup sqlalchemy logging; redirect to metrique logger self._debug_setup_sqlalchemy_logging() if not self._object_cls: from metrique.core_api import metrique_object self._object_cls = metrique_object
def __init__(self, name=None, db=None, config_file=None, config=None, config_key=None, cube_pkgs=None, cube_paths=None, debug=None, log_file=None, log2file=None, log2stdout=None, log_format=None, workers=None, log_dir=None, cache_dir=None, etc_dir=None, tmp_dir=None, container=None, container_config=None, container_config_key=None, proxy=None, proxy_config=None, proxy_config_key=None, version=None, schema=None): super(Metrique, self).__init__() self.name = name or self.name or Metrique.name # cube class defined name # FIXME: this is ugly... and not obvious... # only used currently in sqldata.Generic self._cube = type(self).name options = dict(cache_dir=cache_dir, cube_pkgs=cube_pkgs, cube_paths=cube_paths, db=db, debug=debug, etc_dir=etc_dir, log_dir=log_dir, log_file=log_file, log_format=log_format, log2file=log2file, log2stdout=log2stdout, name=self.name, schema=schema, tmp_dir=tmp_dir, version=version, workers=workers) defaults = dict(cache_dir=CACHE_DIR, cube_pkgs=['cubes'], cube_paths=[], db=getuser(), debug=None, etc_dir=ETC_DIR, log_file='metrique.log', log_dir=LOG_DIR, log_format=None, log2file=True, log2stdout=False, name=None, schema={}, tmp_dir=TMP_DIR, version=0, workers=2) if not self.config: self.config = {} if not self.config.get(self.config_key): self.config[self.config_key] = {} # FIXME: update os.environ LOG_DIR, ETC_DIR, etc to config'd value # if config is passed in, set it, otherwise start # with class assigned default or empty dict self.config.update(copy(config or Metrique.config or {})) self.config_file = config_file or Metrique.config_file self.config_key = config_key or Metrique.config_key # load defaults + set args passed in self.config = configure(options, defaults, config_file=self.config_file, section_key=self.config_key, update=self.config) level = self.lconfig.get('debug') log2stdout = self.lconfig.get('log2stdout') log_format = self.lconfig.get('log_format') log2file = self.lconfig.get('log2file') log_dir = self.lconfig.get('log_dir') log_file = self.lconfig.get('log_file') if self.name: log_file = filename_append(log_file, '.%s' % self.name) self.lconfig['log_file'] = log_file debug_setup(logger='metrique', level=level, log2stdout=log2stdout, log_format=log_format, log2file=log2file, log_dir=log_dir, log_file=log_file) if not schema: # schema (and more) might be defined within self.fields attr schema = getattr(self, 'fields') or {} # filter out invalid schema keys (eg, if derived from .fields) schema = self._schema_filter(schema) self.container_config_key = (container_config_key or Metrique.container_config_key) container_config = dict(container_config or {}) container_config.setdefault('name', self.name) container_config.setdefault('config_file', self.config_file) container_config.setdefault('schema', schema) self.config[self.container_config_key].update(container_config) self.proxy_config_key = proxy_config_key or Metrique.proxy_config_key proxy_config = dict(proxy_config or {}) proxy_config.setdefault('table', self.name) proxy_config.setdefault('config_file', self.config_file) self.config.setdefault(self.proxy_config_key, {}).update(proxy_config) self._proxy = proxy self._container = container if self._container_cls is None: from metrique.core_api import MetriqueContainer self._container_cls = MetriqueContainer if self._proxy_cls is None: from metrique.sqlalchemy import SQLAlchemyProxy self._proxy_cls = SQLAlchemyProxy
def test_configure(): from metrique.utils import configure assert configure() == {} config = dict( debug=100, OK='OK') defaults = dict( debug=False, log2file=False) options = dict( debug=20, log2file=None) # when None, should be ignored config_file = os.path.join(etc, 'test_conf.json') # contents: #{ "file": true # "debug": true, # "log2file": true } # first, only defaults x = configure(defaults=defaults) assert is_in(x, 'debug', False) assert is_in(x, 'log2file', False) # then, where opt is not None, override x = configure(defaults=defaults, options=options) assert is_in(x, 'debug', 20) assert is_in(x, 'log2file', False) # ignored options:None value # update acts as 'template config' in place of {} # but options will override values set already... # so, except that we have a new key, this should # be same as the one above x = configure(update=config, defaults=defaults, options=options) assert is_in(x, 'debug', 20) assert is_in(x, 'log2file', False) # ignored options:None value assert is_in(x, 'OK', 'OK') # only in the template config # first thing loaded is values from disk, then updated # with 'update' config template # since log2file is set in config_file to True, it will # take that value x = configure(config_file=config_file, update=config, defaults=defaults, options=options) assert is_in(x, 'debug', 20) assert is_in(x, 'log2file', True) # ignored options:None value assert is_in(x, 'OK', 'OK') # only in the template config assert is_in(x, 'file', True) # only in the config_file config # cf is loaded first and update config template applied on top x = configure(config_file=config_file, update=config) assert is_in(x, 'debug', 100) assert is_in(x, 'log2file', True) # ignored options:None value assert is_in(x, 'OK', 'OK') # only in the template config assert is_in(x, 'file', True) # only in the config_file config # cf is loaded first and update config template applied on top x = configure(config_file=config_file, options=options) assert is_in(x, 'debug', 20) assert is_in(x, 'log2file', True) # ignored options:None value assert is_in(x, 'file', True) # only in the config_file config # cf is loaded first and where key:values aren't set or set to # None defaults will be applied x = configure(config_file=config_file, defaults=defaults) assert is_in(x, 'debug', True) assert is_in(x, 'log2file', True) # ignored options:None value assert is_in(x, 'file', True) # only in the config_file config config_file = os.path.join(etc, 'test_conf_nested.json') # Contents are same, but one level nested under key 'metrique' x = configure(config_file=config_file, defaults=defaults, section_key='metrique', section_only=True) assert is_in(x, 'debug', True) assert is_in(x, 'log2file', True) # ignored options:None value assert is_in(x, 'file', True) # only in the config_file config _x = x.copy() config_file = os.path.join(etc, 'test_conf_nested.json') # Contents are same, but one level nested under key 'metrique' x = configure(config_file=config_file, defaults=defaults, section_key='metrique') assert is_in(x, 'metrique', _x) try: # should fail x = configure(config_file='I_DO_NOT_EXIST') except IOError: pass else: assert False # shouldn't fail, but instead, returns empty dict x = configure(config_file=config_file, section_key='I_DO_NOT_EXIST', section_only=True) assert x == {} for arg in ('update', 'options', 'defaults'): try: x = configure(**{arg: 'I_SHOULD_BE_A_DICT'}) except AttributeError: pass else: assert False
def __init__(self, db=None, table=None, debug=None, config=None, dialect=None, driver=None, host=None, port=None, username=None, password=None, connect_args=None, batch_size=None, cache_dir=None, db_schema=None, log_file=None, log_dir=None, log2file=None, log2stdout=None, log_format=None, schema=None, retries=None, **kwargs): ''' Accept additional kwargs, but ignore them. ''' is_true(HAS_SQLALCHEMY, '`pip install sqlalchemy` required') # use copy of class default value self.RESERVED_USERNAMES = copy(SQLAlchemyProxy.RESERVED_USERNAMES) self.type_map = copy(SQLAlchemyProxy.type_map) # default _start, _end is epoch timestamp options = dict( batch_size=batch_size, cache_dir=cache_dir, connect_args=connect_args, db=db, db_schema=db_schema, default_fields=None, debug=debug, dialect=dialect, driver=driver, host=host, log_dir=log_dir, log_file=log_file, log_format=log_format, log2file=log2file, log2stdout=log2stdout, password=password, port=None, retries=retries, schema=schema, table=table, username=username) defaults = dict( batch_size=999, cache_dir=CACHE_DIR, connect_args=None, db=None, db_schema=None, default_fields={'_start': 1, '_end': 1, '_oid': 1}, debug=logging.INFO, dialect='sqlite', driver=None, host='127.0.0.1', log_file='metrique.log', log_dir=LOG_DIR, log_format=None, log2file=True, log2stdout=False, password=None, port=5432, retries=1, schema=None, table=None, username=getuser()) self.config = copy(config or self.config or {}) # FIXME: config expected to come from caller as kwarg or defaults # will be used. This is because loading from file causes problems # at the moment such as when container is loaded, it tries to # load top-level 'proxy' key from config_file, which is incorrect, # since that config key is meant for the data source proxy rather # than container proxy. self.config = configure(options, defaults, section_only=True, update=self.config) # db is required; default db is db username else local username self.config['db'] = self.config['db'] or self.config['username'] is_defined(self.config.get('db'), 'db can not be null') # setup sqlalchemy logging; redirect to metrique logger self._debug_setup_sqlalchemy_logging() if not self._object_cls: from metrique.core_api import metrique_object self._object_cls = metrique_object
def test_configure(): from metrique.utils import configure assert configure() == {} config = dict(debug=100, OK='OK') defaults = dict(debug=False, log2file=False) options = dict(debug=20, log2file=None) # when None, should be ignored config_file = os.path.join(etc, 'test_conf.json') # contents: #{ "file": true # "debug": true, # "log2file": true } # first, only defaults x = configure(defaults=defaults) assert is_in(x, 'debug', False) assert is_in(x, 'log2file', False) # then, where opt is not None, override x = configure(defaults=defaults, options=options) assert is_in(x, 'debug', 20) assert is_in(x, 'log2file', False) # ignored options:None value # update acts as 'template config' in place of {} # but options will override values set already... # so, except that we have a new key, this should # be same as the one above x = configure(update=config, defaults=defaults, options=options) assert is_in(x, 'debug', 20) assert is_in(x, 'log2file', False) # ignored options:None value assert is_in(x, 'OK', 'OK') # only in the template config # first thing loaded is values from disk, then updated # with 'update' config template # since log2file is set in config_file to True, it will # take that value x = configure(config_file=config_file, update=config, defaults=defaults, options=options) assert is_in(x, 'debug', 20) assert is_in(x, 'log2file', True) # ignored options:None value assert is_in(x, 'OK', 'OK') # only in the template config assert is_in(x, 'file', True) # only in the config_file config # cf is loaded first and update config template applied on top x = configure(config_file=config_file, update=config) assert is_in(x, 'debug', 100) assert is_in(x, 'log2file', True) # ignored options:None value assert is_in(x, 'OK', 'OK') # only in the template config assert is_in(x, 'file', True) # only in the config_file config # cf is loaded first and update config template applied on top x = configure(config_file=config_file, options=options) assert is_in(x, 'debug', 20) assert is_in(x, 'log2file', True) # ignored options:None value assert is_in(x, 'file', True) # only in the config_file config # cf is loaded first and where key:values aren't set or set to # None defaults will be applied x = configure(config_file=config_file, defaults=defaults) assert is_in(x, 'debug', True) assert is_in(x, 'log2file', True) # ignored options:None value assert is_in(x, 'file', True) # only in the config_file config config_file = os.path.join(etc, 'test_conf_nested.json') # Contents are same, but one level nested under key 'metrique' x = configure(config_file=config_file, defaults=defaults, section_key='metrique', section_only=True) assert is_in(x, 'debug', True) assert is_in(x, 'log2file', True) # ignored options:None value assert is_in(x, 'file', True) # only in the config_file config _x = x.copy() config_file = os.path.join(etc, 'test_conf_nested.json') # Contents are same, but one level nested under key 'metrique' x = configure(config_file=config_file, defaults=defaults, section_key='metrique') assert is_in(x, 'metrique', _x) try: # should fail x = configure(config_file='I_DO_NOT_EXIST') except IOError: pass else: assert False # shouldn't fail, but instead, returns empty dict x = configure(config_file=config_file, section_key='I_DO_NOT_EXIST', section_only=True) assert x == {} for arg in ('update', 'options', 'defaults'): try: x = configure(**{arg: 'I_SHOULD_BE_A_DICT'}) except AttributeError: pass else: assert False
def test_postgresql(): from metrique.sqlalchemy import SQLAlchemyProxy from metrique.utils import rand_chars, configure config = configure(config_file=default_config, section_key='proxy', section_only=True) _db = config['db'] = 'test' _table = config['table'] = 'bla' config['dialect'] = 'postgresql' p = SQLAlchemyProxy(**config) _u = p.config.get('username') _p = p.config.get('password') _po = p.config.get('port') _expected_engine = 'postgresql://%s:%[email protected]:%s/%s' % (_u, _p, _po, _db) p.initialize() assert p._engine_uri == _expected_engine # FIXME: DROP ALL db_tester(p) schema = { 'col_1': { 'type': int }, 'col_3': { 'type': datetime }, } # FIXME: remove user! before test completes # new user new_u = rand_chars(chars='asdfghjkl') new_p = rand_chars(8) p.user_register(username=new_u, password=new_p) assert p.user_exists(new_u) is True # Sharing p.share(new_u) _new_table = 'blabla' # switch to the new users db p.config['username'] = new_u p.config['password'] = new_p p.config['db'] = new_u p.config['table'] = _new_table p.initialize() assert p.ls() == [] q = 'SELECT current_user;' assert p.execute(q)[0] == {'current_user': new_u} p.autotable(name=_new_table, schema=schema, create=True) assert p.ls() == [_new_table] # switch to admin's db p.config['username'] = _u p.config['password'] = _p p.config['db'] = _u p.config['table'] = _table p.initialize() p.autotable(schema=schema) assert p.ls() == [_table] p.drop() try: assert p.count() except RuntimeError: pass else: assert False assert p.ls() == []