def test___setitem(self): lazy_dict = LazyDict() lazy_dict.__setitem__('foo', 'bar') lazy_dict.__setitem__('foo2', 'bar2') lazy_dict.__setitem__('foo3', 'bar3') self.assertEqual(lazy_dict['foo'], 'bar') self.assertEqual(lazy_dict['foo3'], 'bar3') self.assertEqual(lazy_dict['foo2'], 'bar2')
def test_itervalues(self): populate = CallCounter(lambda: {'foo': 'bar', 1: 11, 'empty': None}) lazy_dict = LazyDict(populate) iterator = lazy_dict.itervalues() self.assertEqual(iterator.next(), 11) self.assertEqual(iterator.next(), 'bar') self.assertEqual(iterator.next(), None) self.assertRaises(StopIteration, iterator.next)
def run(output): """Convert *bfo* to *yml*.""" from invenio.legacy.dbquery import run_sql from flask_registry import PkgResourcesDirDiscoveryRegistry, \ ModuleAutoDiscoveryRegistry, RegistryProxy from invenio_utils.datastructures import LazyDict output_formats_directories = RegistryProxy( 'legacy_output_formats_directories', ModuleAutoDiscoveryRegistry, 'output_formats') output_formats = RegistryProxy( 'legacy_output_formats', PkgResourcesDirDiscoveryRegistry, '.', registry_namespace=output_formats_directories) def create_output_formats_lookup(): """Create output formats.""" out = {} for f in output_formats: of = os.path.basename(f).lower() if not of.endswith('.bfo'): continue of = of[:-4] if of in out: continue out[of] = f return out output_formats_lookup = LazyDict(create_output_formats_lookup) for row in run_sql('SELECT id, name, code, description, content_type, ' ' mime_type, visibility FROM format'): code = row[2].lower() out = { 'name': row[1], 'description': row[3], 'visibility': 1 if row[6] == '1' else 0, 'content_type': row[4], } if row[5]: out['mime_type'] = row[5] try: out.update(get_output_format(output_formats_lookup[code])) with open(os.path.join(output, code + '.yml'), 'w') as f: yaml.dump(out, stream=f) # print('echo "{0}" > {1}/{2}.yml'.format( # yaml.dump(out), output, code)) except Exception: current_app.logger.exception( "Could not convert '{0}'".format(code))
def test_lazy_dictionary(self): """Checks content of lazy dictionary and number of evaluations.""" populate = CallCounter(lambda: {'foo': 'bar', 1: 11, 'empty': None}) lazy_dict = LazyDict(populate) self.assertEqual(populate.counter, 0) self.assertEqual(lazy_dict['foo'], 'bar') self.assertEqual(populate.counter, 1) self.assertEqual(lazy_dict[1], 11) self.assertEqual(populate.counter, 1) self.assertEqual(lazy_dict['empty'], None) self.assertEqual(populate.counter, 1) # clear the cache lazy_dict.expunge() self.assertEqual(lazy_dict['foo'], 'bar') self.assertEqual(populate.counter, 2) del lazy_dict['foo'] self.assertEqual(populate.counter, 2) assert 'foo' not in lazy_dict
cur.execute("UNLOCK TABLES") except Exception: pass return app def _db_conn(): current_app.teardown_appcontext_funcs.append(unlock_all) out = {} out[cfg['CFG_DATABASE_HOST']] = {} out[cfg['CFG_DATABASE_SLAVE']] = {} return out connect = DBConnect() _DB_CONN = LazyDict(_db_conn) def _get_password_from_database_password_file(user): """Parse CFG_DATABASE_PASSWORD_FILE and return password for user.""" pwfile = cfg.get("CFG_DATABASE_PASSWORD_FILE", None) if pwfile and os.path.exists(pwfile): for row in open(pwfile): if row.strip(): a_user, pwd = row.strip().split(" // ") if user == a_user: return pwd raise ValueError("user '%s' not found in database password file '%s'" % (user, pwfile)) raise IOError("No password defined for user '%s' but database password " "file is not available" % user)
def to_pathdict(self, test): """Return LazyDict representation.""" return LazyDict(lambda: dict( (os.path.basename(f), f) for f in self if test(f)))
if level > 4: return normpath = os.path.normpath(path) if os.path.isdir(normpath): for p in os.listdir(normpath): _register(os.path.join(normpath, p), level=level + 1) else: parts = normpath.split(os.path.sep) out[os.path.sep.join(parts[-level:])] = normpath for t in reversed(format_templates): _register(t) return out format_templates_lookup = LazyDict(create_format_templates_lookup) def create_output_formats_lookup(): """Create output formats.""" out = {} for f in output_formats_files: of = os.path.basename(f).lower() data = {'names': {}} if of.endswith('.yml'): of = of[:-4] with open(f, 'r') as f: data.update(yaml.load(f) or {}) data['code'] = of else:
'id': (int, -1), 'idb': (int, -1), 'sysnb': (str, ""), 'action': (str, "search"), 'action_search': (str, ""), 'action_browse': (str, ""), 'd1': (str, ""), 'd1y': (int, 0), 'd1m': (int, 0), 'd1d': (int, 0), 'd2': (str, ""), 'd2y': (int, 0), 'd2m': (int, 0), 'd2d': (int, 0), 'dt': (str, ""), 'ap': (int, 1), 'verbose': (int, 0), 'ec': (list, []), 'wl': (int, cfg['CFG_WEBSEARCH_WILDCARD_LIMIT']), } search_results_default_urlargd = LazyDict(get_search_results_default_urlargd) def wash_pattern(p): """Wash pattern passed in URL. Check for sanity of the wildcard by removing wildcards if they are appended to extremely short words (1-3 letters). """ # strip accents: # p = strip_accents(p) # FIXME: when available, strip accents all the time # add leading/trailing whitespace for the two following wildcard-sanity # checking regexps: p = " " + p + " " # replace spaces within quotes by __SPACE__ temporarily: p = re_pattern_single_quotes.sub(
format_templates_lookup = LazyDict(create_format_templates_lookup) def create_output_formats_lookup(): """Create output formats.""" out = {} for f in output_formats_files: of = os.path.basename(f).lower() data = {'names': {}} if of.endswith('.yml'): of = of[:-4] with open(f, 'r') as f: data.update(yaml.load(f) or {}) data['code'] = of else: continue # unknown filetype if of in out: continue out[of] = data return out output_formats = LazyDict(create_output_formats_lookup) export_formats = LazyDict(lambda: dict( (code, of) for code, of in output_formats.items() if of.get('content_type', '') != 'text/html' and of.get('visibility', 0) ))
legacy_modules = RegistryProxy('legacy', ImportPathRegistry, initial=['invenio.legacy.*']) webadmin_proxy = RegistryProxy('legacy.webadmin', \ ModuleAutoDiscoverySubRegistry, 'web.admin', registry_namespace=legacy_modules) def _admin_handler_name(name): parts = name.split('.') return '%s/%s' % (parts[2], parts[5]) webadmin = LazyDict(lambda: dict((_admin_handler_name(module.__name__), module) for module in webadmin_proxy)) webinterface_proxy = RegistryProxy('legacy.webinterface', ModuleAutoDiscoveryRegistry, 'webinterface', registry_namespace=legacy_modules) def _webinterface(module): from invenio.ext.legacy.handler import WebInterfaceDirectory parts = module.__name__.split('.') for value in dir(module): webinterface = getattr(module, value) if inspect.isclass(webinterface) and \ issubclass(webinterface, WebInterfaceDirectory) and \ webinterface.__module__ == module.__name__:
# along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """Registries for redirector module.""" from flask_registry import RegistryProxy from invenio_ext.registry import ModuleAutoDiscoverySubRegistry from invenio_utils.datastructures import LazyDict redirector_proxy = RegistryProxy('redirect_methods', ModuleAutoDiscoverySubRegistry, 'redirect_methods') def register_redirect_methods(): """Register redirect methods.""" out = {} for module in redirector_proxy: if hasattr(module, 'goto'): out[module.__name__.split('.')[-1]] = module.goto return out def get_redirect_method(plugin_name): """Return the function from the plugin name.""" return redirect_methods[plugin_name] redirect_methods = LazyDict(register_redirect_methods)
import os from flask_registry import PkgResourcesDirDiscoveryRegistry, \ ModuleAutoDiscoveryRegistry, RegistryProxy from invenio_utils.datastructures import LazyDict classifierext = RegistryProxy('classifierext', ModuleAutoDiscoveryRegistry, 'classifierext') taxonomies_proxy = RegistryProxy('classifierext.taxonomies', PkgResourcesDirDiscoveryRegistry, 'taxonomies', registry_namespace=classifierext) taxonomies = LazyDict(lambda: dict( (os.path.basename(f), f) for f in taxonomies_proxy)) kb = LazyDict(lambda: dict((os.path.basename(f), f) for f in RegistryProxy( 'converterext.kb', PkgResourcesDirDiscoveryRegistry, 'kb', registry_namespace=classifierext))) templates = LazyDict(lambda: dict((os.path.basename( f), f) for f in RegistryProxy('converterext.templates', PkgResourcesDirDiscoveryRegistry, 'templates', registry_namespace=classifierext))) __all__ = ('classfierext', 'taxonomies_proxy', 'taxonomies', 'kb', 'templates')
# -*- coding: utf-8 -*- # # This file is part of Invenio. # Copyright (C) 2013, 2015 CERN. # # Invenio is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 2 of the # License, or (at your option) any later version. # # Invenio is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. from flask_registry import RegistryProxy from invenio_ext.registry import ModuleAutoDiscoverySubRegistry from invenio_utils.datastructures import LazyDict mixers_proxy = RegistryProxy('mixer', ModuleAutoDiscoverySubRegistry, 'mixer') mixers = LazyDict(lambda: dict( (getattr(module, mixer).__model__.__tablename__, getattr(module, mixer)) for module in mixers_proxy for mixer in module.__all__))
def _queries(): """Preprocess collection queries.""" from invenio_ext.sqlalchemy import db from invenio_collections.models import Collection return dict( (collection.name, dict(query=Query( COLLECTIONS_DELETED_RECORDS.format(dbquery=collection.dbquery)), ancestors=set(c.name for c in collection.ancestors if c.dbquery is None))) for collection in Collection.query.filter( Collection.dbquery.isnot(None), db.not_(Collection.dbquery.like('hostedcollection:%'))).all()) queries = LazyDict(_queries) def get_record_collections(record): """Return list of collections to which record belongs to. :record: Record instance :returns: list of collection names """ output = set() for name, data in iteritems(queries): if data['query'].match(record): output.add(name) output |= data['ancestors'] return list(output)
class PidProvider(object): """Abstract class for persistent identifier provider classes. Subclasses must implement register, update, delete and is_provider_for_pid methods and register itself: class MyProvider(PidProvider): pid_type = "mypid" def reserve(self, pid, *args, **kwargs): return True def register(self, pid, *args, **kwargs): return True def update(self, pid, *args, **kwargs): return True def delete(self, pid, *args, **kwargs): try: ... except Exception as e: pid.log("DELETE","Deletion failed") return False else: pid.log("DELETE","Successfully deleted") return True def is_provider_for_pid(self, pid_str): pass PidProvider.register_provider(MyProvider) The provider is responsible for handling of errors, as well as logging of actions happening to the pid. See example above as well as the DataCitePidProvider. Each method takes variable number of argument and keywords arguments. This can be used to pass additional information to the provider when registering a persistent identifier. E.g. a DOI requires URL and metadata to be able to register the DOI. """ def __load_providers(): registry = dict() for provider_str in cfg['PIDSTORE_PROVIDERS']: provider = import_string(provider_str) if not issubclass(provider, PidProvider): raise TypeError("Argument not an instance of PidProvider.") pid_type = getattr(provider, 'pid_type', None) if pid_type is None: raise AttributeError( "Provider must specify class variable pid_type." ) pid_type = pid_type.lower() if pid_type not in registry: registry[pid_type] = [] # Prevent double registration if provider not in registry[pid_type]: registry[pid_type].append(provider) return registry registry = LazyDict(__load_providers) """Registry of possible providers.""" pid_type = None """ Must be overwritten in subcleass and specified as a string (max len 6) """ @staticmethod def create(pid_type, pid_str, pid_provider, *args, **kwargs): """Create a new instance for the given type and pid.""" providers = PidProvider.registry.get(pid_type.lower(), None) for p in providers: if p.is_provider_for_pid(pid_str): return p(*args, **kwargs) return None # # API methods which must be implemented by each provider. # def reserve(self, pid, *args, **kwargs): """Reserve a new persistent identifier. This might or might not be useful depending on the service of the provider. """ raise NotImplementedError def register(self, pid, *args, **kwargs): """Register a new persistent identifier.""" raise NotImplementedError def update(self, pid, *args, **kwargs): """Update information about a persistent identifier.""" raise NotImplementedError def delete(self, pid, *args, **kwargs): """Delete a persistent identifier.""" raise NotImplementedError def sync_status(self, pid, *args, **kwargs): """Synchronize PIDstatus with remote service provider.""" return True @classmethod def is_provider_for_pid(cls, pid_str): raise NotImplementedError # # API methods which might need to be implemented depending on each # provider. # def create_new_pid(self, pid_value): """Some PidProvider might have the ability to create new values.""" return pid_value
# WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """Registry definition for fixture datasets.""" from flask_registry import RegistryProxy from invenio_ext.registry import ModuleAutoDiscoveryRegistry from invenio_utils.datastructures import LazyDict fixtures_proxy = RegistryProxy( 'fixtures', ModuleAutoDiscoveryRegistry, 'fixtures') def fixtures_loader(): """Load fixtures datasets.""" out = {} for fixture in fixtures_proxy: for data in getattr(fixture, '__all__', dir(fixture)): if data[-4:] != 'Data' or data in out: continue out[data] = getattr(fixture, data) return out fixtures = LazyDict(fixtures_loader)