def test___setitem(self): lazy_dict = LazyDict() lazy_dict.__setitem__('foo', 'bar') lazy_dict.__setitem__('foo2', 'bar2') lazy_dict.__setitem__('foo3', 'bar3') self.assertEqual(lazy_dict['foo'], 'bar') self.assertEqual(lazy_dict['foo3'], 'bar3') self.assertEqual(lazy_dict['foo2'], 'bar2')
def test_itervalues(self): populate = CallCounter(lambda: {'foo': 'bar', 1: 11, 'empty': None}) lazy_dict = LazyDict(populate) iterator = lazy_dict.itervalues() self.assertEqual(iterator.next(), 11) self.assertEqual(iterator.next(), 'bar') self.assertEqual(iterator.next(), None) self.assertRaises(StopIteration, iterator.next)
def test_lazy_dictionary(self): """Checks content of lazy dictionary and number of evaluations.""" populate = CallCounter(lambda: {'foo': 'bar', 1: 11, 'empty': None}) lazy_dict = LazyDict(populate) self.assertEqual(populate.counter, 0) self.assertEqual(lazy_dict['foo'], 'bar') self.assertEqual(populate.counter, 1) self.assertEqual(lazy_dict[1], 11) self.assertEqual(populate.counter, 1) self.assertEqual(lazy_dict['empty'], None) self.assertEqual(populate.counter, 1) # clear the cache lazy_dict.expunge() self.assertEqual(lazy_dict['foo'], 'bar') self.assertEqual(populate.counter, 2) del lazy_dict['foo'] self.assertEqual(populate.counter, 2) assert 'foo' not in lazy_dict
# You should have received a copy of the GNU General Public License # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """Registries for redirector module.""" from flask_registry import RegistryProxy from invenio.ext.registry import ModuleAutoDiscoverySubRegistry from invenio.utils.datastructures import LazyDict redirector_proxy = RegistryProxy('redirect_methods', ModuleAutoDiscoverySubRegistry, 'redirect_methods') def register_redirect_methods(): """Register redirect methods.""" out = {} for module in redirector_proxy: if hasattr(module, 'goto'): out[module.__name__.split('.')[-1]] = module.goto return out def get_redirect_method(plugin_name): """Return the function from the plugin name.""" return redirect_methods[plugin_name] redirect_methods = LazyDict(register_redirect_methods)
# # Invenio is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA import os from flask_registry import PkgResourcesDirDiscoveryRegistry, \ ModuleAutoDiscoveryRegistry, RegistryProxy from invenio.utils.datastructures import LazyDict converterext = RegistryProxy('converterext', ModuleAutoDiscoveryRegistry, 'converterext') kb = LazyDict(lambda: dict((os.path.basename(f), f) for f in RegistryProxy( 'converterext.kb', PkgResourcesDirDiscoveryRegistry, 'kb', registry_namespace=converterext))) templates = LazyDict(lambda: dict((os.path.basename( f), f) for f in RegistryProxy('converterext.templates', PkgResourcesDirDiscoveryRegistry, 'templates', registry_namespace=converterext)))
cur = db.cur() cur.execute("UNLOCK TABLES") except Exception: pass return app def _db_conn(): current_app.teardown_appcontext_funcs.append(unlock_all) out = {} out[cfg['CFG_DATABASE_HOST']] = {} out[cfg['CFG_DATABASE_SLAVE']] = {} return out connect = DBConnect() _DB_CONN = LazyDict(_db_conn) def _get_password_from_database_password_file(user): """Parse CFG_DATABASE_PASSWORD_FILE and return password for user.""" pwfile = cfg.get("CFG_DATABASE_PASSWORD_FILE", None) if pwfile and os.path.exists(pwfile): for row in open(pwfile): if row.strip(): a_user, pwd = row.strip().split(" // ") if user == a_user: return pwd raise ValueError("user '%s' not found in database password file '%s'" % (user, pwfile)) raise IOError("No password defined for user '%s' but database password " "file is not available" % user)
'id': (int, -1), 'idb': (int, -1), 'sysnb': (str, ""), 'action': (str, "search"), 'action_search': (str, ""), 'action_browse': (str, ""), 'd1': (str, ""), 'd1y': (int, 0), 'd1m': (int, 0), 'd1d': (int, 0), 'd2': (str, ""), 'd2y': (int, 0), 'd2m': (int, 0), 'd2d': (int, 0), 'dt': (str, ""), 'ap': (int, 1), 'verbose': (int, 0), 'ec': (list, []), 'wl': (int, cfg['CFG_WEBSEARCH_WILDCARD_LIMIT']), } search_results_default_urlargd = LazyDict(get_search_results_default_urlargd) def wash_search_urlargd(form): """ Create canonical search arguments from those passed via web form. """ from invenio.ext.legacy.handler import wash_urlargd argd = wash_urlargd(form, search_results_default_urlargd) if 'as' in argd: argd['aas'] = argd['as'] del argd['as'] if argd.get('aas', cfg['CFG_WEBSEARCH_DEFAULT_SEARCH_INTERFACE']) \ not in cfg['CFG_WEBSEARCH_ENABLED_SEARCH_INTERFACES']: argd['aas'] = cfg['CFG_WEBSEARCH_DEFAULT_SEARCH_INTERFACE']
# Invenio is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 2 of the # License, or (at your option) any later version. # # Invenio is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA import os from flask_registry import PkgResourcesDirDiscoveryRegistry, \ ModuleAutoDiscoveryRegistry, RegistryProxy from invenio.utils.datastructures import LazyDict exporterext = RegistryProxy( 'exporterext', ModuleAutoDiscoveryRegistry, 'exportext' ) configurations = LazyDict( lambda: dict((os.path.basename(f), f) for f in RegistryProxy('exporterext.configurations', PkgResourcesDirDiscoveryRegistry, 'configurations', registry_namespace=exporterext)))
from __future__ import print_function __required_plugin_API_version__ = "WebSubmit File Metadata Plugin API 1.0" import sys from optparse import OptionParser from six import iteritems from invenio.legacy.bibdocfile.api import decompose_file from invenio.legacy.websubmit.config import ( InvenioWebSubmitFileMetadataRuntimeError) from invenio.legacy.websubmit.registry import file_metadata_plugins from invenio.utils.datastructures import LazyDict metadata_extractor_plugins = LazyDict(lambda: dict( filter(None, map(plugin_builder_function, file_metadata_plugins)))) def read_metadata(inputfile, force=None, remote=False, loginpw=None, verbose=0): """Return metadata extracted from given file as dictionary. Availability depends on input file format and installed plugins (return C{TypeError} if unsupported file format). @param inputfile: path to a file @type inputfile: string @param verbose: verbosity
"stopwords.kb" : {... ,'of':1, ... } It will always map the default stopwords knowledge base given by CFG_BIBRANK_PATH_TO_STOPWORDS_FILE. It is useful for bibrank module. """ stopwords_kb_map = {} stopwords_kb_map[ cfg['CFG_BIBRANK_PATH_TO_STOPWORDS_FILE']] = create_stopwords() index_stopwords = get_all_index_names_and_column_values("remove_stopwords") for index, stopwords in index_stopwords: if stopwords and stopwords != 'No': if stopwords not in stopwords_kb_map: stopwords_kb_map[stopwords] = create_stopwords(stopwords) return stopwords_kb_map stopwords_kb = LazyDict(map_stopwords_names_to_stopwords_kb) def is_stopword(word, stopwords=None): """Return true if WORD is found among stopwords for given index, false otherwise. It searches in the default stopwords knowledge base if stopwords_path is not specified which is useful for bibrank module. If one wants to search in diffrent stopwords knowledge base he must specify the path to stopwords file. :param word: word we want to check if it's stopword or not :param stopwords: name of stopwords knowledge base we want to search in """ # note: input word is assumed to be in lowercase if stopwords in stopwords_kb: if word in stopwords_kb[stopwords]: return True return False
if level > 4: return normpath = os.path.normpath(path) if os.path.isdir(normpath): for p in os.listdir(normpath): _register(os.path.join(normpath, p), level=level + 1) else: parts = normpath.split(os.path.sep) out[os.path.sep.join(parts[-level:])] = normpath for t in reversed(format_templates): _register(t) return out format_templates_lookup = LazyDict(create_format_templates_lookup) def create_output_formats_lookup(): """Create output formats.""" out = {} for f in output_formats: of = os.path.basename(f) if of in out: continue out[of] = f return out output_formats_lookup = LazyDict(create_output_formats_lookup)
from invenio.legacy.miscutil.data_cacher import DataCacher from invenio.utils.datastructures import LazyDict from .models import BsrMETHOD class BibSortDataCacher(DataCacher): """Cache holding all structures created by sorter.""" def __init__(self, method_name): """Initialize data cacher for given method.""" self.method_name = method_name def cache_filler(): """Return data to populate cache.""" method = BsrMETHOD.query.filter_by(name=self.method_name).first() return dict(method.get_cache()) if method is not None else {} def timestamp_verifier(): """Return string representing last update datetime.""" return BsrMETHOD.timestamp_verifier( self.method_name).strftime("%Y-%m-%d %H:%M:%S") DataCacher.__init__(self, cache_filler, timestamp_verifier) SORTING_METHODS = LazyDict(BsrMETHOD.get_sorting_methods) CACHE_SORTED_DATA = LazyDict( lambda: dict([(sorting_method, BibSortDataCacher(sorting_method)) for sorting_method in SORTING_METHODS]))
from invenio.ext.registry import ModuleAutoDiscoverySubRegistry from invenio.utils.datastructures import LazyDict legacy_modules = RegistryProxy('legacy', ImportPathRegistry, initial=['invenio.legacy.*']) webadmin_proxy = RegistryProxy('legacy.webadmin', \ ModuleAutoDiscoverySubRegistry, 'web.admin', registry_namespace=legacy_modules) def _admin_handler_name(name): parts = name.split('.') return '%s/%s' % (parts[2], parts[5]) webadmin = LazyDict(lambda: dict((_admin_handler_name(module.__name__), module) for module in webadmin_proxy)) webinterface_proxy = RegistryProxy( 'legacy.webinterface', ModuleAutoDiscoveryRegistry, 'webinterface', registry_namespace=legacy_modules) def _webinterface(module): from invenio.ext.legacy.handler import WebInterfaceDirectory parts = module.__name__.split('.') for value in dir(module): webinterface = getattr(module, value) if inspect.isclass(webinterface) and \ issubclass(webinterface, WebInterfaceDirectory) and \ webinterface.__module__ == module.__name__: yield webinterface.__name__, webinterface
# You should have received a copy of the GNU General Public License # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """Utility functions.""" import re from invenio.ext.cache import cache from invenio.ext.sqlalchemy import db from invenio.modules.knowledge.api import get_kbr_values from invenio.utils.datastructures import LazyDict from .models import IdxINDEX, IdxINDEXField from .registry import tokenizers field_tokenizer_cache = LazyDict( lambda: dict(IdxINDEXField.get_field_tokenizers())) @cache.memoize() def get_idx_indexer(name): """Return indexer field value.""" return db.session.query(IdxINDEX.indexer).filter_by(name=name).scalar() def load_tokenizers(): """Load all the bibindex tokenizers and returns it.""" return dict((module.__name__.split('.')[-1], getattr(module, module.__name__.split('.')[-1], '')) for module in tokenizers)
if level > 4: return normpath = os.path.normpath(path) if os.path.isdir(normpath): for p in os.listdir(normpath): _register(os.path.join(normpath, p), level=level + 1) else: parts = normpath.split(os.path.sep) out[os.path.sep.join(parts[-level:])] = normpath for t in reversed(format_templates): _register(t) return out format_templates_lookup = LazyDict(create_format_templates_lookup) def create_output_formats_lookup(): """Create output formats.""" out = {} for f in output_formats_files: of = os.path.basename(f).lower() data = {'names': {}} if of.endswith('.yml'): of = of[:-4] with open(f, 'r') as f: data.update(yaml.load(f) or {}) data['code'] = of else:
modules = autodiscover_user_settings() user_settings = {} for module in modules: candidates = getattr(module, 'settings') if candidates is not None: if type(candidates) is not list: candidates = [candidates] for candidate in candidates: if issubclass(candidate, Settings): if candidate.__name__ in user_settings: raise Exception(candidate.__name__, 'duplicate user settings') user_settings[candidate.__name__] = candidate return user_settings _USER_SETTINGS = LazyDict(load_user_settings) @blueprint.route('/', methods=['GET', 'POST']) @blueprint.route('/display', methods=['GET', 'POST']) @login_required @register_menu(blueprint, 'personalize', _('Personalize')) @register_breadcrumb(blueprint, '.', _('Your account')) def index(): # load plugins plugins = filter(lambda x: x.is_authorized and x.widget, map(lambda x: x(), _USER_SETTINGS.values())) closed_plugins = [] plugin_sort = (lambda w, x: x.index(w.name) if w.name in x else len(x)) dashboard_settings = current_user.get('dashboard_settings', {})
# This file is part of Invenio. # Copyright (C) 2014, 2015 CERN. # # Invenio is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 2 of the # License, or (at your option) any later version. # # Invenio is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """Scheduler registry definition.""" from flask_registry import RegistryProxy from invenio.ext.registry import ModuleAutoDiscoverySubRegistry from invenio.utils.datastructures import LazyDict tasklets_proxy = RegistryProxy('tasklets', ModuleAutoDiscoverySubRegistry, 'tasklets') tasklets = LazyDict(lambda: dict((module.__name__.split('.')[-1], getattr(module, module.__name__.split('.')[-1])) for module in tasklets_proxy))
# -*- coding: utf-8 -*- # # This file is part of Invenio. # Copyright (C) 2013, 2015 CERN. # # Invenio is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 2 of the # License, or (at your option) any later version. # # Invenio is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. from flask_registry import RegistryProxy from invenio.ext.registry import ModuleAutoDiscoverySubRegistry from invenio.utils.datastructures import LazyDict mixers_proxy = RegistryProxy('mixer', ModuleAutoDiscoverySubRegistry, 'mixer') mixers = LazyDict(lambda: dict( (getattr(module, mixer).__model__.__tablename__, getattr(module, mixer)) for module in mixers_proxy for mixer in module.__all__))
def _queries(): """Preprocess collection queries.""" from invenio.ext.sqlalchemy import db from invenio_collections.models import Collection return dict( (collection.name, dict(query=Query( COLLECTIONS_DELETED_RECORDS.format(dbquery=collection.dbquery)), ancestors=set(c.name for c in collection.ancestors if c.dbquery is None))) for collection in Collection.query.filter( Collection.dbquery.isnot(None), db.not_(Collection.dbquery.like('hostedcollection:%'))).all()) queries = LazyDict(_queries) def get_record_collections(record): """Return list of collections to which record belongs to. :record: Record instance :returns: list of collection names """ output = set() for name, data in iteritems(queries): if data['query'].match(record): output.add(name) output |= data['ancestors'] return list(output)
module.__name__.split('.')[-1])) for module in function_proxy(namespace)) return funcs parsers = RegistryProxy('jsonext.parsers', ModuleAutoDiscoverySubRegistry, 'parsers', registry_namespace=jsonext('jsonext')) producers_proxy = RegistryProxy('jsonext.producers', ModuleAutoDiscoverySubRegistry, 'producers', registry_namespace=jsonext('jsonext')) producers = LazyDict(lambda: dict((module.__name__.split('.')[-1], module. produce) for module in producers_proxy)) readers_proxy = RegistryProxy('jsonext.readers', ModuleAutoDiscoverySubRegistry, 'readers', registry_namespace=jsonext('jsonext')) readers = LazyDict(lambda: dict((module.reader.__master_format__, module.reader ) for module in readers_proxy)) contexts_proxy = lambda namespace: RegistryProxy( namespace + '.contexts', ModuleAutoDiscoverySubRegistry, 'contexts', registry_namespace=jsonext(namespace))
# General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. __all__ = ['previewerext', 'previewers'] from invenio.ext.registry import ModuleAutoDiscoverySubRegistry from flask_registry import RegistryProxy from invenio.utils.datastructures import LazyDict previewerext = RegistryProxy('previewerext', ModuleAutoDiscoverySubRegistry, 'previewerext') def dummy_can_preview(f): pass def dummy_preview(f, emb=None): pass previewers = LazyDict(lambda: dict( map( lambda m: (m.__name__.split('.')[-1], dict(can_preview=getattr(m, 'can_preview', dummy_can_preview), preview=getattr(m, 'preview', dummy_preview))), previewerext)))
from invenio.modules.knowledge import api as kapi from invenio.modules.search.models import Field from invenio.utils.datastructures import LazyDict from werkzeug.local import LocalProxy from wtforms import HiddenField from wtforms.fields import BooleanField from wtforms.fields import SelectField from wtforms_sqlalchemy.fields import QuerySelectMultipleField from .models import IdxINDEX from .utils import load_tokenizers _TOKENIZERS = LazyDict(load_tokenizers) tokenizer_implemented = LocalProxy(lambda: [ tokenizer for tokenizer in _TOKENIZERS if _TOKENIZERS[tokenizer]().implemented ]) class IdxINDEXAdmin(ModelView): """Flask-Admin module to manage index list.""" _can_create = True _can_edit = True _can_delete = True acc_view_action = 'cfgbibindex'
format_templates_lookup = LazyDict(create_format_templates_lookup) def create_output_formats_lookup(): """Create output formats.""" out = {} for f in output_formats_files: of = os.path.basename(f).lower() data = {'names': {}} if of.endswith('.yml'): of = of[:-4] with open(f, 'r') as f: data.update(yaml.load(f) or {}) data['code'] = of else: continue # unknown filetype if of in out: continue out[of] = data return out output_formats = LazyDict(create_output_formats_lookup) export_formats = LazyDict(lambda: dict( (code, of) for code, of in output_formats.items() if of.get('content_type', '') != 'text/html' and of.get('visibility', 0) ))
class PidProvider(object): """ Abstract class for persistent identifier provider classes. Subclasses must implement register, update, delete and is_provider_for_pid methods and register itself: class MyProvider(PidProvider): pid_type = "mypid" def reserve(self, pid, *args, **kwargs): return True def register(self, pid, *args, **kwargs): return True def update(self, pid, *args, **kwargs): return True def delete(self, pid, *args, **kwargs): try: ... except Exception as e: pid.log("DELETE","Deletion failed") return False else: pid.log("DELETE","Successfully deleted") return True def is_provider_for_pid(self, pid_str): pass PidProvider.register_provider(MyProvider) The provider is responsible for handling of errors, as well as logging of actions happening to the pid. See example above as well as the DataCitePidProvider. Each method takes variable number of argument and keywords arguments. This can be used to pass additional information to the provider when registering a persistent identifier. E.g. a DOI requires URL and metadata to be able to register the DOI. """ def __load_providers(): registry = dict() for provider_str in cfg['PIDSTORE_PROVIDERS']: provider = import_string(provider_str) if not issubclass(provider, PidProvider): raise TypeError("Argument not an instance of PidProvider.") pid_type = getattr(provider, 'pid_type', None) if pid_type is None: raise AttributeError( "Provider must specify class variable pid_type." ) pid_type = pid_type.lower() if pid_type not in registry: registry[pid_type] = [] # Prevent double registration if provider not in registry[pid_type]: registry[pid_type].append(provider) return registry registry = LazyDict(__load_providers) """ Registry of possible providers """ pid_type = None """ Must be overwritten in subcleass and specified as a string (max len 6) """ @staticmethod def create(pid_type, pid_str, pid_provider, *args, **kwargs): """ Create a new instance of a PidProvider for the given type and pid. """ providers = PidProvider.registry.get(pid_type.lower(), None) for p in providers: if p.is_provider_for_pid(pid_str): return p(*args, **kwargs) return None # # API methods which must be implemented by each provider. # def reserve(self, pid, *args, **kwargs): """ Reserve a new persistent identifier This might or might not be useful depending on the service of the provider. """ raise NotImplementedError def register(self, pid, *args, **kwargs): """ Register a new persistent identifier """ raise NotImplementedError def update(self, pid, *args, **kwargs): """ Update information about a persistent identifier """ raise NotImplementedError def delete(self, pid, *args, **kwargs): """ Delete a persistent identifier """ raise NotImplementedError def sync_status(self, pid, *args, **kwargs): """ Synchronize persistent identifier status with remote service provider. """ return True @classmethod def is_provider_for_pid(cls, pid_str): raise NotImplementedError # # API methods which might need to be implemented depending on each provider. # def create_new_pid(self, pid_value): """ Some PidProvider might have the ability to create new values """ return pid_value