def test___setitem(self):
lazy_dict = LazyDict()
lazy_dict.__setitem__('foo', 'bar')
lazy_dict.__setitem__('foo2', 'bar2')
lazy_dict.__setitem__('foo3', 'bar3')
self.assertEqual(lazy_dict['foo'], 'bar')
self.assertEqual(lazy_dict['foo3'], 'bar3')
self.assertEqual(lazy_dict['foo2'], 'bar2')
def test_itervalues(self):
populate = CallCounter(lambda: {'foo': 'bar', 1: 11, 'empty': None})
lazy_dict = LazyDict(populate)
iterator = lazy_dict.itervalues()
self.assertEqual(iterator.next(), 11)
self.assertEqual(iterator.next(), 'bar')
self.assertEqual(iterator.next(), None)
self.assertRaises(StopIteration, iterator.next)
def test_lazy_dictionary(self):
"""Checks content of lazy dictionary and number of evaluations."""
populate = CallCounter(lambda: {'foo': 'bar', 1: 11, 'empty': None})
lazy_dict = LazyDict(populate)
self.assertEqual(populate.counter, 0)
self.assertEqual(lazy_dict['foo'], 'bar')
self.assertEqual(populate.counter, 1)
self.assertEqual(lazy_dict[1], 11)
self.assertEqual(populate.counter, 1)
self.assertEqual(lazy_dict['empty'], None)
self.assertEqual(populate.counter, 1)
# clear the cache
lazy_dict.expunge()
self.assertEqual(lazy_dict['foo'], 'bar')
self.assertEqual(populate.counter, 2)
del lazy_dict['foo']
self.assertEqual(populate.counter, 2)
assert 'foo' not in lazy_dict
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""Registries for redirector module."""
from flask_registry import RegistryProxy
from invenio.ext.registry import ModuleAutoDiscoverySubRegistry
from invenio.utils.datastructures import LazyDict
redirector_proxy = RegistryProxy('redirect_methods',
ModuleAutoDiscoverySubRegistry,
'redirect_methods')
def register_redirect_methods():
"""Register redirect methods."""
out = {}
for module in redirector_proxy:
if hasattr(module, 'goto'):
out[module.__name__.split('.')[-1]] = module.goto
return out
def get_redirect_method(plugin_name):
"""Return the function from the plugin name."""
return redirect_methods[plugin_name]
redirect_methods = LazyDict(register_redirect_methods)
#
# Invenio is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA
import os
from flask_registry import PkgResourcesDirDiscoveryRegistry, \
ModuleAutoDiscoveryRegistry, RegistryProxy
from invenio.utils.datastructures import LazyDict
converterext = RegistryProxy('converterext', ModuleAutoDiscoveryRegistry,
'converterext')
kb = LazyDict(lambda: dict((os.path.basename(f), f) for f in RegistryProxy(
'converterext.kb',
PkgResourcesDirDiscoveryRegistry,
'kb',
registry_namespace=converterext)))
templates = LazyDict(lambda: dict((os.path.basename(
f), f) for f in RegistryProxy('converterext.templates',
PkgResourcesDirDiscoveryRegistry,
'templates',
registry_namespace=converterext)))
cur = db.cur()
cur.execute("UNLOCK TABLES")
except Exception:
pass
return app
def _db_conn():
current_app.teardown_appcontext_funcs.append(unlock_all)
out = {}
out[cfg['CFG_DATABASE_HOST']] = {}
out[cfg['CFG_DATABASE_SLAVE']] = {}
return out
connect = DBConnect()
_DB_CONN = LazyDict(_db_conn)
def _get_password_from_database_password_file(user):
"""Parse CFG_DATABASE_PASSWORD_FILE and return password for user."""
pwfile = cfg.get("CFG_DATABASE_PASSWORD_FILE", None)
if pwfile and os.path.exists(pwfile):
for row in open(pwfile):
if row.strip():
a_user, pwd = row.strip().split(" // ")
if user == a_user:
return pwd
raise ValueError("user '%s' not found in database password file '%s'"
% (user, pwfile))
raise IOError("No password defined for user '%s' but database password "
"file is not available" % user)
'id': (int, -1), 'idb': (int, -1), 'sysnb': (str, ""),
'action': (str, "search"),
'action_search': (str, ""),
'action_browse': (str, ""),
'd1': (str, ""),
'd1y': (int, 0), 'd1m': (int, 0), 'd1d': (int, 0),
'd2': (str, ""),
'd2y': (int, 0), 'd2m': (int, 0), 'd2d': (int, 0),
'dt': (str, ""),
'ap': (int, 1),
'verbose': (int, 0),
'ec': (list, []),
'wl': (int, cfg['CFG_WEBSEARCH_WILDCARD_LIMIT']),
}
search_results_default_urlargd = LazyDict(get_search_results_default_urlargd)
def wash_search_urlargd(form):
"""
Create canonical search arguments from those passed via web form.
"""
from invenio.ext.legacy.handler import wash_urlargd
argd = wash_urlargd(form, search_results_default_urlargd)
if 'as' in argd:
argd['aas'] = argd['as']
del argd['as']
if argd.get('aas', cfg['CFG_WEBSEARCH_DEFAULT_SEARCH_INTERFACE']) \
not in cfg['CFG_WEBSEARCH_ENABLED_SEARCH_INTERFACES']:
argd['aas'] = cfg['CFG_WEBSEARCH_DEFAULT_SEARCH_INTERFACE']
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA
import os
from flask_registry import PkgResourcesDirDiscoveryRegistry, \
ModuleAutoDiscoveryRegistry, RegistryProxy
from invenio.utils.datastructures import LazyDict
exporterext = RegistryProxy(
'exporterext', ModuleAutoDiscoveryRegistry, 'exportext'
)
configurations = LazyDict(
lambda: dict((os.path.basename(f), f)
for f in RegistryProxy('exporterext.configurations',
PkgResourcesDirDiscoveryRegistry,
'configurations',
registry_namespace=exporterext)))
from __future__ import print_function
__required_plugin_API_version__ = "WebSubmit File Metadata Plugin API 1.0"
import sys
from optparse import OptionParser
from six import iteritems
from invenio.legacy.bibdocfile.api import decompose_file
from invenio.legacy.websubmit.config import (
InvenioWebSubmitFileMetadataRuntimeError)
from invenio.legacy.websubmit.registry import file_metadata_plugins
from invenio.utils.datastructures import LazyDict
metadata_extractor_plugins = LazyDict(lambda: dict(
filter(None, map(plugin_builder_function, file_metadata_plugins))))
def read_metadata(inputfile,
force=None,
remote=False,
loginpw=None,
verbose=0):
"""Return metadata extracted from given file as dictionary.
Availability depends on input file format and installed plugins
(return C{TypeError} if unsupported file format).
@param inputfile: path to a file
@type inputfile: string
@param verbose: verbosity
"stopwords.kb" : {... ,'of':1, ... }
It will always map the default stopwords knowledge base given by
CFG_BIBRANK_PATH_TO_STOPWORDS_FILE. It is useful for bibrank module.
"""
stopwords_kb_map = {}
stopwords_kb_map[
cfg['CFG_BIBRANK_PATH_TO_STOPWORDS_FILE']] = create_stopwords()
index_stopwords = get_all_index_names_and_column_values("remove_stopwords")
for index, stopwords in index_stopwords:
if stopwords and stopwords != 'No':
if stopwords not in stopwords_kb_map:
stopwords_kb_map[stopwords] = create_stopwords(stopwords)
return stopwords_kb_map
stopwords_kb = LazyDict(map_stopwords_names_to_stopwords_kb)
def is_stopword(word, stopwords=None):
"""Return true if WORD is found among stopwords for given index, false otherwise.
It searches in the default stopwords knowledge base if stopwords_path is not specified
which is useful for bibrank module. If one wants to search in diffrent stopwords knowledge base
he must specify the path to stopwords file.
:param word: word we want to check if it's stopword or not
:param stopwords: name of stopwords knowledge base we want to search in
"""
# note: input word is assumed to be in lowercase
if stopwords in stopwords_kb:
if word in stopwords_kb[stopwords]:
return True
return False
if level > 4:
return
normpath = os.path.normpath(path)
if os.path.isdir(normpath):
for p in os.listdir(normpath):
_register(os.path.join(normpath, p), level=level + 1)
else:
parts = normpath.split(os.path.sep)
out[os.path.sep.join(parts[-level:])] = normpath
for t in reversed(format_templates):
_register(t)
return out
format_templates_lookup = LazyDict(create_format_templates_lookup)
def create_output_formats_lookup():
"""Create output formats."""
out = {}
for f in output_formats:
of = os.path.basename(f)
if of in out:
continue
out[of] = f
return out
output_formats_lookup = LazyDict(create_output_formats_lookup)
from invenio.legacy.miscutil.data_cacher import DataCacher
from invenio.utils.datastructures import LazyDict
from .models import BsrMETHOD
class BibSortDataCacher(DataCacher):
"""Cache holding all structures created by sorter."""
def __init__(self, method_name):
"""Initialize data cacher for given method."""
self.method_name = method_name
def cache_filler():
"""Return data to populate cache."""
method = BsrMETHOD.query.filter_by(name=self.method_name).first()
return dict(method.get_cache()) if method is not None else {}
def timestamp_verifier():
"""Return string representing last update datetime."""
return BsrMETHOD.timestamp_verifier(
self.method_name).strftime("%Y-%m-%d %H:%M:%S")
DataCacher.__init__(self, cache_filler, timestamp_verifier)
SORTING_METHODS = LazyDict(BsrMETHOD.get_sorting_methods)
CACHE_SORTED_DATA = LazyDict(
lambda: dict([(sorting_method, BibSortDataCacher(sorting_method))
for sorting_method in SORTING_METHODS]))
from invenio.ext.registry import ModuleAutoDiscoverySubRegistry
from invenio.utils.datastructures import LazyDict
legacy_modules = RegistryProxy('legacy', ImportPathRegistry,
initial=['invenio.legacy.*'])
webadmin_proxy = RegistryProxy('legacy.webadmin', \
ModuleAutoDiscoverySubRegistry, 'web.admin',
registry_namespace=legacy_modules)
def _admin_handler_name(name):
parts = name.split('.')
return '%s/%s' % (parts[2], parts[5])
webadmin = LazyDict(lambda: dict((_admin_handler_name(module.__name__), module)
for module in webadmin_proxy))
webinterface_proxy = RegistryProxy(
'legacy.webinterface', ModuleAutoDiscoveryRegistry, 'webinterface',
registry_namespace=legacy_modules)
def _webinterface(module):
from invenio.ext.legacy.handler import WebInterfaceDirectory
parts = module.__name__.split('.')
for value in dir(module):
webinterface = getattr(module, value)
if inspect.isclass(webinterface) and \
issubclass(webinterface, WebInterfaceDirectory) and \
webinterface.__module__ == module.__name__:
yield webinterface.__name__, webinterface
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""Utility functions."""
import re
from invenio.ext.cache import cache
from invenio.ext.sqlalchemy import db
from invenio.modules.knowledge.api import get_kbr_values
from invenio.utils.datastructures import LazyDict
from .models import IdxINDEX, IdxINDEXField
from .registry import tokenizers
field_tokenizer_cache = LazyDict(
lambda: dict(IdxINDEXField.get_field_tokenizers()))
@cache.memoize()
def get_idx_indexer(name):
"""Return indexer field value."""
return db.session.query(IdxINDEX.indexer).filter_by(name=name).scalar()
def load_tokenizers():
"""Load all the bibindex tokenizers and returns it."""
return dict((module.__name__.split('.')[-1],
getattr(module,
module.__name__.split('.')[-1], ''))
for module in tokenizers)
if level > 4:
return
normpath = os.path.normpath(path)
if os.path.isdir(normpath):
for p in os.listdir(normpath):
_register(os.path.join(normpath, p), level=level + 1)
else:
parts = normpath.split(os.path.sep)
out[os.path.sep.join(parts[-level:])] = normpath
for t in reversed(format_templates):
_register(t)
return out
format_templates_lookup = LazyDict(create_format_templates_lookup)
def create_output_formats_lookup():
"""Create output formats."""
out = {}
for f in output_formats_files:
of = os.path.basename(f).lower()
data = {'names': {}}
if of.endswith('.yml'):
of = of[:-4]
with open(f, 'r') as f:
data.update(yaml.load(f) or {})
data['code'] = of
else:
modules = autodiscover_user_settings()
user_settings = {}
for module in modules:
candidates = getattr(module, 'settings')
if candidates is not None:
if type(candidates) is not list:
candidates = [candidates]
for candidate in candidates:
if issubclass(candidate, Settings):
if candidate.__name__ in user_settings:
raise Exception(candidate.__name__,
'duplicate user settings')
user_settings[candidate.__name__] = candidate
return user_settings
_USER_SETTINGS = LazyDict(load_user_settings)
@blueprint.route('/', methods=['GET', 'POST'])
@blueprint.route('/display', methods=['GET', 'POST'])
@login_required
@register_menu(blueprint, 'personalize', _('Personalize'))
@register_breadcrumb(blueprint, '.', _('Your account'))
def index():
# load plugins
plugins = filter(lambda x: x.is_authorized and x.widget,
map(lambda x: x(), _USER_SETTINGS.values()))
closed_plugins = []
plugin_sort = (lambda w, x: x.index(w.name) if w.name in x else len(x))
dashboard_settings = current_user.get('dashboard_settings', {})
# This file is part of Invenio.
# Copyright (C) 2014, 2015 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""Scheduler registry definition."""
from flask_registry import RegistryProxy
from invenio.ext.registry import ModuleAutoDiscoverySubRegistry
from invenio.utils.datastructures import LazyDict
tasklets_proxy = RegistryProxy('tasklets', ModuleAutoDiscoverySubRegistry,
'tasklets')
tasklets = LazyDict(lambda: dict((module.__name__.split('.')[-1],
getattr(module,
module.__name__.split('.')[-1]))
for module in tasklets_proxy))
# -*- coding: utf-8 -*-
#
# This file is part of Invenio.
# Copyright (C) 2013, 2015 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
from flask_registry import RegistryProxy
from invenio.ext.registry import ModuleAutoDiscoverySubRegistry
from invenio.utils.datastructures import LazyDict
mixers_proxy = RegistryProxy('mixer', ModuleAutoDiscoverySubRegistry, 'mixer')
mixers = LazyDict(lambda: dict(
(getattr(module, mixer).__model__.__tablename__, getattr(module, mixer))
for module in mixers_proxy for mixer in module.__all__))
def _queries():
"""Preprocess collection queries."""
from invenio.ext.sqlalchemy import db
from invenio_collections.models import Collection
return dict(
(collection.name,
dict(query=Query(
COLLECTIONS_DELETED_RECORDS.format(dbquery=collection.dbquery)),
ancestors=set(c.name for c in collection.ancestors
if c.dbquery is None)))
for collection in Collection.query.filter(
Collection.dbquery.isnot(None),
db.not_(Collection.dbquery.like('hostedcollection:%'))).all())
queries = LazyDict(_queries)
def get_record_collections(record):
"""Return list of collections to which record belongs to.
:record: Record instance
:returns: list of collection names
"""
output = set()
for name, data in iteritems(queries):
if data['query'].match(record):
output.add(name)
output |= data['ancestors']
return list(output)
module.__name__.split('.')[-1]))
for module in function_proxy(namespace))
return funcs
parsers = RegistryProxy('jsonext.parsers',
ModuleAutoDiscoverySubRegistry,
'parsers',
registry_namespace=jsonext('jsonext'))
producers_proxy = RegistryProxy('jsonext.producers',
ModuleAutoDiscoverySubRegistry,
'producers',
registry_namespace=jsonext('jsonext'))
producers = LazyDict(lambda: dict((module.__name__.split('.')[-1], module.
produce) for module in producers_proxy))
readers_proxy = RegistryProxy('jsonext.readers',
ModuleAutoDiscoverySubRegistry,
'readers',
registry_namespace=jsonext('jsonext'))
readers = LazyDict(lambda: dict((module.reader.__master_format__, module.reader
) for module in readers_proxy))
contexts_proxy = lambda namespace: RegistryProxy(
namespace + '.contexts',
ModuleAutoDiscoverySubRegistry,
'contexts',
registry_namespace=jsonext(namespace))
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
__all__ = ['previewerext', 'previewers']
from invenio.ext.registry import ModuleAutoDiscoverySubRegistry
from flask_registry import RegistryProxy
from invenio.utils.datastructures import LazyDict
previewerext = RegistryProxy('previewerext', ModuleAutoDiscoverySubRegistry,
'previewerext')
def dummy_can_preview(f):
pass
def dummy_preview(f, emb=None):
pass
previewers = LazyDict(lambda: dict(
map(
lambda m:
(m.__name__.split('.')[-1],
dict(can_preview=getattr(m, 'can_preview', dummy_can_preview),
preview=getattr(m, 'preview', dummy_preview))), previewerext)))
from invenio.modules.knowledge import api as kapi
from invenio.modules.search.models import Field
from invenio.utils.datastructures import LazyDict
from werkzeug.local import LocalProxy
from wtforms import HiddenField
from wtforms.fields import BooleanField
from wtforms.fields import SelectField
from wtforms_sqlalchemy.fields import QuerySelectMultipleField
from .models import IdxINDEX
from .utils import load_tokenizers
_TOKENIZERS = LazyDict(load_tokenizers)
tokenizer_implemented = LocalProxy(lambda: [
tokenizer for tokenizer in _TOKENIZERS
if _TOKENIZERS[tokenizer]().implemented
])
class IdxINDEXAdmin(ModelView):
"""Flask-Admin module to manage index list."""
_can_create = True
_can_edit = True
_can_delete = True
acc_view_action = 'cfgbibindex'
format_templates_lookup = LazyDict(create_format_templates_lookup)
def create_output_formats_lookup():
"""Create output formats."""
out = {}
for f in output_formats_files:
of = os.path.basename(f).lower()
data = {'names': {}}
if of.endswith('.yml'):
of = of[:-4]
with open(f, 'r') as f:
data.update(yaml.load(f) or {})
data['code'] = of
else:
continue # unknown filetype
if of in out:
continue
out[of] = data
return out
output_formats = LazyDict(create_output_formats_lookup)
export_formats = LazyDict(lambda: dict(
(code, of) for code, of in output_formats.items()
if of.get('content_type', '') != 'text/html' and of.get('visibility', 0)
))
class PidProvider(object):
"""
Abstract class for persistent identifier provider classes.
Subclasses must implement register, update, delete and is_provider_for_pid
methods and register itself:
class MyProvider(PidProvider):
pid_type = "mypid"
def reserve(self, pid, *args, **kwargs):
return True
def register(self, pid, *args, **kwargs):
return True
def update(self, pid, *args, **kwargs):
return True
def delete(self, pid, *args, **kwargs):
try:
...
except Exception as e:
pid.log("DELETE","Deletion failed")
return False
else:
pid.log("DELETE","Successfully deleted")
return True
def is_provider_for_pid(self, pid_str):
pass
PidProvider.register_provider(MyProvider)
The provider is responsible for handling of errors, as well as logging of
actions happening to the pid. See example above as well as the
DataCitePidProvider.
Each method takes variable number of argument and keywords arguments. This
can be used to pass additional information to the provider when registering
a persistent identifier. E.g. a DOI requires URL and metadata to be able
to register the DOI.
"""
def __load_providers():
registry = dict()
for provider_str in cfg['PIDSTORE_PROVIDERS']:
provider = import_string(provider_str)
if not issubclass(provider, PidProvider):
raise TypeError("Argument not an instance of PidProvider.")
pid_type = getattr(provider, 'pid_type', None)
if pid_type is None:
raise AttributeError(
"Provider must specify class variable pid_type."
)
pid_type = pid_type.lower()
if pid_type not in registry:
registry[pid_type] = []
# Prevent double registration
if provider not in registry[pid_type]:
registry[pid_type].append(provider)
return registry
registry = LazyDict(__load_providers)
""" Registry of possible providers """
pid_type = None
"""
Must be overwritten in subcleass and specified as a string (max len 6)
"""
@staticmethod
def create(pid_type, pid_str, pid_provider, *args, **kwargs):
"""
Create a new instance of a PidProvider for the
given type and pid.
"""
providers = PidProvider.registry.get(pid_type.lower(), None)
for p in providers:
if p.is_provider_for_pid(pid_str):
return p(*args, **kwargs)
return None
#
# API methods which must be implemented by each provider.
#
def reserve(self, pid, *args, **kwargs):
"""
Reserve a new persistent identifier
This might or might not be useful depending on the service of the
provider.
"""
raise NotImplementedError
def register(self, pid, *args, **kwargs):
""" Register a new persistent identifier """
raise NotImplementedError
def update(self, pid, *args, **kwargs):
""" Update information about a persistent identifier """
raise NotImplementedError
def delete(self, pid, *args, **kwargs):
""" Delete a persistent identifier """
raise NotImplementedError
def sync_status(self, pid, *args, **kwargs):
"""
Synchronize persistent identifier status with remote service provider.
"""
return True
@classmethod
def is_provider_for_pid(cls, pid_str):
raise NotImplementedError
#
# API methods which might need to be implemented depending on each provider.
#
def create_new_pid(self, pid_value):
""" Some PidProvider might have the ability to create new values """
return pid_value
