def test___setitem(self):
lazy_dict = LazyDict()
lazy_dict.__setitem__('foo', 'bar')
lazy_dict.__setitem__('foo2', 'bar2')
lazy_dict.__setitem__('foo3', 'bar3')
self.assertEqual(lazy_dict['foo'], 'bar')
self.assertEqual(lazy_dict['foo3'], 'bar3')
self.assertEqual(lazy_dict['foo2'], 'bar2')
def test_itervalues(self):
populate = CallCounter(lambda: {'foo': 'bar', 1: 11, 'empty': None})
lazy_dict = LazyDict(populate)
iterator = lazy_dict.itervalues()
self.assertEqual(iterator.next(), 11)
self.assertEqual(iterator.next(), 'bar')
self.assertEqual(iterator.next(), None)
self.assertRaises(StopIteration, iterator.next)
def run(output):
"""Convert *bfo* to *yml*."""
from invenio.legacy.dbquery import run_sql
from flask_registry import PkgResourcesDirDiscoveryRegistry, \
ModuleAutoDiscoveryRegistry, RegistryProxy
from invenio_utils.datastructures import LazyDict
output_formats_directories = RegistryProxy(
'legacy_output_formats_directories', ModuleAutoDiscoveryRegistry,
'output_formats')
output_formats = RegistryProxy(
'legacy_output_formats',
PkgResourcesDirDiscoveryRegistry,
'.',
registry_namespace=output_formats_directories)
def create_output_formats_lookup():
"""Create output formats."""
out = {}
for f in output_formats:
of = os.path.basename(f).lower()
if not of.endswith('.bfo'):
continue
of = of[:-4]
if of in out:
continue
out[of] = f
return out
output_formats_lookup = LazyDict(create_output_formats_lookup)
for row in run_sql('SELECT id, name, code, description, content_type, '
' mime_type, visibility FROM format'):
code = row[2].lower()
out = {
'name': row[1],
'description': row[3],
'visibility': 1 if row[6] == '1' else 0,
'content_type': row[4],
}
if row[5]:
out['mime_type'] = row[5]
try:
out.update(get_output_format(output_formats_lookup[code]))
with open(os.path.join(output, code + '.yml'), 'w') as f:
yaml.dump(out, stream=f)
# print('echo "{0}" > {1}/{2}.yml'.format(
# yaml.dump(out), output, code))
except Exception:
current_app.logger.exception(
"Could not convert '{0}'".format(code))
def test_lazy_dictionary(self):
"""Checks content of lazy dictionary and number of evaluations."""
populate = CallCounter(lambda: {'foo': 'bar', 1: 11, 'empty': None})
lazy_dict = LazyDict(populate)
self.assertEqual(populate.counter, 0)
self.assertEqual(lazy_dict['foo'], 'bar')
self.assertEqual(populate.counter, 1)
self.assertEqual(lazy_dict[1], 11)
self.assertEqual(populate.counter, 1)
self.assertEqual(lazy_dict['empty'], None)
self.assertEqual(populate.counter, 1)
# clear the cache
lazy_dict.expunge()
self.assertEqual(lazy_dict['foo'], 'bar')
self.assertEqual(populate.counter, 2)
del lazy_dict['foo']
self.assertEqual(populate.counter, 2)
assert 'foo' not in lazy_dict
cur.execute("UNLOCK TABLES")
except Exception:
pass
return app
def _db_conn():
current_app.teardown_appcontext_funcs.append(unlock_all)
out = {}
out[cfg['CFG_DATABASE_HOST']] = {}
out[cfg['CFG_DATABASE_SLAVE']] = {}
return out
connect = DBConnect()
_DB_CONN = LazyDict(_db_conn)
def _get_password_from_database_password_file(user):
"""Parse CFG_DATABASE_PASSWORD_FILE and return password for user."""
pwfile = cfg.get("CFG_DATABASE_PASSWORD_FILE", None)
if pwfile and os.path.exists(pwfile):
for row in open(pwfile):
if row.strip():
a_user, pwd = row.strip().split(" // ")
if user == a_user:
return pwd
raise ValueError("user '%s' not found in database password file '%s'" %
(user, pwfile))
raise IOError("No password defined for user '%s' but database password "
"file is not available" % user)
def to_pathdict(self, test):
"""Return LazyDict representation."""
return LazyDict(lambda: dict(
(os.path.basename(f), f) for f in self if test(f)))
if level > 4:
return
normpath = os.path.normpath(path)
if os.path.isdir(normpath):
for p in os.listdir(normpath):
_register(os.path.join(normpath, p), level=level + 1)
else:
parts = normpath.split(os.path.sep)
out[os.path.sep.join(parts[-level:])] = normpath
for t in reversed(format_templates):
_register(t)
return out
format_templates_lookup = LazyDict(create_format_templates_lookup)
def create_output_formats_lookup():
"""Create output formats."""
out = {}
for f in output_formats_files:
of = os.path.basename(f).lower()
data = {'names': {}}
if of.endswith('.yml'):
of = of[:-4]
with open(f, 'r') as f:
data.update(yaml.load(f) or {})
data['code'] = of
else:
'id': (int, -1), 'idb': (int, -1), 'sysnb': (str, ""),
'action': (str, "search"),
'action_search': (str, ""),
'action_browse': (str, ""),
'd1': (str, ""),
'd1y': (int, 0), 'd1m': (int, 0), 'd1d': (int, 0),
'd2': (str, ""),
'd2y': (int, 0), 'd2m': (int, 0), 'd2d': (int, 0),
'dt': (str, ""),
'ap': (int, 1),
'verbose': (int, 0),
'ec': (list, []),
'wl': (int, cfg['CFG_WEBSEARCH_WILDCARD_LIMIT']),
}
search_results_default_urlargd = LazyDict(get_search_results_default_urlargd)
def wash_pattern(p):
"""Wash pattern passed in URL.
Check for sanity of the wildcard by removing wildcards if they are appended
to extremely short words (1-3 letters).
"""
# strip accents:
# p = strip_accents(p) # FIXME: when available, strip accents all the time
# add leading/trailing whitespace for the two following wildcard-sanity
# checking regexps:
p = " " + p + " "
# replace spaces within quotes by __SPACE__ temporarily:
p = re_pattern_single_quotes.sub(
format_templates_lookup = LazyDict(create_format_templates_lookup)
def create_output_formats_lookup():
"""Create output formats."""
out = {}
for f in output_formats_files:
of = os.path.basename(f).lower()
data = {'names': {}}
if of.endswith('.yml'):
of = of[:-4]
with open(f, 'r') as f:
data.update(yaml.load(f) or {})
data['code'] = of
else:
continue # unknown filetype
if of in out:
continue
out[of] = data
return out
output_formats = LazyDict(create_output_formats_lookup)
export_formats = LazyDict(lambda: dict(
(code, of) for code, of in output_formats.items()
if of.get('content_type', '') != 'text/html' and of.get('visibility', 0)
))
legacy_modules = RegistryProxy('legacy',
ImportPathRegistry,
initial=['invenio.legacy.*'])
webadmin_proxy = RegistryProxy('legacy.webadmin', \
ModuleAutoDiscoverySubRegistry, 'web.admin',
registry_namespace=legacy_modules)
def _admin_handler_name(name):
parts = name.split('.')
return '%s/%s' % (parts[2], parts[5])
webadmin = LazyDict(lambda: dict((_admin_handler_name(module.__name__), module)
for module in webadmin_proxy))
webinterface_proxy = RegistryProxy('legacy.webinterface',
ModuleAutoDiscoveryRegistry,
'webinterface',
registry_namespace=legacy_modules)
def _webinterface(module):
from invenio.ext.legacy.handler import WebInterfaceDirectory
parts = module.__name__.split('.')
for value in dir(module):
webinterface = getattr(module, value)
if inspect.isclass(webinterface) and \
issubclass(webinterface, WebInterfaceDirectory) and \
webinterface.__module__ == module.__name__:
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""Registries for redirector module."""
from flask_registry import RegistryProxy
from invenio_ext.registry import ModuleAutoDiscoverySubRegistry
from invenio_utils.datastructures import LazyDict
redirector_proxy = RegistryProxy('redirect_methods',
ModuleAutoDiscoverySubRegistry,
'redirect_methods')
def register_redirect_methods():
"""Register redirect methods."""
out = {}
for module in redirector_proxy:
if hasattr(module, 'goto'):
out[module.__name__.split('.')[-1]] = module.goto
return out
def get_redirect_method(plugin_name):
"""Return the function from the plugin name."""
return redirect_methods[plugin_name]
redirect_methods = LazyDict(register_redirect_methods)
import os
from flask_registry import PkgResourcesDirDiscoveryRegistry, \
ModuleAutoDiscoveryRegistry, RegistryProxy
from invenio_utils.datastructures import LazyDict
classifierext = RegistryProxy('classifierext', ModuleAutoDiscoveryRegistry,
'classifierext')
taxonomies_proxy = RegistryProxy('classifierext.taxonomies',
PkgResourcesDirDiscoveryRegistry,
'taxonomies',
registry_namespace=classifierext)
taxonomies = LazyDict(lambda: dict(
(os.path.basename(f), f) for f in taxonomies_proxy))
kb = LazyDict(lambda: dict((os.path.basename(f), f) for f in RegistryProxy(
'converterext.kb',
PkgResourcesDirDiscoveryRegistry,
'kb',
registry_namespace=classifierext)))
templates = LazyDict(lambda: dict((os.path.basename(
f), f) for f in RegistryProxy('converterext.templates',
PkgResourcesDirDiscoveryRegistry,
'templates',
registry_namespace=classifierext)))
__all__ = ('classfierext', 'taxonomies_proxy', 'taxonomies', 'kb', 'templates')
# -*- coding: utf-8 -*-
#
# This file is part of Invenio.
# Copyright (C) 2013, 2015 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
from flask_registry import RegistryProxy
from invenio_ext.registry import ModuleAutoDiscoverySubRegistry
from invenio_utils.datastructures import LazyDict
mixers_proxy = RegistryProxy('mixer', ModuleAutoDiscoverySubRegistry, 'mixer')
mixers = LazyDict(lambda: dict(
(getattr(module, mixer).__model__.__tablename__, getattr(module, mixer))
for module in mixers_proxy for mixer in module.__all__))
def _queries():
"""Preprocess collection queries."""
from invenio_ext.sqlalchemy import db
from invenio_collections.models import Collection
return dict(
(collection.name,
dict(query=Query(
COLLECTIONS_DELETED_RECORDS.format(dbquery=collection.dbquery)),
ancestors=set(c.name for c in collection.ancestors
if c.dbquery is None)))
for collection in Collection.query.filter(
Collection.dbquery.isnot(None),
db.not_(Collection.dbquery.like('hostedcollection:%'))).all())
queries = LazyDict(_queries)
def get_record_collections(record):
"""Return list of collections to which record belongs to.
:record: Record instance
:returns: list of collection names
"""
output = set()
for name, data in iteritems(queries):
if data['query'].match(record):
output.add(name)
output |= data['ancestors']
return list(output)
class PidProvider(object):
"""Abstract class for persistent identifier provider classes.
Subclasses must implement register, update, delete and is_provider_for_pid
methods and register itself:
class MyProvider(PidProvider):
pid_type = "mypid"
def reserve(self, pid, *args, **kwargs):
return True
def register(self, pid, *args, **kwargs):
return True
def update(self, pid, *args, **kwargs):
return True
def delete(self, pid, *args, **kwargs):
try:
...
except Exception as e:
pid.log("DELETE","Deletion failed")
return False
else:
pid.log("DELETE","Successfully deleted")
return True
def is_provider_for_pid(self, pid_str):
pass
PidProvider.register_provider(MyProvider)
The provider is responsible for handling of errors, as well as logging of
actions happening to the pid. See example above as well as the
DataCitePidProvider.
Each method takes variable number of argument and keywords arguments. This
can be used to pass additional information to the provider when registering
a persistent identifier. E.g. a DOI requires URL and metadata to be able to
register the DOI.
"""
def __load_providers():
registry = dict()
for provider_str in cfg['PIDSTORE_PROVIDERS']:
provider = import_string(provider_str)
if not issubclass(provider, PidProvider):
raise TypeError("Argument not an instance of PidProvider.")
pid_type = getattr(provider, 'pid_type', None)
if pid_type is None:
raise AttributeError(
"Provider must specify class variable pid_type."
)
pid_type = pid_type.lower()
if pid_type not in registry:
registry[pid_type] = []
# Prevent double registration
if provider not in registry[pid_type]:
registry[pid_type].append(provider)
return registry
registry = LazyDict(__load_providers)
"""Registry of possible providers."""
pid_type = None
"""
Must be overwritten in subcleass and specified as a string (max len 6)
"""
@staticmethod
def create(pid_type, pid_str, pid_provider, *args, **kwargs):
"""Create a new instance for the given type and pid."""
providers = PidProvider.registry.get(pid_type.lower(), None)
for p in providers:
if p.is_provider_for_pid(pid_str):
return p(*args, **kwargs)
return None
#
# API methods which must be implemented by each provider.
#
def reserve(self, pid, *args, **kwargs):
"""Reserve a new persistent identifier.
This might or might not be useful depending on the service of the
provider.
"""
raise NotImplementedError
def register(self, pid, *args, **kwargs):
"""Register a new persistent identifier."""
raise NotImplementedError
def update(self, pid, *args, **kwargs):
"""Update information about a persistent identifier."""
raise NotImplementedError
def delete(self, pid, *args, **kwargs):
"""Delete a persistent identifier."""
raise NotImplementedError
def sync_status(self, pid, *args, **kwargs):
"""Synchronize PIDstatus with remote service provider."""
return True
@classmethod
def is_provider_for_pid(cls, pid_str):
raise NotImplementedError
#
# API methods which might need to be implemented depending on each
# provider.
#
def create_new_pid(self, pid_value):
"""Some PidProvider might have the ability to create new values."""
return pid_value
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""Registry definition for fixture datasets."""
from flask_registry import RegistryProxy
from invenio_ext.registry import ModuleAutoDiscoveryRegistry
from invenio_utils.datastructures import LazyDict
fixtures_proxy = RegistryProxy(
'fixtures', ModuleAutoDiscoveryRegistry, 'fixtures')
def fixtures_loader():
"""Load fixtures datasets."""
out = {}
for fixture in fixtures_proxy:
for data in getattr(fixture, '__all__', dir(fixture)):
if data[-4:] != 'Data' or data in out:
continue
out[data] = getattr(fixture, data)
return out
fixtures = LazyDict(fixtures_loader)
