def build_and_run_web_test_suite():
"""Build and run the web tests.
Detect all Invenio modules with names ending by
'*_web_tests.py', build a complete test suite of them, and
run it. Called by 'inveniocfg --run-web-tests'.
"""
from invenio.config import CFG_PYLIBDIR
from invenio.pluginutils import PluginContainer
test_modules_map = PluginContainer(
os.path.join(CFG_PYLIBDIR, 'invenio', '*_web_tests.py'),
lambda plugin_name, plugin_code: getattr(plugin_code, "TEST_SUITE"))
test_modules = test_modules_map.values()
broken_tests = test_modules_map.get_broken_plugins()
broken_web_tests = ['%s (reason: %s)' % (name, broken_tests[name][1])
for name in broken_tests]
if broken_web_tests:
warn("Broken web tests suites found: %s" % ', '.join(broken_web_tests))
warn_user_about_tests()
complete_suite = unittest.TestSuite(test_modules)
runner = unittest.TextTestRunner(descriptions=False, verbosity=2)
return runner.run(complete_suite).wasSuccessful()
def test_plugin_container_wrapping_external_authentications(self):
"""pluginutils - wrapping external authentications"""
def plugin_builder(plugin_name, plugin_code):
for name in dir(plugin_code):
candidate = getattr(plugin_code, name)
try:
if issubclass(candidate, ExternalAuth):
return candidate
except TypeError:
pass
raise ValueError(
'%s is not a valid external authentication plugin' %
plugin_name)
external_authentications = PluginContainer(
os.path.join(CFG_PYLIBDIR, 'invenio',
'external_authentication_*.py'),
plugin_signature=ExternalAuth,
plugin_builder=plugin_builder)
self.failUnless(
issubclass(external_authentications['external_authentication_sso'],
ExternalAuth))
self.failIf(
external_authentications.get(
'external_authentication_cern_wrapper'))
self.failUnless(len(external_authentications) >= 1)
self.failUnless(
len(external_authentications.get_broken_plugins()) >= 2)
def build_and_run_web_test_suite():
"""Build and run the web tests.
Detect all Invenio modules with names ending by
'*_web_tests.py', build a complete test suite of them, and
run it. Called by 'inveniocfg --run-web-tests'.
"""
from invenio.config import CFG_PYLIBDIR
from invenio.pluginutils import PluginContainer
test_modules_map = PluginContainer(
os.path.join(CFG_PYLIBDIR, 'invenio', '*_web_tests.py'),
lambda plugin_name, plugin_code: getattr(plugin_code, "TEST_SUITE"))
test_modules = test_modules_map.values()
broken_tests = test_modules_map.get_broken_plugins()
broken_web_tests = [
'%s (reason: %s)' % (name, broken_tests[name][1])
for name in broken_tests
]
if broken_web_tests:
warn("Broken web tests suites found: %s" % ', '.join(broken_web_tests))
warn_user_about_tests()
complete_suite = unittest.TestSuite(test_modules)
runner = unittest.TextTestRunner(descriptions=False, verbosity=2)
return runner.run(complete_suite).wasSuccessful()
def load_ticket_plugins():
"""
Will load all the ticket plugins found under CFG_BIBCATALOG_PLUGIN_DIR.
Returns a tuple of plugin_object, list of errors.
"""
# TODO add to configfile
CFG_BIBCATALOG_PLUGIN_DIR = os.path.join(CFG_PYLIBDIR,
"invenio",
"bibcatalog_ticket_templates",
"*.py")
# Load plugins
plugins = PluginContainer(CFG_BIBCATALOG_PLUGIN_DIR,
plugin_builder=_bibcatalog_plugin_builder)
# Remove __init__ if applicable
try:
plugins.disable_plugin("__init__")
except KeyError:
pass
error_messages = []
# Check for broken plug-ins
broken = plugins.get_broken_plugins()
if broken:
error_messages = []
for plugin, info in broken.items():
error_messages.append("Failed to load %s:\n"
" %s" % (plugin, "".join(traceback.format_exception(*info))))
return plugins, error_messages
def load_ticket_plugins():
"""
Will load all the ticket plugins found under CFG_BIBEDIT_PLUGIN_DIR.
Returns a tuple of plugin_object, list of errors.
"""
# TODO add to configfile
CFG_BIBEDIT_PLUGIN_DIR = os.path.join(CFG_PYLIBDIR, "invenio",
"bibedit_ticket_templates", "*.py")
# Load plugins
plugins = PluginContainer(CFG_BIBEDIT_PLUGIN_DIR,
plugin_builder=_bibedit_plugin_builder)
# Remove __init__ if applicable
try:
plugins.disable_plugin("__init__")
except KeyError:
pass
error_messages = []
# Check for broken plug-ins
broken = plugins.get_broken_plugins()
if broken:
error_messages = []
for plugin, info in broken.items():
error_messages.append(
"Failed to load %s:\n"
" %s" % (plugin, "".join(traceback.format_exception(*info))))
return plugins, error_messages
def build_and_run_unit_test_suite():
"""
Detect all Invenio modules with names ending by '*_unit_tests.py', build
a complete test suite of them, and run it.
Called by 'inveniocfg --run-unit-tests'.
"""
from invenio.pluginutils import PluginContainer
test_modules_map = PluginContainer(
os.path.join(CFG_PYLIBDIR, 'invenio', '*_unit_tests.py'),
lambda plugin_name, plugin_code: getattr(plugin_code, "TEST_SUITE"))
test_modules = [test_modules_map[name] for name in test_modules_map]
broken_tests = test_modules_map.get_broken_plugins()
broken_unit_tests = [
'%s (reason: %s)' % (name, broken_tests[name][1])
for name in broken_tests
]
if broken_unit_tests:
warn("Broken unit tests suites found: %s" %
', '.join(broken_unit_tests))
complete_suite = unittest.TestSuite(test_modules)
res = unittest.TextTestRunner(verbosity=2).run(complete_suite)
return res.wasSuccessful()
def setUp(self):
"""Load plugin."""
search_service_plugins = PluginContainer(
os.path.join(CFG_SEARCH_SERVICES_PATH, '*Service.py'),
api_version=__required_plugin_API_version__,
plugin_signature=SearchService
)
self.plugin = search_service_plugins.get('JournalHintService')()
def setUp(self):
"""Load plugin"""
search_service_plugins = PluginContainer(
os.path.join(CFG_SEARCH_SERVICES_PATH, '*Service.py'),
api_version=__required_plugin_API_version__,
plugin_signature=SearchService)
self.plugin = search_service_plugins.get(
'SubmissionNameSearchService')()
def workflows():
loaded_workflows = PluginContainer(os.path.join(CFG_PYLIBDIR, 'invenio',
'bibworkflow', 'workflows', '*.py'))
open(os.path.join(CFG_LOGDIR, 'broken-bibworkflow-workflows.log'), 'w').\
write(pformat(loaded_workflows.get_broken_plugins()))
return dict(workflows=loaded_workflows.get_enabled_plugins(),
broken_workflows=loaded_workflows.get_broken_plugins())
def test_plugin_container_module_reloading(self):
"""pluginutils - plugin reloading"""
websubmit_functions = PluginContainer(os.path.join(CFG_PYLIBDIR, 'invenio', 'websubmit_functions', '*.py'))
self.assertNotEqual(websubmit_functions['Is_Referee'].__doc__, "test_reloading")
websubmit_functions['Is_Referee'].__doc__ = "test_reloading"
self.assertEqual(websubmit_functions['Is_Referee'].__doc__, "test_reloading")
websubmit_functions.reload_plugins(reload=True)
self.assertNotEqual(websubmit_functions['Is_Referee'].__doc__, "test_reloading")
def test_plugin_container_module_reloading(self):
"""pluginutils - plugin reloading"""
websubmit_functions = PluginContainer(os.path.join(CFG_PYLIBDIR, 'invenio', 'websubmit_functions', '*.py'))
self.assertNotEqual(websubmit_functions['Is_Referee'].__doc__, "test_reloading")
websubmit_functions['Is_Referee'].__doc__ = "test_reloading"
self.assertEqual(websubmit_functions['Is_Referee'].__doc__, "test_reloading")
websubmit_functions.reload_plugins(reload=True)
self.assertNotEqual(websubmit_functions['Is_Referee'].__doc__, "test_reloading")
def test_plugin_container_module_caching(self):
"""pluginutils - plugin caching"""
websubmit_functions = PluginContainer(os.path.join(CFG_PYLIBDIR, "invenio", "websubmit_functions", "*.py"))
self.assertNotEqual(websubmit_functions["Is_Referee"].__doc__, "test_caching")
websubmit_functions["Is_Referee"].__doc__ = "test_caching"
self.assertEqual(websubmit_functions["Is_Referee"].__doc__, "test_caching")
websubmit_functions.reload_plugins()
websubmit_functions_new = PluginContainer(os.path.join(CFG_PYLIBDIR, "invenio", "websubmit_functions", "*.py"))
self.assertEqual(websubmit_functions_new["Is_Referee"].__doc__, "test_caching")
def load_facet_builders():
## Let's load all facets.
_FACETS = PluginContainer(
os.path.join(CFG_PYLIBDIR, 'invenio', 'websearch_facets', 'facet_*.py'),
plugin_builder=_invenio_facet_plugin_builder)
FACET_DICTS = dict((f.name, f) for f in _FACETS.values())
FACET_SORTED_LIST = sorted(FACET_DICTS.values(), key=lambda x: x.order)
current_app.config['FACET_DICTS'] = FACET_DICTS
current_app.config['FACET_SORTED_LIST'] = FACET_SORTED_LIST
def load_bfe_elements():
BFE_ELEMENTS = PluginContainer(os.path.join(CFG_PYLIBDIR, 'invenio',
'bibformat_elements',
'bfe_*.py'),
plugin_builder=plugin_builder)
## Let's report about broken plugins
open(os.path.join(CFG_LOGDIR, 'broken-bibformat-elements.log'), 'w').write(
pformat(BFE_ELEMENTS.get_broken_plugins()))
current_app.config['BFE_ELEMENTS'] = BFE_ELEMENTS
def test_plugin_container_wrapping_websubmit_functions(self):
"""pluginutils - wrapping websubmit functions"""
websubmit_functions = PluginContainer(os.path.join(CFG_PYLIBDIR, 'invenio', 'websubmit_functions', '*.py'))
self.failUnless(websubmit_functions['Is_Referee'])
self.failUnless(websubmit_functions['CaseEDS'])
self.failUnless(callable(websubmit_functions['CaseEDS']))
self.failUnless(len(websubmit_functions) >= 62, 'There should exist at least 62 websubmit_functions. Found: %s' % len(websubmit_functions))
## Retrieve_Data and Shared_Functions are not real plugins
self.failUnless(len(websubmit_functions.get_broken_plugins()) >= 2)
self.failIf(websubmit_functions.get('Shared_Functions'))
self.failUnless('Shared_Functions' in websubmit_functions.get_broken_plugins())
def test_plugin_container_wrapping_websubmit_functions(self):
"""pluginutils - wrapping websubmit functions"""
websubmit_functions = PluginContainer(os.path.join(CFG_PYLIBDIR, 'invenio', 'websubmit_functions', '*.py'))
self.failUnless(websubmit_functions['Is_Referee'])
self.failUnless(websubmit_functions['CaseEDS'])
self.failUnless(callable(websubmit_functions['CaseEDS']))
self.failUnless(len(websubmit_functions) >= 62)
## Retrieve_Data and Shared_Functions are not real plugins
self.failUnless(len(websubmit_functions.get_broken_plugins()) >= 2)
self.failIf(websubmit_functions.get('Shared_Functions'))
self.failUnless('Shared_Functions' in websubmit_functions.get_broken_plugins())
def test_no_broken_search_services_(self):
"""websearch - no broken search services"""
error_messages = []
search_service_plugins = PluginContainer(os.path.join(CFG_SEARCH_SERVICES_PATH, '*Service.py'),
api_version=__required_plugin_API_version__,
plugin_signature=SearchService)
for name, error in search_service_plugins.get_broken_plugins().iteritems():
error_messages.append("Service '%s' could not be loaded:\n%s" % \
(name, repr(error[0]) + " " + repr(error[1]) + "\n" + "\n".join(traceback.format_tb(error[2]))))
if error_messages:
self.fail(merge_error_messages(error_messages))
def write_metadata(inputfile, outputfile, metadata_dictionary,
force=None, verbose=0):
"""
Writes metadata to given file.
Availability depends on input file format and installed plugins
(return C{TypeError} if unsupported file format).
@param inputfile: path to a file
@type inputfile: string
@param outputfile: path to the resulting file.
@type outputfile: string
@param verbose: verbosity
@type verbose: int
@param metadata_dictionary: keys and values of metadata to update.
@type metadata_dictionary: dict
@param force: name of plugin to use, to skip plugin auto-discovery
@type force: string
@return: output of the plugin
@rtype: string
@raise TypeError: if file format is not supported.
@raise RuntimeError: if required library to process file is missing.
@raise InvenioWebSubmitFileMetadataRuntimeError: when metadata cannot be updated.
"""
# Check file type (0 base, 1 name, 2 ext)
ext = decompose_file(inputfile)[2]
if verbose > 5:
print ext.lower(), 'extension to write to'
# Plugins
metadata_extractor_plugins = PluginContainer(
os.path.join(CFG_PYLIBDIR,
'invenio', 'websubmit_file_metadata_plugins', 'wsm_*.py'),
plugin_builder=plugin_builder_function,
api_version=__required_plugin_API_version__
)
# Loop through the plugins to find a good one to ext
for plugin_name, plugin in metadata_extractor_plugins.iteritems():
if plugin.has_key('can_write_local') and \
plugin['can_write_local'](inputfile) and \
(not force or plugin_name == force):
if verbose > 5:
print 'Using ' + plugin_name
return plugin['write_metadata_local'](inputfile,
outputfile,
metadata_dictionary,
verbose)
# Case of no plugin found, raise
raise TypeError, 'Unsupported file type'
def write_metadata(inputfile, outputfile, metadata_dictionary,
force=None, verbose=0):
"""
Writes metadata to given file.
Availability depends on input file format and installed plugins
(return C{TypeError} if unsupported file format).
@param inputfile: path to a file
@type inputfile: string
@param outputfile: path to the resulting file.
@type outputfile: string
@param verbose: verbosity
@type verbose: int
@param metadata_dictionary: keys and values of metadata to update.
@type metadata_dictionary: dict
@param force: name of plugin to use, to skip plugin auto-discovery
@type force: string
@return: output of the plugin
@rtype: string
@raise TypeError: if file format is not supported.
@raise RuntimeError: if required library to process file is missing.
@raise InvenioWebSubmitFileMetadataRuntimeError: when metadata cannot be updated.
"""
# Check file type (0 base, 1 name, 2 ext)
ext = decompose_file(inputfile)[2]
if verbose > 5:
print ext.lower(), 'extension to write to'
# Plugins
metadata_extractor_plugins = PluginContainer(
os.path.join(CFG_PYLIBDIR,
'invenio', 'websubmit_file_metadata_plugins', 'wsm_*.py'),
plugin_builder=plugin_builder_function,
api_version=__required_plugin_API_version__
)
# Loop through the plugins to find a good one to ext
for plugin_name, plugin in metadata_extractor_plugins.iteritems():
if plugin.has_key('can_write_local') and \
plugin['can_write_local'](inputfile) and \
(not force or plugin_name == force):
if verbose > 5:
print 'Using ' + plugin_name
return plugin['write_metadata_local'](inputfile,
outputfile,
metadata_dictionary,
verbose)
# Case of no plugin found, raise
raise TypeError, 'Unsupported file type'
def test_no_broken_search_services_(self):
"""websearch - no broken search services"""
error_messages = []
search_service_plugins = PluginContainer(
os.path.join(CFG_SEARCH_SERVICES_PATH, '*Service.py'),
api_version=__required_plugin_API_version__,
plugin_signature=SearchService)
for name, error in search_service_plugins.get_broken_plugins(
).iteritems():
error_messages.append("Service '%s' could not be loaded:\n%s" % \
(name, repr(error[0]) + " " + repr(error[1]) + "\n" + "\n".join(traceback.format_tb(error[2]))))
if error_messages:
self.fail(merge_error_messages(error_messages))
def test_plugin_container_wrapping_websubmit_functions(self):
"""pluginutils - wrapping websubmit functions"""
websubmit_functions = PluginContainer(os.path.join(CFG_PYLIBDIR, "invenio", "websubmit_functions", "*.py"))
self.failUnless(websubmit_functions["Is_Referee"])
self.failUnless(websubmit_functions["CaseEDS"])
self.failUnless(callable(websubmit_functions["CaseEDS"]))
self.failUnless(
len(websubmit_functions) >= 62,
"There should exist at least 62 websubmit_functions. Found: %s" % len(websubmit_functions),
)
## Retrieve_Data and Shared_Functions are not real plugins
self.failUnless(len(websubmit_functions.get_broken_plugins()) >= 2)
self.failIf(websubmit_functions.get("Shared_Functions"))
self.failUnless("Shared_Functions" in websubmit_functions.get_broken_plugins())
def test_plugin_container_wrapping_external_authentications(self):
"""pluginutils - wrapping external authentications"""
def plugin_builder(plugin_name, plugin_code):
for name in dir(plugin_code):
candidate = getattr(plugin_code, name)
try:
if issubclass(candidate, ExternalAuth):
return candidate
except TypeError:
pass
raise ValueError('%s is not a valid external authentication plugin' % plugin_name)
external_authentications = PluginContainer(os.path.join(CFG_PYLIBDIR, 'invenio', 'external_authentication_*.py'), plugin_signature=ExternalAuth, plugin_builder=plugin_builder)
self.failUnless(issubclass(external_authentications['external_authentication_sso'], ExternalAuth))
self.failIf(external_authentications.get('external_authentication_cern_wrapper'))
self.failUnless(len(external_authentications) >= 1)
self.failUnless(len(external_authentications.get_broken_plugins()) >= 2)
def build_and_run_unit_test_suite():
"""
Detect all Invenio modules with names ending by '*_unit_tests.py', build
a complete test suite of them, and run it.
Called by 'inveniocfg --run-unit-tests'.
"""
test_modules_map = PluginContainer(
os.path.join(CFG_PYLIBDIR, 'invenio', '*_unit_tests.py'),
lambda plugin_name, plugin_code: getattr(plugin_code, "TEST_SUITE"))
test_modules = [test_modules_map[name] for name in test_modules_map]
broken_tests = test_modules_map.get_broken_plugins()
broken_unit_tests = ['%s (reason: %s)' % (name, broken_tests[name][1]) for name in broken_tests]
if broken_unit_tests:
warn("Broken unit tests suites found: %s" % ', '.join(broken_unit_tests))
complete_suite = unittest.TestSuite(test_modules)
res = unittest.TextTestRunner(verbosity=2).run(complete_suite)
return res.wasSuccessful()
def load_plugins():
"""
Will load all the plugins found under the bibcheck_plugins folder.
Returns a list of plugin objects.
"""
plugin_dir = os.path.join(CFG_PYLIBDIR, "invenio/bibcheck_plugins/*.py")
# Load plugins
plugins = PluginContainer(plugin_dir,
plugin_builder=_bibcheck_plugin_builder)
# Check for broken plug-ins
broken = plugins.get_broken_plugins()
if broken:
for plugin, info in broken.items():
print "Failed to load %s:\n" % plugin
print "".join(traceback.format_exception(*info))
enabled = plugins.get_enabled_plugins()
enabled.pop("__init__", None)
return enabled
def load_plugins():
"""
Will load all the plugins found under the bibcheck_plugins folder.
Returns a list of plugin objects.
"""
plugin_dir = os.path.join(CFG_PYLIBDIR, "invenio/bibcheck_plugins/*.py")
# Load plugins
plugins = PluginContainer(plugin_dir,
plugin_builder=_bibcheck_plugin_builder)
# Check for broken plug-ins
broken = plugins.get_broken_plugins()
if broken:
for plugin, info in broken.items():
print "Failed to load %s:\n" % plugin
print "".join(traceback.format_exception(*info))
enabled = plugins.get_enabled_plugins()
enabled.pop("__init__", None)
return enabled
def test_plugin_container_wrapping_bibformat_elements(self):
"""pluginutils - wrapping bibformat elements"""
def bibformat_signature(bfo, *args, **argd):
pass
bibformat_elements = PluginContainer(
os.path.join(CFG_BIBFORMAT_ELEMENTS_PATH, 'bfe_*.py'),
plugin_signature=bibformat_signature,
plugin_builder=lambda plugin_name, plugin_code: getattr(
plugin_code, 'format_element'))
self.failUnless(bibformat_elements['bfe_fulltext'])
self.failUnless(callable(bibformat_elements['bfe_fulltext']))
self.failUnless(len(bibformat_elements) >= 50)
def metadata_info(verbose=0):
"""Shows information about the available plugins"""
print 'Plugin APIs version: %s' % str(__required_plugin_API_version__)
# Plugins
print 'Available plugins:'
metadata_extractor_plugins = PluginContainer(
os.path.join(CFG_PYLIBDIR, 'invenio',
'websubmit_file_metadata_plugins', 'wsm_*.py'),
plugin_builder=plugin_builder_function,
api_version=__required_plugin_API_version__)
# Print each operation on each plugin
for plugin_name, plugin_funcs in metadata_extractor_plugins.iteritems():
if len(plugin_funcs) > 0:
print '-- Name: ' + plugin_name
print ' Supported operation%s: ' % \
(len(plugin_funcs) > 1 and 's' or '') + \
', '.join(plugin_funcs)
# Are there any unloaded plugins?
broken_plugins = metadata_extractor_plugins.get_broken_plugins()
if len(broken_plugins.keys()) > 0:
print 'Could not load the following plugin%s:' % \
(len(broken_plugins.keys()) > 1 and 's' or '')
for broken_plugin_name, broken_plugin_trace_info in broken_plugins.iteritems(
):
print '-- Name: ' + broken_plugin_name
if verbose > 5:
formatted_traceback = \
traceback.format_exception(broken_plugin_trace_info[0],
broken_plugin_trace_info[1],
broken_plugin_trace_info[2])
print ' ' + ''.join(formatted_traceback).replace(
'\n', '\n ')
elif verbose > 0:
print ' ' + str(broken_plugin_trace_info[1])
def get_search_services():
"""
Return an iterator on all enabled search services.
Then each service can be queried with method C{answer}.
"""
global _SERVICES
if _SERVICES is None:
_SERVICES = [
plugin() for plugin in PluginContainer(
os.path.join(CFG_SEARCH_SERVICES_PATH, '*Service.py'),
api_version=__required_plugin_API_version__,
plugin_signature=SearchService).values()
]
return _SERVICES
def metadata_info(verbose=0):
"""Shows information about the available plugins"""
print 'Plugin APIs version: %s' % str(__required_plugin_API_version__)
# Plugins
print 'Available plugins:'
metadata_extractor_plugins = PluginContainer(
os.path.join(CFG_PYLIBDIR,
'invenio', 'websubmit_file_metadata_plugins', 'wsm_*.py'),
plugin_builder=plugin_builder_function,
api_version=__required_plugin_API_version__
)
# Print each operation on each plugin
for plugin_name, plugin_funcs in metadata_extractor_plugins.iteritems():
if len(plugin_funcs) > 0:
print '-- Name: ' + plugin_name
print ' Supported operation%s: ' % \
(len(plugin_funcs) > 1 and 's' or '') + \
', '.join(plugin_funcs)
# Are there any unloaded plugins?
broken_plugins = metadata_extractor_plugins.get_broken_plugins()
if len(broken_plugins.keys()) > 0:
print 'Could not load the following plugin%s:' % \
(len(broken_plugins.keys()) > 1 and 's' or '')
for broken_plugin_name, broken_plugin_trace_info in broken_plugins.iteritems():
print '-- Name: ' + broken_plugin_name
if verbose > 5:
formatted_traceback = \
traceback.format_exception(broken_plugin_trace_info[0],
broken_plugin_trace_info[1],
broken_plugin_trace_info[2])
print ' ' + ''.join(formatted_traceback).replace('\n', '\n ')
elif verbose > 0:
print ' ' + str(broken_plugin_trace_info[1])
def check_field_type(record,
field,
field_type,
subfield=None,
continuable=True):
"""
Checks if record[field.subfield] is of type "field_type"
@note: If record[field.subfield] is a list or a dictionary then it checks
every single element inside is type is a "system type"
@param record: BibFieldDict where the record is stored
@param field: Main json ID or field name to make test on
@param field_type: Field_Type defined by the user inside bibfield_types or a system type
i.e.: "datetime.datetime"
@param subfield: If this parameter is present, instead of applying the checker
to the field, it is applied to record['field.subfield']
"""
field = '[n]' in field and field[:-3] or field
key = subfield and "%s.%s" % (field, subfield) or field
if not key in record:
return
import os
from invenio.config import CFG_PYLIBDIR
from invenio.pluginutils import PluginContainer
CFG_BIBFIELD_TYPES = PluginContainer(
os.path.join(CFG_PYLIBDIR, 'invenio', 'bibfield_functions',
'is_type_*.py'))
from invenio.bibfield_utils import InvenioBibFieldContinuableError, \
InvenioBibFieldError
error = continuable and InvenioBibFieldContinuableError or InvenioBibFieldError
new_type = 'is_type_%s' % (field_type, )
if new_type in CFG_BIBFIELD_TYPES:
globals()[new_type] = CFG_BIBFIELD_TYPES[new_type]
if not eval('%s(record[key])' % (new_type, )):
raise error("Field %s should be of type '%s'" % (key, field_type))
else:
if not check_field_sys_type(record[key], field_type):
raise error("Field %s should be of type '%s'" % (key, field_type))
def test_plugin_container_wrapping_bibformat_elements(self):
"""pluginutils - wrapping bibformat elements"""
def format_signature(bfo, *dummy_args, **dummy_argd):
pass
def escape_values_signature(bfo):
pass
plugin_builder = create_enhanced_plugin_builder(
compulsory_objects={
'format_element' : format_signature,
},
optional_objects={
'escape_values' : escape_values_signature,
})
bibformat_elements = PluginContainer(os.path.join(CFG_BIBFORMAT_ELEMENTS_PATH, 'bfe_*.py'),
plugin_builder=plugin_builder)
self.failUnless(bibformat_elements['bfe_fulltext'])
self.failUnless(callable(bibformat_elements['bfe_fulltext']['format_element']))
self.failUnless(len(bibformat_elements) >= 50)
def _load_upgrades(self, remove_applied=True):
"""
Load upgrade modules
Upgrade modules are loaded using pluginutils. The pluginutils module
is either loaded from site-packages via normal or via a user-loaded
module supplied in the __init__ method. This is useful when the engine
is running before actually being installed into site-packages.
@param remove_applied: if True, already applied upgrades will not
be included, if False the entire upgrade graph will be
returned.
"""
if self.pluginutils is None:
from invenio.pluginutils import PluginContainer
from invenio.pluginutils import create_enhanced_plugin_builder
else:
PluginContainer = self.pluginutils.PluginContainer
create_enhanced_plugin_builder = self.pluginutils.create_enhanced_plugin_builder
if remove_applied:
self.load_history()
builder = create_enhanced_plugin_builder(
compulsory_objects={
'do_upgrade': dummy_signgature,
'info': dummy_signgature,
},
optional_objects={
'estimate': dummy_signgature,
'pre_upgrade': dummy_signgature,
'post_upgrade': dummy_signgature,
},
other_data={
'depends_on': (list, []),
},
)
# Load all upgrades
plugins = PluginContainer(
[os.path.join(p, '*.py') for p in self._paths],
plugin_builder=builder,
external=True,
)
# Check for broken plug-ins
broken = plugins.get_broken_plugins()
if broken:
messages = []
import traceback
for plugin, info in broken.items():
messages.append(
"Failed to load %s:\n"
" %s" %
(plugin, "".join(traceback.format_exception(*info))))
raise RuntimeError(*messages)
# Store name and doc of plug-in in itself
for plugin_id in plugins.keys():
plugins[plugin_id]['id'] = plugin_id
plugins[plugin_id]['repository'] = self._parse_plugin_id(plugin_id)
return plugins
def read_metadata(inputfile, force=None, remote=False,
loginpw=None, verbose=0):
"""
Returns metadata extracted from given file as dictionary.
Availability depends on input file format and installed plugins
(return C{TypeError} if unsupported file format).
@param inputfile: path to a file
@type inputfile: string
@param verbose: verbosity
@type verbose: int
@param force: name of plugin to use, to skip plugin auto-discovery
@type force: string
@param remote: if the file is accessed remotely or not
@type remote: boolean
@param loginpw: credentials to access secure servers (username:password)
@type loginpw: string
@return: dictionary of metadata tags as keys, and (interpreted)
value as value
@rtype: dict
@raise TypeError: if file format is not supported.
@raise RuntimeError: if required library to process file is missing.
@raise InvenioWebSubmitFileMetadataRuntimeError: when metadata cannot be read.
"""
metadata = None
# Check file type (0 base, 1 name, 2 ext)
ext = decompose_file(inputfile)[2]
if verbose > 5:
print ext.lower(), 'extension to extract from'
# Load plugins
metadata_extractor_plugins = PluginContainer(
os.path.join(CFG_PYLIBDIR,
'invenio', 'websubmit_file_metadata_plugins', 'wsm_*.py'),
plugin_builder=plugin_builder_function,
api_version=__required_plugin_API_version__)
# Loop through the plugins to find a good one for given file
for plugin_name, plugin in metadata_extractor_plugins.iteritems():
# Local file
if plugin.has_key('can_read_local') and \
plugin['can_read_local'](inputfile) and not remote and \
(not force or plugin_name == force):
if verbose > 5:
print 'Using ' + plugin_name
fetched_metadata = plugin['read_metadata_local'](inputfile,
verbose)
if not metadata:
metadata = fetched_metadata
else:
metadata.update(fetched_metadata)
# Remote file
elif remote and plugin.has_key('can_read_remote') and \
plugin['can_read_remote'](inputfile) and \
(not force or plugin_name == force):
if verbose > 5:
print 'Using ' + plugin_name
fetched_metadata = plugin['read_metadata_remote'](inputfile,
loginpw,
verbose)
if not metadata:
metadata = fetched_metadata
else:
metadata.update(fetched_metadata)
# Return in case we have something
if metadata is not None:
return metadata
# Case of no plugin found, raise
raise TypeError, 'Unsupported file type'
import os
from pprint import pformat
from invenio.config import CFG_PYLIBDIR, CFG_LOGDIR
from invenio.pluginutils import PluginContainer
def plugin_builder(plugin_name, plugin_code):
if plugin_name == '__init__':
return
all = getattr(plugin_code, '__all__')
for name in all:
return getattr(plugin_code, name)
CFG_DOC_METADATA = PluginContainer(os.path.join(CFG_PYLIBDIR, 'invenio',
'webdeposit_workflows',
'*_metadata.py'),
plugin_builder=plugin_builder)
"""
Create a dict with groups, names and deposition types
in order to render the deposition type chooser page
e.g.
deposition_types = {"Articles & Preprints": \
[{"name": "Articles", "dep_type": "Article"}, \
{"name": "Preprints", "dep_type": "Preprint"}, \
{"name": "Theses", "dep_type": "Thesis"}]}
Used to load all the definitions of webdeposit workflow.
They must be defined in the deposition_types folder with filname '*_metadata.py'
Also if you want to rename the workflow, you can redefine the __all__ variable
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""
Redirecting engine.
"""
import os
import optparse
from invenio.config import CFG_PYLIBDIR
from invenio.dbquery import run_sql, IntegrityError
from invenio.jsonutils import json, json_unicode_to_utf8
from invenio.pluginutils import PluginContainer, get_callable_documentation
from invenio.textutils import wait_for_user
CFG_GOTO_PLUGINS = PluginContainer(os.path.join(CFG_PYLIBDIR, 'invenio',
'goto_plugins', 'goto_plugin_*.py'), plugin_builder=lambda name, code: code.goto)
def register_redirection(label, plugin, parameters=None, update_on_duplicate=False):
"""
Register a redirection from /goto/<LABEL> to the URL returned by running the
given plugin (as available in CFG_GOTO_PLUGINS), with the given parameters.
@param label: the uniquely identifying label for this redirection
@type label: string
@param plugin: the algorithm that should resolve the redirection, usually:
"goto_plugin_FOO"
@type plugin: string
@param parameters: further parameters that should be passed to the plugin.
This should be a dictionary or None. Note that these parameters could
def plugin_builder(plugin_name, plugin_code):
if plugin_name == '__init__':
return
plugin_candidate = getattr(plugin_code, plugin_name, None)
if plugin_candidate is None:
all_plugins = getattr(plugin_code, '__all__')
for name in all_plugins:
candidate = getattr(plugin_code, name)
return candidate
else:
return plugin_candidate
loaded_workflows = PluginContainer(os.path.join(CFG_PYLIBDIR, 'invenio',
'*_workflows', '*.py'),
plugin_builder=plugin_builder)
workflows = {}
for workflow in loaded_workflows.values():
if workflow is not None:
workflows[workflow.__name__] = workflow
# Let's report about broken plugins
open(os.path.join(CFG_LOGDIR, 'broken-workflows.log'),
'w').write(pformat(loaded_workflows.get_broken_plugins()))
__all__ = ['workflows']
def _load_tasklets():
"""
Load all the bibsched tasklets into the global variable _TASKLETS.
"""
return PluginContainer(os.path.join(CFG_BIBTASK_TASKLETS_PATH, 'bst_*.py'))
def getWorkflowDefinition(name):
workflows = PluginContainer(
os.path.join(CFG_PYLIBDIR, 'invenio', 'bibworkflow', 'workflows',
'*.py'))
return workflows.get_enabled_plugins()[name]().get_definition()
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""
Redirecting engine.
"""
import os
import optparse
from invenio.config import CFG_PYLIBDIR
from invenio.dbquery import run_sql, IntegrityError
from invenio.jsonutils import json, json_unicode_to_utf8
from invenio.pluginutils import PluginContainer, get_callable_documentation
from invenio.textutils import wait_for_user
CFG_GOTO_PLUGINS = PluginContainer(os.path.join(CFG_PYLIBDIR, 'invenio',
'goto_plugins',
'goto_plugin_*.py'),
plugin_builder=lambda name, code: code.goto)
def register_redirection(label,
plugin,
parameters=None,
update_on_duplicate=False):
"""
Register a redirection from /goto/<LABEL> to the URL returned by running the
given plugin (as available in CFG_GOTO_PLUGINS), with the given parameters.
@param label: the uniquely identifying label for this redirection
@type label: string
@param plugin: the algorithm that should resolve the redirection, usually:
def create_invenio_flask_app(**kwargs_config):
"""
Prepare WSGI Invenio application based on Flask.
Invenio consists of a new Flask application with legacy support for
the old WSGI legacy application and the old Python legacy
scripts (URLs to *.py files).
An incoming request is processed in the following manner:
* The Flask application first routes request via its URL routing
system (see LegacyAppMiddleware.__call__()).
* One route in the Flask system, will match Python legacy
scripts (see static_handler_with_legacy_publisher()).
* If the Flask application aborts the request with a 404 error, the request
is passed on to the WSGI legacy application (see page_not_found()). E.g.
either the Flask application did not find a route, or a view aborted the
request with a 404 error.
"""
def decorate_build(f):
@wraps(f)
def decorator(*args, **kwargs):
scheme_url = {
'http': current_app.config['CFG_SITE_URL'],
'https': current_app.config['CFG_SITE_SECURE_URL']
}
force_external = kwargs.get('force_external', False)
url_scheme = getattr(f.im_self, 'url_scheme', 'http')
kwargs['force_external'] = False
url = f(*args, **kwargs)
if force_external:
url = scheme_url.get(url_scheme) + url
return url
return decorator
class InvenioFlask(Flask):
def create_url_adapter(self, request):
url_adapter = super(InvenioFlask, self).create_url_adapter(request)
if url_adapter is not None and hasattr(url_adapter, 'build'):
url_adapter.build = decorate_build(url_adapter.build)
return url_adapter
## The Flask application instance
_app = InvenioFlask(
__name__,
## Static files are usually handled directly by the webserver (e.g. Apache)
## However in case WSGI is required to handle static files too (such
## as when running simple server), then this flag can be
## turned on (it is done automatically by wsgi_handler_test).
## We assume anything under '/' which is static to be server directly
## by the webserver from CFG_WEBDIR. In order to generate independent
## url for static files use func:`url_for('static', filename='test')`.
static_url_path='',
static_folder=CFG_WEBDIR)
from urlparse import urlparse
_app.config['SERVER_NAME'] = urlparse(CFG_SITE_URL).netloc
## Update application config from parameters.
_app.config.update(kwargs_config)
if 'SQLALCHEMY_DATABASE_URI' not in _app.config:
from sqlalchemy.engine.url import URL
# Global variables
from invenio.dbquery import CFG_DATABASE_HOST, CFG_DATABASE_PORT,\
CFG_DATABASE_NAME, CFG_DATABASE_USER, CFG_DATABASE_PASS, \
CFG_DATABASE_TYPE
_app.config['SQLALCHEMY_DATABASE_URI'] = URL(
CFG_DATABASE_TYPE,
username=CFG_DATABASE_USER,
password=CFG_DATABASE_PASS,
host=CFG_DATABASE_HOST,
database=CFG_DATABASE_NAME,
port=CFG_DATABASE_PORT,
)
## Let's initialize database.
from invenio.sqlalchemyutils import db
db.init_app(_app)
## Make sure that all tables are loaded in `db.metadata.tables`.
from invenio.importutils import autodiscover_modules
autodiscover_modules(['invenio'], related_name_re=".+_model\.py")
## First check that you have all rights to logs
from invenio.bibtask import check_running_process_user
check_running_process_user()
from invenio.pluginutils import PluginContainer
from invenio.session_flask import InvenioSessionInterface
from invenio.webuser_flask import InvenioLoginManager, current_user
from invenio.messages import wash_language, gettext_set_language, \
language_list_long, is_language_rtl
from invenio.urlutils import create_url, get_canonical_and_alternates_urls
from invenio.cache import cache
from invenio.jinja2utils import CollectionExtension, \
LangExtension, hack_jinja2_utf8decoding, \
extend_application_template_filters
from flask.ext.assets import Environment, Bundle
from invenio.webinterface_handler_flask_utils import unicodifier, InvenioRequest
from flaskext.gravatar import Gravatar
from werkzeug.wrappers import BaseResponse
from werkzeug.exceptions import HTTPException
from invenio.flask_sslify import SSLify
from invenio.webinterface_handler_wsgi import application as legacy_application
from invenio.webinterface_handler_wsgi import is_mp_legacy_publisher_path, \
mp_legacy_publisher
# See note on Jinja2 string decoding using ASCII codec instead of UTF8 in
# function documentation
hack_jinja2_utf8decoding()
# Handle both url with and without trailing slashe by Flask.
# @blueprint.route('/test')
# @blueprint.route('/test/') -> not necessary when strict_slashes == False
_app.url_map.strict_slashes = False
# SECRET_KEY is needed by Flask Debug Toolbar
SECRET_KEY = _app.config.get('SECRET_KEY') or CFG_SITE_SECRET_KEY
if not SECRET_KEY or SECRET_KEY == '':
fill_secret_key = """
Set variable CFG_SITE_SECRET_KEY with random string in invenio-local.conf.
You can use following commands:
$ %s
$ %s
""" % (CFG_BINDIR + os.sep + 'inveniocfg --create-secret-key',
CFG_BINDIR + os.sep + 'inveniocfg --update-config-py')
try:
raise Exception(fill_secret_key)
except Exception:
register_exception(alert_admin=True,
subject="Missing CFG_SITE_SECRET_KEY")
raise Exception(fill_secret_key)
_app.config["SECRET_KEY"] = SECRET_KEY
# Enable Flask Debug Toolbar early to also catch HTTPS redirects
if 'debug-toolbar' in getattr(config, 'CFG_DEVEL_TOOLS', []):
_app.config["DEBUG_TB_ENABLED"] = True
_app.config[
"DEBUG_TB_INTERCEPT_REDIRECTS"] = 'intercept-redirects' in getattr(
config, 'CFG_DEVEL_TOOLS', [])
from flask_debugtoolbar import DebugToolbarExtension
DebugToolbarExtension(_app)
# Set email backend for Flask-Email plugin
from invenio.mailutils import initialize_email_backend
initialize_email_backend(_app)
if CFG_HAS_HTTPS_SUPPORT:
# Makes request always run over HTTPS.
_sslify = SSLify(_app)
if not CFG_FULL_HTTPS:
@_sslify.criteria_handler
def criteria():
"""Extends criteria when to stay on HTTP site."""
_force_https = False
if request.blueprint in current_app.blueprints:
_force_https = current_app.blueprints[request.blueprint].\
_force_https
view_func = current_app.view_functions.get(request.endpoint)
if view_func is not None and hasattr(view_func,
'_force_https'):
_force_https = view_func._force_https
return not (_force_https or session.need_https())
class LegacyAppMiddleware(object):
def __init__(self, app):
self.app = app
def __call__(self, environ, start_response):
if remote_debugger:
remote_debugger.start()
with self.app.request_context(environ):
g.start_response = start_response
try:
response = self.app.full_dispatch_request()
except Exception as e:
register_exception(req=request, alert_admin=True)
response = self.app.handle_exception(e)
return response(environ, start_response)
_app.wsgi_app = LegacyAppMiddleware(_app)
@_app.errorhandler(404)
def page_not_found(error):
try:
response = legacy_application(request.environ, g.start_response)
if not isinstance(response, BaseResponse):
response = current_app.make_response(str(response))
return response
except HTTPException:
return render_template("404.html"), 404
@_app.errorhandler(401)
def do_login_first(error=401):
"""Displays login page when user is not authorised."""
if request.is_xhr:
return _("Authorization failure"), 401
flash(_("Authorization failure"), 'error')
from invenio.webaccount_blueprint import login
return login(referer=request.url), 401
@_app.endpoint('static')
@_app.route(_app.static_url_path + '/<path:filename>',
methods=['POST', 'PUT'])
def static_handler_with_legacy_publisher(*args, **kwargs):
"""
Adds support for legacy publisher.
NOTE: It changes order of url page lookup. First, the invenio_handler
will be called and on 404 error the mp_legacy_publisher is called.
"""
possible_module, possible_handler = is_mp_legacy_publisher_path(
request.environ['PATH_INFO'])
if possible_module is not None:
legacy_publisher = lambda req: \
mp_legacy_publisher(req, possible_module, possible_handler)
return legacy_application(request.environ,
g.start_response,
handler=legacy_publisher)
# Static file serving for devserver
# ---------------------------------
# Apache normally serve all static files, but if we are using the
# devserver we need to serve static files here. Werkzeugs default
# behaviour is to return a '405 Method not allowed' for POST requests
# to static files. However, if we abort all POST requests with 405, the
# legacy_application (see page_not_found()) will not be given a chance
# to serve static files as it only get's invokved when we abort with a
# 404. Hence, on POST requests, we first check if the static file exists,
# and if it does we return we abort the request with a 405.
if not CFG_FLASK_SERVE_STATIC_FILES:
abort(404)
else:
static_file_response = _app.send_static_file(*args, **kwargs)
if request.method in ['POST', 'PUT']:
abort(405)
else:
return static_file_response
if CFG_FLASK_CACHE_TYPE not in [None, 'null']:
_app.jinja_options = dict(_app.jinja_options,
auto_reload=False,
cache_size=-1,
bytecode_cache=MemcachedBytecodeCache(
cache, prefix="jinja::", timeout=3600))
## Let's customize the template loader to first look into
## /opt/invenio/etc-local/templates and then into
## /opt/invenio/etc/templates
_app.jinja_loader = FileSystemLoader([
join(CFG_ETCDIR + '-local', 'templates'),
join(CFG_ETCDIR, 'templates')
])
## Let's attach our session handling (which is bridging with the native
## Invenio session handling
_app.session_interface = InvenioSessionInterface()
## Set custom request class
_app.request_class = InvenioRequest
## Let's load the whole invenio.config into Flask :-) ...
_app.config.from_object(config)
## ... and map certain common parameters
_app.config['SESSION_COOKIE_NAME'] = CFG_WEBSESSION_COOKIE_NAME
_app.config['PERMANENT_SESSION_LIFETIME'] = \
CFG_WEBSESSION_EXPIRY_LIMIT_REMEMBER * CFG_WEBSESSION_ONE_DAY
_app.config['USE_X_SENDFILE'] = CFG_BIBDOCFILE_USE_XSENDFILE
_app.config['DEBUG'] = CFG_DEVEL_SITE > 0
_app.debug = CFG_DEVEL_SITE > 0
_app.config['CFG_LANGUAGE_LIST_LONG'] = [
(lang, longname.decode('utf-8'))
for (lang, longname) in language_list_long()
]
## Invenio is all using str objects. Let's change them to unicode
_app.config.update(unicodifier(dict(_app.config)))
## Cache
_app.config['CACHE_TYPE'] = CFG_FLASK_CACHE_TYPE
# FIXME problem in Flask-Cache==0.11.1
cache.app = _app
cache.init_app(_app)
if CFG_FLASK_CACHE_TYPE == 'redis':
def with_try_except_block(f):
def decorator(*args, **kwargs):
try:
return f(*args, **kwargs)
except Exception:
register_exception(alert_admin=True)
pass
return decorator
## When the redis is down, we would like to keep the site running.
cache.cache._client.execute_command = with_try_except_block(
cache.cache._client.execute_command)
# FIXME problem in Flask-Cache==0.11.1
cache.app = current_app
_flask_log_handler = RotatingFileHandler(
os.path.join(CFG_LOGDIR, 'flask.log'))
_flask_log_handler.setFormatter(
Formatter('%(asctime)s %(levelname)s: %(message)s '
'[in %(pathname)s:%(lineno)d]'))
_app.logger.addHandler(_flask_log_handler)
# Let's create login manager.
_login_manager = InvenioLoginManager()
_login_manager.login_view = 'webaccount.login'
_login_manager.setup_app(_app)
_login_manager.unauthorized_handler(do_login_first)
# Let's create main menu.
class Menu(object):
def __init__(self,
id='',
title='',
url='',
order=None,
children=None,
display=lambda: True):
self.id = id
self.title = title
self.url = url
self.children = children or {}
self.order = order or 100
self.display = display
# Let's create assets environment.
_assets = Environment(_app)
_assets.debug = 'assets-debug' in getattr(config, 'CFG_DEVEL_TOOLS', [])
_assets.directory = config.CFG_WEBDIR
def _jinja2_new_bundle(tag, collection, name=None):
if not _assets.debug:
files = [
f for f in collection
if os.path.isfile(os.path.join(_assets.directory, f))
]
if len(files) != len(collection):
## Turn on debuging to generate 404 request on missing files.
_assets.debug = True
current_app.logger.error(
'Missing files: ' + ','.join(set(collection) - set(files)))
if len(collection):
return Bundle(output="%s/%s-%s.%s" %
(tag, 'invenio' if name is None else name,
hash('|'.join(collection)), tag),
*collection)
_app.jinja_env.extend(new_bundle=_jinja2_new_bundle,
default_bundle_name='90-invenio')
_app.jinja_env.add_extension(CollectionExtension)
_app.jinja_env.add_extension(LangExtension)
_app.jinja_env.add_extension('jinja2.ext.do')
# Let's extend application with custom template filters.
extend_application_template_filters(_app)
# Let's create Gravatar bridge.
_gravatar = Gravatar(_app,
size=100,
rating='g',
default='retro',
force_default=False,
force_lower=False)
del _gravatar
# Let's set the user language
from invenio.webinterface_handler_flask_utils import guess_language
_app.before_request(guess_language)
# Let's extend application with more custom templete filters
from invenio.jinja2utils import inject_utils
_app.context_processor(inject_utils)
@_login_manager.user_loader
def _load_user(uid):
"""
Function should not raise an exception if uid is not valid
or User was not found in database.
"""
from invenio.webuser_flask import UserInfo
return UserInfo(uid)
@_app.before_request
def reset_template_context_processor():
g._template_context_processor = []
@_app.context_processor
def _inject_template_context():
context = {}
if not hasattr(g, '_template_context_processor'):
reset_template_context_processor()
for func in g._template_context_processor:
context.update(func())
return context
def _invenio_blueprint_plugin_builder(plugin_name, plugin_code):
"""
Handy function to bridge pluginutils with (Invenio) blueprints.
"""
if plugin_name in CFG_FLASK_DISABLED_BLUEPRINTS:
raise ValueError(
'%s is excluded by CFG_FLASK_DISABLED_BLUEPRINTS' %
plugin_name)
from invenio.webinterface_handler_flask_utils import InvenioBlueprint
if 'blueprint' in dir(plugin_code):
candidate = getattr(plugin_code, 'blueprint')
if isinstance(candidate, InvenioBlueprint):
return candidate
raise ValueError('%s is not a valid blueprint plugin' % plugin_name)
## Let's load all the blueprints that are composing this Invenio instance
_BLUEPRINTS = PluginContainer(
os.path.join(CFG_PYLIBDIR, 'invenio', '*_blueprint.py'),
plugin_builder=_invenio_blueprint_plugin_builder)
## Let's report about broken plugins
open(join(CFG_LOGDIR, 'broken-blueprints.log'),
'w').write(pformat(_BLUEPRINTS.get_broken_plugins()))
_app.config['breadcrumbs_map'] = {}
_app.config['menubuilder_map'] = {}
## Let's attach all the blueprints
from invenio.webinterface_handler_flask_utils import _
for plugin in _BLUEPRINTS.values():
_app.register_blueprint(plugin)
if plugin.config:
## Let's include the configuration parameters of the config file.
## E.g. if the blueprint specify the config string
## 'invenio.webmessage_config' any uppercase variable defined in
## the module invenio.webmessage_config is loaded into the system.
_app.config.from_object(plugin.config)
if plugin.breadcrumbs:
_app.config['breadcrumbs_map'][plugin.name] = plugin.breadcrumbs
_app.config['breadcrumbs_map'].update(plugin.breadcrumbs_map)
## Let's build global menu. Each blueprint can plug its own menu items.
if plugin.menubuilder:
_app.config['menubuilder_map'].update(
(m[0], Menu(*m)) for m in plugin.menubuilder)
_app.config['menubuilder_map'].update(plugin.menubuilder_map)
_app.config['menubuilder_map'].update({
'main.admin':
Menu('main.admin', _('Administration'), 'help.admin', 9998, [],
lambda: current_user.is_admin),
'main.help':
Menu('main.help', _('Help'), 'help', 9999)
})
menu = {
'main': Menu('main', '', ''),
'personalize': Menu('personalize', '', '')
}
for key, item in _app.config['menubuilder_map'].iteritems():
start = menu
if '.' not in key:
if key in menu:
menu[key] = item.children.update(menu[key].children)
else:
menu[key] = item
continue
keys = key.split('.')
for k in keys[:-1]:
try:
start = start[k].children
except:
start[k] = Menu()
start = start[k].children
if keys[-1] in start:
item.children.update(start[keys[-1]].children)
start[keys[-1]] = item
_app.config['menubuilder_map'] = menu
# Flask-Admin
from invenio.adminutils import register_admin
register_admin(_app)
try:
## When deploying Invenio, one can prepare a module called
## webinterface_handler_local.py to be deployed under
## CFG_PYLIBDIR/invenio directory, and containing a function called
## customize_app which should accept a Flask application.
## This function has a chance to modify the application as needed
## including changing the URL routing map.
# pylint: disable=E0611
from invenio.webinterface_handler_local import customize_app
# pylint: enable=E0611
customize_app(_app)
except ImportError:
## No customization needed.
pass
return _app
def plugin_builder(plugin_name, plugin_code):
if 'reader' in dir(plugin_code):
candidate = getattr(plugin_code, 'reader')
try:
if issubclass(candidate, Reader):
return candidate
except:
pass
raise ValueError('%s is not a valid external authentication plugin' %
plugin_name)
CFG_BIBFIELD_READERS = PluginContainer(os.path.join(CFG_PYLIBDIR, 'invenio',
'bibfield_*reader.py'),
plugin_builder=plugin_builder)
# end Plug-in utils
def create_record(blob, master_format='marc', verbose=0, **additional_info):
"""
Creates a record object from the blob description using the apropiate reader
for it.
@return Record object
"""
reader = CFG_BIBFIELD_READERS['bibfield_%sreader.py' % (master_format, )](
blob, **additional_info)
def plugin_builder(plugin_name, plugin_code):
if plugin_name == '__init__':
return
try:
all = getattr(plugin_code, '__all__')
for name in all:
candidate = getattr(plugin_code, name)
if issubclass(candidate, Field):
return candidate
except AttributeError:
pass
CFG_FIELDS = PluginContainer(os.path.join(CFG_PYLIBDIR, 'invenio',
'webdeposit_deposition_fields',
'*_field.py'),
plugin_builder=plugin_builder)
class Fields(object):
pass
fields = Fields()
for field in CFG_FIELDS.itervalues():
## Change the names of the fields from the file names to the class names.
if field is not None:
fields.__setattr__(field.__name__, field)
## Let's report about broken plugins
open(os.path.join(CFG_LOGDIR, 'broken-deposition-fields.log'), 'w').write(
def _post_process_json(self):
"""
If needed this method will post process the json structure, e.g. pruning
the json to delete None values
"""
def remove_none_values(d):
if d is None or not isinstance(d, dict):
return
for key, value in d.items():
if value is None:
del d[key]
if isinstance(value, dict):
remove_none_values(value)
if isinstance(value, list):
for element in value:
if element is None:
value.remove(element)
else:
remove_none_values(element)
remove_none_values(self.rec_json)
for key, value in PluginContainer(
os.path.join(CFG_PYLIBDIR, 'invenio', 'bibfield_functions',
'produce_json_for_*.py')).iteritems():
setattr(JsonReader, key, value)
## Compulsory plugin interface
readers = JsonReader
def _load_upgrades(self, remove_applied=True):
"""
Load upgrade modules
Upgrade modules are loaded using pluginutils. The pluginutils module
is either loaded from site-packages via normal or via a user-loaded
module supplied in the __init__ method. This is useful when the engine
is running before actually being installed into site-packages.
@param remove_applied: if True, already applied upgrades will not
be included, if False the entire upgrade graph will be
returned.
"""
if self.pluginutils is None:
from invenio.pluginutils import PluginContainer
from invenio.pluginutils import create_enhanced_plugin_builder
else:
PluginContainer = self.pluginutils.PluginContainer
create_enhanced_plugin_builder = self.pluginutils.create_enhanced_plugin_builder
if remove_applied:
self.load_history()
builder = create_enhanced_plugin_builder(
compulsory_objects={
'do_upgrade': dummy_signgature,
'info': dummy_signgature,
},
optional_objects={
'estimate': dummy_signgature,
'pre_upgrade': dummy_signgature,
'post_upgrade': dummy_signgature,
},
other_data={
'depends_on': (list, []),
},
)
# Load all upgrades
plugins = PluginContainer(
[os.path.join(p, '*.py') for p in self._paths],
plugin_builder=builder,
external=True,
)
# Check for broken plug-ins
broken = plugins.get_broken_plugins()
if broken:
messages = []
import traceback
for plugin, info in broken.items():
messages.append("Failed to load %s:\n"
" %s" % (plugin, "".join(traceback.format_exception(*info))))
raise RuntimeError(*messages)
# Store name and doc of plug-in in itself
for plugin_id in plugins.keys():
plugins[plugin_id]['id'] = plugin_id
plugins[plugin_id]['repository'] = self._parse_plugin_id(plugin_id)
return plugins
BibField Utils
Helper classes and functions to work with BibField
"""
import re
__revision__ = "$Id$"
import os
import datetime
from invenio.config import CFG_PYLIBDIR
from invenio.pluginutils import PluginContainer
CFG_BIBFIELD_FUNCTIONS = PluginContainer(os.path.join(CFG_PYLIBDIR, 'invenio', 'bibfield_functions', '*.py'))
class BibFieldException(Exception):
"""
General exception to use within BibField
"""
pass
class BibFieldCheckerException(Exception):
"""
Exception raised when some error happens during checking
"""
pass
def load_tokenizers():
"""
Load all the bibindex tokenizers and returns it.
"""
return PluginContainer(
os.path.join(CFG_BIBINDEX_TOKENIZERS_PATH, 'BibIndex*.py'))
"""
def plugin_builder(plugin_name, plugin_code):
if plugin_name == '__init__':
return
try:
all_widgets = getattr(plugin_code, '__all__')
for name in all_widgets:
candidate = getattr(plugin_code, name)
if issubclass(candidate, Form):
return candidate
except AttributeError:
pass
CFG_WIDGETS = PluginContainer(os.path.join(CFG_PYLIBDIR, 'invenio',
'bibworkflow_widgets',
'*_widget.py'),
plugin_builder=plugin_builder)
widgets = {}
for widget in CFG_WIDGETS.itervalues():
widgets[widget.__name__] = widget
# ## Let's report about broken plugins
# open(os.path.join(CFG_LOGDIR, 'broken-deposition-fields.log'), 'w').write(
# pformat(CFG_FIELDS.get_broken_plugins()))
__all__ = ['widgets']
def read_metadata(inputfile, force=None, remote=False,
loginpw=None, verbose=0):
"""
Returns metadata extracted from given file as dictionary.
Availability depends on input file format and installed plugins
(return C{TypeError} if unsupported file format).
@param inputfile: path to a file
@type inputfile: string
@param verbose: verbosity
@type verbose: int
@param force: name of plugin to use, to skip plugin auto-discovery
@type force: string
@param remote: if the file is accessed remotely or not
@type remote: boolean
@param loginpw: credentials to access secure servers (username:password)
@type loginpw: string
@return: dictionary of metadata tags as keys, and (interpreted)
value as value
@rtype: dict
@raise TypeError: if file format is not supported.
@raise RuntimeError: if required library to process file is missing.
@raise InvenioWebSubmitFileMetadataRuntimeError: when metadata cannot be read.
"""
metadata = None
# Check file type (0 base, 1 name, 2 ext)
ext = decompose_file(inputfile)[2]
if verbose > 5:
print ext.lower(), 'extension to extract from'
# Load plugins
metadata_extractor_plugins = PluginContainer(
os.path.join(CFG_PYLIBDIR,
'invenio', 'websubmit_file_metadata_plugins', 'wsm_*.py'),
plugin_builder=plugin_builder_function,
api_version=__required_plugin_API_version__)
# Loop through the plugins to find a good one for given file
for plugin_name, plugin in metadata_extractor_plugins.iteritems():
# Local file
if plugin.has_key('can_read_local') and \
plugin['can_read_local'](inputfile) and not remote and \
(not force or plugin_name == force):
if verbose > 5:
print 'Using ' + plugin_name
fetched_metadata = plugin['read_metadata_local'](inputfile,
verbose)
if not metadata:
metadata = fetched_metadata
else:
metadata.update(fetched_metadata)
# Remote file
elif remote and plugin.has_key('can_read_remote') and \
plugin['can_read_remote'](inputfile) and \
(not force or plugin_name == force):
if verbose > 5:
print 'Using ' + plugin_name
fetched_metadata = plugin['read_metadata_remote'](inputfile,
loginpw,
verbose)
if not metadata:
metadata = fetched_metadata
else:
metadata.update(fetched_metadata)
# Return in case we have something
if metadata is not None:
return metadata
# Case of no plugin found, raise
raise TypeError, 'Unsupported file type'