def corp_freqs_cache_path(corp, attrname):
"""
Generates an absolute path to an 'attribute' directory/file. The path
consists of two parts: 1) absolute path to corpus indexed data
2) filename given by the 'attrname' argument. It is also dependent
on whether you pass a subcorpus (files are created in user's assigned directory)
or a regular corpus (files are created in the 'cache' directory).
arguments:
corp -- manatee corpus instance
attrname -- name of an attribute
returns:
a path encoded as an 8-bit string (i.e. unicode paths are encoded)
"""
if hasattr(corp, "spath"):
ans = corp.spath.decode("utf-8")[:-4] + attrname
else:
if settings.contains("corpora", "freqs_cache_dir"):
cache_dir = os.path.abspath(settings.get("corpora", "freqs_cache_dir"))
subdirs = (corp.corpname,)
else:
cache_dir = os.path.abspath(settings.get("corpora", "cache_dir"))
subdirs = (corp.corpname, "freqs")
for d in subdirs:
cache_dir = "%s/%s" % (cache_dir, d)
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)
ans = "%s/%s" % (cache_dir, attrname)
return ans.encode("utf-8")
def init_plugin(name, module=None, optional=False):
"""
Installs a plug-in specified by the supplied name (or name and module).
arguments:
name -- a name of the plugin
module -- if supplied then name->module inference is skipped and init_plugin
uses this module as a source of the plug-in
optional -- if True then the module is installed only if it is configured
"""
if not optional or has_configured_plugin(name):
try:
if module is None:
if not settings.contains('plugins', name):
raise PluginException('Missing configuration for the "%s" plugin' % name)
plugin_module = plugins.load_plugin_module(settings.get('plugins', name)['module'])
else:
plugin_module = module
plugins.install_plugin(name, plugin_module, settings)
except ImportError as e:
logging.getLogger(__name__).warn('Plugin [%s] configured but following error occurred: %r'
% (name, e))
except (PluginException, Exception) as e:
from controller.errors import get_traceback
logging.getLogger(__name__).critical('Failed to initiate plug-in %s: %s' % (name, e))
logging.getLogger(__name__).error(''.join(get_traceback()))
raise e
else:
plugins.add_missing_plugin(name)
def init_plugin(name, module=None, optional=False):
"""
Installs a plug-in specified by the supplied name (or name and module).
arguments:
name -- a name of the plugin
module -- if supplied then name->module inference is skipped and init_plugin
uses this module as a source of the plug-in
optional -- if True then the module is installed only if it is configured
"""
if not optional or has_configured_plugin(name):
try:
if module is None:
if not settings.contains('plugins', name):
raise PluginException(
'Missing configuration for the "%s" plugin' % name)
plugin_module = plugins.load_plugin_module(
settings.get('plugins', name)['module'])
else:
plugin_module = module
plugins.install_plugin(name, plugin_module, settings)
except ImportError as e:
logging.getLogger(__name__).warn(
'Plugin [%s] configured but following error occurred: %r' %
(name, e))
except (PluginException, Exception) as e:
logging.getLogger(__name__).critical(
'Failed to initiate plug-in %s: %s' % (name, e))
raise e
else:
plugins.add_missing_plugin(name)
def has_configured_plugin(name):
"""
Tests whether there is a properly configured plugin of a specified name. Only
config.xml is tested (i.e. no actual python modules are involved).
arguments:
name -- name of the plugin
"""
return settings.contains('plugins', name) and settings.get('plugins', name).get('module', None)
def has_configured_plugin(name):
"""
Tests whether there is a properly configured plugin of a specified name. Only
config.xml is tested (i.e. no actual python modules are involved).
arguments:
name -- name of the plugin
"""
return settings.contains('plugins', name) and settings.get(
'plugins', name).get('module', None)
def _apply_theme(self):
theme_name = settings.get('theme', 'name')
theme_css = settings.get('theme', 'css', None)
if theme_css is None:
theme_css = []
elif not hasattr(theme_css, '__iter__'):
theme_css = [theme_css]
logo_img = settings.get('theme', 'logo')
if settings.contains('theme', 'logo_mouseover'):
logo_alt_img = settings.get('theme', 'logo_mouseover')
else:
logo_alt_img = logo_img
logo_href = ''
if theme_name == 'default':
logo_title = _('Click to enter a new query')
else:
logo_title = logo_href
fonts = settings.get('theme', 'fonts', None)
if fonts is None:
fonts = []
elif not hasattr(fonts, '__iter__'):
fonts = [fonts]
return {
'theme_name':
settings.get('theme', 'name'),
'theme_css': [
os.path.normpath('../files/themes/%s/%s' % (theme_name, p))
for p in theme_css
],
'theme_logo_path':
os.path.normpath('../files/themes/%s/%s' % (theme_name, logo_img)),
'theme_logo_mouseover_path':
os.path.normpath('../files/themes/%s/%s' %
(theme_name, logo_alt_img)),
'theme_logo_href':
logo_href,
'theme_logo_title':
logo_title,
'theme_logo_inline_css':
settings.get('theme', 'logo_inline_css', ''),
'theme_fonts':
fonts
}
def init_plugin(name, module=None, optional=False):
"""
Installs a plug-in specified by the supplied name (or name and module).
arguments:
name -- a name of the plugin
module -- if supplied then name->module inference is skipped and init_plugin
uses this module as a source of the plug-in
optional -- if True then the module is installed only if it is configured
"""
if not optional or has_configured_plugin(name):
try:
if module is None:
if not settings.contains('plugins', name):
raise PluginException(
'Missing configuration for the "%s" plugin' % name)
plugin_module = plugins.load_plugin_module(
settings.get('plugins', name)['module'])
else:
plugin_module = module
plg = plugins.install_plugin(name, plugin_module, settings)
if hasattr(plg, 'wait_for_environment') and callable(
plg.wait_for_environment):
logging.getLogger(__name__).info(
f'Plug-in {plg.__class__.__name__} is waiting for environment'
)
err = plg.wait_for_environment()
if err:
logging.getLogger(__name__).error(
f'{plg.__class__.__name__}: {err}')
else:
logging.getLogger(__name__).info(
f'Plug-in {plg.__class__.__name__} environment OK')
except ImportError as e:
logging.getLogger(__name__).warn(
'Plugin [%s] configured but following error occurred: %r' %
(name, e))
except (PluginException, Exception) as e:
from controller.errors import get_traceback
logging.getLogger(__name__).critical(
'Failed to initiate plug-in %s: %s' % (name, e))
logging.getLogger(__name__).error(''.join(get_traceback()))
raise e
else:
plugins.add_missing_plugin(name)
def _apply_theme(self):
theme_name = settings.get('theme', 'name')
theme_css = settings.get('theme', 'css', None)
if theme_css is None:
theme_css = []
elif not hasattr(theme_css, '__iter__'):
theme_css = [theme_css]
logo_img = settings.get('theme', 'logo')
if settings.contains('theme', 'logo_mouseover'):
logo_alt_img = settings.get('theme', 'logo_mouseover')
else:
logo_alt_img = logo_img
logo_href = ''
if theme_name == 'default':
logo_title = _('Click to enter a new query')
else:
logo_title = logo_href
fonts = settings.get('theme', 'fonts', None)
if fonts is None:
fonts = []
elif not hasattr(fonts, '__iter__'):
fonts = [fonts]
return {
'theme_name': settings.get('theme', 'name'),
'theme_css': [os.path.normpath('../files/themes/%s/%s' % (theme_name, p))
for p in theme_css],
'theme_logo_path': os.path.normpath('../files/themes/%s/%s' % (theme_name, logo_img)),
'theme_logo_mouseover_path': os.path.normpath('../files/themes/%s/%s' % (theme_name,
logo_alt_img)),
'theme_logo_href': logo_href,
'theme_logo_title': logo_title,
'theme_logo_inline_css': settings.get('theme', 'logo_inline_css', ''),
'theme_fonts': fonts
}
return data
def test_sections(self):
self.assertTrue(settings.contains('theme'))
self.assertTrue(settings.contains('global'))
self.assertTrue(settings.contains('mailing'))
self.assertTrue(settings.contains('logging'))
self.assertTrue(settings.contains('corpora'))
self.assertTrue(settings.contains('plugins'))
self.assertTrue(settings.contains('plugins', 'application_bar'))
self.assertTrue(settings.contains('plugins', 'auth'))
self.assertTrue(settings.contains('plugins', 'conc_cache'))
self.assertTrue(settings.contains('plugins', 'query_persistence'))
self.assertTrue(settings.contains('plugins', 'corparch'))
self.assertTrue(settings.contains('plugins', 'db'))
self.assertTrue(settings.contains('plugins', 'export'))
self.assertTrue(settings.contains('plugins', 'footer_bar'))
self.assertTrue(settings.contains('plugins', 'getlang'))
self.assertTrue(settings.contains('plugins', 'live_attributes'))
self.assertTrue(settings.contains('plugins', 'menu_items'))
self.assertTrue(settings.contains('plugins', 'query_history'))
self.assertTrue(settings.contains('plugins', 'settings_storage'))
self.assertTrue(settings.contains('plugins', 'sessions'))
self.assertTrue(settings.contains('plugins', 'subcmixer'))
self.assertTrue(settings.contains('plugins', 'subc_restore'))
self.assertTrue(settings.contains('plugins', 'syntax_viewer'))
self.assertTrue(settings.contains('plugins', 'taghelper'))
self.assertTrue(settings.contains('plugins', 'user_items'))
def test_set(self):
settings.set('local', 'test', '1')
self.assertTrue(settings.contains('local'))
self.assertEqual(settings.get('local', 'test'), '1')
# creates [plugin_name]__[task_name] tasks
for p in plugins.runtime:
if callable(getattr(p.instance, 'export_tasks', None)):
for tsk in p.instance.export_tasks():
globals()[f'{p.name}__{tsk.__name__}'] = tsk
if __name__ == "__main__":
with Connection(redis.Redis(
host=settings.get('calc_backend', 'rq_redis_host'),
port=settings.get('calc_backend', 'rq_redis_port'),
db=settings.get('calc_backend', 'rq_redis_db')
)):
logging_handlers = []
if settings.contains('logging', 'stderr'):
handler = logging.StreamHandler(sys.stderr)
elif settings.contains('logging', 'stdout'):
handler = logging.StreamHandler(sys.stdout)
elif settings.contains('calc_backend', 'rq_log_path'):
handler = logging.FileHandler(settings.get('calc_backend', 'rq_log_path'))
if handler:
handler.setFormatter(logging.Formatter(
fmt='%(asctime)s [%(name)s] %(levelname)s: %(message)s'))
logging_handlers.append(handler)
logging.basicConfig(
handlers=logging_handlers,
level=logging.INFO if not settings.is_debug_mode() else logging.DEBUG
)
if request.session.should_save:
sessions.save(request.session)
response.set_cookie(sessions.get_cookie_name(), request.session.sid)
return response(environ, start_response)
settings.load(path=CONF_PATH)
if settings.get('global', 'manatee_path', None):
sys.path.insert(0, settings.get('global', 'manatee_path'))
# please note that some environments may provide umask setting themselves
if settings.get('global', 'umask', None):
os.umask(int(settings.get('global', 'umask'), 8))
if not settings.contains('corpora', 'calc_pid_dir'):
raise Exception('Missing configuration: calc_pid_dir')
elif not os.path.exists(settings.get('corpora', 'calc_pid_dir')):
os.makedirs(settings.get('corpora', 'calc_pid_dir'))
if not settings.get_bool('global', 'maintenance'):
application = App()
else:
application = MaintenanceApp()
robots_path = os.path.join(os.path.dirname(__file__), 'files/robots.txt')
if os.path.isfile(robots_path):
from werkzeug.wsgi import SharedDataMiddleware
application = SharedDataMiddleware(application, {
'/robots.txt': robots_path
})
def test_sections(self):
self.assertTrue(settings.contains('theme'))
self.assertTrue(settings.contains('global'))
self.assertTrue(settings.contains('mailing'))
self.assertTrue(settings.contains('logging'))
self.assertTrue(settings.contains('corpora'))
self.assertTrue(settings.contains('plugins'))
self.assertTrue(settings.contains('plugins', 'application_bar'))
self.assertTrue(settings.contains('plugins', 'auth'))
self.assertTrue(settings.contains('plugins', 'conc_cache'))
self.assertTrue(settings.contains('plugins', 'conc_persistence'))
self.assertTrue(settings.contains('plugins', 'corparch'))
self.assertTrue(settings.contains('plugins', 'db'))
self.assertTrue(settings.contains('plugins', 'export'))
self.assertTrue(settings.contains('plugins', 'footer_bar'))
self.assertTrue(settings.contains('plugins', 'getlang'))
self.assertTrue(settings.contains('plugins', 'live_attributes'))
self.assertTrue(settings.contains('plugins', 'menu_items'))
self.assertTrue(settings.contains('plugins', 'query_storage'))
self.assertTrue(settings.contains('plugins', 'settings_storage'))
self.assertTrue(settings.contains('plugins', 'sessions'))
self.assertTrue(settings.contains('plugins', 'subcmixer'))
self.assertTrue(settings.contains('plugins', 'subc_restore'))
self.assertTrue(settings.contains('plugins', 'syntax_viewer'))
self.assertTrue(settings.contains('plugins', 'taghelper'))
self.assertTrue(settings.contains('plugins', 'user_items'))
def test_set(self):
settings.set('local', 'test', '1')
self.assertTrue(settings.contains('local'))
self.assertEqual(settings.get('local', 'test'), '1')
sessions.save(request.session)
response.set_cookie(sessions.get_cookie_name(),
request.session.sid)
return response(environ, start_response)
settings.load(path=CONF_PATH)
if settings.get('global', 'manatee_path', None):
sys.path.insert(0, settings.get('global', 'manatee_path'))
# please note that some environments may provide umask setting themselves
if settings.get('global', 'umask', None):
os.umask(int(settings.get('global', 'umask'), 8))
if not settings.contains('corpora', 'calc_pid_dir'):
raise Exception('Missing configuration: calc_pid_dir')
elif not os.path.exists(settings.get('corpora', 'calc_pid_dir')):
os.makedirs(settings.get('corpora', 'calc_pid_dir'))
if not settings.get_bool('global', 'maintenance'):
application = App()
else:
application = MaintenanceApp()
robots_path = os.path.join(os.path.dirname(__file__), 'files/robots.txt')
if os.path.isfile(robots_path):
from werkzeug.wsgi import SharedDataMiddleware
application = SharedDataMiddleware(application,
{'/robots.txt': robots_path})
import re
import sqlite3
import vertparser
SCRIPT_PATH = os.path.realpath(os.path.dirname(os.path.abspath(__file__)))
APP_PATH = os.path.realpath('%s/../../..' % SCRIPT_PATH)
sys.path.insert(0, '%s/lib' % APP_PATH)
import corplib
import settings
settings.load('%s/conf/config.xml' % APP_PATH)
if settings.contains('global', 'manatee_path'):
sys.path.insert(0, settings.get('global', 'manatee_path'))
os.environ['MANATEE_REGISTRY'] = settings.get('corpora', 'manatee_registry')
def get_registry_attrs(corpus_id):
"""
Returns all the structures and their attributes of a specified corpus.
arguments:
corpus_id -- a corpus identifier
returns:
a list of 2-tuples (structure, attribute)
"""
response = Response(response=body, status=status, headers=headers)
if not sid_is_valid:
curr_data = dict(request.session)
request.session = sessions.new()
request.session.update(curr_data)
request.session.modified = True
if request.session.should_save:
sessions.save(request.session)
response.set_cookie(sessions.get_cookie_name(), request.session.sid)
start_response(status, headers)
return response(environ, start_response)
settings.load(path=CONF_PATH)
if settings.contains('global', 'manatee_path'):
sys.path.insert(0, settings.get('global', 'manatee_path'))
if not settings.contains('corpora', 'calc_pid_dir'):
raise Exception('Missing configuration: calc_pid_dir')
elif not os.path.exists(settings.get('corpora', 'calc_pid_dir')):
os.makedirs(settings.get('corpora', 'calc_pid_dir'))
application = App()
robots_path = os.path.join(os.path.dirname(__file__), 'files/robots.txt')
if os.path.isfile(robots_path):
from werkzeug.wsgi import SharedDataMiddleware
application = SharedDataMiddleware(application, {
'/robots.txt': robots_path
})