Beispiel #1
0
def corp_freqs_cache_path(corp, attrname):
    """
    Generates an absolute path to an 'attribute' directory/file. The path
    consists of two parts: 1) absolute path to corpus indexed data
    2) filename given by the 'attrname' argument. It is also dependent
    on whether you pass a subcorpus (files are created in user's assigned directory)
    or a regular corpus (files are created in the 'cache' directory).

    arguments:
    corp -- manatee corpus instance
    attrname -- name of an attribute

    returns:
    a path encoded as an 8-bit string (i.e. unicode paths are encoded)
    """
    if hasattr(corp, "spath"):
        ans = corp.spath.decode("utf-8")[:-4] + attrname
    else:
        if settings.contains("corpora", "freqs_cache_dir"):
            cache_dir = os.path.abspath(settings.get("corpora", "freqs_cache_dir"))
            subdirs = (corp.corpname,)
        else:
            cache_dir = os.path.abspath(settings.get("corpora", "cache_dir"))
            subdirs = (corp.corpname, "freqs")
        for d in subdirs:
            cache_dir = "%s/%s" % (cache_dir, d)
            if not os.path.exists(cache_dir):
                os.makedirs(cache_dir)
        ans = "%s/%s" % (cache_dir, attrname)
    return ans.encode("utf-8")
Beispiel #2
0
def init_plugin(name, module=None, optional=False):
    """
    Installs a plug-in specified by the supplied name (or name and module).

    arguments:
    name -- a name of the plugin
    module -- if supplied then name->module inference is skipped and init_plugin
              uses this module as a source of the plug-in
    optional -- if True then the module is installed only if it is configured
    """
    if not optional or has_configured_plugin(name):
        try:
            if module is None:
                if not settings.contains('plugins', name):
                    raise PluginException('Missing configuration for the "%s" plugin' % name)
                plugin_module = plugins.load_plugin_module(settings.get('plugins', name)['module'])
            else:
                plugin_module = module
            plugins.install_plugin(name, plugin_module, settings)
        except ImportError as e:
            logging.getLogger(__name__).warn('Plugin [%s] configured but following error occurred: %r'
                                             % (name, e))
        except (PluginException, Exception) as e:
            from controller.errors import get_traceback
            logging.getLogger(__name__).critical('Failed to initiate plug-in %s: %s' % (name, e))
            logging.getLogger(__name__).error(''.join(get_traceback()))
            raise e
    else:
        plugins.add_missing_plugin(name)
Beispiel #3
0
def init_plugin(name, module=None, optional=False):
    """
    Installs a plug-in specified by the supplied name (or name and module).

    arguments:
    name -- a name of the plugin
    module -- if supplied then name->module inference is skipped and init_plugin
              uses this module as a source of the plug-in
    optional -- if True then the module is installed only if it is configured
    """
    if not optional or has_configured_plugin(name):
        try:
            if module is None:
                if not settings.contains('plugins', name):
                    raise PluginException(
                        'Missing configuration for the "%s" plugin' % name)
                plugin_module = plugins.load_plugin_module(
                    settings.get('plugins', name)['module'])
            else:
                plugin_module = module
            plugins.install_plugin(name, plugin_module, settings)
        except ImportError as e:
            logging.getLogger(__name__).warn(
                'Plugin [%s] configured but following error occurred: %r' %
                (name, e))
        except (PluginException, Exception) as e:
            logging.getLogger(__name__).critical(
                'Failed to initiate plug-in %s: %s' % (name, e))
            raise e
    else:
        plugins.add_missing_plugin(name)
Beispiel #4
0
def has_configured_plugin(name):
    """
    Tests whether there is a properly configured plugin of a specified name. Only
    config.xml is tested (i.e. no actual python modules are involved).

    arguments:
    name -- name of the plugin
    """
    return settings.contains('plugins', name) and settings.get('plugins', name).get('module', None)
Beispiel #5
0
def has_configured_plugin(name):
    """
    Tests whether there is a properly configured plugin of a specified name. Only
    config.xml is tested (i.e. no actual python modules are involved).

    arguments:
    name -- name of the plugin
    """
    return settings.contains('plugins', name) and settings.get(
        'plugins', name).get('module', None)
Beispiel #6
0
    def _apply_theme(self):
        theme_name = settings.get('theme', 'name')
        theme_css = settings.get('theme', 'css', None)
        if theme_css is None:
            theme_css = []
        elif not hasattr(theme_css, '__iter__'):
            theme_css = [theme_css]

        logo_img = settings.get('theme', 'logo')
        if settings.contains('theme', 'logo_mouseover'):
            logo_alt_img = settings.get('theme', 'logo_mouseover')
        else:
            logo_alt_img = logo_img

        logo_href = ''

        if theme_name == 'default':
            logo_title = _('Click to enter a new query')
        else:
            logo_title = logo_href

        fonts = settings.get('theme', 'fonts', None)
        if fonts is None:
            fonts = []
        elif not hasattr(fonts, '__iter__'):
            fonts = [fonts]

        return {
            'theme_name':
            settings.get('theme', 'name'),
            'theme_css': [
                os.path.normpath('../files/themes/%s/%s' % (theme_name, p))
                for p in theme_css
            ],
            'theme_logo_path':
            os.path.normpath('../files/themes/%s/%s' % (theme_name, logo_img)),
            'theme_logo_mouseover_path':
            os.path.normpath('../files/themes/%s/%s' %
                             (theme_name, logo_alt_img)),
            'theme_logo_href':
            logo_href,
            'theme_logo_title':
            logo_title,
            'theme_logo_inline_css':
            settings.get('theme', 'logo_inline_css', ''),
            'theme_fonts':
            fonts
        }
Beispiel #7
0
def init_plugin(name, module=None, optional=False):
    """
    Installs a plug-in specified by the supplied name (or name and module).

    arguments:
    name -- a name of the plugin
    module -- if supplied then name->module inference is skipped and init_plugin
              uses this module as a source of the plug-in
    optional -- if True then the module is installed only if it is configured
    """
    if not optional or has_configured_plugin(name):
        try:
            if module is None:
                if not settings.contains('plugins', name):
                    raise PluginException(
                        'Missing configuration for the "%s" plugin' % name)
                plugin_module = plugins.load_plugin_module(
                    settings.get('plugins', name)['module'])
            else:
                plugin_module = module
            plg = plugins.install_plugin(name, plugin_module, settings)
            if hasattr(plg, 'wait_for_environment') and callable(
                    plg.wait_for_environment):
                logging.getLogger(__name__).info(
                    f'Plug-in {plg.__class__.__name__} is waiting for environment'
                )
                err = plg.wait_for_environment()
                if err:
                    logging.getLogger(__name__).error(
                        f'{plg.__class__.__name__}: {err}')
                else:
                    logging.getLogger(__name__).info(
                        f'Plug-in {plg.__class__.__name__} environment OK')

        except ImportError as e:
            logging.getLogger(__name__).warn(
                'Plugin [%s] configured but following error occurred: %r' %
                (name, e))
        except (PluginException, Exception) as e:
            from controller.errors import get_traceback
            logging.getLogger(__name__).critical(
                'Failed to initiate plug-in %s: %s' % (name, e))
            logging.getLogger(__name__).error(''.join(get_traceback()))
            raise e
    else:
        plugins.add_missing_plugin(name)
Beispiel #8
0
    def _apply_theme(self):
        theme_name = settings.get('theme', 'name')
        theme_css = settings.get('theme', 'css', None)
        if theme_css is None:
            theme_css = []
        elif not hasattr(theme_css, '__iter__'):
            theme_css = [theme_css]

        logo_img = settings.get('theme', 'logo')
        if settings.contains('theme', 'logo_mouseover'):
            logo_alt_img = settings.get('theme', 'logo_mouseover')
        else:
            logo_alt_img = logo_img

        logo_href = ''

        if theme_name == 'default':
            logo_title = _('Click to enter a new query')
        else:
            logo_title = logo_href

        fonts = settings.get('theme', 'fonts', None)
        if fonts is None:
            fonts = []
        elif not hasattr(fonts, '__iter__'):
            fonts = [fonts]

        return {
            'theme_name': settings.get('theme', 'name'),
            'theme_css': [os.path.normpath('../files/themes/%s/%s' % (theme_name, p))
                    for p in theme_css],
            'theme_logo_path': os.path.normpath('../files/themes/%s/%s' % (theme_name, logo_img)),
            'theme_logo_mouseover_path': os.path.normpath('../files/themes/%s/%s' % (theme_name,
                                                                               logo_alt_img)),
            'theme_logo_href': logo_href,
            'theme_logo_title': logo_title,
            'theme_logo_inline_css': settings.get('theme', 'logo_inline_css', ''),
            'theme_fonts': fonts
        }
        return data
Beispiel #9
0
 def test_sections(self):
     self.assertTrue(settings.contains('theme'))
     self.assertTrue(settings.contains('global'))
     self.assertTrue(settings.contains('mailing'))
     self.assertTrue(settings.contains('logging'))
     self.assertTrue(settings.contains('corpora'))
     self.assertTrue(settings.contains('plugins'))
     self.assertTrue(settings.contains('plugins', 'application_bar'))
     self.assertTrue(settings.contains('plugins', 'auth'))
     self.assertTrue(settings.contains('plugins', 'conc_cache'))
     self.assertTrue(settings.contains('plugins', 'query_persistence'))
     self.assertTrue(settings.contains('plugins', 'corparch'))
     self.assertTrue(settings.contains('plugins', 'db'))
     self.assertTrue(settings.contains('plugins', 'export'))
     self.assertTrue(settings.contains('plugins', 'footer_bar'))
     self.assertTrue(settings.contains('plugins', 'getlang'))
     self.assertTrue(settings.contains('plugins', 'live_attributes'))
     self.assertTrue(settings.contains('plugins', 'menu_items'))
     self.assertTrue(settings.contains('plugins', 'query_history'))
     self.assertTrue(settings.contains('plugins', 'settings_storage'))
     self.assertTrue(settings.contains('plugins', 'sessions'))
     self.assertTrue(settings.contains('plugins', 'subcmixer'))
     self.assertTrue(settings.contains('plugins', 'subc_restore'))
     self.assertTrue(settings.contains('plugins', 'syntax_viewer'))
     self.assertTrue(settings.contains('plugins', 'taghelper'))
     self.assertTrue(settings.contains('plugins', 'user_items'))
Beispiel #10
0
 def test_set(self):
     settings.set('local', 'test', '1')
     self.assertTrue(settings.contains('local'))
     self.assertEqual(settings.get('local', 'test'), '1')
Beispiel #11
0
# creates [plugin_name]__[task_name] tasks
for p in plugins.runtime:
    if callable(getattr(p.instance, 'export_tasks', None)):
        for tsk in p.instance.export_tasks():
            globals()[f'{p.name}__{tsk.__name__}'] = tsk


if __name__ == "__main__":
    with Connection(redis.Redis(
        host=settings.get('calc_backend', 'rq_redis_host'),
        port=settings.get('calc_backend', 'rq_redis_port'),
        db=settings.get('calc_backend', 'rq_redis_db')
    )):

        logging_handlers = []
        if settings.contains('logging', 'stderr'):
            handler = logging.StreamHandler(sys.stderr)
        elif settings.contains('logging', 'stdout'):
            handler = logging.StreamHandler(sys.stdout)
        elif settings.contains('calc_backend', 'rq_log_path'):
            handler = logging.FileHandler(settings.get('calc_backend', 'rq_log_path'))

        if handler:
            handler.setFormatter(logging.Formatter(
                fmt='%(asctime)s [%(name)s] %(levelname)s: %(message)s'))
            logging_handlers.append(handler)

        logging.basicConfig(
            handlers=logging_handlers,
            level=logging.INFO if not settings.is_debug_mode() else logging.DEBUG
        )
Beispiel #12
0
        if request.session.should_save:
            sessions.save(request.session)
            response.set_cookie(sessions.get_cookie_name(), request.session.sid)
        return response(environ, start_response)


settings.load(path=CONF_PATH)

if settings.get('global', 'manatee_path', None):
    sys.path.insert(0, settings.get('global', 'manatee_path'))

# please note that some environments may provide umask setting themselves
if settings.get('global', 'umask', None):
    os.umask(int(settings.get('global', 'umask'), 8))

if not settings.contains('corpora', 'calc_pid_dir'):
    raise Exception('Missing configuration: calc_pid_dir')
elif not os.path.exists(settings.get('corpora', 'calc_pid_dir')):
    os.makedirs(settings.get('corpora', 'calc_pid_dir'))

if not settings.get_bool('global', 'maintenance'):
    application = App()
else:
    application = MaintenanceApp()

robots_path = os.path.join(os.path.dirname(__file__), 'files/robots.txt')
if os.path.isfile(robots_path):
    from werkzeug.wsgi import SharedDataMiddleware
    application = SharedDataMiddleware(application, {
        '/robots.txt': robots_path
    })
Beispiel #13
0
 def test_sections(self):
     self.assertTrue(settings.contains('theme'))
     self.assertTrue(settings.contains('global'))
     self.assertTrue(settings.contains('mailing'))
     self.assertTrue(settings.contains('logging'))
     self.assertTrue(settings.contains('corpora'))
     self.assertTrue(settings.contains('plugins'))
     self.assertTrue(settings.contains('plugins', 'application_bar'))
     self.assertTrue(settings.contains('plugins', 'auth'))
     self.assertTrue(settings.contains('plugins', 'conc_cache'))
     self.assertTrue(settings.contains('plugins', 'conc_persistence'))
     self.assertTrue(settings.contains('plugins', 'corparch'))
     self.assertTrue(settings.contains('plugins', 'db'))
     self.assertTrue(settings.contains('plugins', 'export'))
     self.assertTrue(settings.contains('plugins', 'footer_bar'))
     self.assertTrue(settings.contains('plugins', 'getlang'))
     self.assertTrue(settings.contains('plugins', 'live_attributes'))
     self.assertTrue(settings.contains('plugins', 'menu_items'))
     self.assertTrue(settings.contains('plugins', 'query_storage'))
     self.assertTrue(settings.contains('plugins', 'settings_storage'))
     self.assertTrue(settings.contains('plugins', 'sessions'))
     self.assertTrue(settings.contains('plugins', 'subcmixer'))
     self.assertTrue(settings.contains('plugins', 'subc_restore'))
     self.assertTrue(settings.contains('plugins', 'syntax_viewer'))
     self.assertTrue(settings.contains('plugins', 'taghelper'))
     self.assertTrue(settings.contains('plugins', 'user_items'))
Beispiel #14
0
 def test_set(self):
     settings.set('local', 'test', '1')
     self.assertTrue(settings.contains('local'))
     self.assertEqual(settings.get('local', 'test'), '1')
Beispiel #15
0
            sessions.save(request.session)
            response.set_cookie(sessions.get_cookie_name(),
                                request.session.sid)
        return response(environ, start_response)


settings.load(path=CONF_PATH)

if settings.get('global', 'manatee_path', None):
    sys.path.insert(0, settings.get('global', 'manatee_path'))

# please note that some environments may provide umask setting themselves
if settings.get('global', 'umask', None):
    os.umask(int(settings.get('global', 'umask'), 8))

if not settings.contains('corpora', 'calc_pid_dir'):
    raise Exception('Missing configuration: calc_pid_dir')
elif not os.path.exists(settings.get('corpora', 'calc_pid_dir')):
    os.makedirs(settings.get('corpora', 'calc_pid_dir'))

if not settings.get_bool('global', 'maintenance'):
    application = App()
else:
    application = MaintenanceApp()

robots_path = os.path.join(os.path.dirname(__file__), 'files/robots.txt')
if os.path.isfile(robots_path):
    from werkzeug.wsgi import SharedDataMiddleware
    application = SharedDataMiddleware(application,
                                       {'/robots.txt': robots_path})
Beispiel #16
0
import re

import sqlite3

import vertparser


SCRIPT_PATH = os.path.realpath(os.path.dirname(os.path.abspath(__file__)))
APP_PATH = os.path.realpath('%s/../../..' % SCRIPT_PATH)
sys.path.insert(0, '%s/lib' % APP_PATH)

import corplib
import settings
settings.load('%s/conf/config.xml' % APP_PATH)

if settings.contains('global', 'manatee_path'):
    sys.path.insert(0, settings.get('global', 'manatee_path'))

os.environ['MANATEE_REGISTRY'] = settings.get('corpora', 'manatee_registry')


def get_registry_attrs(corpus_id):
    """
    Returns all the structures and their attributes of a specified corpus.

    arguments:
    corpus_id -- a corpus identifier

    returns:
    a list of 2-tuples (structure, attribute)
    """
Beispiel #17
0
        response = Response(response=body, status=status, headers=headers)
        if not sid_is_valid:
            curr_data = dict(request.session)
            request.session = sessions.new()
            request.session.update(curr_data)
            request.session.modified = True
        if request.session.should_save:
            sessions.save(request.session)
            response.set_cookie(sessions.get_cookie_name(), request.session.sid)
        start_response(status, headers)
        return response(environ, start_response)


settings.load(path=CONF_PATH)

if settings.contains('global', 'manatee_path'):
    sys.path.insert(0, settings.get('global', 'manatee_path'))

if not settings.contains('corpora', 'calc_pid_dir'):
    raise Exception('Missing configuration: calc_pid_dir')
elif not os.path.exists(settings.get('corpora', 'calc_pid_dir')):
    os.makedirs(settings.get('corpora', 'calc_pid_dir'))

application = App()

robots_path = os.path.join(os.path.dirname(__file__), 'files/robots.txt')
if os.path.isfile(robots_path):
    from werkzeug.wsgi import SharedDataMiddleware
    application = SharedDataMiddleware(application, {
        '/robots.txt': robots_path
    })