def set_favorite_item(self, request, conc_args):
"""
"""
main_corp = self.cm.get_Corpus(request.form['corpus_id'],
request.form['subcorpus_id'])
corp_size = main_corp.search_size()
data = {
'corpora': [],
'canonical_id': request.form['canonical_id'],
'corpus_id': request.form['corpus_id'],
'subcorpus_id': request.form['subcorpus_id'],
'name': request.form['name'],
'size': corp_size,
'size_info': l10n.simplify_num(corp_size),
'type': request.form['type']
}
aligned_corpnames = request.form.getlist('corpora[]')
for ac in aligned_corpnames:
data['corpora'].append({
'name': ac, # TODO fetch real name??
'corpus_id': ac,
'canonical_id': self._canonical_corpname(ac),
'type': 'corpus'
})
item = plugins.get('user_items').from_dict(data)
plugins.get('user_items').add_user_item(
self._session_get('user', 'id'), item)
return {'id': item.id}
def _create_subcorpus(self, request):
"""
req. arguments:
subcname -- name of new subcorpus
create -- bool, sets whether to create new subcorpus
cql -- custom within condition
"""
subcname = request.form['subcname']
within_json = request.form.get('within_json')
raw_cql = request.form.get('cql')
corp_encoding = self._corp().get_conf('ENCODING')
if raw_cql:
tt_query = ()
within_cql = raw_cql
full_cql = 'aword,[] %s' % raw_cql
imp_cql = (full_cql,)
elif within_json: # user entered a subcorpus query manually
tt_query = ()
within_cql = self._deserialize_custom_within(json.loads(within_json))
full_cql = 'aword,[] %s' % within_cql
imp_cql = (full_cql,)
else:
tt_query = TextTypeCollector(self._corp(), request).get_query()
full_cql = ' within '.join(['<%s %s />' % item for item in tt_query])
full_cql = 'aword,[] within %s' % full_cql
full_cql = import_string(full_cql, from_encoding=corp_encoding)
imp_cql = (full_cql,)
basecorpname = self.args.corpname.split(':')[0]
if not subcname:
raise UserActionException(_('No subcorpus name specified!'))
path = self.prepare_subc_path(basecorpname, subcname)
if type(path) == unicode:
path = path.encode('utf-8')
if len(tt_query) == 1:
result = corplib.create_subcorpus(path, self._corp(), tt_query[0][0], tt_query[0][1])
elif len(tt_query) > 1 or within_cql:
conc = conclib.get_conc(self._corp(), self._session_get('user', 'user'), q=imp_cql)
conc.sync()
struct = self._corp().get_struct(tt_query[0][0]) if len(tt_query) == 1 else None
result = corplib.subcorpus_from_conc(path, conc, struct)
else:
raise UserActionException(_('Nothing specified!'))
if result:
if plugins.has_plugin('subc_restore'):
try:
plugins.get('subc_restore').store_query(user_id=self._session_get('user', 'id'),
corpname=self.args.corpname,
subcname=subcname,
cql=full_cql.split('[]')[-1])
except Exception as e:
logging.getLogger(__name__).warning('Failed to store subcorpus query: %s' % e)
self.add_system_message('warning',
_('Subcorpus created but there was a problem saving a backup copy.'))
return {}
else:
raise ConcError(_('Empty subcorpus!'))
def set_favorite_item(self, request, conc_args):
"""
"""
main_corp = self.cm.get_Corpus(request.form['corpus_id'], request.form['subcorpus_id'])
corp_size = main_corp.search_size()
data = {
'corpora': [],
'canonical_id': request.form['canonical_id'],
'corpus_id': request.form['corpus_id'],
'subcorpus_id': request.form['subcorpus_id'],
'name': request.form['name'],
'size': corp_size,
'size_info': l10n.simplify_num(corp_size),
'type': request.form['type']
}
aligned_corpnames = request.form.getlist('corpora[]')
for ac in aligned_corpnames:
data['corpora'].append({
'name': ac, # TODO fetch real name??
'corpus_id': ac,
'canonical_id': self._canonical_corpname(ac),
'type': 'corpus'
})
item = plugins.get('user_items').from_dict(data)
plugins.get('user_items').add_user_item(self._session_get('user', 'id'), item)
return {'id': item.id}
def showPlugin(pluginsGet, pluginsProc, identifier):
for id, plugin in plugins.get(pluginsGet, identifier).items():
print('GET Plugin {}'.format(id))
print(' {}'.format(plugin.DESCRIPTION))
showParameterList(plugin.PARAMETERS)
for id, plugin in plugins.get(pluginsProc, identifier).items():
print('PROCESS Plugin {}'.format(id))
print(' {}'.format(plugin.DESCRIPTION))
showParameterList(plugin.PARAMETERS)
def corplist(self, request):
self.disabled_menu_items = self.CONCORDANCE_ACTIONS
return dict(
corplist_params=plugins.get('corparch').initial_search_params(request.args.get('query'),
request.args),
corplist_data=plugins.get('corparch').search(self._session_get('user', 'id'),
request.args.get('query'),
offset=0,
filter_dict=request.args)
)
def logoutx(self, request):
self.disabled_menu_items = (MainMenu.NEW_QUERY, MainMenu.VIEW,
MainMenu.SAVE, MainMenu.CORPORA,
MainMenu.CONCORDANCE, MainMenu.FILTER,
MainMenu.FREQUENCY, MainMenu.COLLOCATIONS)
plugins.get('auth').logout(self._session)
self._init_session()
self.refresh_session_id()
return {'message': ('info', _('You have been logged out'))}
def _create_subcorpus(self, request):
"""
req. arguments:
subcname -- name of new subcorpus
create -- bool, sets whether to create new subcorpus
within_condition -- custom within condition; if non-empty then clickable form is omitted
within_struct -- a structure the within_condition will be applied to
"""
subcname = request.form['subcname']
within_condition = request.form['within_condition']
within_struct = request.form['within_struct']
corp_encoding = self._corp().get_conf('ENCODING')
if within_condition and within_struct: # user entered a subcorpus query manually
tt_query = [(export_string(within_struct, to_encoding=corp_encoding),
export_string(within_condition, to_encoding=corp_encoding))]
else:
tt_query = self._texttype_query(request)
within_struct = import_string(tt_query[0][0], from_encoding=corp_encoding)
within_condition = import_string(tt_query[0][1], from_encoding=corp_encoding)
basecorpname = self.args.corpname.split(':')[0]
if not subcname:
raise ConcError(_('No subcorpus name specified!'))
path = os.path.join(self.subcpath[-1], basecorpname)
if not os.path.isdir(path):
os.makedirs(path)
path = os.path.join(path, subcname) + '.subc'
if not tt_query:
raise ConcError(_('Nothing specified!'))
# Even if _texttype_query() parsed multiple structures into tt_query,
# Manatee can accept directly only one (but with arbitrarily complex attribute
# condition).
# For this reason, we choose only the first struct+condition pair.
# It is up to the user interface to deal with it.
structname, subquery = tt_query[0]
if type(path) == unicode:
path = path.encode("utf-8")
if corplib.create_subcorpus(path, self._corp(), structname, subquery):
if plugins.has_plugin('subc_restore'):
try:
plugins.get('subc_restore').store_query(user_id=self._session_get('user', 'id'),
corpname=self.args.corpname,
subcname=subcname,
structname=within_struct,
condition=within_condition)
except Exception as e:
logging.getLogger(__name__).warning('Failed to store subcorpus query: %s' % e)
self.add_system_message('warning',
_('Subcorpus created but there was a problem saving a backup copy.'))
return {}
else:
raise ConcError(_('Empty subcorpus!'))
def ajax_wipe_subcorpus(self, request):
if plugins.has_plugin('subc_restore'):
corpus_id = request.form['corpname']
subcorp_name = request.form['subcname']
plugins.get('subc_restore').delete_query(self._session_get('user', 'id'),
corpus_id, subcorp_name)
self.add_system_message('info',
_('Subcorpus %s has been deleted permanently.') % subcorp_name)
else:
self.add_system_message('error', _('Unsupported operation (plug-in not present)'))
return {}
def ajax_wipe_subcorpus(self, request):
if plugins.has_plugin('subc_restore'):
corpus_id = request.form['corpname']
subcorp_name = request.form['subcname']
plugins.get('subc_restore').delete_query(self._session_get('user', 'id'),
corpus_id, subcorp_name)
self.add_system_message('info',
_('Subcorpus %s has been deleted permanently.') % subcorp_name)
else:
self.add_system_message('error', _('Unsupported operation (plug-in not present)'))
return {}
def logoutx(self, request):
self.disabled_menu_items = (MainMenu.NEW_QUERY, MainMenu.VIEW,
MainMenu.SAVE, MainMenu.CORPORA, MainMenu.CONCORDANCE,
MainMenu.FILTER, MainMenu.FREQUENCY, MainMenu.COLLOCATIONS)
plugins.get('auth').logout(self._session)
self._init_session()
self.refresh_session_id()
return {
'message': ('info', _('You have been logged out'))
}
def corplist(self, request):
self.disabled_menu_items = self.CONCORDANCE_ACTIONS
return dict(
corplist_params=plugins.get('corparch').initial_search_params(
request.args.get('query'), request.args),
corplist_data=plugins.get('corparch').search(
plugin_api=self._plugin_api,
user_id=self._session_get('user', 'id'),
query=False,
offset=0,
filter_dict=request.args))
def export_with_norms(self, subcorpattrs='', format_num=True, ret_nums=True, subcnorm='tokens'):
"""
Returns a text types table containing also an information about
total occurrences of respective attribute values.
See corplib.texttype_values for arguments and returned value
"""
ans = {}
if not subcorpattrs:
subcorpattrs = self._corp.get_conf('SUBCORPATTRS')
if not subcorpattrs:
subcorpattrs = self._corp.get_conf('FULLREF')
if not subcorpattrs or subcorpattrs == '#':
raise TextTypesException(
_('Missing display configuration of structural attributes (SUBCORPATTRS or FULLREF).'))
corpus_info = plugins.get('corparch').get_corpus_info(self._corpname)
maxlistsize = settings.get_int('global', 'max_attr_list_size')
# if 'live_attributes' are installed then always shrink bibliographical
# entries even if their count is < maxlistsize
subcorp_attr_list = re.split(r'\s*[,|]\s*', subcorpattrs)
if plugins.has_plugin('live_attributes'):
ans['bib_attr'] = corpus_info['metadata']['label_attr']
list_none = (ans['bib_attr'], )
tmp = [s for s in subcorp_attr_list] # making copy here
if ans['bib_attr'] and ans['bib_attr'] not in tmp: # if bib type is not in subcorpattrs
tmp.append(ans['bib_attr']) # we add it there
subcorpattrs = '|'.join(tmp) # we ignore NoSkE '|' vs. ',' stuff deliberately here
else:
ans['bib_attr'] = None
list_none = ()
tt = self._tt_cache.get_values(corp=self._corp, subcorpattrs=subcorpattrs, maxlistsize=maxlistsize,
shrink_list=list_none, collator_locale=corpus_info.collator_locale)
self._add_tt_custom_metadata(tt)
if ret_nums:
struct_calc = collections.OrderedDict()
for item in subcorp_attr_list:
k = item.split('.')[0]
struct_calc[k] = CachedStructNormsCalc(self._corp, k, subcnorm, db=plugins.get('db'))
for col in reduce(lambda p, c: p + c['Line'], tt, []):
if 'textboxlength' not in col:
structname, attrname = col['name'].split('.')
for val in col['Values']:
v = struct_calc[structname].compute_norm(attrname, val['v'])
val['xcnt'] = l10n.format_number(v) if format_num else v
ans['Blocks'] = tt
ans['Normslist'] = self._get_normslist(struct_calc.keys()[0])
else:
ans['Blocks'] = tt
ans['Normslist'] = []
return ans
def _discovery(use_plugins, builds, start_dir, cfg):
use_plugins = set(use_plugins)
cfg = cfg.copy()
# Legge il file di configurazione locale (se c'è), e sovrascrive le
# vecchie configurazioni.
cfg.update(parseBuildCfg())
# Se il discovery è bloccato, non considera questa directory.
if not cfg["discovery"]:
return
# Applica l'attivazione o disattivazione dei plugin da considerare in
# questo tree.
if cfg["disabled"] or cfg["enabled"]:
for p in cfg["disabled"]:
use_plugins.discard(plugins.get(p))
for p in cfg["enabled"]:
use_plugins.add(plugins.get(p))
# Le opzioni a linea di comando vincono sempre su tutto il resto,
# quindi qualunque modifica sia stata fatta, riapplichiamo quanto
# richiesto dall'utente.
for p in opts.disabled:
use_plugins.discard(p)
for p in opts.enabled:
use_plugins.add(p)
sys.stdout.write("."); sys.stdout.flush()
# Per ogni plugin attivo, fa il find() per trovare i file interessanti
for p in use_plugins:
if os.getcwd().startswith(start_dir) or hasattr(p, "solid"):
files = map(abspath, p.find())
if files:
priority = cfg.get("priority", p.priority)
builds[(priority, p)] += files
# Va in ricorsione su eventuali sotto-directory
for fn in map(abspath, os.listdir(".")):
if isdir(fn) and split(fn)[1] not in ["CVS", ".svn", ".hg", ".git"]:
# Applica il trucco della modalità deploy: sostituisce la directory
# dei sorgenti (/src) con quella specificata a linea di comando
# (--source, di solito src_mirror). In questo modo continua il
# discovery lì dentro.
if opts.deploy and normpath(fn) == normpath(src_dir):
fn = opts.source
os.chdir(fn)
_discovery(use_plugins, builds, start_dir, cfg)
def _get_async_conc(corp, user_id, q, save, subchash, samplesize, fullsize, minsize):
"""
Note: 'save' argument is present because of bonito-open-3.45.11 compatibility but it is
currently not used ----- TODO remove it
"""
backend, conf = settings.get_full('corpora', 'conc_calc_backend')
if backend == 'multiprocessing':
from concworker.default import BackgroundCalc, NotifierFactory
receiver, sender = NotifierFactory()()
calc = BackgroundCalc(notification_sender=sender)
proc = Process(target=calc, args=(corp, subchash, q, samplesize,))
proc.start()
elif backend == 'celery':
from concworker.wcelery import NotifierFactory
import task
app = task.get_celery_app(conf['conf'])
res = app.send_task('worker.register', (user_id, corp.corpname, getattr(corp, 'subcname', None),
subchash, q, samplesize))
receiver, sender = NotifierFactory(res)()
else:
raise ValueError('Unknown concordance calculation backend: %s' % (backend,))
cachefile, pidfile = receiver.receive()
try:
_wait_for_conc(corp=corp, q=q, subchash=subchash, cachefile=cachefile,
cache_map=plugins.get('conc_cache').get_mapping(corp), pidfile=pidfile, minsize=minsize)
if not os.path.exists(cachefile):
raise RuntimeError('Concordance cache file [%s] not created. PID file: %s' %
(cachefile, pidfile))
except Exception as e:
if os.path.exists(pidfile):
os.remove(pidfile)
raise e
return PyConc(corp, 'l', cachefile)
def __init__(self, plugin_names):
for plugin_name in plugin_names:
plg = plugins.get(plugin_name)
if callable(getattr(plg, 'export_tasks', None)):
for tsk in plg.export_tasks():
setattr(self, tsk.__name__,
app.task(tsk, name='%s.%s' % (plugin_name, tsk.__name__,)))
def _add_tt_custom_metadata(self, tt):
metadata = plugins.get('corparch').get_corpus_info(
self._corpname, language=self._ui_lang)['metadata']
for line in tt:
for item in line.get('Line', ()):
item['is_interval'] = int(
item['label'] in metadata.get('interval_attrs', []))
def call_plugin(cmd, bot, update):
plugin = plugins.get(cmd)
if plugin == None:
update.message.reply_markdown("Yep... not yet....")
return
text = update.message.text[len(cmd) + 1:].strip()
res = None
try:
res = plugin.process(cmd, text)
except Exception as inst:
#logger.warning("Failed to lock file: is seems to be script already running")
logger.error(type(inst))
logger.error(inst.args)
logger.error(inst)
update.message.reply_text("Failed to call plugin. Check logs")
raise
if not isinstance(res, list):
res = [res]
for r in res:
if isinstance(r, str):
update.message.reply_markdown(r)
elif isinstance(r, tuple) and r[0] == "photo":
with open(r[1], 'rb') as f:
update.message.reply_photo(photo=f, timeout=50)
else:
update.message.reply_markdown("Unsupported result: {}".format(
type(r)))
return
def ajax_list_corpora(self, request):
return plugins.get('corparch').search(plugin_api=self._plugin_api,
user_id=self._session_get('user', 'id'),
query=request.args['query'],
offset=request.args.get('offset', None),
limit=request.args.get('limit', None),
filter_dict=request.args)
def _get_cached_conc(corp, subchash, q, pid_dir, minsize):
"""
Loads a concordance from cache
"""
start_time = time.time()
q = tuple(q)
if not os.path.isdir(pid_dir):
os.makedirs(pid_dir, mode=0o775)
cache_map = plugins.get('conc_cache').get_mapping(corp)
cache_map.refresh_map()
if _contains_shuffle_seq(q):
srch_from = 1
else:
srch_from = len(q)
ans = (0, None)
for i in range(srch_from, 0, -1):
cachefile = cache_map.cache_file_path(subchash, q[:i])
if cachefile:
pidfile = cache_map.get_stored_pidfile(subchash, q[:i])
_wait_for_conc(corp=corp,
q=q,
subchash=subchash,
cachefile=cachefile,
cache_map=cache_map,
pidfile=pidfile,
minsize=minsize)
if not os.path.exists(cachefile): # broken cache
cache_map.del_entry(subchash, q)
try:
os.remove(pidfile)
except OSError:
pass
continue
conccorp = corp
for qq in reversed(q[:i]): # find the right main corp, if aligned
if qq.startswith('x-'):
conccorp = manatee.Corpus(qq[2:])
break
conc = PyConc(conccorp, 'l', cachefile, orig_corp=corp)
if not _is_conc_alive(pidfile, minsize) and not conc.finished():
# unfinished and dead concordance
cache_map.del_entry(subchash, q)
try:
os.remove(cachefile)
except OSError:
pass
try:
os.remove(pidfile)
except OSError:
pass
continue
ans = (i, conc)
break
logging.getLogger(__name__).debug(
'get_cached_conc(%s, [%s]) -> %s, %01.4f' %
(corp.corpname, ','.join(q), 'hit' if ans[1] else 'miss',
time.time() - start_time))
return ans
def stats(self, request):
from_date = request.args.get('from_date')
to_date = request.args.get('to_date')
min_occur = request.args.get('min_occur')
if plugins.get('auth').is_administrator(self._session_get(
'user', 'id')):
import system_stats
data = system_stats.load(settings.get('logging', 'path'),
from_date=from_date,
to_date=to_date,
min_occur=min_occur)
maxmin = {}
for label, section in data.items():
maxmin[label] = system_stats.get_max_min(section)
out = {
'stats': data,
'minmax': maxmin,
'from_date': from_date,
'to_date': to_date,
'min_occur': min_occur
}
else:
out = {
'message':
('error',
_('You don\'t have enough privileges to see this page.'))
}
return out
def _load_query_history(self, offset, limit, from_date, to_date,
query_type, current_corpus):
if plugins.has_plugin('query_storage'):
from query_history import Export
if current_corpus:
corpname = self.args.corpname
else:
corpname = None
exporter = Export(corpus_manager=self.cm,
corpname_canonizer=self._canonical_corpname,
url_creator=self.create_url)
rows = plugins.get('query_storage').get_user_queries(
self._session_get('user', 'id'),
offset=offset,
limit=limit,
query_type=query_type,
corpname=corpname,
from_date=from_date,
to_date=to_date)
rows = filter(lambda x: x is not None,
[exporter.export_row(row) for row in rows])
else:
rows = ()
return rows
def ajax_get_tag_variants(self, pattern=''):
"""
"""
try:
tag_loader = plugins.get('taghelper').loader(
self.args.corpname,
plugins.get('corparch').get_corpus_info(self.args.corpname)['tagset'],
self.ui_lang)
except IOError:
raise UserActionException(_('Corpus %s is not supported by this widget.') % self.args.corpname)
if len(pattern) > 0:
ans = tag_loader.get_variant(pattern)
else:
ans = tag_loader.get_initial_values()
return ans
def ajax_user_info(self, request):
user_info = plugins.get('auth').get_user_info(
self._session_get('user', 'id'))
if not self.user_is_anonymous():
return {'user': user_info}
else:
return {'user': {'username': user_info['username']}}
def ajax_get_corp_details(self, request):
"""
"""
corp_conf_info = plugins.get('corparch').get_corpus_info(request.args['corpname'])
corpus = self.cm.get_Corpus(request.args['corpname'])
encoding = corpus.get_conf('ENCODING')
ans = {
'corpname': l10n.import_string(self._canonical_corpname(corpus.get_conf('NAME')),
from_encoding=encoding),
'description': l10n.import_string(corpus.get_info(), from_encoding=encoding),
'size': l10n.format_number(int(corpus.size())),
'attrlist': [],
'structlist': [],
'web_url': corp_conf_info['web'] if corp_conf_info is not None else ''
}
try:
ans['attrlist'] = [{'name': item, 'size': l10n.format_number(int(corpus.get_attr(item).id_range()))}
for item in corpus.get_conf('ATTRLIST').split(',')]
except RuntimeError as e:
logging.getLogger(__name__).warn('%s' % e)
ans['attrlist'] = {'error': _('Failed to load')}
ans['structlist'] = [{'name': item, 'size': l10n.format_number(int(corpus.get_struct(item).size()))}
for item in corpus.get_conf('STRUCTLIST').split(',')]
return ans
def user_password(self, request):
auth = plugins.get('auth')
try:
curr_passwd = request.form['curr_passwd']
new_passwd = request.form['new_passwd']
new_passwd2 = request.form['new_passwd2']
if not self._uses_internal_user_pages():
raise UserActionException(_('This function is disabled.'))
logged_in = auth.validate_user(self._session_get('user', 'user'),
curr_passwd)
if self._is_anonymous_id(logged_in['id']):
raise UserActionException(_('Invalid user or password'))
if new_passwd != new_passwd2:
raise UserActionException(
_('New password and its confirmation do not match.'))
if not auth.validate_new_password(new_passwd):
raise UserActionException(
auth.get_required_password_properties())
auth.update_user_password(self._session_get('user', 'id'),
new_passwd)
except UserActionException as e:
self.add_system_message('error', e)
return {}
def stats(self, request):
from_date = request.args.get('from_date')
to_date = request.args.get('to_date')
min_occur = request.args.get('min_occur')
if plugins.get('auth').is_administrator(self._session_get('user', 'id')):
import system_stats
data = system_stats.load(settings.get('logging', 'path'), from_date=from_date,
to_date=to_date, min_occur=min_occur)
maxmin = {}
for label, section in data.items():
maxmin[label] = system_stats.get_max_min(section)
out = {
'stats': data,
'minmax': maxmin,
'from_date': from_date,
'to_date': to_date,
'min_occur': min_occur
}
else:
out = {'message': ('error', _('You don\'t have enough privileges to see this page.'))}
return out
def get_lang(environ):
"""
Detects user's preferred language (either via the 'getlang' plugin or from HTTP_ACCEPT_LANGUAGE env value)
arguments:
environ -- WSGI environment variable
returns:
underscore-separated ISO 639 language code and ISO 3166 country code
"""
installed = dict([
(x.split('_')[0], x)
for x in os.listdir('%s/../locale' % os.path.dirname(__file__))
])
if plugins.has_plugin('getlang'):
lgs_string = plugins.get('getlang').fetch_current_language(
KonTextCookie(environ.get('HTTP_COOKIE', '')))
else:
lgs_string = parse_accept_header(
environ.get('HTTP_ACCEPT_LANGUAGE')).best
if len(lgs_string
) == 2: # in case we obtain just an ISO 639 language code
lgs_string = installed.get(lgs_string)
else:
lgs_string = lgs_string.replace('-', '_')
if lgs_string is None:
lgs_string = 'en_US'
return lgs_string
def ajax_list_corpora(self, request):
return plugins.get('corparch').search(
plugin_api=self._plugin_api,
user_id=self._session_get('user', 'id'),
query=request.args['query'],
offset=request.args.get('offset', None),
limit=request.args.get('limit', None),
filter_dict=request.args)
def viewattrs(self):
"""
attrs, refs, structs form
"""
from collections import defaultdict
self.disabled_menu_items = (MainMenu.SAVE, MainMenu.CONCORDANCE,
MainMenu.FILTER, MainMenu.FREQUENCY, MainMenu.COLLOCATIONS)
out = {}
if self.args.maincorp:
corp = corplib.manatee.Corpus(self.args.maincorp)
out['AttrList'] = [{'label': corp.get_conf(n + '.LABEL') or n, 'n': n}
for n in corp.get_conf('ATTRLIST').split(',')
if n]
else:
corp = self._corp()
availstruct = corp.get_conf('STRUCTLIST').split(',')
structlist = self.args.structs.split(',')
out['Availstructs'] = [{'n': n,
'sel': (((n in structlist)
and 'selected') or ''),
'label': corp.get_conf(n + '.LABEL')}
for n in availstruct if n and n != '#']
availref = corp.get_conf('STRUCTATTRLIST').split(',')
structattrs = defaultdict(list)
reflist = self.args.refs.split(',')
ref_is_allowed = lambda r: r and r not in (
'#', plugins.get('corparch').get_corpus_info(self.args.corpname).get('speech_segment'))
for item in availref:
if ref_is_allowed(item):
k, v = item.split('.', 1)
structattrs[k].append(v)
if not k in reflist:
reflist.append(k)
out['Availrefs'] = [{
'n': '#',
'label': _('Token number'),
'sel': ((('#' in reflist) and 'selected') or '')
}] + \
[{
'n': '=' + n,
'sel': ((('=' + n in reflist) and 'selected') or ''),
'label': (corp.get_conf(n + '.LABEL') or n)
}
for n in availref if ref_is_allowed(n)
]
doc = corp.get_conf('DOCSTRUCTURE')
if doc in availstruct:
out['Availrefs'].insert(1, {'n': doc, 'label': _('Document number'),
'sel': (doc in reflist and 'selected' or '')})
out['newctxsize'] = self.args.kwicleftctx[1:]
out['structattrs'] = structattrs
out['curr_structattrs'] = self.args.structattrs
return out
def ajax_get_tag_variants(ctrl, pattern=''):
"""
"""
try:
tag_loader = plugins.get('taghelper').loader(
ctrl.args.corpname,
plugins.get('corparch').get_corpus_info(
ctrl.args.corpname)['tagset'], ctrl.ui_lang)
except IOError:
raise UserActionException(
_('Corpus %s is not supported by this widget.') %
ctrl.args.corpname)
if len(pattern) > 0:
ans = tag_loader.get_variant(pattern)
else:
ans = tag_loader.get_initial_values()
return ans
def subcmixer_run_calc(ctrl, request):
try:
subc_path = ctrl.prepare_subc_path(request.args['corpname'], request.form['subcname'])
stats = plugins.get('subcmixer').process(subc_path, ctrl.corp, request.args['corpname'],
json.loads(request.form['expression']))
return {'status': 'OK', 'stats': stats}
except Exception as e:
ctrl.add_system_message('error', unicode(e))
return {}
def ask_corpus_access(controller, request):
ans = {}
status = plugins.get('corparch').send_request_email(corpus_id=request.form['corpusId'],
plugin_api=getattr(controller, '_plugin_api'),
custom_message=request.form['customMessage'])
if status is False:
ans['error'] = _(
'Failed to send e-mail. Please try again later or contact system administrator')
return ans
def _init_session(self):
"""
Starts/reloads user's web session data. It can be called even
if there is no 'sessions' plugin installed (in such case, it just
creates an empty dictionary with some predefined keys to allow other
parts of the application to operate properly)
"""
auth = plugins.get('auth')
if 'user' not in self._session:
self._session['user'] = auth.anonymous_user()
if hasattr(plugins.get('auth'), 'revalidate'):
try:
auth.revalidate(self._plugin_api)
except Exception as ex:
self._session['user'] = auth.anonymous_user()
logging.getLogger(__name__).error('Revalidation error: %s' % ex)
self.add_system_message('error', _('User authentication error. Please try to reload the page or '
'contact system administrator.'))
def ask_corpus_access(controller, request):
ans = {}
status = plugins.get('corparch').send_request_email(
corpus_id=request.form['corpusId'],
plugin_api=getattr(controller, '_plugin_api'),
custom_message=request.form['customMessage'])
if status is False:
ans['error'] = _(
'Failed to send e-mail. Please try again later or contact system administrator'
)
return ans
def __init__(self, corp, corpname, ui_lang):
"""
arguments:
corp -- a manatee.Corpus instance (enriched version returned by corplib.CorpusManager)
corpname -- a corpus ID
ui_lang -- a language of the current user interface
"""
self._corp = corp
self._corpname = corpname
self._ui_lang = ui_lang
self._tt_cache = TextTypesCache(plugins.get('db'))
def _get_sync_conc(corp, q, save, subchash, samplesize):
from concworker import GeneralWorker
conc = GeneralWorker().compute_conc(corp, q, samplesize)
conc.sync() # wait for the computation to finish
if save:
cache_map = plugins.get('conc_cache').get_mapping(corp)
cachefile, stored_pidfile = cache_map.add_to_map(subchash, q[:1], conc.size())
conc.save(cachefile)
# update size in map file
cache_map.add_to_map(subchash, q[:1], conc.size())
return conc
def _get_cached_conc(corp, subchash, q, pid_dir, minsize):
"""
Loads a concordance from cache
"""
start_time = time.time()
q = tuple(q)
if not os.path.isdir(pid_dir):
os.makedirs(pid_dir, mode=0o775)
cache_map = plugins.get('conc_cache').get_mapping(corp)
cache_map.refresh_map()
if _contains_shuffle_seq(q):
srch_from = 1
else:
srch_from = len(q)
ans = (0, None)
for i in range(srch_from, 0, -1):
cachefile = cache_map.cache_file_path(subchash, q[:i])
if cachefile:
pidfile = cache_map.get_stored_pidfile(subchash, q[:i])
_wait_for_conc(corp=corp, q=q, subchash=subchash, cachefile=cachefile,
cache_map=cache_map, pidfile=pidfile, minsize=minsize)
if not os.path.exists(cachefile): # broken cache
cache_map.del_entry(subchash, q)
try:
os.remove(pidfile)
except OSError:
pass
continue
conccorp = corp
for qq in reversed(q[:i]): # find the right main corp, if aligned
if qq.startswith('x-'):
conccorp = manatee.Corpus(qq[2:])
break
conc = PyConc(conccorp, 'l', cachefile, orig_corp=corp)
if not _is_conc_alive(pidfile, minsize) and not conc.finished():
# unfinished and dead concordance
cache_map.del_entry(subchash, q)
try:
os.remove(cachefile)
except OSError:
pass
try:
os.remove(pidfile)
except OSError:
pass
continue
ans = (i, conc)
break
logging.getLogger(__name__).debug('get_cached_conc(%s, [%s]) -> %s, %01.4f'
% (corp.corpname, ','.join(q), 'hit' if ans[1] else 'miss',
time.time() - start_time))
return ans
def __init__(self, corp, corpname, ui_lang):
"""
arguments:
corp -- a manatee.Corpus instance (enriched version returned by corplib.CorpusManager)
corpname -- a corpus ID
ui_lang -- a language of the current user interface
"""
self._corp = corp
self._corpname = corpname
self._ui_lang = ui_lang
self._tt_cache = TextTypesCache(plugins.get('db'))
def load_topbar(self):
if plugins.has_plugin('application_bar'):
import urlparse
html = plugins.get('application_bar').get_contents(self.cookies, 'en', '/')
parts = urlparse.urlparse(self.environ['REQUEST_URI'])
css = '<link rel="stylesheet" type="text/css" href="%s://www.korpus.cz/toolbar/css/cnc-toolbar.css" />' \
% parts.scheme
else:
html = ''
css = ''
return html, css
def __init__(self, plugin_names):
for plugin_name in plugin_names:
plg = plugins.get(plugin_name)
if callable(getattr(plg, 'export_tasks', None)):
for tsk in plg.export_tasks():
setattr(
self, tsk.__name__,
app.task(tsk,
name='%s.%s' % (
plugin_name,
tsk.__name__,
)))
def ajax_get_structs_details(self):
"""
Provides a map (struct_name=>[list of attributes]). This is used
by 'insert within' widget.
"""
speech_segment = plugins.get('corparch').get_corpus_info(self.args.corpname).speech_segment
ans = defaultdict(lambda: [])
for item in self._corp().get_conf('STRUCTATTRLIST').split(','):
if item != speech_segment:
k, v = item.split('.')
ans[k].append(v)
return dict((k, v) for k, v in ans.items() if len(v) > 0)
def _get_sync_conc(corp, q, save, subchash, samplesize):
from concworker import GeneralWorker
conc = GeneralWorker().compute_conc(corp, q, samplesize)
conc.sync() # wait for the computation to finish
if save:
cache_map = plugins.get('conc_cache').get_mapping(corp)
cachefile, stored_pidfile = cache_map.add_to_map(
subchash, q[:1], conc.size())
conc.save(cachefile)
# update size in map file
cache_map.add_to_map(subchash, q[:1], conc.size())
return conc
def ajax_get_structs_details(self):
"""
Provides a map (struct_name=>[list of attributes]). This is used
by 'insert within' widget.
"""
speech_segment = plugins.get('corparch').get_corpus_info(
self.args.corpname).speech_segment
ans = defaultdict(lambda: [])
for item in self._corp().get_conf('STRUCTATTRLIST').split(','):
if item != speech_segment:
k, v = item.split('.')
ans[k].append(v)
return dict((k, v) for k, v in ans.items() if len(v) > 0)
def load_topbar(self):
if plugins.has_plugin('application_bar'):
import urlparse
html = plugins.get('application_bar').get_contents(
self.cookies, 'en', '/')
parts = urlparse.urlparse(self.environ['REQUEST_URI'])
css = '<link rel="stylesheet" type="text/css" href="%s://www.korpus.cz/toolbar/css/cnc-toolbar.css" />' \
% parts.scheme
else:
html = ''
css = ''
return html, css
def loginx(self, request):
ans = {}
self._session['user'] = plugins.get('auth').validate_user(request.form['username'],
request.form['password'])
if self._session['user'].get('id', None):
self._redirect('%sfirst_form' % (self.get_root_url(), ))
else:
self.disabled_menu_items = (MainMenu.NEW_QUERY, MainMenu.VIEW,
MainMenu.SAVE, MainMenu.CORPORA, MainMenu.CONCORDANCE,
MainMenu.FILTER, MainMenu.FREQUENCY, MainMenu.COLLOCATIONS)
self.add_system_message('error', _('Incorrect username or password'))
self.refresh_session_id()
return ans
def _get_async_conc(corp, user_id, q, save, subchash, samplesize, fullsize,
minsize):
"""
Note: 'save' argument is present because of bonito-open-3.45.11 compatibility but it is
currently not used ----- TODO remove it
"""
backend, conf = settings.get_full('corpora', 'conc_calc_backend')
if backend == 'multiprocessing':
from concworker.default import BackgroundCalc, NotifierFactory
receiver, sender = NotifierFactory()()
calc = BackgroundCalc(notification_sender=sender)
proc = Process(target=calc, args=(
corp,
subchash,
q,
samplesize,
))
proc.start()
elif backend == 'celery':
from concworker.wcelery import NotifierFactory
import task
app = task.get_celery_app(conf['conf'])
res = app.send_task(
'worker.register',
(user_id, corp.corpname, getattr(corp, 'subcname',
None), subchash, q, samplesize))
receiver, sender = NotifierFactory(res)()
else:
raise ValueError('Unknown concordance calculation backend: %s' %
(backend, ))
cachefile, pidfile = receiver.receive()
try:
_wait_for_conc(corp=corp,
q=q,
subchash=subchash,
cachefile=cachefile,
cache_map=plugins.get('conc_cache').get_mapping(corp),
pidfile=pidfile,
minsize=minsize)
if not os.path.exists(cachefile):
raise RuntimeError(
'Concordance cache file [%s] not created. PID file: %s' %
(cachefile, pidfile))
except Exception as e:
if os.path.exists(pidfile):
os.remove(pidfile)
raise e
return PyConc(corp, 'l', cachefile)
def ajax_subcorp_info(self, subcname=''):
sc = self.cm.get_Corpus(self.args.corpname, subcname)
ans = {
'corpusName': self._canonical_corpname(self.args.corpname),
'subCorpusName': subcname,
'corpusSize': format_number(sc.size()),
'subCorpusSize': format_number(sc.search_size()),
'extended_info': {}
}
if plugins.has_plugin('subc_restore'):
tmp = plugins.get('subc_restore').get_info(self._session_get('user', 'id'),
self.args.corpname, subcname)
if tmp:
ans['extended_info'].update(tmp)
return ans
def __call__(self, environ, start_response):
"""
Works as specified by the WSGI
"""
ui_lang = get_lang(environ)
translation.activate(ui_lang)
l10n.activate(ui_lang)
environ['REQUEST_URI'] = wsgiref.util.request_uri(
environ) # TODO remove?
sessions = plugins.get('sessions')
request = Request(environ)
sid = request.cookies.get(sessions.get_cookie_name())
if sid is None:
request.session = sessions.new()
else:
request.session = sessions.get(sid)
sid_is_valid = True
if environ['PATH_INFO'] in ('/', ''):
url = environ['REQUEST_URI'].split('?')[0]
if not url.endswith('/'):
url += '/'
status = '303 See Other'
headers = [('Location', '%sfirst_form' % url)]
body = ''
elif '/run.cgi/' in environ[
'REQUEST_URI']: # old-style (CGI version) URLs are redirected to new ones
status = '301 Moved Permanently'
headers = [('Location',
environ['REQUEST_URI'].replace('/run.cgi/', '/'))]
body = ''
else:
controller_class = load_controller_class(environ['PATH_INFO'])
app = controller_class(request=request, ui_lang=ui_lang)
status, headers, sid_is_valid, body = app.run()
response = Response(response=body, status=status, headers=headers)
if not sid_is_valid:
curr_data = dict(request.session)
request.session = sessions.new()
request.session.update(curr_data)
request.session.modified = True
if request.session.should_save:
sessions.save(request.session)
response.set_cookie(sessions.get_cookie_name(),
request.session.sid)
return response(environ, start_response)
def loginx(self, request):
ans = {}
self._session['user'] = plugins.get('auth').validate_user(
request.form['username'], request.form['password'])
if self._session['user'].get('id', None):
self._redirect('%sfirst_form' % (self.get_root_url(), ))
else:
self.disabled_menu_items = (MainMenu.NEW_QUERY, MainMenu.VIEW,
MainMenu.SAVE, MainMenu.CORPORA,
MainMenu.CONCORDANCE, MainMenu.FILTER,
MainMenu.FREQUENCY,
MainMenu.COLLOCATIONS)
self.add_system_message('error',
_('Incorrect username or password'))
self.refresh_session_id()
return ans
def get_syntax_data(ctrl, request):
"""
This is the actual controller method exported by the plug-in.
To be able to export a JSON with custom encoder this method
returns a callable which ensures that controller.Controller
skips its simple JSON serialization.
"""
try:
corp = getattr(ctrl, '_corp')()
canonical_corpname = getattr(ctrl, '_canonical_corpname')(corp.corpname)
data = plugins.get('syntax_viewer').search_by_token_id(corp, canonical_corpname,
int(request.args.get('kwic_id')))
except MaximumContextExceeded:
data = dict(contains_errors=True,
error=_('Failed to get the syntax tree due to limited permissions'
' to the corpus (too long sentence).'))
return data
def __call__(self, environ, start_response):
"""
Works as specified by the WSGI
"""
ui_lang = get_lang(environ)
translation.activate(ui_lang)
l10n.activate(ui_lang)
environ['REQUEST_URI'] = wsgiref.util.request_uri(environ) # TODO remove?
sessions = plugins.get('sessions')
request = Request(environ)
sid = request.cookies.get(sessions.get_cookie_name())
if sid is None:
request.session = sessions.new()
else:
request.session = sessions.get(sid)
sid_is_valid = True
if environ['PATH_INFO'] in ('/', ''):
url = environ['REQUEST_URI']
if not url.endswith('/'):
url += '/'
status = '303 See Other'
headers = [('Location', '%sfirst_form' % url)]
body = ''
elif '/run.cgi/' in environ['REQUEST_URI']: # old-style (CGI version) URLs are redirected to new ones
status = '301 Moved Permanently'
headers = [('Location', environ['REQUEST_URI'].replace('/run.cgi/', '/'))]
body = ''
else:
controller_class = load_controller_class(environ['PATH_INFO'])
app = controller_class(request=request, ui_lang=ui_lang)
status, headers, sid_is_valid, body = app.run(request)
response = Response(response=body, status=status, headers=headers)
if not sid_is_valid:
curr_data = dict(request.session)
request.session = sessions.new()
request.session.update(curr_data)
request.session.modified = True
if request.session.should_save:
sessions.save(request.session)
response.set_cookie(sessions.get_cookie_name(), request.session.sid)
start_response(status, headers)
return response(environ, start_response)
def ajax_get_corp_details(self, request):
"""
"""
corp_conf_info = plugins.get('corparch').get_corpus_info(
request.args['corpname'])
corpus = self.cm.get_Corpus(request.args['corpname'])
citation_info = corp_conf_info.get('citation_info', None)
citation_info = citation_info.to_dict() if citation_info else {}
import_str = partial(l10n.import_string,
from_encoding=corpus.get_conf('ENCODING'))
if corpus.get_conf('NAME'):
corpus_name = corpus.get_conf('NAME')
else:
corpus_name = self._canonical_corpname(corpus.corpname)
ans = {
'corpname': import_str(corpus_name),
'description': import_str(corpus.get_info()),
'size': l10n.format_number(int(corpus.size())),
'attrlist': [],
'structlist': [],
'web_url':
corp_conf_info['web'] if corp_conf_info is not None else '',
'citation_info': citation_info
}
try:
ans['attrlist'] = [{
'name':
item,
'size':
l10n.format_number(int(corpus.get_attr(item).id_range()))
} for item in corpus.get_conf('ATTRLIST').split(',')]
except RuntimeError as e:
logging.getLogger(__name__).warn('%s' % e)
ans['attrlist'] = {'error': _('Failed to load')}
ans['structlist'] = [{
'name':
item,
'size':
l10n.format_number(int(corpus.get_struct(item).size()))
} for item in corpus.get_conf('STRUCTLIST').split(',')]
return ans
def get_syntax_data(ctrl, request):
"""
This is the actual controller method exported by the plug-in.
To be able to export a JSON with custom encoder this method
returns a callable which ensures that controller.Controller
skips its simple JSON serialization.
"""
try:
corp = getattr(ctrl, '_corp')()
canonical_corpname = getattr(ctrl,
'_canonical_corpname')(corp.corpname)
data = plugins.get('syntax_viewer').search_by_token_id(
corp, canonical_corpname, int(request.args.get('kwic_id')))
except MaximumContextExceeded:
data = dict(
contains_errors=True,
error=_('Failed to get the syntax tree due to limited permissions'
' to the corpus (too long sentence).'))
return data
def _load_query_history(self, offset, limit, from_date, to_date, query_type, current_corpus):
if plugins.has_plugin('query_storage'):
from query_history import Export
if current_corpus:
corpname = self.args.corpname
else:
corpname = None
exporter = Export(corpus_manager=self.cm, corpname_canonizer=self._canonical_corpname,
url_creator=self.create_url)
rows = plugins.get('query_storage').get_user_queries(
self._session_get('user', 'id'),
offset=offset, limit=limit,
query_type=query_type, corpname=corpname,
from_date=from_date, to_date=to_date)
rows = [exporter.export_row(row) for row in rows]
else:
rows = ()
return rows
def __init__(self, dev, server):
self.dev = dev
self.failed_counter =0
self.server = server
self.pipeline = Gst.Pipeline()
self.bus = self.pipeline.get_bus()
self.bus.add_signal_watch()
self.bus.connect("message", self.on_message)
self.converter = plugins.python.videofilter.OpenCVVideoFilter()
self.src = plugins.python.device.VirtualDevice()
self.src.set_property('uri', dev['instream'])
vc = Gst.ElementFactory.make('videoconvert', None)
vc2 = Gst.ElementFactory.make('videoconvert', None)
sink = GstOutputStream(dev['outstream'], split=False)
# self.destination = self.pipeline.get_by_name("destination")
# self.destination.connect("new-buffer", self.on_new_buffer)
self.pipeline.add(self.src)
self.pipeline.add(vc)
self.pipeline.add(vc2)
self.pipeline.add(sink)
self.src.link(vc)
src = vc
# fixed to ocrplate plugin.
dev['modules'] = [{'id':5}]
for m in dev['modules']:
plug = plugins.get(int(m['id']))
plug.set_property('roi_x', int(dev['roi_x']))
plug.set_property('roi_y', int(dev['roi_y']))
plug.set_property('roi_w', int(dev['roi_w']))
plug.set_property('roi_h', int(dev['roi_h']))
self.pipeline.add(plug)
src.link(plug)
src = plug
src.link(vc2)
vc2.link(sink)
def subcorp_form(self, request, conc_args):
"""
Displays a form to create a new subcorpus
"""
self.disabled_menu_items = self.CONCORDANCE_ACTIONS
method = request.form.get('method', 'gui')
within_json = request.form.get('within_json', 'null')
subcname = request.form.get('subcname', None)
subcnorm = request.args.get('subcnorm', 'tokens')
try:
tt_sel = get_tt(self._corp(), self.ui_lang).export_with_norms(subcnorm=subcnorm)
except UserActionException as e:
tt_sel = {'Normslist': [], 'Blocks': []}
self.add_system_message('warning', e)
structs_and_attrs = {}
for s, a in [t.split('.') for t in self._corp().get_conf('STRUCTATTRLIST').split(',')]:
if s not in structs_and_attrs:
structs_and_attrs[s] = []
structs_and_attrs[s].append(a)
out = {'SubcorpList': ()}
if self.environ['REQUEST_METHOD'] == 'POST':
self._store_checked_text_types(request.form, out)
if plugins.has_plugin('subcmixer'):
out['subcmixer_form_data'] = plugins.get('subcmixer').form_data(self._plugin_api)
else:
out['subcmixer_form_data'] = {}
out.update({
'TextTypeSel': tt_sel,
'structs_and_attrs': structs_and_attrs,
'method': method,
'within_json': within_json,
'subcname': subcname,
'subcnorm': subcnorm
})
return out
