def user_password(self, request):
with plugins.runtime.AUTH as auth:
try:
curr_passwd = request.form['curr_passwd']
new_passwd = request.form['new_passwd']
new_passwd2 = request.form['new_passwd2']
if not self._uses_internal_user_pages():
raise UserActionException(_('This function is disabled.'))
logged_in = auth.validate_user(
self._plugin_api, self.session_get('user', 'user'),
curr_passwd)
if self._is_anonymous_id(logged_in['id']):
raise UserActionException(_('Invalid user or password'))
if new_passwd != new_passwd2:
raise UserActionException(
_('New password and its confirmation do not match.'))
if not auth.validate_new_password(new_passwd):
raise UserActionException(
auth.get_required_password_properties())
auth.update_user_password(self.session_get('user', 'id'),
new_passwd)
except UserActionException as e:
self.add_system_message('error', e)
return {}
def _load_conc_queries(
plugin_ctx: PluginCtx, conc_ids: List[str], corpus_id: str,
form_type: str) -> Tuple[Dict[str, Any], Dict[str, Any]]:
"""
Load both conc. query forms and respective raw Manatee queries
form_type is either 'query' or 'filter'
"""
forms = {}
raw_queries = {}
with plugins.runtime.QUERY_PERSISTENCE as qs:
for conc_id in conc_ids:
data = qs.open(conc_id)
if data is None:
raise UserActionException(
'Source concordance query does not exist: {}'.format(
conc_id))
if qs.stored_form_type(data) != form_type:
raise UserActionException(
'Invalid source query used: {}'.format(conc_id))
if form_type == 'query':
args = QueryFormArgs(plugin_ctx=plugin_ctx,
corpora=[corpus_id],
persist=True).updated(
data['lastop_form'], conc_id)
elif form_type == 'filter':
args = FilterFormArgs(plugin_ctx=plugin_ctx,
maincorp=corpus_id,
persist=True).updated(
data['lastop_form'], conc_id)
forms[args.op_key] = args.to_dict()
raw_queries[args.op_key] = data['q']
return forms, raw_queries
def _normalize_error(self, err):
"""
This method is intended to extract as much details as possible
from a broad range of errors and rephrase them in a more
specific ones (including exception object type).
It is quite a lame solution but it appears that in case of
syntax errors, attribute errors etc. Manatee raises only RuntimeError
without further type distinction.
Please note that some of the decoding is dependent on how Manatee
outputs phrases its errors which may change between versions
(as it probably happened in 2.150.x).
arguments:
err -- an instance of Exception
returns:
a (possibly different) instance of Exception with
(possibly) rephrased error message.
"""
if isinstance(err, UserActionException):
return err
if err.message:
if type(err.message) == unicode:
text = err.message
else:
text = str(err.message).decode(self.corp_encoding,
errors='replace')
else:
text = unicode(err)
err.message = text # in case we return the original error
if 'syntax error' in text.lower():
srch = re.match(r'.+ position (\d+)', text)
if srch:
text = translate('Query failed: Syntax error at position %s.'
) % srch.groups()[0]
else:
text = translate('Query failed: Syntax error.')
new_err = UserActionException(
translate(
'%s Please make sure the query and selected query type are correct.'
) % text)
elif 'AttrNotFound' in text:
srch = re.match(r'AttrNotFound\s+\(([^)]+)\)', text)
if srch:
text = translate(
'Attribute "%s" not found.') % srch.groups()[0]
else:
text = translate('Attribute not found.')
new_err = UserActionException(text)
elif 'EvalQueryException' in text:
new_err = UserActionException(
translate(
'Failed to evaluate the query. Please check the syntax and used attributes.'
))
else:
new_err = err
return new_err
def convert_chart_svg(self, request: Request):
vert_bar_chart_max_label = int(
request.args.get('vertBarChartMaxLabel', '10'))
chart_type = request.args.get('chartType')
if chart_type in ('bar', 'time', 'timescatter'):
svg_src = normalize_bar_chart_svg(request.get_data(),
vert_bar_chart_max_label)
else:
svg_src = normalize_wcloud_svg(request.get_data())
if request.args.get('outFormat', '') == 'png':
self._response.set_header('Content-Type', 'image/png')
return svg2png(bytestring=svg_src,
output_width=1200,
background_color='#FFFFFF')
elif request.args.get('outFormat', '') == 'png-print':
self._response.set_header('Content-Type', 'image/png')
return svg2png(bytestring=svg_src,
output_width=4961,
background_color='#FFFFFF')
elif request.args.get('outFormat', '') == 'svg':
self._response.set_header('Content-Type', 'image/svg+xml')
return svg2svg(bytestring=svg_src,
scale=5,
background_color='#FFFFFF')
elif request.args.get('outFormat', '') == 'pdf':
self._response.set_header('Content-Type', 'application/pdf')
return svg2pdf(bytestring=svg_src,
scale=5,
background_color='#FFFFFF')
else:
raise UserActionException('Invalid data format', code=422)
def set_user_password(self, request):
with plugins.runtime.AUTH as auth:
curr_passwd = request.form['curr_passwd']
new_passwd = request.form['new_passwd']
new_passwd2 = request.form['new_passwd2']
fields = dict(curr_passwd=True, new_passwd=True, new_passwd2=True)
ans = dict(fields=fields, messages=[])
if not self._uses_internal_user_pages():
raise UserActionException(_('This function is disabled.'))
logged_in = auth.validate_user(self._plugin_api,
self.session_get('user', 'user'),
curr_passwd)
if self._is_anonymous_id(logged_in['id']):
fields['curr_passwd'] = False
ans['messages'].append(_('Invalid user or password'))
return ans
if new_passwd != new_passwd2:
fields['new_passwd'] = False
fields['new_passwd2'] = False
ans['messages'].append(
_('New password and its confirmation do not match.'))
return ans
if not auth.validate_new_password(new_passwd):
ans['messages'].append(auth.get_required_password_properties())
fields['new_passwd'] = False
fields['new_passwd2'] = False
return ans
auth.update_user_password(self.session_get('user', 'id'),
new_passwd)
return ans
def calculate_freqs_ct(args):
"""
note: this is called by webserver
"""
backend, conf = settings.get_full('global', 'calc_backend')
if backend == 'celery':
import task
try:
app = task.get_celery_app(conf['conf'])
res = app.send_task('worker.calculate_freqs_ct',
args=(args.to_dict(), ),
time_limit=TASK_TIME_LIMIT)
calc_result = res.get()
except Exception as ex:
if is_celery_user_error(ex):
raise UserActionException(ex.message)
else:
raise ex
elif backend == 'multiprocessing':
raise NotImplementedError(
'Multi-processing backend is not yet supported for freq_ct calculation'
)
else:
raise ValueError('Invalid backend')
return calc_result
def sign_up_form(self, request):
ans = dict(credentials_form={}, username_taken=False, user_registered=False)
with plugins.runtime.AUTH as auth:
token_key = request.args.get('key')
username_taken = bool(int(request.args.get('username_taken', '0')))
if token_key:
credentials = auth.get_form_props_from_token(token_key)
if not credentials:
raise UserActionException('Invalid confirmation token')
del credentials['password']
ans['credentials_form'] = credentials
ans['username_taken'] = username_taken
if not self.user_is_anonymous():
raise UserActionException('You are already registered')
else:
ans['user'] = dict(username=None)
return ans
def pre_dispatch(self, action_name, action_metadata=None) -> Union[RequestArgsProxy, JSONRequestArgsProxy]:
ans = super().pre_dispatch(action_name, action_metadata)
if self._active_q_data is not None:
if self._active_q_data.get('form', {}).get('form_type') != 'wlist':
raise UserActionException('Invalid search session for a word-list')
self._curr_wlform_args = WordlistFormArgs.from_dict(
self._active_q_data['form'], id=self._active_q_data['id'])
return ans
def ajax_get_freq_dispersion(
self, request: werkzeug.Request) -> List[FreqDispersionBin]:
conc = require_existing_conc(self.corp, self.args.q)
resolution = request.args.get('resolution', 100, type=int)
if 0 < resolution < 1000:
return self._get_freq_dispersion(conc, resolution)
raise UserActionException(
'Invalid dispersion resolution. Acceptable values [1, 1000].')
def profile(self, request):
if not self._uses_internal_user_pages():
raise UserActionException(_('This function is disabled.'))
with plugins.runtime.AUTH as auth:
user_info = auth.get_user_info(self._plugin_api)
if not self.user_is_anonymous():
return {'user': user_info}
else:
return {'user': {'username': user_info['username']}}
def profile(self, request):
if not self._uses_internal_user_pages():
raise UserActionException(_('This function is disabled.'))
with plugins.runtime.AUTH as auth:
user_info = auth.get_user_info(self._plugin_api)
if not self.user_is_anonymous():
return dict(credentials_form=user_info, user_registered=True)
else:
return dict(credentials_form=dict(username=user_info['username']), user_registered=False)
def publish_subcorpus(self, request):
subcname = request.form['subcname']
corpname = request.form['corpname']
description = request.form['description']
curr_subc = os.path.join(self.subcpath[0], corpname, subcname + '.subc')
public_subc = self.prepare_subc_path(corpname, subcname, True)
if os.path.isfile(curr_subc):
corplib.mk_publish_links(curr_subc, public_subc,
self.session_get('user', 'fullname'), description)
return dict(code=os.path.splitext(os.path.basename(public_subc))[0])
else:
raise UserActionException('Subcorpus {0} not found'.format(subcname))
def on_forbidden_corpus(self, plugin_api, corpname, corp_variant):
"""
Optional method run in case KonText finds out that user
does not have access rights to a corpus specified by 'corpname'.
There are two main action types you can perform here:
1) Redirect to a different page or set 'not found'.
2) Set some system message user can read.
"""
if corpname:
raise CorpusForbiddenException(corpname, corp_variant)
else:
raise UserActionException()
def ct_dist(self, crit, limit_type, limit=1):
"""
Calculate join distribution (contingency table).
"""
words = manatee.StrVector()
freqs = manatee.NumVector()
norms = manatee.NumVector()
abs_limit = 1 # we always fetch all the values to be able to filter by percentiles and provide misc. info
self._corp.freq_dist(self._conc.RS(), crit, abs_limit, words, freqs,
norms)
crit_lx = re.split(r'\s+', crit)
attrs = []
for i in range(0, len(crit_lx), 2):
attrs.append(crit_lx[i])
if len(attrs) > 2:
raise CTCalculationError(
'Exactly two attributes (either positional or structural) can be used'
)
words = [tuple(w.split('\t')) for w in words]
num_structattrs = self._get_num_structattrs(attrs)
if num_structattrs == 2:
norms = [1e6] * len(words) # this is not really needed
elif num_structattrs == 1:
sattr_idx = 0 if '.' in attrs[0] else 1
norms = self._calc_1sattr_norms(words,
sattr=attrs[sattr_idx],
sattr_idx=sattr_idx)
else:
norms = [self._corp.size()] * len(words)
mans = list(zip(words, freqs, norms))
if limit_type == 'abs':
ans = [v for v in mans if v[1] >= limit]
elif limit_type == 'ipm':
ans = [v for v in mans if v[1] / float(v[2]) * 1e6 >= limit]
elif limit_type == 'pabs':
values = sorted(mans, key=lambda v: v[1])
plimit = int(math.floor(limit / 100. * len(values)))
ans = values[plimit:]
elif limit_type == 'pipm':
values = sorted(mans, key=lambda v: v[1] / float(v[2]) * 1e6)
# math.floor(x) == math.ceil(x) - 1 (indexing from 0)
plimit = math.floor(limit / 100. * len(values))
ans = values[plimit:]
if len(ans) > 1000:
raise UserActionException(
'The result size is too high. Please try to increase the minimum frequency.'
)
return ans, len(mans)
def sign_up(self, request):
with plugins.runtime.AUTH as auth:
errors = auth.sign_up_user(self._plugin_api, dict(
username=request.form['username'],
firstname=request.form['firstname'],
lastname=request.form['lastname'],
email=request.form['email'],
password=request.form['password'],
password2=request.form['password2']
))
if len(errors) == 0:
return dict(ok=True)
else:
raise UserActionException(_('Failed to sign up user'), error_args=errors)
def pre_dispatch(self, action_name, action_metadata=None):
ans = super().pre_dispatch(action_name, action_metadata)
if self._active_q_data is not None:
if self._active_q_data.get('form',
{}).get('form_type') != 'pquery':
raise UserActionException(
'Invalid search session for a paradimatic query')
self._curr_pquery_args = PqueryFormArgs(
corpname=self.corp.corpname,
attr=self._get_default_attr(),
position='0<0~0>0')
self._curr_pquery_args.from_dict(self._active_q_data['form'])
return ans
def get_syntax_data(ctrl, request):
"""
This is the actual controller method exported by the plug-in.
To be able to export a JSON with custom encoder this method
returns a callable which ensures that controller.Controller
skips its simple JSON serialization.
"""
try:
with plugins.runtime.SYNTAX_VIEWER as sv:
return sv.search_by_token_id(ctrl.corp, ctrl.corp.corpname,
int(request.args.get('kwic_id')),
int(request.args.get('kwic_len')))
except MaximumContextExceeded:
raise UserActionException(
_('Failed to get the syntax tree due to limited KWIC context (too long sentence).'))
def archive(self, user_id, conc_id, revoke=False):
key = self._mk_key(conc_id)
data = self.open(conc_id)
if data is None:
raise UserActionException('Concordance key \'%s\' not found.' % (conc_id,))
stored_user_id = data.get('user_id', None)
if user_id != stored_user_id:
raise ForbiddenException(
'Cannot change status of a concordance belonging to another user')
if revoke:
self._db.set(key, data)
self._archive_backend.revoke(key)
else:
self._archive_backend.archive(data, key)
def calculate_freqs_ct(args):
"""
note: this is called by webserver
"""
try:
app = bgcalc.calc_backend_client(settings)
res = app.send_task('calculate_freqs_ct',
args=(args.to_dict(), ),
time_limit=TASK_TIME_LIMIT)
calc_result = res.get()
except Exception as ex:
if is_celery_user_error(ex):
raise UserActionException(str(ex)) from ex
else:
raise ex
return calc_result
def on_forbidden_corpus(self, plugin_api: 'PluginApi', corpname: str,
corp_variant: str):
"""
Optional method run in case KonText finds out that user
does not have access rights to a corpus specified by 'corpname'.
There are two main action types you can perform here:
1) Redirect to a different page or set 'not found'.
2) Set some system message user can read.
"""
if corpname:
raise CorpusForbiddenException(corpname, corp_variant)
else:
# normally, this happens only with flawed configuration
# (e.g. no default accessible corpus for the current user)
raise UserActionException(
'Cannot find any usable corpus for the user.')
def ajax_get_tag_variants(ctrl, pattern=''):
"""
"""
try:
tag_loader = plugins.runtime.TAGHELPER.instance.loader(
ctrl.args.corpname,
ctrl.get_corpus_info(ctrl.args.corpname)['tagset'],
ctrl.ui_lang)
except IOError:
raise UserActionException(
_('Corpus %s is not supported by this widget.') % ctrl.args.corpname)
if len(pattern) > 0:
ans = tag_loader.get_variant(pattern)
else:
ans = tag_loader.get_initial_values()
return ans
def ajax_get_tag_variants(ctrl: Controller, request):
"""
"""
corpname = request.args['corpname']
tagset_name = request.args['tagset']
values_selection = plugins.runtime.TAGHELPER.instance.fetcher(
ctrl._plugin_ctx, corpname, tagset_name).fetch(request)
try:
tag_loader = plugins.runtime.TAGHELPER.instance.loader(
ctrl._plugin_ctx, corpname, tagset_name)
except IOError:
raise UserActionException(
_('Corpus %s is not supported by this widget.') % corpname)
if plugins.runtime.TAGHELPER.instance.fetcher(ctrl._plugin_ctx, corpname, tagset_name).is_empty(values_selection):
ans = tag_loader.get_initial_values(ctrl.ui_lang)
else:
ans = tag_loader.get_variant(values_selection, ctrl.ui_lang)
return ans
def subcorpus_info(self, _) -> Dict[str, Any]:
if not self.corp.is_subcorpus:
raise UserActionException('Not a subcorpus')
ans = dict(corpusId=self.corp.corpname,
corpusName=self._human_readable_corpname(),
subCorpusName=self.corp.subcname,
origSubCorpusName=self.corp.orig_subcname,
corpusSize=self.corp.size,
subCorpusSize=self.corp.search_size,
created=time.mktime(self.corp.created.timetuple()),
description=self.corp.description,
published=self.corp.is_published,
extended_info={})
if plugins.runtime.SUBC_RESTORE.exists:
with plugins.runtime.SUBC_RESTORE as sr:
tmp = sr.get_info(self.session_get('user', 'id'),
self.args.corpname, self.corp.subcname)
if tmp:
ans['extended_info'].update(tmp.to_dict())
return ans
def calculate_freqs_ct(args):
"""
note: this is called by webserver
"""
backend = settings.get('calc_backend', 'type')
if backend in ('celery', 'konserver'):
import bgcalc
try:
app = bgcalc.calc_backend_client(settings)
res = app.send_task('worker.calculate_freqs_ct', args=(args.to_dict(),),
time_limit=TASK_TIME_LIMIT)
calc_result = res.get()
except Exception as ex:
if is_celery_user_error(ex):
raise UserActionException(ex.message)
else:
raise ex
elif backend == 'multiprocessing':
raise NotImplementedError(
'Multi-processing backend is not yet supported for freq_ct calculation')
else:
raise ValueError('Invalid backend')
return calc_result
def archive_concordance(ctrl, request):
conc_key = request.args.get('conc_key')
if request.method == 'POST':
data = concdb.get(mk_key(conc_key))
if data:
save_time = int(round(time.time()))
cursor = archdb.cursor()
cursor.execute(
'INSERT OR IGNORE INTO archive (id, data, created, num_access) VALUES (?, ?, ?, ?)',
[conc_key, json.dumps(data), save_time, 0])
archdb.commit()
else:
raise UserActionException('Concordance key \'%s\' not found.' %
(conc_key, ))
return dict(save_time=save_time, conc_key=conc_key)
elif request.method == 'GET':
cursor = archdb.cursor()
cursor.execute('SELECT * FROM archive WHERE id = ?', [conc_key])
row = cursor.fetchone()
return dict(data=json.loads(row[1]) if row is not None else None)
else:
ctrl.set_not_found()
return {}
def _validate_http_method(self, action_metadata):
if 'http_method' in action_metadata and (
self.get_http_method().lower() !=
action_metadata['http_method'].lower()):
raise UserActionException(_('Unknown action'), code=404)
def _create_subcorpus(self, request: Request) -> Dict[str, Any]:
"""
req. arguments:
subcname -- name of new subcorpus
create -- bool, sets whether to create new subcorpus
cql -- custom within condition
"""
within_cql = None
form_type = request.json['form_type']
if form_type == 'tt-sel':
data = CreateSubcorpusArgs(**request.json)
corpus_info = self.get_corpus_info(data.corpname)
if (plugins.runtime.LIVE_ATTRIBUTES.exists and
plugins.runtime.LIVE_ATTRIBUTES.instance.is_enabled_for(
self._plugin_ctx, [data.corpname]
) # TODO here we skip aligned corpora which is debatable
and len(data.aligned_corpora) > 0):
if corpus_info.metadata.label_attr and corpus_info.metadata.id_attr:
within_cql = None
sel_match = plugins.runtime.LIVE_ATTRIBUTES.instance.get_attr_values(
self._plugin_ctx,
corpus=self.corp,
attr_map=data.text_types,
aligned_corpora=data.aligned_corpora,
limit_lists=False)
sel_attrs = {}
for k, vals in sel_match.attr_values.items():
if k == corpus_info.metadata.label_attr:
k = corpus_info.metadata.id_attr
if '.' in k:
sel_attrs[k] = [v[1] for v in vals]
tt_query = TextTypeCollector(self.corp,
sel_attrs).get_query()
tmp = ['<%s %s />' % item for item in tt_query]
full_cql = ' within '.join(tmp)
full_cql = 'aword,[] within %s' % full_cql
imp_cql = (full_cql, )
else:
raise FunctionNotSupported(
'Corpus must have a bibliography item defined to support this function'
)
else:
tt_query = TextTypeCollector(self.corp,
data.text_types).get_query()
tmp = ['<%s %s />' % item for item in tt_query]
full_cql = ' within '.join(tmp)
full_cql = 'aword,[] within %s' % full_cql
imp_cql = (full_cql, )
elif form_type == 'within':
data = CreateSubcorpusWithinArgs(**request.json)
tt_query = ()
within_cql = self._deserialize_custom_within(data.within)
full_cql = 'aword,[] %s' % within_cql
imp_cql = (full_cql, )
elif form_type == 'cql':
data = CreateSubcorpusRawCQLArgs(**request.json)
tt_query = ()
within_cql = data.cql
full_cql = f'aword,[] {data.cql}'
imp_cql = (full_cql, )
else:
raise UserActionException(
f'Invalid form type provided - "{form_type}"')
if not data.subcname:
raise UserActionException(
translate('No subcorpus name specified!'))
if data.publish and not data.description:
raise UserActionException(translate('No description specified'))
path = self.prepare_subc_path(self.args.corpname,
data.subcname,
publish=False)
publish_path = self.prepare_subc_path(
self.args.corpname, data.subcname,
publish=True) if data.publish else None
if len(tt_query) == 1 and not data.has_aligned_corpora():
result = corplib.create_subcorpus(path, self.corp, tt_query[0][0],
tt_query[0][1])
if result and publish_path:
corplib.mk_publish_links(path, publish_path,
self.session_get('user', 'fullname'),
data.description)
elif len(tt_query) > 1 or within_cql or data.has_aligned_corpora():
worker = bgcalc.calc_backend_client(settings)
res = worker.send_task(
'create_subcorpus',
object.__class__,
(self.session_get('user', 'id'), self.args.corpname, path,
publish_path, tt_query, imp_cql,
self.session_get('user', 'fullname'), data.description),
time_limit=TASK_TIME_LIMIT)
self._store_async_task(
AsyncTaskStatus(status=res.status,
ident=res.id,
category=AsyncTaskStatus.CATEGORY_SUBCORPUS,
label=f'{self.args.corpname}/{data.subcname}',
args=dict(subcname=data.subcname,
corpname=self.args.corpname)))
result = {}
else:
raise UserActionException(translate('Nothing specified!'))
if result is not False:
with plugins.runtime.SUBC_RESTORE as sr:
try:
sr.store_query(user_id=self.session_get('user', 'id'),
corpname=self.args.corpname,
subcname=data.subcname,
cql=full_cql.strip().split('[]', 1)[-1])
except Exception as e:
logging.getLogger(__name__).warning(
'Failed to store subcorpus query: %s' % e)
self.add_system_message(
'warning',
translate(
'Subcorpus created but there was a problem saving a backup copy.'
))
unfinished_corpora = [
at for at in self.get_async_tasks(
category=AsyncTaskStatus.CATEGORY_SUBCORPUS)
if not at.is_finished()
]
return dict(
processed_subc=[uc.to_dict() for uc in unfinished_corpora])
else:
raise SubcorpusError(translate('Empty subcorpus!'))
def parse_viewbox(vb: str) -> Tuple[float, float, float, float]:
items = [float(v) for v in vb.split(' ')]
try:
return items[0], items[1], items[2], items[3]
except IndexError:
raise UserActionException(f'Invalid SVG viewBox: {vb}', code=422)
def create(self, request: Request) -> Dict[str, Any]:
try:
return self._create_subcorpus(request)
except (SubcorpusError, RuntimeError) as e:
raise UserActionException(str(e)) from e
def update_public_desc(self, request: Request) -> Dict[str, Any]:
if not self.corp.is_published:
raise UserActionException(
'Corpus is not published - cannot change description')
self.corp.save_subc_description(request.form['description'])
return {}
