def parse_ints_to_list_flash(arg, name):
try:
return parse_ints_to_list(arg)
except ValueError:
flash_error(
"Error: <span style='color:black'>%s</span> is not a valid input for <span style='color:black'>%s</span>. It needs to be an integer (such as 25), a range of integers (such as 2-10 or 2..10), or a comma-separated list of these (such as 4,9,16 or 4-25, 81-121).",
arg, name)
raise
def raw_parsing_error(self, info, query, err, err_title, template,
template_kwds):
flash_error('Error parsing %s.', str(err))
info['err'] = str(err)
info['query'] = dict(query)
return render_template(template,
info=info,
title=self.err_title,
**template_kwds)
def oob_error(self, info, query, err, err_title, template, template_kwds):
# The error string is long and ugly, so we just describe the type of issue
flash_error(
'Input number larger than allowed by integer type in database.')
info['err'] = str(err)
info['query'] = dict(query)
return render_template(template,
info=info,
title=self.err_title,
**template_kwds)
def query_cancelled_error(self, info, query, err, err_title, template,
template_kwds):
ctx = ctx_proc_userdata()
flash_error(
'The search query took longer than expected! Please help us improve by reporting this error <a href="%s" target=_blank>here</a>.'
% ctx['feedbackpage'])
info['err'] = str(err)
info['query'] = dict(query)
return render_template(template,
info=info,
title=self.err_title,
**template_kwds)
def __call__(self,
info,
query,
field=None,
name=None,
qfield=None,
*args,
**kwds):
try:
if field is None: field = self.default_field
inp = info.get(field)
if not inp: return
if name is None:
if self.default_name is None:
name = field.replace('_', ' ').capitalize()
else:
name = self.default_name
inp = str(inp)
if SPACES_RE.search(inp):
raise SearchParsingError(
"You have entered spaces in between digits. Please add a comma or delete the spaces."
)
inp = clean_input(inp, self.clean_spaces)
if qfield is None:
if field is None:
qfield = self.default_qfield
else:
qfield = field
if self.prep_ranges:
inp = prep_ranges(inp)
if self.prep_plus:
inp = inp.replace('+', '')
if self.pass_name:
self.f(inp, query, name, qfield, *args, **kwds)
else:
self.f(inp, query, qfield, *args, **kwds)
if self.clean_info:
info[field] = inp
except (ValueError, AttributeError, TypeError) as err:
if self.error_is_safe:
flash_error(
"<span style='color:black'>%s</span> is not a valid input for <span style='color:black'>%s</span>. "
+ str(err) + ".", inp, name)
else:
flash_error(
"<span style='color:black'>%s</span> is not a valid input for <span style='color:black'>%s</span>. %s",
inp, name, str(err))
info['err'] = ''
raise
def __call__(self, info):
info = to_dict(info,
exclude=["bread"
]) # I'm not sure why this is required...
data = self.make_query(info)
if not isinstance(data, tuple):
return data # error page
query, sort, table, title, err_title, template = data
template_kwds = {
key: info.get(key, val())
for key, val in self.kwds.items()
}
try:
if query:
res = table.count(query, groupby=self.groupby)
else:
res = table.stats.column_counts(self.groupby)
except QueryCanceledError as err:
return self.query_cancelled_error(info, query, err, err_title,
template, template_kwds)
else:
try:
if self.postprocess is not None:
res = self.postprocess(res, info, query)
else:
for row in info["row_heads"]:
for col in info["col_heads"]:
if (row, col) not in res:
if (row, col) in self.overall:
res[row, col] = 0
else:
res[row, col] = None
info[
'count'] = 50 # put count back in so that it doesn't show up as none in url
except ValueError as err:
# Errors raised in postprocessing
flash_error(str(err))
info["err"] = str(err)
return render_template(template,
info=info,
title=err_title,
**template_kwds)
info["results"] = res
return render_template(template,
info=info,
title=title,
**template_kwds)
def __call__(self, info):
info = to_dict(info,
exclude=['bread'
]) # I'm not sure why this is required...
data = self.make_query(info)
if not isinstance(data, tuple):
return data # error page
query, template_kwds, sort, table, title, err_title, template = data
try:
if query:
res = table.count(query, groupby=self.groupby)
else:
res = table.stats.column_counts(self.groupby)
except QueryCanceledError as err:
return self.query_cancelled_error(info, query, err, err_title,
template, template_kwds)
else:
try:
if self.postprocess is not None:
res = self.postprocess(res, info, query)
else:
for row in info['row_heads']:
for col in info['col_heads']:
if (row, col) not in res:
if (row, col) in self.overall:
res[row, col] = 0
else:
res[row, col] = None
except ValueError as err:
# Errors raised in postprocessing
flash_error(str(err))
info['err'] = str(err)
return render_template(template,
info=info,
title=err_title,
**template_kwds)
info['results'] = res
return render_template(template,
info=info,
title=title,
**template_kwds)
def __call__(self, info, random=False):
# If random is True, returns a random label
info = to_dict(info,
exclude=['bread'
]) # I'm not sure why this is required...
for key, func in self.shortcuts.items():
if info.get(key, '').strip():
return func(info)
data = self.make_query(info, random)
if not isinstance(data, tuple):
return data
query, template_kwds, sort, table, title, err_title, template = data
if random:
query.pop('__projection__', None)
proj = query.pop('__projection__', self.projection)
if 'result_count' in info:
nres = table.count(query)
return jsonify({"nres": str(nres)})
count = parse_count(info, self.per_page)
start = parse_start(info)
try:
split_ors = use_split_ors(info, query, self.split_ors, start,
table)
if random:
# Ignore __projection__: it's intended for searches
if split_ors:
queries = table._split_ors(query)
else:
queries = [query]
if len(queries) > 1:
# The following method won't produce a uniform distribution
# if there's overlap between queries. But it's simple,
# and in many use cases (e.g. galois group for number fields)
# the subqueries are disjoint.
counts = [table.count(Q) for Q in queries]
pick = randrange(sum(counts))
accum = 0
for Q, cnt in zip(queries, counts):
accum += cnt
if pick < accum:
query = Q
break
label = table.random(query, projection=0)
if label is None:
res = []
# ugh; we have to set these manually
info['query'] = dict(query)
info['number'] = 0
info['count'] = count
info['start'] = start
info['exact_count'] = True
else:
return redirect(self.url_for_label(label), 307)
else:
res = table.search(query,
proj,
limit=count,
offset=start,
sort=sort,
info=info,
split_ors=split_ors)
except QueryCanceledError as err:
return self.query_cancelled_error(info, query, err, err_title,
template, template_kwds)
else:
try:
if self.cleaners:
for v in res:
for name, func in self.cleaners.items():
v[name] = func(v)
if self.postprocess is not None:
res = self.postprocess(res, info, query)
except ValueError as err:
# Errors raised in postprocessing
flash_error(str(err))
info['err'] = str(err)
return render_template(template,
info=info,
title=err_title,
**template_kwds)
for key, func in self.longcuts.items():
if info.get(key, '').strip():
return func(res, info, query)
info['results'] = res
return render_template(template,
info=info,
title=title,
**template_kwds)
def __call__(self, info):
info = to_dict(info, exclude=["bread"]) # I'm not sure why this is required...
# if search_type starts with 'Random' returns a random label
info["search_type"] = info.get("search_type", info.get("hst", "List"))
random = info["search_type"].startswith("Random")
template_kwds = {key: info.get(key, val()) for key, val in self.kwds.items()}
for key, func in self.shortcuts.items():
if info.get(key, "").strip():
try:
return func(info)
except Exception as err:
# Errors raised in jump box, for example
# Using the search results is an okay default, though some
# jump boxes will use their own error processing
if "%s" in str(err):
flash_error(str(err), info[key])
else:
flash_error(str(err))
info["err"] = str(err)
return render_template(
self.template, info=info, title=self.err_title, **template_kwds
)
data = self.make_query(info, random)
if not isinstance(data, tuple):
return data
query, sort, table, title, err_title, template = data
if random:
query.pop("__projection__", None)
proj = query.pop("__projection__", self.projection)
if "result_count" in info:
nres = table.count(query)
return jsonify({"nres": str(nres)})
count = parse_count(info, self.per_page)
start = parse_start(info)
try:
split_ors = use_split_ors(info, query, self.split_ors, start, table)
if random:
# Ignore __projection__: it's intended for searches
if split_ors:
queries = table._split_ors(query)
else:
queries = [query]
if len(queries) > 1:
# The following method won't produce a uniform distribution
# if there's overlap between queries. But it's simple,
# and in many use cases (e.g. galois group for number fields)
# the subqueries are disjoint.
counts = [table.count(Q) for Q in queries]
pick = randrange(sum(counts))
accum = 0
for Q, cnt in zip(queries, counts):
accum += cnt
if pick < accum:
query = Q
break
label = table.random(query, projection=self.random_projection)
if label is None:
res = []
# ugh; we have to set these manually
info["query"] = dict(query)
info["number"] = 0
info["count"] = count
info["start"] = start
info["exact_count"] = True
else:
return redirect(self.url_for_label(label), 307)
else:
res = table.search(
query,
proj,
limit=count,
offset=start,
sort=sort,
info=info,
split_ors=split_ors,
)
except QueryCanceledError as err:
return self.query_cancelled_error(info, query, err, err_title, template, template_kwds)
except SearchParsingError as err:
# These can be raised when the query includes $raw keys.
return self.raw_parsing_error(info, query, err, err_title, template, template_kwds)
else:
try:
if self.cleaners:
for v in res:
for name, func in self.cleaners.items():
v[name] = func(v)
if self.postprocess is not None:
res = self.postprocess(res, info, query)
except ValueError as err:
# Errors raised in postprocessing
flash_error(str(err))
info["err"] = str(err)
return render_template(
template, info=info, title=err_title, **template_kwds
)
for key, func in self.longcuts.items():
if info.get(key, "").strip():
return func(res, info, query)
info["results"] = res
return render_template(template, info=info, title=title, **template_kwds)
def __call__(self, info, random=False):
# If random is True, returns a random label
info = to_dict(info,
exclude=['bread'
]) # I'm not sure why this is required...
for key, func in self.shortcuts.items():
if info.get(key, '').strip():
return func(info)
query = {}
template_kwds = {}
for key in self.kwds:
template_kwds[key] = info.get(key, self.kwds[key]())
try:
errpage = self.f(info, query)
except ValueError as err:
# Errors raised in parsing
info['err'] = str(err)
err_title = query.pop('__err_title__', self.err_title)
return render_template(self.template,
info=info,
title=err_title,
**template_kwds)
else:
err_title = query.pop('__err_title__', self.err_title)
if errpage is not None:
return errpage
sort = query.pop('__sort__', None)
table = query.pop('__table__', self.table)
# We want to pop __title__ even if overridden by info.
title = query.pop('__title__', self.title)
title = info.get('title', title)
template = query.pop('__template__', self.template)
if random:
query.pop('__projection__', None)
proj = query.pop('__projection__', self.projection)
if 'result_count' in info:
nres = table.count(query)
return jsonify({"nres": str(nres)})
count = parse_count(info, self.per_page)
start = parse_start(info)
try:
if random:
# Ignore __projection__: it's intended for searches
label = table.random(query, projection=0)
if label is None:
res = []
# ugh; we have to set these manually
info['query'] = dict(query)
info['number'] = 0
info['count'] = count
info['start'] = start
info['exact_count'] = True
else:
return redirect(self.url_for_label(label), 307)
else:
res = table.search(query,
proj,
limit=count,
offset=start,
sort=sort,
info=info)
except QueryCanceledError as err:
ctx = ctx_proc_userdata()
flash_error(
'The search query took longer than expected! Please help us improve by reporting this error <a href="%s" target=_blank>here</a>.'
% ctx['feedbackpage'])
info['err'] = str(err)
info['query'] = dict(query)
return render_template(self.template,
info=info,
title=self.err_title,
**template_kwds)
else:
try:
if self.cleaners:
for v in res:
for name, func in self.cleaners.items():
v[name] = func(v)
if self.postprocess is not None:
res = self.postprocess(res, info, query)
except ValueError as err:
# Errors raised in postprocessing
flash_error(str(err))
info['err'] = str(err)
return render_template(self.template,
info=info,
title=err_title,
**template_kwds)
for key, func in self.longcuts.items():
if info.get(key, '').strip():
return func(res, info, query)
info['results'] = res
return render_template(template,
info=info,
title=title,
**template_kwds)
def __call__(self, info, random=False):
# If random is True, returns a random label
info = to_dict(info,
exclude=['bread'
]) # I'm not sure why this is required...
for key, func in self.shortcuts.items():
if info.get(key, '').strip():
return func(info)
query = {}
template_kwds = {}
for key in self.kwds:
template_kwds[key] = info.get(key, self.kwds[key]())
try:
errpage = self.f(info, query)
except ValueError as err:
# Errors raised in parsing
info['err'] = str(err)
err_title = query.pop('__err_title__', self.err_title)
return render_template(self.template,
info=info,
title=err_title,
**template_kwds)
else:
err_title = query.pop('__err_title__', self.err_title)
if errpage is not None:
return errpage
sort = query.pop('__sort__', None)
table = query.pop('__table__', self.table)
# We want to pop __title__ even if overridden by info.
title = query.pop('__title__', self.title)
title = info.get('title', title)
template = query.pop('__template__', self.template)
if random:
query.pop('__projection__', None)
proj = query.pop('__projection__', self.projection)
if 'result_count' in info:
nres = table.count(query)
return jsonify({"nres": str(nres)})
count = parse_count(info, self.per_page)
start = parse_start(info)
try:
split_ors = use_split_ors(info, query, self.split_ors, start,
table)
if random:
# Ignore __projection__: it's intended for searches
if split_ors:
queries = table._split_ors(query)
else:
queries = [query]
if len(queries) > 1:
# The following method won't produce a uniform distribution
# if there's overlap between queries. But it's simple,
# and in many use cases (e.g. galois group for number fields)
# the subqueries are disjoint.
counts = [table.count(Q) for Q in queries]
pick = randrange(sum(counts))
accum = 0
for Q, cnt in zip(queries, counts):
accum += cnt
if pick < accum:
query = Q
break
label = table.random(query, projection=0)
if label is None:
res = []
# ugh; we have to set these manually
info['query'] = dict(query)
info['number'] = 0
info['count'] = count
info['start'] = start
info['exact_count'] = True
else:
return redirect(self.url_for_label(label), 307)
else:
res = table.search(query,
proj,
limit=count,
offset=start,
sort=sort,
info=info,
split_ors=split_ors)
except QueryCanceledError as err:
ctx = ctx_proc_userdata()
flash_error(
'The search query took longer than expected! Please help us improve by reporting this error <a href="%s" target=_blank>here</a>.'
% ctx['feedbackpage'])
info['err'] = str(err)
info['query'] = dict(query)
return render_template(self.template,
info=info,
title=self.err_title,
**template_kwds)
else:
try:
if self.cleaners:
for v in res:
for name, func in self.cleaners.items():
v[name] = func(v)
if self.postprocess is not None:
res = self.postprocess(res, info, query)
except ValueError as err:
# Errors raised in postprocessing
flash_error(str(err))
info['err'] = str(err)
return render_template(self.template,
info=info,
title=err_title,
**template_kwds)
for key, func in self.longcuts.items():
if info.get(key, '').strip():
return func(res, info, query)
info['results'] = res
return render_template(template,
info=info,
title=title,
**template_kwds)
def __call__(self, info):
info = to_dict(info,
exclude=["bread"
]) # I'm not sure why this is required...
# if search_type starts with 'Random' returns a random label
info["search_type"] = info.get("search_type", info.get("hst", "List"))
info["columns"] = self.columns
random = info["search_type"].startswith("Random")
template_kwds = {
key: info.get(key, val())
for key, val in self.kwds.items()
}
for key, func in self.shortcuts.items():
if info.get(key, "").strip():
try:
return func(info)
except Exception as err:
# Errors raised in jump box, for example
# Using the search results is an okay default, though some
# jump boxes will use their own error processing
if "%s" in str(err):
flash_error(str(err), info[key])
else:
flash_error(str(err))
info["err"] = str(err)
return render_template(self.template,
info=info,
title=self.err_title,
**template_kwds)
data = self.make_query(info, random)
if not isinstance(data, tuple):
return data
query, sort, table, title, err_title, template, one_per = data
if random:
query.pop("__projection__", None)
proj = query.pop("__projection__", self.projection)
if "result_count" in info:
if one_per:
nres = table.count_distinct(one_per, query)
else:
nres = table.count(query)
return jsonify({"nres": str(nres)})
count = parse_count(info, self.per_page)
start = parse_start(info)
try:
split_ors = not one_per and use_split_ors(
info, query, self.split_ors, start, table)
if random:
# Ignore __projection__: it's intended for searches
if split_ors:
queries = table._split_ors(query)
else:
queries = [query]
if len(queries) > 1:
# The following method won't produce a uniform distribution
# if there's overlap between queries. But it's simple,
# and in many use cases (e.g. galois group for number fields)
# the subqueries are disjoint.
counts = [table.count(Q) for Q in queries]
pick = randrange(sum(counts))
accum = 0
for Q, cnt in zip(queries, counts):
accum += cnt
if pick < accum:
query = Q
break
label = table.random(query, projection=self.random_projection)
if label is None:
res = []
# ugh; we have to set these manually
info["query"] = dict(query)
info["number"] = 0
info["count"] = count
info["start"] = start
info["exact_count"] = True
else:
return redirect(self.url_for_label(label), 307)
else:
res = table.search(
query,
proj,
limit=count,
offset=start,
sort=sort,
info=info,
one_per=one_per,
split_ors=split_ors,
)
except QueryCanceledError as err:
return self.query_cancelled_error(info, query, err, err_title,
template, template_kwds)
except SearchParsingError as err:
# These can be raised when the query includes $raw keys.
return self.raw_parsing_error(info, query, err, err_title,
template, template_kwds)
except NumericValueOutOfRange as err:
# This is caused when a user inputs a number that's too large for a column search type
return self.oob_error(info, query, err, err_title, template,
template_kwds)
else:
try:
if self.cleaners:
for v in res:
for name, func in self.cleaners.items():
v[name] = func(v)
if self.postprocess is not None:
res = self.postprocess(res, info, query)
except ValueError as err:
# Errors raised in postprocessing
flash_error(str(err))
info["err"] = str(err)
return render_template(template,
info=info,
title=err_title,
**template_kwds)
for key, func in self.longcuts.items():
if info.get(key, "").strip():
return func(res, info, query)
info["results"] = res
# Display warning message if user searched on column(s) with null values
if query:
nulls = table.stats.null_counts()
if nulls:
search_columns = table._columns_searched(query)
nulls = {
col: cnt
for (col, cnt) in nulls.items()
if col in search_columns
}
col_display = {}
if "search_array" in info:
for row in info["search_array"].refine_array:
if isinstance(row, (list, tuple)):
for item in row:
if hasattr(item, "name") and hasattr(
item, "label"):
col_display[item.name] = item.label
for col, cnt in list(nulls.items()):
override = info[
"search_array"].null_column_explanations.get(
col)
if override is False:
del nulls[col]
elif override:
nulls[col] = override
else:
nulls[
col] = f"{col_display.get(col, col)} ({cnt} objects)"
else:
for col, cnt in list(nulls.items()):
nulls[col] = f"{col} ({cnt} objects)"
if nulls:
msg = 'Search results may be incomplete due to <a href="Completeness">uncomputed quantities</a>: '
msg += ", ".join(nulls.values())
flash_info(msg)
return render_template(template,
info=info,
title=title,
**template_kwds)
def __call__(self, info, random=False):
# If random is True, returns a random label
info = to_dict(info, exclude =['bread']) # I'm not sure why this is required...
for key, func in self.shortcuts.items():
if info.get(key,'').strip():
return func(info)
query = {}
template_kwds = {}
for key in self.kwds:
template_kwds[key] = info.get(key, self.kwds[key]())
try:
errpage = self.f(info, query)
except ValueError as err:
# Errors raised in parsing
info['err'] = str(err)
err_title = query.pop('__err_title__', self.err_title)
return render_template(self.template, info=info, title=err_title, **template_kwds)
else:
err_title = query.pop('__err_title__', self.err_title)
if errpage is not None:
return errpage
sort = query.pop('__sort__', None)
table = query.pop('__table__', self.table)
# We want to pop __title__ even if overridden by info.
title = query.pop('__title__', self.title)
title = info.get('title', title)
template = query.pop('__template__', self.template)
if random:
query.pop('__projection__', None)
proj = query.pop('__projection__', self.projection)
if 'result_count' in info:
nres = table.count(query)
return jsonify({"nres":str(nres)})
count = parse_count(info, self.per_page)
start = parse_start(info)
try:
if random:
# Ignore __projection__: it's intended for searches
label = table.random(query, projection=0)
if label is None:
res = []
# ugh; we have to set these manually
info['query'] = dict(query)
info['number'] = 0
info['count'] = count
info['start'] = start
info['exact_count'] = True
else:
return redirect(self.url_for_label(label), 307)
else:
res = table.search(query, proj, limit=count, offset=start, sort=sort, info=info)
except QueryCanceledError as err:
ctx = ctx_proc_userdata()
flash_error('The search query took longer than expected! Please help us improve by reporting this error <a href="%s" target=_blank>here</a>.' % ctx['feedbackpage'])
info['err'] = str(err)
info['query'] = dict(query)
return render_template(self.template, info=info, title=self.err_title, **template_kwds)
else:
try:
if self.cleaners:
for v in res:
for name, func in self.cleaners.items():
v[name] = func(v)
if self.postprocess is not None:
res = self.postprocess(res, info, query)
except ValueError as err:
# Errors raised in postprocessing
flash_error(str(err))
info['err'] = str(err)
return render_template(self.template, info=info, title=err_title, **template_kwds)
for key, func in self.longcuts.items():
if info.get(key,'').strip():
return func(res, info, query)
info['results'] = res
return render_template(template, info=info, title=title, **template_kwds)
