def __call__(self, info): info = to_dict(info, exclude=["bread" ]) # I'm not sure why this is required... data = self.make_query(info) if not isinstance(data, tuple): return data # error page query, sort, table, title, err_title, template = data template_kwds = { key: info.get(key, val()) for key, val in self.kwds.items() } try: if query: res = table.count(query, groupby=self.groupby) else: res = table.stats.column_counts(self.groupby) except QueryCanceledError as err: return self.query_cancelled_error(info, query, err, err_title, template, template_kwds) else: try: if self.postprocess is not None: res = self.postprocess(res, info, query) else: for row in info["row_heads"]: for col in info["col_heads"]: if (row, col) not in res: if (row, col) in self.overall: res[row, col] = 0 else: res[row, col] = None info[ 'count'] = 50 # put count back in so that it doesn't show up as none in url except ValueError as err: # Errors raised in postprocessing flash_error(str(err)) info["err"] = str(err) return render_template(template, info=info, title=err_title, **template_kwds) info["results"] = res return render_template(template, info=info, title=title, **template_kwds)
def __call__(self, info): info = to_dict(info, exclude=['bread' ]) # I'm not sure why this is required... data = self.make_query(info) if not isinstance(data, tuple): return data # error page query, template_kwds, sort, table, title, err_title, template = data try: if query: res = table.count(query, groupby=self.groupby) else: res = table.stats.column_counts(self.groupby) except QueryCanceledError as err: return self.query_cancelled_error(info, query, err, err_title, template, template_kwds) else: try: if self.postprocess is not None: res = self.postprocess(res, info, query) else: for row in info['row_heads']: for col in info['col_heads']: if (row, col) not in res: if (row, col) in self.overall: res[row, col] = 0 else: res[row, col] = None except ValueError as err: # Errors raised in postprocessing flash_error(str(err)) info['err'] = str(err) return render_template(template, info=info, title=err_title, **template_kwds) info['results'] = res return render_template(template, info=info, title=title, **template_kwds)
def __call__(self, info, random=False): # If random is True, returns a random label info = to_dict(info, exclude=['bread' ]) # I'm not sure why this is required... for key, func in self.shortcuts.items(): if info.get(key, '').strip(): return func(info) data = self.make_query(info, random) if not isinstance(data, tuple): return data query, template_kwds, sort, table, title, err_title, template = data if random: query.pop('__projection__', None) proj = query.pop('__projection__', self.projection) if 'result_count' in info: nres = table.count(query) return jsonify({"nres": str(nres)}) count = parse_count(info, self.per_page) start = parse_start(info) try: split_ors = use_split_ors(info, query, self.split_ors, start, table) if random: # Ignore __projection__: it's intended for searches if split_ors: queries = table._split_ors(query) else: queries = [query] if len(queries) > 1: # The following method won't produce a uniform distribution # if there's overlap between queries. But it's simple, # and in many use cases (e.g. galois group for number fields) # the subqueries are disjoint. counts = [table.count(Q) for Q in queries] pick = randrange(sum(counts)) accum = 0 for Q, cnt in zip(queries, counts): accum += cnt if pick < accum: query = Q break label = table.random(query, projection=0) if label is None: res = [] # ugh; we have to set these manually info['query'] = dict(query) info['number'] = 0 info['count'] = count info['start'] = start info['exact_count'] = True else: return redirect(self.url_for_label(label), 307) else: res = table.search(query, proj, limit=count, offset=start, sort=sort, info=info, split_ors=split_ors) except QueryCanceledError as err: return self.query_cancelled_error(info, query, err, err_title, template, template_kwds) else: try: if self.cleaners: for v in res: for name, func in self.cleaners.items(): v[name] = func(v) if self.postprocess is not None: res = self.postprocess(res, info, query) except ValueError as err: # Errors raised in postprocessing flash_error(str(err)) info['err'] = str(err) return render_template(template, info=info, title=err_title, **template_kwds) for key, func in self.longcuts.items(): if info.get(key, '').strip(): return func(res, info, query) info['results'] = res return render_template(template, info=info, title=title, **template_kwds)
def __call__(self, info): info = to_dict(info, exclude=["bread"]) # I'm not sure why this is required... # if search_type starts with 'Random' returns a random label info["search_type"] = info.get("search_type", info.get("hst", "List")) random = info["search_type"].startswith("Random") template_kwds = {key: info.get(key, val()) for key, val in self.kwds.items()} for key, func in self.shortcuts.items(): if info.get(key, "").strip(): try: return func(info) except Exception as err: # Errors raised in jump box, for example # Using the search results is an okay default, though some # jump boxes will use their own error processing if "%s" in str(err): flash_error(str(err), info[key]) else: flash_error(str(err)) info["err"] = str(err) return render_template( self.template, info=info, title=self.err_title, **template_kwds ) data = self.make_query(info, random) if not isinstance(data, tuple): return data query, sort, table, title, err_title, template = data if random: query.pop("__projection__", None) proj = query.pop("__projection__", self.projection) if "result_count" in info: nres = table.count(query) return jsonify({"nres": str(nres)}) count = parse_count(info, self.per_page) start = parse_start(info) try: split_ors = use_split_ors(info, query, self.split_ors, start, table) if random: # Ignore __projection__: it's intended for searches if split_ors: queries = table._split_ors(query) else: queries = [query] if len(queries) > 1: # The following method won't produce a uniform distribution # if there's overlap between queries. But it's simple, # and in many use cases (e.g. galois group for number fields) # the subqueries are disjoint. counts = [table.count(Q) for Q in queries] pick = randrange(sum(counts)) accum = 0 for Q, cnt in zip(queries, counts): accum += cnt if pick < accum: query = Q break label = table.random(query, projection=self.random_projection) if label is None: res = [] # ugh; we have to set these manually info["query"] = dict(query) info["number"] = 0 info["count"] = count info["start"] = start info["exact_count"] = True else: return redirect(self.url_for_label(label), 307) else: res = table.search( query, proj, limit=count, offset=start, sort=sort, info=info, split_ors=split_ors, ) except QueryCanceledError as err: return self.query_cancelled_error(info, query, err, err_title, template, template_kwds) except SearchParsingError as err: # These can be raised when the query includes $raw keys. return self.raw_parsing_error(info, query, err, err_title, template, template_kwds) else: try: if self.cleaners: for v in res: for name, func in self.cleaners.items(): v[name] = func(v) if self.postprocess is not None: res = self.postprocess(res, info, query) except ValueError as err: # Errors raised in postprocessing flash_error(str(err)) info["err"] = str(err) return render_template( template, info=info, title=err_title, **template_kwds ) for key, func in self.longcuts.items(): if info.get(key, "").strip(): return func(res, info, query) info["results"] = res return render_template(template, info=info, title=title, **template_kwds)
def __call__(self, info, random=False): # If random is True, returns a random label info = to_dict(info, exclude=['bread' ]) # I'm not sure why this is required... for key, func in self.shortcuts.items(): if info.get(key, '').strip(): return func(info) query = {} template_kwds = {} for key in self.kwds: template_kwds[key] = info.get(key, self.kwds[key]()) try: errpage = self.f(info, query) except ValueError as err: # Errors raised in parsing info['err'] = str(err) err_title = query.pop('__err_title__', self.err_title) return render_template(self.template, info=info, title=err_title, **template_kwds) else: err_title = query.pop('__err_title__', self.err_title) if errpage is not None: return errpage sort = query.pop('__sort__', None) table = query.pop('__table__', self.table) # We want to pop __title__ even if overridden by info. title = query.pop('__title__', self.title) title = info.get('title', title) template = query.pop('__template__', self.template) if random: query.pop('__projection__', None) proj = query.pop('__projection__', self.projection) if 'result_count' in info: nres = table.count(query) return jsonify({"nres": str(nres)}) count = parse_count(info, self.per_page) start = parse_start(info) try: if random: # Ignore __projection__: it's intended for searches label = table.random(query, projection=0) if label is None: res = [] # ugh; we have to set these manually info['query'] = dict(query) info['number'] = 0 info['count'] = count info['start'] = start info['exact_count'] = True else: return redirect(self.url_for_label(label), 307) else: res = table.search(query, proj, limit=count, offset=start, sort=sort, info=info) except QueryCanceledError as err: ctx = ctx_proc_userdata() flash_error( 'The search query took longer than expected! Please help us improve by reporting this error <a href="%s" target=_blank>here</a>.' % ctx['feedbackpage']) info['err'] = str(err) info['query'] = dict(query) return render_template(self.template, info=info, title=self.err_title, **template_kwds) else: try: if self.cleaners: for v in res: for name, func in self.cleaners.items(): v[name] = func(v) if self.postprocess is not None: res = self.postprocess(res, info, query) except ValueError as err: # Errors raised in postprocessing flash_error(str(err)) info['err'] = str(err) return render_template(self.template, info=info, title=err_title, **template_kwds) for key, func in self.longcuts.items(): if info.get(key, '').strip(): return func(res, info, query) info['results'] = res return render_template(template, info=info, title=title, **template_kwds)
def __call__(self, info, random=False): # If random is True, returns a random label info = to_dict(info, exclude=['bread' ]) # I'm not sure why this is required... for key, func in self.shortcuts.items(): if info.get(key, '').strip(): return func(info) query = {} template_kwds = {} for key in self.kwds: template_kwds[key] = info.get(key, self.kwds[key]()) try: errpage = self.f(info, query) except ValueError as err: # Errors raised in parsing info['err'] = str(err) err_title = query.pop('__err_title__', self.err_title) return render_template(self.template, info=info, title=err_title, **template_kwds) else: err_title = query.pop('__err_title__', self.err_title) if errpage is not None: return errpage sort = query.pop('__sort__', None) table = query.pop('__table__', self.table) # We want to pop __title__ even if overridden by info. title = query.pop('__title__', self.title) title = info.get('title', title) template = query.pop('__template__', self.template) if random: query.pop('__projection__', None) proj = query.pop('__projection__', self.projection) if 'result_count' in info: nres = table.count(query) return jsonify({"nres": str(nres)}) count = parse_count(info, self.per_page) start = parse_start(info) try: split_ors = use_split_ors(info, query, self.split_ors, start, table) if random: # Ignore __projection__: it's intended for searches if split_ors: queries = table._split_ors(query) else: queries = [query] if len(queries) > 1: # The following method won't produce a uniform distribution # if there's overlap between queries. But it's simple, # and in many use cases (e.g. galois group for number fields) # the subqueries are disjoint. counts = [table.count(Q) for Q in queries] pick = randrange(sum(counts)) accum = 0 for Q, cnt in zip(queries, counts): accum += cnt if pick < accum: query = Q break label = table.random(query, projection=0) if label is None: res = [] # ugh; we have to set these manually info['query'] = dict(query) info['number'] = 0 info['count'] = count info['start'] = start info['exact_count'] = True else: return redirect(self.url_for_label(label), 307) else: res = table.search(query, proj, limit=count, offset=start, sort=sort, info=info, split_ors=split_ors) except QueryCanceledError as err: ctx = ctx_proc_userdata() flash_error( 'The search query took longer than expected! Please help us improve by reporting this error <a href="%s" target=_blank>here</a>.' % ctx['feedbackpage']) info['err'] = str(err) info['query'] = dict(query) return render_template(self.template, info=info, title=self.err_title, **template_kwds) else: try: if self.cleaners: for v in res: for name, func in self.cleaners.items(): v[name] = func(v) if self.postprocess is not None: res = self.postprocess(res, info, query) except ValueError as err: # Errors raised in postprocessing flash_error(str(err)) info['err'] = str(err) return render_template(self.template, info=info, title=err_title, **template_kwds) for key, func in self.longcuts.items(): if info.get(key, '').strip(): return func(res, info, query) info['results'] = res return render_template(template, info=info, title=title, **template_kwds)
def __call__(self, info): info = to_dict(info, exclude=["bread" ]) # I'm not sure why this is required... # if search_type starts with 'Random' returns a random label info["search_type"] = info.get("search_type", info.get("hst", "List")) info["columns"] = self.columns random = info["search_type"].startswith("Random") template_kwds = { key: info.get(key, val()) for key, val in self.kwds.items() } for key, func in self.shortcuts.items(): if info.get(key, "").strip(): try: return func(info) except Exception as err: # Errors raised in jump box, for example # Using the search results is an okay default, though some # jump boxes will use their own error processing if "%s" in str(err): flash_error(str(err), info[key]) else: flash_error(str(err)) info["err"] = str(err) return render_template(self.template, info=info, title=self.err_title, **template_kwds) data = self.make_query(info, random) if not isinstance(data, tuple): return data query, sort, table, title, err_title, template, one_per = data if random: query.pop("__projection__", None) proj = query.pop("__projection__", self.projection) if "result_count" in info: if one_per: nres = table.count_distinct(one_per, query) else: nres = table.count(query) return jsonify({"nres": str(nres)}) count = parse_count(info, self.per_page) start = parse_start(info) try: split_ors = not one_per and use_split_ors( info, query, self.split_ors, start, table) if random: # Ignore __projection__: it's intended for searches if split_ors: queries = table._split_ors(query) else: queries = [query] if len(queries) > 1: # The following method won't produce a uniform distribution # if there's overlap between queries. But it's simple, # and in many use cases (e.g. galois group for number fields) # the subqueries are disjoint. counts = [table.count(Q) for Q in queries] pick = randrange(sum(counts)) accum = 0 for Q, cnt in zip(queries, counts): accum += cnt if pick < accum: query = Q break label = table.random(query, projection=self.random_projection) if label is None: res = [] # ugh; we have to set these manually info["query"] = dict(query) info["number"] = 0 info["count"] = count info["start"] = start info["exact_count"] = True else: return redirect(self.url_for_label(label), 307) else: res = table.search( query, proj, limit=count, offset=start, sort=sort, info=info, one_per=one_per, split_ors=split_ors, ) except QueryCanceledError as err: return self.query_cancelled_error(info, query, err, err_title, template, template_kwds) except SearchParsingError as err: # These can be raised when the query includes $raw keys. return self.raw_parsing_error(info, query, err, err_title, template, template_kwds) except NumericValueOutOfRange as err: # This is caused when a user inputs a number that's too large for a column search type return self.oob_error(info, query, err, err_title, template, template_kwds) else: try: if self.cleaners: for v in res: for name, func in self.cleaners.items(): v[name] = func(v) if self.postprocess is not None: res = self.postprocess(res, info, query) except ValueError as err: # Errors raised in postprocessing flash_error(str(err)) info["err"] = str(err) return render_template(template, info=info, title=err_title, **template_kwds) for key, func in self.longcuts.items(): if info.get(key, "").strip(): return func(res, info, query) info["results"] = res # Display warning message if user searched on column(s) with null values if query: nulls = table.stats.null_counts() if nulls: search_columns = table._columns_searched(query) nulls = { col: cnt for (col, cnt) in nulls.items() if col in search_columns } col_display = {} if "search_array" in info: for row in info["search_array"].refine_array: if isinstance(row, (list, tuple)): for item in row: if hasattr(item, "name") and hasattr( item, "label"): col_display[item.name] = item.label for col, cnt in list(nulls.items()): override = info[ "search_array"].null_column_explanations.get( col) if override is False: del nulls[col] elif override: nulls[col] = override else: nulls[ col] = f"{col_display.get(col, col)} ({cnt} objects)" else: for col, cnt in list(nulls.items()): nulls[col] = f"{col} ({cnt} objects)" if nulls: msg = 'Search results may be incomplete due to <a href="Completeness">uncomputed quantities</a>: ' msg += ", ".join(nulls.values()) flash_info(msg) return render_template(template, info=info, title=title, **template_kwds)
def __call__(self, info, random=False): # If random is True, returns a random label info = to_dict(info, exclude =['bread']) # I'm not sure why this is required... for key, func in self.shortcuts.items(): if info.get(key,'').strip(): return func(info) query = {} template_kwds = {} for key in self.kwds: template_kwds[key] = info.get(key, self.kwds[key]()) try: errpage = self.f(info, query) except ValueError as err: # Errors raised in parsing info['err'] = str(err) err_title = query.pop('__err_title__', self.err_title) return render_template(self.template, info=info, title=err_title, **template_kwds) else: err_title = query.pop('__err_title__', self.err_title) if errpage is not None: return errpage sort = query.pop('__sort__', None) table = query.pop('__table__', self.table) # We want to pop __title__ even if overridden by info. title = query.pop('__title__', self.title) title = info.get('title', title) template = query.pop('__template__', self.template) if random: query.pop('__projection__', None) proj = query.pop('__projection__', self.projection) if 'result_count' in info: nres = table.count(query) return jsonify({"nres":str(nres)}) count = parse_count(info, self.per_page) start = parse_start(info) try: if random: # Ignore __projection__: it's intended for searches label = table.random(query, projection=0) if label is None: res = [] # ugh; we have to set these manually info['query'] = dict(query) info['number'] = 0 info['count'] = count info['start'] = start info['exact_count'] = True else: return redirect(self.url_for_label(label), 307) else: res = table.search(query, proj, limit=count, offset=start, sort=sort, info=info) except QueryCanceledError as err: ctx = ctx_proc_userdata() flash_error('The search query took longer than expected! Please help us improve by reporting this error <a href="%s" target=_blank>here</a>.' % ctx['feedbackpage']) info['err'] = str(err) info['query'] = dict(query) return render_template(self.template, info=info, title=self.err_title, **template_kwds) else: try: if self.cleaners: for v in res: for name, func in self.cleaners.items(): v[name] = func(v) if self.postprocess is not None: res = self.postprocess(res, info, query) except ValueError as err: # Errors raised in postprocessing flash_error(str(err)) info['err'] = str(err) return render_template(self.template, info=info, title=err_title, **template_kwds) for key, func in self.longcuts.items(): if info.get(key,'').strip(): return func(res, info, query) info['results'] = res return render_template(template, info=info, title=title, **template_kwds)