}
VALUE_TYPE_MAP = {
'string[]': 'string',
'words': 'string',
}
TIMEZONE = pendulum.local_timezone().name
# Order corresponds to marv.model.STATUS OrderedDict
STATUS_ICON = ['fire', 'eye-close', 'warning-sign', 'time']
STATUS_JSON = [
jsondumps({
'icon': STATUS_ICON[i],
'title': x
}, separators=(',', ':')) for i, x in enumerate(STATUS.values())
]
# TODO: reconsider in case we get a couple of more states
STATUS_STRS = {
bitmask: ','.join(
filter(None, [
STATUS_JSON[i] if bitmask & 2**i else None
for i in range(len(STATUS_JSON))
]))
for bitmask in range(2**(len(STATUS_JSON)))
}
class FilterParseError(Exception):
pass
def collection(collection_id):
collections = current_app.site.collections
collection_id = collections.default_id if collection_id is None else collection_id
try:
collection = collections[collection_id]
except KeyError:
flask.abort(404)
try:
filters = parse_filters(collection.filter_specs,
request.args.get('filter', '{}'))
except (KeyError, ValueError):
current_app.logger.warn('Bad request', exc_info=True)
return flask.abort(400)
try:
stmt = collection.filtered_listing(filters)
except UnknownOperator:
current_app.logger.warn('Bad request', exc_info=True)
flask.abort(400)
fspecs = collection.filter_specs
all_known = {
name: [
x[0] for x in db.session.execute(
select([rel.value]).order_by(rel.value)).fetchall()
]
for name, rel in collection.model.relations.items()
if {'any', 'all'}.intersection(fspecs[name].operators)
}
all_known['status'] = STATUS.keys()
all_known['tags'] = [
x[0] for x in db.session.execute(
select([Tag.value]).where(Tag.collection == collection.name).
order_by(Tag.value)).fetchall()
]
filters = [{
'key': x.name,
'constraints': all_known.get(x.name),
'title': x.title,
'operators': x.operators,
'value_type': VALUE_TYPE_MAP.get(x.value_type, x.value_type)
} for x in fspecs.values()]
rows = db.session.execute(stmt).fetchall()
rowdata = ',\n'.join([
x.replace('["#TAGS#"]', tags if tags != '[null]' else '[]').replace(
'"#TAGS#"', tags[1:-1] if tags != '[null]' else '').replace(
'[,', '[').replace('"#STATUS#"', STATUS_STRS[status])
for x, status, tags in rows
])
dct = {
'all_known': all_known,
'compare': bool(collection.compare),
'filters': {
'title': 'Filter',
'widget': {
'type': 'filter',
'filters': filters
}
},
'summary_config': {
'title': 'Summary',
'items': collection.summary_items
},
'listing': {
'title': 'Data sets ({} found)'.format(len(rows)),
'widget': {
'data': {
'columns': [{
'align':
ALIGN.get(x.formatter, 'left'),
'formatter':
x.formatter,
'sortkey':
'title' if x.formatter == 'route' else None,
'id':
x.name,
'title':
x.heading,
'list':
x.islist,
} for x in collection.listing_columns],
'rows': ['#ROWS#'],
'sortcolumn':
collection.sortcolumn,
'sortorder':
collection.sortorder
},
'type': 'table',
}
},
}
indent = None
separators = (',', ':')
if current_app.config['JSONIFY_PRETTYPRINT_REGULAR'] or current_app.debug:
indent = 2
separators = (', ', ': ')
jsondata = jsondumps(dct,
indent=indent,
separators=separators,
sort_keys=True)
jsondata = jsondata.replace('"#ROWS#"', rowdata)
resp = current_app.response_class(
(jsondata, '\n'), mimetype=current_app.config['JSONIFY_MIMETYPE'])
resp.headers['Cache-Control'] = 'no-cache'
return resp
def filtered_listing(self, filters):
model = self.model
Listing = model.Listing
listing = Listing.__table__
relations = model.relations
secondaries = model.secondaries
stmt = select([Listing.id.label('id'),
Listing.row.label('row'),
Dataset.status.label('status'),
Tag.value.label('tag_value')])\
.select_from(listing.outerjoin(Dataset)
.outerjoin(dataset_tag)
.outerjoin(Tag))\
.where(Dataset.discarded.isnot(True))
for name, value, op in filters:
if isinstance(value, long):
value = min([value, sys.maxsize])
if name == 'comments':
containstext = Comment.text.like('%{}%'.format(esc(value)),
escape='$')
commentquery = db.session.query(Comment.dataset_id)\
.filter(containstext)\
.group_by(Comment.dataset_id)
stmt = stmt.where(Listing.id.in_(commentquery.subquery()))
continue
elif name == 'status':
status_ids = STATUS.keys()
bitmasks = [2**status_ids.index(x) for x in value]
if op == 'any':
stmt = stmt.where(
reduce(lambda x, y: x | y,
(Dataset.status.op('&')(x) for x in bitmasks)))
elif op == 'all':
bitmask = sum(bitmasks)
stmt = stmt.where(
Dataset.status.op('&')(bitmask) == bitmask)
else:
raise UnknownOperator(op)
continue
elif name == 'tags':
if op == 'any':
relquery = db.session.query(dataset_tag.c.dataset_id)\
.join(Tag)\
.filter(Tag.value.in_(value))
stmt = stmt.where(Listing.id.in_(relquery.subquery()))
elif op == 'all':
relquery = Tag.query\
.join(dataset_tag)\
.filter(reduce(lambda x, y: x|y, (Tag.value == x for x in value)))\
.group_by(dataset_tag.c.dataset_id)\
.having(db.func.count('*') == len(value))\
.with_entities(dataset_tag.c.dataset_id)
stmt = stmt.where(Listing.id.in_(relquery.subquery()))
else:
raise UnknownOperator(op)
continue
col = getattr(Listing, name)
if op == 'lt':
stmt = stmt.where(col < value)
elif op == 'le':
stmt = stmt.where(col <= value)
elif op == 'eq':
stmt = stmt.where(col == value)
elif op == 'ne':
stmt = stmt.where(col != value)
elif op == 'ge':
stmt = stmt.where(col >= value)
elif op == 'gt':
stmt = stmt.where(col > value)
elif op == 'substring':
stmt = stmt.where(
col.like('%{}%'.format(esc(value)), escape='$'))
elif op == 'startswith':
stmt = stmt.where(
col.like('{}%'.format(esc(value)), escape='$'))
elif op == 'any':
rel = relations[name]
sec = secondaries[name]
relquery = rel.query\
.join(sec)\
.filter(rel.value.in_(value))\
.with_entities(sec.c.listing_id)
stmt = stmt.where(Listing.id.in_(relquery.subquery()))
elif op == 'all':
rel = relations[name]
sec = secondaries[name]
relquery = rel.query\
.join(sec)\
.filter(reduce(lambda x, y: x|y, (rel.value == x for x in value)))\
.group_by(sec.c.listing_id)\
.having(db.func.count('*') == len(value))\
.with_entities(sec.c.listing_id)
stmt = stmt.where(Listing.id.in_(relquery.subquery()))
elif op == 'substring_any':
rel = relations[name]
stmt = stmt.where(
col.any(
rel.value.like('%{}%'.format(esc(value)), escape='$')))
elif op == 'words':
stmt = reduce(
lambda stmt, x: stmt.where(
col.like('%{}%'.format(esc(x)), escape='$')), value,
stmt)
else:
raise UnknownOperator(op)
stmt = select([column('row'), column('status'), func.json_group_array(column('tag_value'))])\
.select_from(stmt.order_by(column('tag_value')))\
.group_by('id')
return stmt