def cast_value(_filter):
# range (is between)
if hasattr(_filter.value, 'max'):
if _filter.type == 'Number':
_filter.value.min = float(_filter.value.min)
_filter.value.max = float(_filter.value.max)
elif _filter.type == 'Datetime':
_filter.value.min = datetime.strptime(_filter.value.min, DATETIME_FORMAT)
_filter.value.max = datetime.strptime(_filter.value.max, DATETIME_FORMAT)
# one value
else:
if _filter.type == 'Number':
_filter.value = float(_filter.value)
elif _filter.type == 'Datetime':
_filter.value = datetime.strptime(_filter.value, DATETIME_FORMAT)
elif _filter.type == 'Boolean':
_filter.value = cast_bool_from_str(_filter.value)
def apply_filters(queryset, filters, only_undefined_field=False):
if not filters:
return queryset
# convert conjunction to orm statement
filter_expressions = []
for _filter in filters.items:
# we can also have annotations filters
if not _filter.filter.startswith(
"filter:tasks:") or _filter.value is None:
continue
# django orm loop expression attached to column name
preprocess_field_name = load_func(settings.PREPROCESS_FIELD_NAME)
field_name, _ = preprocess_field_name(_filter.filter,
only_undefined_field)
# filter preprocessing, value type conversion, etc..
preprocess_filter = load_func(settings.DATA_MANAGER_PREPROCESS_FILTER)
_filter = preprocess_filter(_filter, field_name)
# custom expressions for enterprise
custom_filter_expressions = load_func(
settings.DATA_MANAGER_CUSTOM_FILTER_EXPRESSIONS)
filter_expression = custom_filter_expressions(_filter, field_name)
if filter_expression:
filter_expressions.append(filter_expression)
continue
# annotators
if field_name == 'annotators' and _filter.operator == Operator.CONTAINS:
filter_expressions.append(
Q(annotations__completed_by=int(_filter.value)))
continue
elif field_name == 'annotators' and _filter.operator == Operator.NOT_CONTAINS:
filter_expressions.append(~Q(
annotations__completed_by=int(_filter.value)))
continue
elif field_name == 'annotators' and _filter.operator == Operator.EMPTY:
value = cast_bool_from_str(_filter.value)
filter_expressions.append(
Q(annotations__completed_by__isnull=value))
continue
# annotations results & predictions results
if field_name in ['annotations_results', 'predictions_results']:
name = 'annotations__result' if field_name == 'annotations_results' else 'predictions__result'
if _filter.operator in [Operator.EQUAL, Operator.NOT_EQUAL]:
try:
value = json.loads(_filter.value)
except:
return queryset.none()
q = Q(**{name: value})
filter_expressions.append(q if _filter.operator ==
Operator.EQUAL else ~q)
continue
elif _filter.operator == Operator.CONTAINS:
filter_expressions.append(
Q(**{name + '__icontains': _filter.value}))
continue
elif _filter.operator == Operator.NOT_CONTAINS:
filter_expressions.append(~Q(
**{name + '__icontains': _filter.value}))
continue
# annotation ids
if field_name == 'annotations_ids':
field_name = 'annotations__id'
if 'contains' in _filter.operator:
# convert string like "1 2,3" => [1,2,3]
_filter.value = [
int(value) for value in re.split(',|;| ', _filter.value)
if value and value.isdigit()
]
_filter.operator = 'in_list' if _filter.operator == 'contains' else 'not_in_list'
elif 'equal' in _filter.operator:
if not _filter.value.isdigit():
_filter.value = 0
# annotators
if field_name == 'annotators' and _filter.operator == Operator.CONTAINS:
filter_expressions.append(
Q(annotations__completed_by=int(_filter.value)))
continue
elif field_name == 'annotators' and _filter.operator == Operator.NOT_CONTAINS:
filter_expressions.append(~Q(
annotations__completed_by=int(_filter.value)))
continue
elif field_name == 'annotators' and _filter.operator == Operator.EMPTY:
value = cast_bool_from_str(_filter.value)
filter_expressions.append(
Q(annotations__completed_by__isnull=value))
continue
# predictions model versions
if field_name == 'predictions_model_versions' and _filter.operator == Operator.CONTAINS:
q = Q()
for value in _filter.value:
q |= Q(predictions__model_version__contains=value)
filter_expressions.append(q)
continue
elif field_name == 'predictions_model_versions' and _filter.operator == Operator.NOT_CONTAINS:
q = Q()
for value in _filter.value:
q &= ~Q(predictions__model_version__contains=value)
filter_expressions.append(q)
continue
elif field_name == 'predictions_model_versions' and _filter.operator == Operator.EMPTY:
value = cast_bool_from_str(_filter.value)
filter_expressions.append(
Q(predictions__model_version__isnull=value))
continue
# use other name because of model names conflict
if field_name == 'file_upload':
field_name = 'file_upload_field'
# annotate with cast to number if need
if _filter.type == 'Number' and field_name.startswith('data__'):
json_field = field_name.replace('data__', '')
queryset = queryset.annotate(
**{
f'filter_{json_field.replace("$undefined$", "undefined")}':
Cast(KeyTextTransform(json_field, 'data'),
output_field=FloatField())
})
clean_field_name = f'filter_{json_field.replace("$undefined$", "undefined")}'
else:
clean_field_name = field_name
# special case: predictions, annotations, cancelled --- for them 0 is equal to is_empty=True
if clean_field_name in ('total_predictions', 'total_annotations', 'cancelled_annotations') and \
_filter.operator == 'empty':
_filter.operator = 'equal' if cast_bool_from_str(
_filter.value) else 'not_equal'
_filter.value = 0
# get type of annotated field
value_type = 'str'
if queryset.exists():
value_type = type(queryset.values_list(field_name,
flat=True)[0]).__name__
if (value_type == 'list'
or value_type == 'tuple') and 'equal' in _filter.operator:
raise Exception('Not supported filter type')
# special case: for strings empty is "" or null=True
if _filter.type in ('String',
'Unknown') and _filter.operator == 'empty':
value = cast_bool_from_str(_filter.value)
if value: # empty = true
q = Q(
Q(**{field_name: None})
| Q(**{field_name + '__isnull': True}))
if value_type == 'str':
q |= Q(**{field_name: ''})
if value_type == 'list':
q = Q(**{field_name: [None]})
else: # empty = false
q = Q(~Q(**{field_name: None})
& ~Q(**{field_name + '__isnull': True}))
if value_type == 'str':
q &= ~Q(**{field_name: ''})
if value_type == 'list':
q = ~Q(**{field_name: [None]})
filter_expressions.append(q)
continue
# regex pattern check
elif _filter.operator == 'regex':
try:
re.compile(pattern=str(_filter.value))
except Exception as e:
logger.info('Incorrect regex for filter: %s: %s',
_filter.value, str(e))
return queryset.none()
# append operator
field_name = f"{clean_field_name}{operators.get(_filter.operator, '')}"
# in
if _filter.operator == "in":
cast_value(_filter)
filter_expressions.append(
Q(
**{
f"{field_name}__gte": _filter.value.min,
f"{field_name}__lte": _filter.value.max,
}), )
# not in
elif _filter.operator == "not_in":
cast_value(_filter)
filter_expressions.append(
~Q(
**{
f"{field_name}__gte": _filter.value.min,
f"{field_name}__lte": _filter.value.max,
}), )
# in list
elif _filter.operator == "in_list":
filter_expressions.append(
Q(**{f"{field_name}__in": _filter.value}), )
# not in list
elif _filter.operator == "not_in_list":
filter_expressions.append(
~Q(**{f"{field_name}__in": _filter.value}), )
# empty
elif _filter.operator == 'empty':
if cast_bool_from_str(_filter.value):
filter_expressions.append(Q(**{field_name: True}))
else:
filter_expressions.append(~Q(**{field_name: True}))
# starting from not_
elif _filter.operator.startswith("not_"):
cast_value(_filter)
filter_expressions.append(~Q(**{field_name: _filter.value}))
# all others
else:
cast_value(_filter)
filter_expressions.append(Q(**{field_name: _filter.value}))
logger.debug(f'Apply filter: {filter_expressions}')
if filters.conjunction == ConjunctionEnum.OR:
result_filter = Q()
for filter_expression in filter_expressions:
result_filter.add(filter_expression, Q.OR)
queryset = queryset.filter(result_filter)
else:
for filter_expression in filter_expressions:
queryset = queryset.filter(filter_expression)
return queryset
def cast_value(_filter):
if _filter.type == 'Number':
_filter.value = float(_filter.value)
elif _filter.type == 'Boolean':
_filter.value = cast_bool_from_str(_filter.value)
def apply_filters(queryset, filters):
if not filters:
return queryset
# convert conjunction to orm statement
filter_expression = Q()
if filters.conjunction == ConjunctionEnum.OR:
conjunction = Q.OR
else:
conjunction = Q.AND
only_undefined_field = queryset.exists() and queryset.first(
).project.only_undefined_field
for _filter in filters.items:
# we can also have annotations filters
if not _filter.filter.startswith("filter:tasks:") or not _filter.value:
continue
# django orm loop expression attached to column name
field_name = preprocess_field_name(_filter.filter,
only_undefined_field)
# annotate with cast to number if need
if _filter.type == 'Number' and field_name.startswith('data__'):
json_field = field_name.replace('data__', '')
queryset = queryset.annotate(
**{
f'filter_{json_field.replace("$undefined$", "undefined")}':
Cast(KeyTransform(json_field, 'data'),
output_field=FloatField())
})
clean_field_name = f'filter_{json_field.replace("$undefined$", "undefined")}'
else:
clean_field_name = field_name
# special case: predictions, annotations, cancelled --- for them 0 is equal to is_empty=True
if clean_field_name in ('total_predictions', 'total_annotations', 'cancelled_annotations') and \
_filter.operator == 'empty':
_filter.operator = 'equal' if cast_bool_from_str(
_filter.value) else 'not_equal'
_filter.value = 0
# append operator
field_name = f"{clean_field_name}{operators.get(_filter.operator, '')}"
# in
if _filter.operator == "in":
cast_value(_filter)
filter_expression.add(
Q(
**{
f"{field_name}__gte": _filter.value.min,
f"{field_name}__lte": _filter.value.max,
}),
conjunction,
)
# not in
elif _filter.operator == "not_in":
cast_value(_filter)
filter_expression.add(
~Q(
**{
f"{field_name}__gte": _filter.value.min,
f"{field_name}__lte": _filter.value.max,
}),
conjunction,
)
# empty
elif _filter.operator == 'empty':
if cast_bool_from_str(_filter.value):
filter_expression.add(Q(**{field_name: True}), conjunction)
else:
filter_expression.add(~Q(**{field_name: True}), conjunction)
# starting from not_
elif _filter.operator.startswith("not_"):
cast_value(_filter)
filter_expression.add(~Q(**{field_name: _filter.value}),
conjunction)
# all others
else:
cast_value(_filter)
filter_expression.add(Q(**{field_name: _filter.value}),
conjunction)
logger.debug(f'Apply filter: {filter_expression}')
queryset = queryset.filter(filter_expression)
return queryset