def cast_value(_filter): # range (is between) if hasattr(_filter.value, 'max'): if _filter.type == 'Number': _filter.value.min = float(_filter.value.min) _filter.value.max = float(_filter.value.max) elif _filter.type == 'Datetime': _filter.value.min = datetime.strptime(_filter.value.min, DATETIME_FORMAT) _filter.value.max = datetime.strptime(_filter.value.max, DATETIME_FORMAT) # one value else: if _filter.type == 'Number': _filter.value = float(_filter.value) elif _filter.type == 'Datetime': _filter.value = datetime.strptime(_filter.value, DATETIME_FORMAT) elif _filter.type == 'Boolean': _filter.value = cast_bool_from_str(_filter.value)
def apply_filters(queryset, filters, only_undefined_field=False): if not filters: return queryset # convert conjunction to orm statement filter_expressions = [] for _filter in filters.items: # we can also have annotations filters if not _filter.filter.startswith( "filter:tasks:") or _filter.value is None: continue # django orm loop expression attached to column name preprocess_field_name = load_func(settings.PREPROCESS_FIELD_NAME) field_name, _ = preprocess_field_name(_filter.filter, only_undefined_field) # filter preprocessing, value type conversion, etc.. preprocess_filter = load_func(settings.DATA_MANAGER_PREPROCESS_FILTER) _filter = preprocess_filter(_filter, field_name) # custom expressions for enterprise custom_filter_expressions = load_func( settings.DATA_MANAGER_CUSTOM_FILTER_EXPRESSIONS) filter_expression = custom_filter_expressions(_filter, field_name) if filter_expression: filter_expressions.append(filter_expression) continue # annotators if field_name == 'annotators' and _filter.operator == Operator.CONTAINS: filter_expressions.append( Q(annotations__completed_by=int(_filter.value))) continue elif field_name == 'annotators' and _filter.operator == Operator.NOT_CONTAINS: filter_expressions.append(~Q( annotations__completed_by=int(_filter.value))) continue elif field_name == 'annotators' and _filter.operator == Operator.EMPTY: value = cast_bool_from_str(_filter.value) filter_expressions.append( Q(annotations__completed_by__isnull=value)) continue # annotations results & predictions results if field_name in ['annotations_results', 'predictions_results']: name = 'annotations__result' if field_name == 'annotations_results' else 'predictions__result' if _filter.operator in [Operator.EQUAL, Operator.NOT_EQUAL]: try: value = json.loads(_filter.value) except: return queryset.none() q = Q(**{name: value}) filter_expressions.append(q if _filter.operator == Operator.EQUAL else ~q) continue elif _filter.operator == Operator.CONTAINS: filter_expressions.append( Q(**{name + '__icontains': _filter.value})) continue elif _filter.operator == Operator.NOT_CONTAINS: filter_expressions.append(~Q( **{name + '__icontains': _filter.value})) continue # annotation ids if field_name == 'annotations_ids': field_name = 'annotations__id' if 'contains' in _filter.operator: # convert string like "1 2,3" => [1,2,3] _filter.value = [ int(value) for value in re.split(',|;| ', _filter.value) if value and value.isdigit() ] _filter.operator = 'in_list' if _filter.operator == 'contains' else 'not_in_list' elif 'equal' in _filter.operator: if not _filter.value.isdigit(): _filter.value = 0 # annotators if field_name == 'annotators' and _filter.operator == Operator.CONTAINS: filter_expressions.append( Q(annotations__completed_by=int(_filter.value))) continue elif field_name == 'annotators' and _filter.operator == Operator.NOT_CONTAINS: filter_expressions.append(~Q( annotations__completed_by=int(_filter.value))) continue elif field_name == 'annotators' and _filter.operator == Operator.EMPTY: value = cast_bool_from_str(_filter.value) filter_expressions.append( Q(annotations__completed_by__isnull=value)) continue # predictions model versions if field_name == 'predictions_model_versions' and _filter.operator == Operator.CONTAINS: q = Q() for value in _filter.value: q |= Q(predictions__model_version__contains=value) filter_expressions.append(q) continue elif field_name == 'predictions_model_versions' and _filter.operator == Operator.NOT_CONTAINS: q = Q() for value in _filter.value: q &= ~Q(predictions__model_version__contains=value) filter_expressions.append(q) continue elif field_name == 'predictions_model_versions' and _filter.operator == Operator.EMPTY: value = cast_bool_from_str(_filter.value) filter_expressions.append( Q(predictions__model_version__isnull=value)) continue # use other name because of model names conflict if field_name == 'file_upload': field_name = 'file_upload_field' # annotate with cast to number if need if _filter.type == 'Number' and field_name.startswith('data__'): json_field = field_name.replace('data__', '') queryset = queryset.annotate( **{ f'filter_{json_field.replace("$undefined$", "undefined")}': Cast(KeyTextTransform(json_field, 'data'), output_field=FloatField()) }) clean_field_name = f'filter_{json_field.replace("$undefined$", "undefined")}' else: clean_field_name = field_name # special case: predictions, annotations, cancelled --- for them 0 is equal to is_empty=True if clean_field_name in ('total_predictions', 'total_annotations', 'cancelled_annotations') and \ _filter.operator == 'empty': _filter.operator = 'equal' if cast_bool_from_str( _filter.value) else 'not_equal' _filter.value = 0 # get type of annotated field value_type = 'str' if queryset.exists(): value_type = type(queryset.values_list(field_name, flat=True)[0]).__name__ if (value_type == 'list' or value_type == 'tuple') and 'equal' in _filter.operator: raise Exception('Not supported filter type') # special case: for strings empty is "" or null=True if _filter.type in ('String', 'Unknown') and _filter.operator == 'empty': value = cast_bool_from_str(_filter.value) if value: # empty = true q = Q( Q(**{field_name: None}) | Q(**{field_name + '__isnull': True})) if value_type == 'str': q |= Q(**{field_name: ''}) if value_type == 'list': q = Q(**{field_name: [None]}) else: # empty = false q = Q(~Q(**{field_name: None}) & ~Q(**{field_name + '__isnull': True})) if value_type == 'str': q &= ~Q(**{field_name: ''}) if value_type == 'list': q = ~Q(**{field_name: [None]}) filter_expressions.append(q) continue # regex pattern check elif _filter.operator == 'regex': try: re.compile(pattern=str(_filter.value)) except Exception as e: logger.info('Incorrect regex for filter: %s: %s', _filter.value, str(e)) return queryset.none() # append operator field_name = f"{clean_field_name}{operators.get(_filter.operator, '')}" # in if _filter.operator == "in": cast_value(_filter) filter_expressions.append( Q( **{ f"{field_name}__gte": _filter.value.min, f"{field_name}__lte": _filter.value.max, }), ) # not in elif _filter.operator == "not_in": cast_value(_filter) filter_expressions.append( ~Q( **{ f"{field_name}__gte": _filter.value.min, f"{field_name}__lte": _filter.value.max, }), ) # in list elif _filter.operator == "in_list": filter_expressions.append( Q(**{f"{field_name}__in": _filter.value}), ) # not in list elif _filter.operator == "not_in_list": filter_expressions.append( ~Q(**{f"{field_name}__in": _filter.value}), ) # empty elif _filter.operator == 'empty': if cast_bool_from_str(_filter.value): filter_expressions.append(Q(**{field_name: True})) else: filter_expressions.append(~Q(**{field_name: True})) # starting from not_ elif _filter.operator.startswith("not_"): cast_value(_filter) filter_expressions.append(~Q(**{field_name: _filter.value})) # all others else: cast_value(_filter) filter_expressions.append(Q(**{field_name: _filter.value})) logger.debug(f'Apply filter: {filter_expressions}') if filters.conjunction == ConjunctionEnum.OR: result_filter = Q() for filter_expression in filter_expressions: result_filter.add(filter_expression, Q.OR) queryset = queryset.filter(result_filter) else: for filter_expression in filter_expressions: queryset = queryset.filter(filter_expression) return queryset
def cast_value(_filter): if _filter.type == 'Number': _filter.value = float(_filter.value) elif _filter.type == 'Boolean': _filter.value = cast_bool_from_str(_filter.value)
def apply_filters(queryset, filters): if not filters: return queryset # convert conjunction to orm statement filter_expression = Q() if filters.conjunction == ConjunctionEnum.OR: conjunction = Q.OR else: conjunction = Q.AND only_undefined_field = queryset.exists() and queryset.first( ).project.only_undefined_field for _filter in filters.items: # we can also have annotations filters if not _filter.filter.startswith("filter:tasks:") or not _filter.value: continue # django orm loop expression attached to column name field_name = preprocess_field_name(_filter.filter, only_undefined_field) # annotate with cast to number if need if _filter.type == 'Number' and field_name.startswith('data__'): json_field = field_name.replace('data__', '') queryset = queryset.annotate( **{ f'filter_{json_field.replace("$undefined$", "undefined")}': Cast(KeyTransform(json_field, 'data'), output_field=FloatField()) }) clean_field_name = f'filter_{json_field.replace("$undefined$", "undefined")}' else: clean_field_name = field_name # special case: predictions, annotations, cancelled --- for them 0 is equal to is_empty=True if clean_field_name in ('total_predictions', 'total_annotations', 'cancelled_annotations') and \ _filter.operator == 'empty': _filter.operator = 'equal' if cast_bool_from_str( _filter.value) else 'not_equal' _filter.value = 0 # append operator field_name = f"{clean_field_name}{operators.get(_filter.operator, '')}" # in if _filter.operator == "in": cast_value(_filter) filter_expression.add( Q( **{ f"{field_name}__gte": _filter.value.min, f"{field_name}__lte": _filter.value.max, }), conjunction, ) # not in elif _filter.operator == "not_in": cast_value(_filter) filter_expression.add( ~Q( **{ f"{field_name}__gte": _filter.value.min, f"{field_name}__lte": _filter.value.max, }), conjunction, ) # empty elif _filter.operator == 'empty': if cast_bool_from_str(_filter.value): filter_expression.add(Q(**{field_name: True}), conjunction) else: filter_expression.add(~Q(**{field_name: True}), conjunction) # starting from not_ elif _filter.operator.startswith("not_"): cast_value(_filter) filter_expression.add(~Q(**{field_name: _filter.value}), conjunction) # all others else: cast_value(_filter) filter_expression.add(Q(**{field_name: _filter.value}), conjunction) logger.debug(f'Apply filter: {filter_expression}') queryset = queryset.filter(filter_expression) return queryset