class ExportGQLFilterSet(django_filters.rest_framework.FilterSet):
type = MultipleInputFilter(ExportDataTypeEnum)
format = MultipleInputFilter(ExportFormatEnum)
status = MultipleInputFilter(ExportStatusEnum)
search = django_filters.CharFilter(field_name='title',
lookup_expr='icontains')
exported_at = DateTimeFilter()
exported_at_gte = DateTimeGteFilter(field_name='exported_at')
exported_at_lte = DateTimeLteFilter(field_name='exported_at')
class Meta:
model = Export
fields = ()
class ConnectorSourceGQFilterSet(OrderEnumMixin, django_filters.FilterSet):
search = django_filters.CharFilter(field_name='title',
lookup_expr='icontains')
ordering = MultipleInputFilter(ConnectorSourceOrderingEnum,
method='ordering_filter')
sources = MultipleInputFilter(ConnectorSourceSourceEnum,
field_name='source')
statuses = MultipleInputFilter(ConnectorSourceStatusEnum,
field_name='status')
unified_connectors = IDListFilter(field_name='unified_connector')
class Meta:
model = ConnectorSource
fields = ()
class UnifiedConnectorGQFilterSet(OrderEnumMixin, django_filters.FilterSet):
search = django_filters.CharFilter(field_name='title',
lookup_expr='icontains')
ordering = MultipleInputFilter(UnifiedConnectorOrderingEnum,
method='ordering_filter')
is_active = django_filters.BooleanFilter()
class Meta:
model = UnifiedConnector
fields = ()
class EntryReviewCommentGQFilterSet(django_filters.FilterSet):
entry = IDFilter()
ordering = MultipleInputFilter(EntryReviewCommentOrderingEnum, method='ordering_filter')
class Meta:
model = EntryReviewComment
fields = ()
def ordering_filter(self, qs, name, value):
return qs.order_by(*value)
class ConnectorSourceLeadGQFilterSet(OrderEnumMixin, django_filters.FilterSet):
ordering = MultipleInputFilter(ConnectorSourceLeadOrderingEnum,
method='ordering_filter')
sources = IDListFilter(field_name='source')
blocked = django_filters.BooleanFilter()
already_added = django_filters.BooleanFilter()
extraction_status = MultipleInputFilter(
ConnectorLeadExtractionStatusEnum,
field_name='connector_lead__extraction_status')
search = django_filters.CharFilter(method='search_filter')
author_organizations = IDListFilter(field_name='connector_lead__authors')
published_on = django_filters.DateFilter(
field_name='connector_lead__published_on')
published_on_gte = DateGteFilter(field_name='connector_lead__published_on')
published_on_lte = DateLteFilter(field_name='connector_lead__published_on')
class Meta:
model = ConnectorSourceLead
fields = ()
def search_filter(self, qs, _, value):
# NOTE: This exists to make it compatible with post filter
if not value:
return qs
return qs.filter(
# By title
models.Q(connector_lead__title__icontains=value) |
# By source
models.Q(connector_lead__source_raw__icontains=value)
| models.Q(connector_lead__source__title__icontains=value)
| models.Q(connector_lead__source__parent__title__icontains=value)
|
# By author
models.Q(connector_lead__author_raw__icontains=value)
| models.Q(connector_lead__authors__title__icontains=value)
| models.Q(connector_lead__authors__parent__title__icontains=value)
|
# By URL
models.Q(connector_lead__url__icontains=value)
| models.Q(connector_lead__website__icontains=value)).distinct()
class GeoAreaGqlFilterSet(OrderEnumMixin, django_filters.rest_framework.FilterSet):
ids = IDListFilter(field_name='id')
search = django_filters.CharFilter(
label='Geo Area Label search',
method='geo_area_label'
)
titles = StringListFilter(
label='Geo Area Label search',
method='filter_titles'
)
ordering = MultipleInputFilter(GeoAreaOrderingEnum, method='ordering_filter')
class Meta:
model = GeoArea
fields = ()
def geo_area_label(self, queryset, name, value):
if value:
return queryset.annotate(
# TODO: Improve this search
label=Concat(
models.F('admin_level__region__title'),
models.Value(' '),
models.F('admin_level__title'),
models.Value(' '),
models.F('title'),
output_field=models.fields.CharField()
),
).filter(label__icontains=value)
return queryset
def filter_titles(self, queryset, name, values):
if values:
# Let's only use 20 max.
_values = values[:20]
return queryset.filter(
reduce(
lambda acc, item: acc | item,
[
models.Q(title__icontains=value)
for value in _values
]
)
)
return queryset
class OrganizationFilterSet(django_filters.FilterSet):
search = django_filters.CharFilter(method='search_filter')
ordering = MultipleInputFilter(
OrganizationOrderingEnum,
method='ordering_filter',
)
class Meta:
model = Organization
fields = ['id']
def search_filter(self, qs, _, value):
if value:
return qs.filter(
models.Q(title__icontains=value)
| models.Q(short_name__icontains=value)
| models.Q(long_name__icontains=value)
| models.Q(related_childs__title__icontains=value)
| models.Q(related_childs__short_name__icontains=value)
| models.Q(
related_childs__long_name__icontains=value)).distinct()
return qs
def ordering_filter(self, qs, _, value):
if value:
if (OrganizationOrderingEnum.ASC_TITLE_LENGTH.value in value or
OrganizationOrderingEnum.DESC_TITLE_LENGTH.value in value):
qs = qs.annotate(
**{
OrganizationOrderingEnum.ASC_TITLE_LENGTH.value:
Length('title'),
})
return qs.order_by(*value)
return qs
@property
def qs(self):
qs = super().qs
if 'ordering' not in self.data:
# Default is Title Length
qs = self.ordering_filter(
qs, None, [OrganizationOrderingEnum.ASC_TITLE_LENGTH.value])
return qs
class EntryGQFilterSet(GrapheneFilterSetMixin, UserResourceGqlFilterSet):
class CommentStatus(models.TextChoices):
RESOLVED = 'resolved', 'Resolved',
UNRESOLVED = 'unresolved', 'Unresolved',
# Lead fields
leads = IDListFilter(field_name='lead')
lead_created_by = IDListFilter(field_name='lead__created_by')
lead_published_on = django_filters.DateFilter()
lead_published_on_gte = DateGteFilter(field_name='lead__published_on')
lead_published_on_lte = DateLteFilter(field_name='lead__published_on')
lead_title = django_filters.CharFilter(lookup_expr='icontains',
field_name='lead__title')
lead_assignees = IDListFilter(label='Lead Assignees',
field_name='lead__assignee')
lead_statuses = MultipleInputFilter(LeadStatusEnum,
field_name='lead__status')
lead_priorities = MultipleInputFilter(LeadPriorityEnum,
field_name='lead__priority')
lead_confidentialities = MultipleInputFilter(
LeadConfidentialityEnum, field_name='lead__confidentiality')
lead_authoring_organization_types = IDListFilter(
method='authoring_organization_types_filter')
lead_author_organizations = IDListFilter(field_name='lead__authors')
lead_source_organizations = IDListFilter(field_name='lead__source')
search = django_filters.CharFilter(method='search_filter')
created_by = IDListFilter()
modified_by = IDListFilter()
comment_status = SimpleInputFilter(
convert_enum_to_graphene_enum(CommentStatus,
name='EntryFilterCommentStatusEnum'),
label='Comment Status',
method='comment_status_filter',
)
entry_types = MultipleInputFilter(EntryTagTypeEnum,
field_name='entry_type')
project_entry_labels = IDListFilter(label='Project Entry Labels',
method='project_entry_labels_filter')
entries_id = IDListFilter(field_name='id')
geo_custom_shape = django_filters.CharFilter(
label='GEO Custom Shapes', method='geo_custom_shape_filter')
# Entry Group Label Filters
lead_group_label = django_filters.CharFilter(
label='Lead Group Label', method='lead_group_label_filter')
# Dynamic filterable data
filterable_data = MultipleInputFilter(EntryFilterDataType,
method='filterable_data_filter')
class Meta:
model = Entry
fields = {
**{x: ['exact']
for x in [
'id',
'excerpt',
'controlled',
]},
}
filter_overrides = {
models.CharField: {
'filter_class': django_filters.CharFilter,
'extra': lambda _: {
'lookup_expr': 'icontains',
},
},
}
def filterable_data_filter(self, queryset, _, value):
if value:
project = self.request and self.request.active_project
if project is None or project.analysis_framework_id is None:
# This needs to be defined
raise Exception(
f'Both should be defined {project=} {project and project.analysis_framework_id=}'
)
filters = Filter.qs_with_widget_type().filter(
analysis_framework_id=project.analysis_framework_id).all()
return get_filtered_entries_using_af_filter(
queryset,
filters,
value,
project=project,
new_query_structure=True)
return queryset
def comment_status_filter(self, queryset, name, value):
if value == self.CommentStatus.UNRESOLVED:
return queryset.filter(
entrycomment__is_resolved=False,
entrycomment__parent__isnull=True,
)
elif value == self.CommentStatus.RESOLVED:
return queryset.filter(
entrycomment__is_resolved=True,
entrycomment__parent__isnull=True,
)
return queryset
def geo_custom_shape_filter(self, queryset, name, value):
if value:
query_params = reduce(
lambda acc, item: acc | item,
[
models.Q(
attribute__widget__widget_id='geoWidget',
attribute__data__value__contains=[{
'type': v
}],
) for v in value.split(',')
],
)
return queryset.filter(query_params)
return queryset
def project_entry_labels_filter(self, queryset, name, value):
if value:
return queryset.filter(entrygrouplabel__label__in=value, )
return queryset
def lead_group_label_filter(self, queryset, name, value):
if value:
return queryset.filter(
entrygrouplabel__group__title__icontains=value)
return queryset
def authoring_organization_types_filter(self, qs, name, value):
if value:
qs = qs.annotate(organization_types=models.functions.Coalesce(
'lead__authors__parent__organization_type',
'lead__authors__organization_type'))
if type(value[0]) == OrganizationType:
return qs.filter(
organization_types__in=[ot.id for ot in value]).distinct()
return qs.filter(organization_types__in=value).distinct()
return qs
def search_filter(self, qs, _, value):
if value:
return qs.filter(
models.Q(lead__title__icontains=value)
| models.Q(excerpt__icontains=value))
return qs
@property
def qs(self):
qs = super().qs
# Note: Since we cannot have `.distinct()` inside a subquery
if self.data.get('from_subquery', False):
return Entry.objects.filter(id__in=qs)
return qs.distinct()
class LeadGQFilterSet(UserResourceGqlFilterSet):
ids = IDListFilter(method='filter_leads_id', help_text='Empty ids are ignored.')
exclude_provided_leads_id = django_filters.BooleanFilter(
method='filter_exclude_provided_leads_id', help_text='Only used when ids are provided.')
created_by = IDListFilter()
modified_by = IDListFilter()
source_types = MultipleInputFilter(LeadSourceTypeEnum, field_name='source_type')
priorities = MultipleInputFilter(LeadPriorityEnum, field_name='priority')
confidentiality = SimpleInputFilter(LeadConfidentialityEnum)
statuses = MultipleInputFilter(LeadStatusEnum, field_name='status')
extraction_status = SimpleInputFilter(LeadExtractionStatusEnum, field_name='extraction_status')
assignees = IDListFilter(field_name='assignee')
authoring_organization_types = IDListFilter(method='authoring_organization_types_filter')
author_organizations = IDListFilter(method='authoring_organizations_filter')
source_organizations = IDListFilter(method='source_organizations_filter')
# Filter-only enum filter
has_entries = django_filters.BooleanFilter(method='filter_has_entries', help_text='Lead has entries.')
has_assessment = django_filters.BooleanFilter(method='filter_has_assessment', help_text='Lead has assessment.')
entries_filter_data = SimpleInputFilter(
type(
'LeadEntriesFilterData',
(graphene.InputObjectType,),
get_filtering_args_from_filterset(EntryGQFilterSet, 'entry.schema.EntryListType')
),
method='filtered_entries_filter_data',
)
search = django_filters.CharFilter(method='search_filter')
published_on = django_filters.DateFilter()
published_on_gte = DateGteFilter(field_name='published_on')
published_on_lte = DateLteFilter(field_name='published_on')
emm_entities = django_filters.CharFilter(method='emm_entities_filter')
emm_keywords = django_filters.CharFilter(method='emm_keywords_filter')
emm_risk_factors = django_filters.CharFilter(method='emm_risk_factors_filter')
ordering = MultipleInputFilter(LeadOrderingEnum, method='ordering_filter')
class Meta:
model = Lead
fields = {
**{
x: ['exact']
for x in ['text', 'url']
},
}
filter_overrides = {
models.CharField: {
'filter_class': django_filters.CharFilter,
'extra': lambda _: {
'lookup_expr': 'icontains',
},
},
}
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.custom_context = {}
@property
def active_project(self) -> Project:
if self.request is None:
raise Exception(f'{self.request=} should be defined')
if self.request.active_project is None:
raise Exception(f'{self.request.active_project=} should be defined')
return self.request.active_project
@staticmethod
def get_dummy_request(project):
"""
Use this if request is not available
"""
return type('DummyRequest', (object,), dict(active_project=project))()
# Filters methods
def search_filter(self, qs, name, value):
# NOTE: This exists to make it compatible with post filter
if not value:
return qs
return qs.filter(
# By title
models.Q(title__icontains=value) |
# By source
models.Q(source_raw__icontains=value) |
models.Q(source__title__icontains=value) |
models.Q(source__parent__title__icontains=value) |
# By author
models.Q(author__title__icontains=value) |
models.Q(author__parent__title__icontains=value) |
models.Q(author_raw__icontains=value) |
models.Q(authors__title__icontains=value) |
models.Q(authors__parent__title__icontains=value) |
# By URL
models.Q(url__icontains=value)
).distinct()
def ordering_filter(self, qs, name, value):
active_entry_count_field = self.custom_context.get('active_entry_count_field')
for ordering in value:
# Custom for entries count (use filter or normal entry count)
if active_entry_count_field and ordering in [
LeadOrderingEnum.ASC_ENTRIES_COUNT,
LeadOrderingEnum.DESC_ENTRIES_COUNT,
]:
if ordering == LeadOrderingEnum.ASC_ENTRIES_COUNT:
qs = qs.order_by(active_entry_count_field)
else:
qs = qs.order_by(f'-{active_entry_count_field}')
# Custom for page count with nulls_last
elif ordering == LeadOrderingEnum.DESC_PAGE_COUNT:
qs = qs.order_by(models.F('leadpreview__page_count').desc(nulls_last=True))
elif ordering == LeadOrderingEnum.ASC_PAGE_COUNT:
qs = qs.order_by(models.F('leadpreview__page_count').asc(nulls_first=True))
# For remaining
else:
qs = qs.order_by(ordering)
return qs
def emm_entities_filter(self, qs, name, value):
splitted = [x for x in value.split(',') if x]
return qs.filter(emm_entities__in=splitted)
def emm_keywords_filter(self, qs, name, value):
splitted = [x for x in value.split(',') if x]
return qs.filter(emm_triggers__emm_keyword__in=splitted)
def emm_risk_factors_filter(self, qs, name, value):
splitted = [x for x in value.split(',') if x]
return qs.filter(emm_triggers__emm_risk_factor__in=splitted)
def authoring_organization_types_filter(self, qs, name, value):
if value:
qs = qs.annotate(
organization_types=Coalesce(
'authors__parent__organization_type',
'authors__organization_type'
)
)
if type(value[0]) == OrganizationType:
return qs.filter(organization_types__in=[ot.id for ot in value]).distinct()
return qs.filter(organization_types__in=value).distinct()
return qs
def authoring_organizations_filter(self, qs, _, value):
if value:
qs = qs.annotate(authoring_organizations=Coalesce('authors__parent_id', 'authors__id'))
return qs.filter(authoring_organizations__in=value).distinct()
return qs
def source_organizations_filter(self, qs, _, value):
if value:
qs = qs.annotate(source_organizations=Coalesce('source__parent_id', 'source__id'))
return qs.filter(source_organizations__in=value).distinct()
return qs
def filter_exclude_provided_leads_id(self, qs, *_):
# NOTE: Used in filter_leads_id
return qs
def filter_leads_id(self, qs, _, value):
if value is None:
return qs
if self.data.get('exclude_provided_leads_id'):
return qs.exclude(id__in=value)
return qs.filter(id__in=value)
def filter_has_entries(self, qs, _, value):
if value is None:
return qs
if value:
return qs.filter(entry_count__gt=0)
return qs.filter(entry_count=0)
def filter_has_assessment(self, qs, _, value):
if value is None:
return qs
return qs.filter(assessment__isnull=not value)
def filtered_entries_filter_data(self, qs, _, value):
if value is None:
return qs
return qs.filter(filtered_entry_count__gt=0)
def filter_queryset(self, qs):
def _entry_subquery(entry_qs: models.QuerySet):
subquery_qs = entry_qs.\
filter(
project=self.active_project,
analysis_framework=self.active_project.analysis_framework_id,
lead=models.OuterRef('pk'),
)\
.values('lead').order_by()\
.annotate(count=models.Count('id'))\
.values('count')
return Coalesce(
models.Subquery(
subquery_qs[:1],
output_field=models.IntegerField()
), 0,
)
# Pre-annotate required fields for entries count (w/wo filters)
entries_filter_data = self.data.get('entries_filter_data')
has_entries = self.data.get('has_entries')
has_entries_count_ordering = any(
ordering in [
LeadOrderingEnum.ASC_ENTRIES_COUNT,
LeadOrderingEnum.DESC_ENTRIES_COUNT,
] for ordering in self.data.get('ordering') or []
)
# With filter
if entries_filter_data is not None:
qs = qs.annotate(
filtered_entry_count=_entry_subquery(
EntryGQFilterSet(
data=entries_filter_data,
request=self.request,
).qs
)
)
self.custom_context['active_entry_count_field'] = 'filtered_entry_count'
# Without filter
if has_entries is not None or (
entries_filter_data is None and has_entries_count_ordering
):
self.custom_context['active_entry_count_field'] = self.custom_context.\
get('active_entry_count_field', 'entry_count')
qs = qs.annotate(
entry_count=_entry_subquery(Entry.objects.all())
)
# Call super function
return super().filter_queryset(qs)
@property
def qs(self):
return super().qs.distinct()