def test_set_operator_specified_tag_filters_or(self): """Test that AND/OR terms are correctly applied to tag filters.""" operator = "or" term = self.mock_tag_key first = FAKE.word() second = FAKE.word() operation = "icontains" url = ( f"?filter[time_scope_value]=-1&" f"filter[or:tag:{term}]={first}&" f"filter[or:tag:{term}]={second}&" f"group_by[or:tag:{term}]={first}&" f"group_by[or:tag:{term}]={second}" ) params = self.mocked_query_params(url, self.mock_view) mapper = {"filter": [{}], "filters": {term: {"field": term, "operation": operation}}} rqh = create_test_handler(params, mapper=mapper) output = rqh._set_operator_specified_tag_filters(QueryFilterCollection(), operator) self.assertIsNotNone(output) expected = QueryFilterCollection( filters=[ QueryFilter(field=term, operation=operation, parameter=second, logical_operator=operator), QueryFilter(field=term, operation=operation, parameter=first, logical_operator=operator), ] ) self.assertIsInstance(output, QueryFilterCollection) assertSameQ(output.compose(), expected.compose())
def _get_filter(self, delta=False): # noqa: C901 """Create dictionary for filter parameters. Args: delta (Boolean): Construct timeframe for delta Returns: (Dict): query filter dictionary """ filters = QueryFilterCollection() for filter_key in self.SUPPORTED_FILTERS: filter_value = self.parameters.get_filter(filter_key) if filter_value and not OrgQueryHandler.has_wildcard(filter_value): filter_obj = self.FILTER_MAP.get(filter_key) for item in filter_value: q_filter = QueryFilter(parameter=item, **filter_obj) filters.add(q_filter) # Update filters that specifiy and or or in the query parameter and_composed_filters = self._set_operator_specified_filters("and") or_composed_filters = self._set_operator_specified_filters("or") composed_filters = filters.compose() filter_list = [composed_filters, and_composed_filters, or_composed_filters] final_filters = None for filter_option in filter_list: if filter_option: if final_filters is not None: final_filters & filter_option else: final_filters = filter_option LOG.debug(f"_get_filter: {final_filters}") return final_filters
def _get_exclusions(self, delta=False): """Create dictionary for filter parameters for exclude clause. Returns: (Dict): query filter dictionary """ exclusions = QueryFilterCollection() tag_column = self._mapper.tag_column tag_group_by = self.get_tag_group_by_keys() if tag_group_by: for tag in tag_group_by: tag_db_name = tag_column + '__' + strip_tag_prefix(tag) filt = { 'field': tag_db_name, 'operation': 'isnull', 'parameter': True } q_filter = QueryFilter(**filt) exclusions.add(q_filter) composed_exclusions = exclusions.compose() LOG.debug(f'_get_exclusions: {composed_exclusions}') return composed_exclusions
def _set_operator_specified_filters(self, operator): """Set any filters using AND instead of OR.""" filters = QueryFilterCollection() composed_filter = Q() for filter_key in self.SUPPORTED_FILTERS: operator_key = operator + ":" + filter_key filter_value = self.parameters.get_filter(operator_key) logical_operator = operator if filter_value and len(filter_value) < 2: logical_operator = "or" if filter_value and not OrgQueryHandler.has_wildcard(filter_value): filter_obj = self.FILTER_MAP.get(filter_key) if isinstance(filter_obj, list): for _filt in filter_obj: filt_filters = QueryFilterCollection() for item in filter_value: q_filter = QueryFilter(parameter=item, logical_operator=logical_operator, **_filt) filt_filters.add(q_filter) composed_filter = composed_filter | filt_filters.compose() else: for item in filter_value: q_filter = QueryFilter(parameter=item, logical_operator=logical_operator, **filter_obj) filters.add(q_filter) if filters: composed_filter = composed_filter & filters.compose() return composed_filter
def _create_accounts_mapping(self): """Returns a mapping of org ids to accounts.""" account_mapping = {} with tenant_context(self.tenant): for source in self.data_sources: # Grab columns for this query account_info = source.get("account_alias_column") # Create filters & Query filters = QueryFilterCollection() no_org_units = QueryFilter(field=f"{account_info}", operation="isnull", parameter=False) filters.add(no_org_units) composed_filters = filters.compose() account_query = source.get("db_table").objects account_query = account_query.filter(composed_filters) account_query = account_query.exclude(deleted_timestamp__lte=self.start_datetime) account_query = account_query.exclude(created_timestamp__gt=self.end_datetime) if self.access: accounts_to_filter = self.access.get("aws.account", {}).get("read", []) if accounts_to_filter and "*" not in accounts_to_filter: account_query = account_query.filter(account_alias__account_id__in=accounts_to_filter) account_query = account_query.order_by(f"{account_info}", "-created_timestamp") account_query = account_query.distinct(f"{account_info}") account_query = account_query.annotate( alias=Coalesce(F(f"{account_info}__account_alias"), F(f"{account_info}__account_id")) ) for account in account_query: org_id = account.org_unit_id alias = account.alias if account_mapping.get(org_id): account_list = account_mapping[org_id] account_list.append(alias) account_mapping[org_id] = account_list else: account_mapping[org_id] = [alias] return account_mapping
def __init__(self, query_params): # noqa: C901 """Class Constructor. Instance Attributes: - cost_summary_table (Model) - aggregates (dict) - filters (QueryFilterCollection) - query_range (tuple) """ self.params = query_params # select appropriate model based on access access = query_params.get("access", {}) access_key = "default" self.cost_summary_table = self.provider_map.views.get("costs").get( access_key) if access: access_key = tuple(access.keys()) filter_fields = self.provider_map.provider_map.get("filters") materialized_view = self.provider_map.views.get("costs").get( access_key) if materialized_view: # We found a matching materialized view, use that self.cost_summary_table = materialized_view else: # We have access constraints, but no view to accomodate, default to daily summary table self.cost_summary_table = self.provider_map.report_type_map.get( "tables", {}).get("query") current_day_of_month = self.dh.today.day yesterday = (self.dh.today - timedelta(days=1)).day last_day_of_month = self.dh.this_month_end.day if current_day_of_month == 1: self.forecast_days_required = last_day_of_month else: self.forecast_days_required = last_day_of_month - yesterday if current_day_of_month <= self.MINIMUM: self.query_range = (self.dh.last_month_start, self.dh.last_month_end) else: self.query_range = (self.dh.this_month_start, self.dh.today - timedelta(days=1)) self.filters = QueryFilterCollection() self.filters.add(field="usage_start", operation="gte", parameter=self.query_range[0]) self.filters.add(field="usage_end", operation="lte", parameter=self.query_range[1]) # filter queries based on access if access_key != "default": for q_param, filt in filter_fields.items(): access = query_params.get_access(q_param, list()) if access: self.set_access_filters(access, filt, self.filters)
def _get_key_filter(self): """ Add new `exact` QueryFilter that filters on the key name. If filtering on value, uses the tags summary table to find the key """ filters = QueryFilterCollection() filters.add(QueryFilter(field="key", operation="exact", parameter=self.key)) return self.query_filter & filters.compose()
def test_add_filter(self): """Test the add() method using a QueryFilter instance.""" filters = [] qf_coll = QueryFilterCollection() for _ in range(0, 3): filt = QueryFilter(self.fake.word(), self.fake.word(), self.fake.word(), self.fake.word()) filters.append(filt) qf_coll.add(query_filter=filt) self.assertEqual(qf_coll._filters, filters)
def _get_sub_ou_list(self, data, org_ids): """Get a list of the sub org units for a org unit.""" level = data.get("level") level = level + 1 unit_path = data.get("org_unit_path") final_sub_ou_list = [] with tenant_context(self.tenant): for source in self.data_sources: # Grab columns for this query account_info = source.get("account_alias_column") level_column = source.get("level_column") org_path = source.get("org_path_column") # Build filters filters = QueryFilterCollection() no_accounts = QueryFilter(field=f"{account_info}", operation="isnull", parameter=True) filters.add(no_accounts) exact_parent_id = QueryFilter(field=f"{level_column}", operation="exact", parameter=level) filters.add(exact_parent_id) path_on_like = QueryFilter(field=f"{org_path}", operation="icontains", parameter=unit_path) filters.add(path_on_like) composed_filters = filters.compose() # Start quering sub_org_query = source.get("db_table").objects sub_org_query = sub_org_query.filter(composed_filters) sub_org_query = sub_org_query.filter(id__in=org_ids) sub_ou_list = sub_org_query.values_list("org_unit_id", flat=True) final_sub_ou_list.extend(sub_ou_list) return final_sub_ou_list
def _set_operator_specified_filters(self, operator): """Set any filters using AND instead of OR.""" fields = self._mapper._provider_map.get("filters") filters = QueryFilterCollection() composed_filter = Q() for q_param, filt in fields.items(): q_param = operator + ":" + q_param group_by = self.parameters.get_group_by(q_param, list()) filter_ = self.parameters.get_filter(q_param, list()) list_ = list(set(group_by + filter_)) # uniquify the list logical_operator = operator # This is a flexibilty feature allowing a user to set # a single and: value and still get a result instead # of erroring on validation if len(list_) < 2: logical_operator = "or" if list_ and not ReportQueryHandler.has_wildcard(list_): if isinstance(filt, list): for _filt in filt: filt_filters = QueryFilterCollection() for item in list_: q_filter = QueryFilter( parameter=item, logical_operator=logical_operator, **_filt) filt_filters.add(q_filter) # List filter are a complex mix of and/or logic # Each filter in the list must be ORed together # regardless of the operator on the item in the filter # Ex: # (OR: # (AND: # ('cluster_alias__icontains', 'ni'), # ('cluster_alias__icontains', 'se') # ), # (AND: # ('cluster_id__icontains', 'ni'), # ('cluster_id__icontains', 'se') # ) # ) composed_filter = composed_filter | filt_filters.compose( ) else: list_ = self._build_custom_filter_list( q_param, filt.get("custom"), list_) for item in list_: q_filter = QueryFilter( parameter=item, logical_operator=logical_operator, **filt) filters.add(q_filter) if filters: composed_filter = composed_filter & filters.compose() return composed_filter
def test_iterable(self): """Test the __iter__() method returns an iterable.""" qf1 = QueryFilter(table=self.fake.word(), field=self.fake.word(), parameter=self.fake.word()) qf2 = QueryFilter(table=self.fake.word(), field=self.fake.word(), parameter=self.fake.word()) qf_coll = QueryFilterCollection([qf1, qf2]) self.assertIsInstance(qf_coll.__iter__(), Iterable)
def get_org_units(self): """Get a list of org keys to build upon.""" org_units = list() org_id_list = list() with tenant_context(self.tenant): for source in self.data_sources: # Grab columns for this query org_id = source.get("org_id_column") org_path = source.get("org_path_column") org_name = source.get("org_name_column") level = source.get("level_column") account_info = source.get("account_alias_column") created_field = source.get("created_time_column") # Create filters & Query account_filter = QueryFilterCollection() no_accounts = QueryFilter(field=f"{account_info}", operation="isnull", parameter=True) account_filter.add(no_accounts) remove_accounts = account_filter.compose() org_unit_query = source.get("db_table").objects org_unit_query = org_unit_query.filter(remove_accounts) org_unit_query = org_unit_query.exclude( deleted_timestamp__lte=self.start_datetime) org_unit_query = org_unit_query.exclude( created_timestamp__gt=self.end_datetime) val_list = [org_id, org_name, org_path, level] org_unit_query = org_unit_query.order_by( f"{org_id}", f"-{created_field}").distinct(f"{org_id}") org_ids = org_unit_query.values_list("id", flat=True) if self.access: acceptable_ous = self.access.get("aws.organizational_unit", {}).get("read", []) if acceptable_ous and "*" not in acceptable_ous: allowed_ids_query = source.get("db_table").objects allowed_ids_query = allowed_ids_query.filter( reduce(operator.or_, (Q(org_unit_path__icontains=rbac) for rbac in acceptable_ous ))).filter(remove_accounts) allowed_ids = allowed_ids_query.values_list("id", flat=True) org_ids = list(set(org_ids) & set(allowed_ids)) org_unit_query = org_unit_query.filter(id__in=org_ids) org_id_list.extend(org_ids) # Note: you want to collect the org_id_list before you implement the self.query_filter # so that way the get_sub_ou list will still work when you do filter[org_unit_id]=OU_002 if self.query_filter: org_unit_query = org_unit_query.filter(self.query_filter) org_unit_query = org_unit_query.values(*val_list) org_units.extend(org_unit_query) return org_units, org_id_list
def test_delete_filter(self): """Test the delete() method works with QueryFilters.""" qf1 = QueryFilter(table=self.fake.word(), field=self.fake.word(), parameter=self.fake.word()) qf2 = QueryFilter(table=self.fake.word(), field=self.fake.word(), parameter=self.fake.word()) qf_coll = QueryFilterCollection([qf1, qf2]) qf_coll.delete(qf1) self.assertEqual([qf2], qf_coll._filters) self.assertNotIn(qf1, qf_coll)
def _get_search_filter(self, filters): """Populate the query filter collection for search filters. Args: filters (QueryFilterCollection): collection of query filters Returns: (QueryFilterCollection): populated collection of query filters """ # define filter parameters using API query params. fields = self._mapper._provider_map.get("filters") access_filters = QueryFilterCollection() for q_param, filt in fields.items(): access = self.parameters.get_access(q_param, list()) group_by = self.parameters.get_group_by(q_param, list()) filter_ = self.parameters.get_filter(q_param, list()) list_ = list(set(group_by + filter_)) # uniquify the list if list_ and not ReportQueryHandler.has_wildcard(list_): if isinstance(filt, list): for _filt in filt: for item in list_: q_filter = QueryFilter(parameter=item, **_filt) filters.add(q_filter) else: list_ = self._build_custom_filter_list( q_param, filt.get("custom"), list_) for item in list_: q_filter = QueryFilter(parameter=item, **filt) filters.add(q_filter) if access: access_filt = copy.deepcopy(filt) self.set_access_filters(access, access_filt, access_filters) # Update filters with tag filters filters = self._set_tag_filters(filters) filters = self._set_operator_specified_tag_filters(filters, "and") filters = self._set_operator_specified_tag_filters(filters, "or") # Update filters that specifiy and or or in the query parameter and_composed_filters = self._set_operator_specified_filters("and") or_composed_filters = self._set_operator_specified_filters("or") multi_field_or_composed_filters = self._set_or_filters() composed_filters = filters.compose() composed_filters = composed_filters & and_composed_filters & or_composed_filters if access_filters: composed_access_filters = access_filters.compose() composed_filters = composed_filters & composed_access_filters if multi_field_or_composed_filters: composed_filters = composed_filters & multi_field_or_composed_filters LOG.debug(f"_get_search_filter: {composed_filters}") return composed_filters
def _set_or_filters(self): """Create a composed filter collection of ORed filters. This is designed to handle specific cases in the provider_map not to accomodate user input via the API. """ filters = QueryFilterCollection() or_filter = self._mapper._report_type_map.get("or_filter", []) for filt in or_filter: q_filter = QueryFilter(**filt) filters.add(q_filter) return filters.compose(logical_operator="or")
def __init__(self, query_params): # noqa: C901 """Class Constructor. Instance Attributes: - cost_summary_table (Model) - aggregates (dict) - filters (QueryFilterCollection) - query_range (tuple) """ self.dh = DateHelper() self.params = query_params # select appropriate model based on access access = query_params.get("access", {}) access_key = "default" self.cost_summary_table = self.provider_map.views.get("costs").get( access_key) if access: access_key = tuple(access.keys()) filter_fields = self.provider_map.provider_map.get("filters") materialized_view = self.provider_map.views.get("costs").get( access_key) if materialized_view: # We found a matching materialized view, use that self.cost_summary_table = materialized_view else: # We have access constraints, but no view to accomodate, default to daily summary table self.cost_summary_table = self.provider_map.query_table self.forecast_days_required = max( (self.dh.this_month_end - self.dh.yesterday).days, 2) # forecasts use a rolling window self.query_range = (self.dh.n_days_ago(self.dh.yesterday, 30), self.dh.yesterday) self.filters = QueryFilterCollection() self.filters.add(field="usage_start", operation="gte", parameter=self.query_range[0]) self.filters.add(field="usage_end", operation="lte", parameter=self.query_range[1]) # filter queries based on access if access_key != "default": for q_param, filt in filter_fields.items(): access = query_params.get_access(q_param, list()) if access: self.set_access_filters(access, filt, self.filters)
def test_compose_with_or_operator(self): """Test the compose() method with or operator on the compose method.""" qf_coll = QueryFilterCollection() operation = self.fake.word() filts = [ QueryFilter(table=self.fake.word(), field=self.fake.word(), operation=operation, parameter=self.fake.word()) for _ in range(2) ] expected = filts[0].composed_Q() | filts[1].composed_Q() qf_coll.add(filts[0]) qf_coll.add(filts[1]) self.assertEqual(qf_coll.compose(logical_operator="or"), expected)
def calculate_total(self, units_value): """Calculate aggregated totals for the query. Args: units_value (str): The unit of the reported total Returns: (dict) The aggregated totals for the query """ filt_collection = QueryFilterCollection() total_filter = self._get_search_filter(filt_collection) time_scope_value = self.get_query_param_data('filter', 'time_scope_value', -10) time_and_report_filter = Q(time_scope_value=time_scope_value) & \ Q(report_type=self._report_type) if total_filter is None: total_filter = time_and_report_filter else: total_filter = total_filter & time_and_report_filter q_table = self._mapper._provider_map.get('tables').get('total') aggregates = self._mapper._report_type_map.get('aggregate') total_query = q_table.objects.filter(total_filter).aggregate( **aggregates) total_query['units'] = units_value return total_query
def test_set_access_filters_with_list(self): """Test that the execute query runs properly with value query and an RBAC restriction on cluster.""" key = "app" value = "b" url = f"/app/?filter[value]={value}" query_params = self.mocked_query_params(url, OCPTagView) # the mocked query parameters dont include the key from the url so it needs to be added query_params.kwargs = {"key": key} handler = OCPTagQueryHandler(query_params) access = ["my-ocp-cluster-2"] filt = [ { "field": "report_period__cluster_id", "operation": "icontains", "composition_key": "cluster_filter" }, { "field": "report_period__cluster_alias", "operation": "icontains", "composition_key": "cluster_filter" }, ] filters = QueryFilterCollection() handler.set_access_filters(access, filt, filters) expected = [] expected.append( QueryFilter(field="report_period__cluster_id", operation="icontains", parameter=["my-ocp-cluster-2"])) expected.append( QueryFilter(field="report_period__cluster_alias", operation="icontains", parameter=["my-ocp-cluster-2"])) self.assertEqual(filters._filters, expected)
def test_add_params(self): """Test the add() method using parameters.""" table = self.fake.word() field = self.fake.word() operation = self.fake.word() parameter = self.fake.word() filt = QueryFilter(table=table, field=field, operation=operation, parameter=parameter) qf_coll = QueryFilterCollection() qf_coll.add(table=table, field=field, operation=operation, parameter=parameter) self.assertEqual(qf_coll._filters[0], filt)
def test_get_fail(self): """Test the get() method fails when no match is found.""" qf1 = QueryFilter(table=self.fake.word(), field=self.fake.word(), parameter=self.fake.word()) qf2 = QueryFilter(table=self.fake.word(), field=self.fake.word(), parameter=self.fake.word()) qf_coll = QueryFilterCollection([qf1, qf2]) response = qf_coll.get({ 'table': self.fake.word(), 'field': self.fake.word(), 'parameter': self.fake.word() }) self.assertIsNone(response)
def test_set_tag_filters(self): """Test that tag filters are created properly.""" filters = QueryFilterCollection() # '?' query_params = FakeQueryParameters({}, tenant=self.tenant) handler = OCPTagQueryHandler(query_params.mock_qp) tag_keys = handler.get_tag_keys(filters=False) filter_key = tag_keys[0] filter_value = 'filter' group_by_key = tag_keys[1] group_by_value = 'group_By' # '?filter[tag:some_key]=some_value&group_by[tag:some_key]=some_value' params = { 'filter': { filter_key: [filter_value] }, 'group_by': { group_by_key: [group_by_value] } } query_params = FakeQueryParameters(params, report_type='cpu', tag_keys=tag_keys, tenant=self.tenant) handler = OCPReportQueryHandler(query_params.mock_qp) filters = handler._set_tag_filters(filters) expected = f"""<class 'api.query_filter.QueryFilterCollection'>: (AND: ('pod_labels__{filter_key}__icontains', '{filter_value}')), (AND: ('pod_labels__{group_by_key}__icontains', '{group_by_value}')), """ # noqa: E501 self.assertEqual(repr(filters), expected)
def test_set_tag_filters(self): """Test that tag filters are created properly.""" filters = QueryFilterCollection() handler = OCPTagQueryHandler('', {}, self.tenant) tag_keys = handler.get_tag_keys(filters=False) filter_key = tag_keys[0] filter_value = 'filter' group_by_key = tag_keys[1] group_by_value = 'group_By' query_params = { 'filter': {filter_key: [filter_value]}, 'group_by': {group_by_key: [group_by_value]} } handler = OCPReportQueryHandler( query_params, '', self.tenant, **{ 'report_type': 'cpu', 'tag_keys': tag_keys } ) filters = handler._set_tag_filters(filters) expected = f"""<class 'api.query_filter.QueryFilterCollection'>: (AND: ('pod_labels__{filter_key}__icontains', '{filter_value}')), (AND: ('pod_labels__{group_by_key}__icontains', '{group_by_value}')), """ # noqa: E501 self.assertEqual(repr(filters), expected)
def test_contains_with_filter(self): """Test the __contains__() method using a QueryFilter.""" qf = QueryFilter(table=self.fake.word(), field=self.fake.word(), parameter=self.fake.word()) qf_coll = QueryFilterCollection([qf]) self.assertIn(qf, qf_coll)
def test_compose(self): """Test the compose() method.""" qf_coll = QueryFilterCollection() table = self.fake.word() field = self.fake.word() operation = self.fake.word() parameter = self.fake.word() filt = QueryFilter(table=table, field=field, operation=operation, parameter=parameter) expected = filt.composed_Q() qf_coll.add(table=table, field=field, operation=operation, parameter=parameter) self.assertEqual(qf_coll.compose(), expected)
def test_compose_with_filter_with_and_operator(self): """Test the compose() method with and operator on the filter.""" qf_coll = QueryFilterCollection() table = self.fake.word() field = self.fake.word() operation = self.fake.word() filts = [ QueryFilter(table=table, field=field, operation=operation, parameter=self.fake.word(), logical_operator='and') for _ in range(2) ] expected = filts[0].composed_Q() & filts[1].composed_Q() qf_coll.add(filts[0]) qf_coll.add(filts[1]) self.assertEqual(qf_coll.compose(), expected)
def test_delete_fail(self): """Test the delete() method works with QueryFilters.""" qf1 = QueryFilter(table=self.fake.word(), field=self.fake.word(), parameter=self.fake.word()) qf2 = QueryFilter(table=self.fake.word(), field=self.fake.word(), parameter=self.fake.word()) qf_coll = QueryFilterCollection([qf1, qf2]) q_dict = { 'table': self.fake.word(), 'field': self.fake.word(), 'parameter': self.fake.word() } with self.assertRaises(AttributeError): qf_coll.delete(qf1, **q_dict)
def test_constructor(self): """Test the constructor using valid QueryFilter instances.""" filters = [] for _ in range(0, 3): filt = QueryFilter(table=self.fake.word(), field=self.fake.word(), operation=self.fake.word(), parameter=self.fake.word()) filters.append(filt) qf_coll = QueryFilterCollection(filters) self.assertEqual(qf_coll._filters, filters)
def test_contains_fail(self): """Test the __contains__() method fails with a non-matching filter.""" qf1 = QueryFilter(table=self.fake.word(), field=self.fake.word(), parameter=self.fake.word()) qf2 = QueryFilter(table=self.fake.word(), field=self.fake.word(), parameter=self.fake.word()) qf_coll = QueryFilterCollection([qf1]) self.assertNotIn(qf2, qf_coll) self.assertFalse(qf2 in qf_coll)
def test_contains_with_dict(self): """Test the __contains__() method using a dict to get a fuzzy match.""" table = self.fake.word() field = self.fake.word() operation = self.fake.word() parameter = self.fake.word() qf = QueryFilter(table=table, field=field, operation=operation, parameter=parameter) qf_coll = QueryFilterCollection([qf]) self.assertIn({'table': table, 'parameter': parameter}, qf_coll)