def _get_probe_events_body(self, probe, **search_dict): self.wait_and_configure_if_necessary() # TODO: doc, better args, ... query_filter = [] # inventory and metadata filters seen_event_types = set([]) for section, attributes in (("inventory", (("terms", "machine.meta_business_units.id", "meta_business_unit_ids"), ("terms", "machine.tags.id", "tag_ids"), ("terms", "machine.platform", "platforms"), ("terms", "machine.type", "types"))), ("metadata", (("type", "value", "event_types"), ("terms", "tags", "event_tags")))): section_should = [] for section_filter in getattr(probe, "{}_filters".format(section)): section_filter_must = [] for query_type, query_attribute, filter_attribute in attributes: values = getattr(section_filter, filter_attribute, None) if values: if section == "metadata": if filter_attribute == "event_types": seen_event_types.update(values) elif filter_attribute == "event_tags": seen_event_types.update( event_cls.event_type for v in values for event_cls in event_tags.get(v, [])) if query_type == "terms": section_filter_must.append( {"terms": { query_attribute: list(values) }}) elif query_type == "type": if len(values) > 1: section_filter_must.append({ 'bool': { 'should': [ self._get_type_filter(t) for t in values ] } }) else: section_filter_must.append( self._get_type_filter(list(values)[0])) else: raise ValueError("Unknown query type") if section_filter_must: if len(section_filter_must) > 1: section_should.append( {'bool': { 'must': section_filter_must }}) else: section_should.append(section_filter_must[0]) if section_should: if len(section_should) > 1: query_filter.append({'bool': {'should': section_should}}) else: query_filter.append(section_should[0]) # payload filters # PB attributes prefixed by event type in ES # we must try all possible prefixes if seen_event_types: # We are lucky, we only have to test some prefixes # because there is at least one metadata filter with an event type or a tag. payload_event_types_for_filters = seen_event_types else: # Bad luck, no metadata filter. We must try all event types. payload_event_types_for_filters = event_types.keys() payload_should = [] for payload_filter in probe.payload_filters: payload_filter_must = [] for attribute, values in payload_filter.items.items(): if values: def make_query_string(v): return { 'query_string': { 'fields': [ '{}.{}'.format(event_type, attribute) for event_type in payload_event_types_for_filters ], 'query': '"{}"'.format(v) } } if len(values) > 1: # TODO: escape value in query string payload_filter_must.append({ 'bool': { 'should': [make_query_string(v) for v in values] } }) else: v = list(values)[0] payload_filter_must.append(make_query_string(v)) if payload_filter_must: if len(payload_filter_must) > 1: payload_should.append( {'bool': { 'must': payload_filter_must }}) else: payload_should.append(payload_filter_must[0]) if payload_should: if len(payload_should) > 1: query_filter.append({'bool': {'should': payload_should}}) else: query_filter.append(payload_should[0]) # search dict if search_dict: event_type = search_dict.pop('event_type') for attribute, values in search_dict.items(): attribute = "{et}.{attr}".format(et=event_type, attr=attribute) if values is None: continue if not isinstance(values, list): values = [values] elif not values: continue if attribute.endswith('__startswith'): attribute = attribute.replace('__startswith', '') if len(values) > 1: sub_query_filter = { 'bool': { 'should': [{ 'prefix': { attribute: v } } for v in values] } } else: sub_query_filter = {'prefix': {attribute: values[0]}} elif attribute.endswith('__regexp'): attribute = attribute.replace('__regexp', '') if len(values) > 1: sub_query_filter = { 'bool': { 'should': [{ 'regexp': { attribute: v } } for v in values] } } else: sub_query_filter = {'regexp': {attribute: values[0]}} else: if len(values) > 1: sub_query_filter = {'terms': {attribute: values}} else: sub_query_filter = {'term': {attribute: values[0]}} query_filter.append(sub_query_filter) return {'query': {'bool': {'filter': query_filter}}}
def _get_probe_events_body(self, probe, **search_dict): self.wait_and_configure_if_necessary() # TODO: doc, better args, ... query_filter = [] # inventory and metadata filters seen_event_types = set([]) for section, attributes in (("inventory", (("terms", "machine.meta_business_units.id", "meta_business_unit_ids"), ("terms", "machine.tags.id", "tag_ids"), ("terms", "machine.platform", "platforms"), ("terms", "machine.type", "types"))), ("metadata", (("type", "value", "event_types"), ("terms", "tags", "event_tags")))): section_should = [] for section_filter in getattr(probe, "{}_filters".format(section)): section_filter_must = [] for query_type, query_attribute, filter_attribute in attributes: values = getattr(section_filter, filter_attribute, None) if values: if section == "metadata": if filter_attribute == "event_types": seen_event_types.update(values) elif filter_attribute == "event_tags": seen_event_types.update( event_cls.event_type for v in values for event_cls in event_tags.get(v, [])) if query_type == "terms": section_filter_must.append( {"terms": { query_attribute: list(values) }}) elif query_type == "type": if len(values) > 1: section_filter_must.append({ 'bool': { 'should': [ self._get_type_filter(t) for t in values ] } }) else: section_filter_must.append( self._get_type_filter(list(values)[0])) else: raise ValueError("Unknown query type") if section_filter_must: if len(section_filter_must) > 1: section_should.append( {'bool': { 'must': section_filter_must }}) else: section_should.append(section_filter_must[0]) if section_should: if len(section_should) > 1: query_filter.append({'bool': {'should': section_should}}) else: query_filter.append(section_should[0]) # payload filters # PB attributes prefixed by event type in ES # we must try all possible prefixes if seen_event_types: # We are lucky, we only have to test some prefixes # because there is at least one metadata filter with an event type or a tag. payload_event_types_for_filters = seen_event_types else: # Bad luck, no metadata filter. We must try all event types. payload_event_types_for_filters = event_types.keys() payload_should = [] for payload_filter in probe.payload_filters: payload_filter_must = [] for attribute, operator, values in payload_filter.items: # the values may be casted to booleans if the field is stored as a boolean field in elasticsearch # only json like boolean strings will be automatically interpreted as booleans processed_values = [ v.lower() if v in ("False", "True") else v for v in values ] attribute_queries = [{ "term": { "{}.{}".format(event_type, attribute): value } } for event_type, value in product( payload_event_types_for_filters, processed_values)] if len(attribute_queries) > 1: # OR for the different values if operator == PayloadFilter.IN: bool_attr = "should" elif operator == PayloadFilter.NOT_IN: bool_attr = "must_not" payload_filter_must.append( {'bool': { bool_attr: attribute_queries }}) else: attribute_query = attribute_queries[0] if operator == PayloadFilter.NOT_IN: attribute_query = { 'bool': { 'must_not': attribute_query } } payload_filter_must.append(attribute_query) if payload_filter_must: # AND for the attributes of the same payload filter if len(payload_filter_must) > 1: payload_should.append( {'bool': { 'must': payload_filter_must }}) else: payload_should.append(payload_filter_must[0]) if payload_should: if len(payload_should) > 1: query_filter.append({'bool': {'should': payload_should}}) else: query_filter.append(payload_should[0]) # search dict if search_dict: event_type = search_dict.pop('event_type') for attribute, values in search_dict.items(): attribute = "{et}.{attr}".format(et=event_type, attr=attribute) if values is None: continue if not isinstance(values, list): values = [values] elif not values: continue if attribute.endswith('__startswith'): attribute = attribute.replace('__startswith', '') if len(values) > 1: sub_query_filter = { 'bool': { 'should': [{ 'prefix': { attribute: v } } for v in values] } } else: sub_query_filter = {'prefix': {attribute: values[0]}} elif attribute.endswith('__regexp'): attribute = attribute.replace('__regexp', '') if len(values) > 1: sub_query_filter = { 'bool': { 'should': [{ 'regexp': { attribute: v } } for v in values] } } else: sub_query_filter = {'regexp': {attribute: values[0]}} else: if len(values) > 1: sub_query_filter = {'terms': {attribute: values}} else: sub_query_filter = {'term': {attribute: values[0]}} query_filter.append(sub_query_filter) return {'query': {'bool': {'filter': query_filter}}}
def _get_probe_events_body(self, probe, **search_dict): self.wait_and_configure_if_necessary() # TODO: doc, better args, ... query_filter = [] # inventory and metadata filters seen_event_types = set([]) for section, attributes in (("inventory", (("terms", "machine.meta_business_units.id", "meta_business_unit_ids"), ("terms", "machine.tags.id", "tag_ids"), ("terms", "machine.platform", "platforms"), ("terms", "machine.type", "types"))), ("metadata", (("type", "value", "event_types"), ("terms", "tags", "event_tags")))): section_should = [] for section_filter in getattr(probe, "{}_filters".format(section)): section_filter_must = [] for query_type, query_attribute, filter_attribute in attributes: values = getattr(section_filter, filter_attribute, None) if values: if section == "metadata": if filter_attribute == "event_types": seen_event_types.update(values) elif filter_attribute == "event_tags": seen_event_types.update(event_cls.event_type for v in values for event_cls in event_tags.get(v, [])) if query_type == "terms": section_filter_must.append({"terms": {query_attribute: list(values)}}) elif query_type == "type": if len(values) > 1: section_filter_must.append( {'bool': {'should': [{'type': {'value': t}} for t in values]}}) else: section_filter_must.append({"type": {"value": list(values)[0]}}) else: raise ValueError("Unknown query type") if section_filter_must: if len(section_filter_must) > 1: section_should.append({'bool': {'must': section_filter_must}}) else: section_should.append(section_filter_must[0]) if section_should: if len(section_should) > 1: query_filter.append({'bool': {'should': section_should}}) else: query_filter.append(section_should[0]) # payload filters # PB attributes prefixed by event type in ES # we must try all possible prefixes if seen_event_types: # We are lucky, we only have to test some prefixes # because there is at least one metadata filter with an event type or a tag. payload_event_types_for_filters = seen_event_types else: # Bad luck, no metadata filter. We must try all event types. payload_event_types_for_filters = event_types.keys() payload_should = [] for payload_filter in probe.payload_filters: payload_filter_must = [] for attribute, values in payload_filter.items.items(): if values: def make_query_string(v): return {'query_string': {'fields': ['{}.{}'.format(event_type, attribute) for event_type in payload_event_types_for_filters], 'query': '"{}"'.format(v)}} if len(values) > 1: # TODO: escape value in query string payload_filter_must.append({'bool': {'should': [make_query_string(v) for v in values]}}) else: v = list(values)[0] payload_filter_must.append(make_query_string(v)) if payload_filter_must: if len(payload_filter_must) > 1: payload_should.append({'bool': {'must': payload_filter_must}}) else: payload_should.append(payload_filter_must[0]) if payload_should: if len(payload_should) > 1: query_filter.append({'bool': {'should': payload_should}}) else: query_filter.append(payload_should[0]) # search dict if search_dict: event_type = search_dict.pop('event_type') for attribute, values in search_dict.items(): attribute = "{et}.{attr}".format(et=event_type, attr=attribute) if not values: continue if not isinstance(values, list): values = [values] if attribute.endswith('__startswith'): attribute = attribute.replace('__startswith', '') if len(values) > 1: query_filter.append({'bool': {'should': [{'prefix': {attribute: v}} for v in values]}}) else: query_filter.append({'prefix': {attribute: values[0]}}) elif attribute.endswith('__regexp'): attribute = attribute.replace('__regexp', '') if len(values) > 1: query_filter.append({'bool': {'should': [{'regexp': {attribute: v}} for v in values]}}) else: query_filter.append({'regexp': {attribute: values[0]}}) else: if len(values) > 1: query_filter.append({'terms': {attribute: values}}) else: query_filter.append({'term': {attribute: values[0]}}) return {'query': {'bool': {'filter': query_filter}}}