def _get_probe_events_body(self, probe, **search_dict):
self.wait_and_configure_if_necessary()
# TODO: doc, better args, ...
query_filter = []
# inventory and metadata filters
seen_event_types = set([])
for section, attributes in (("inventory",
(("terms",
"machine.meta_business_units.id",
"meta_business_unit_ids"),
("terms", "machine.tags.id", "tag_ids"),
("terms", "machine.platform",
"platforms"), ("terms", "machine.type",
"types"))),
("metadata",
(("type", "value", "event_types"),
("terms", "tags", "event_tags")))):
section_should = []
for section_filter in getattr(probe, "{}_filters".format(section)):
section_filter_must = []
for query_type, query_attribute, filter_attribute in attributes:
values = getattr(section_filter, filter_attribute, None)
if values:
if section == "metadata":
if filter_attribute == "event_types":
seen_event_types.update(values)
elif filter_attribute == "event_tags":
seen_event_types.update(
event_cls.event_type for v in values
for event_cls in event_tags.get(v, []))
if query_type == "terms":
section_filter_must.append(
{"terms": {
query_attribute: list(values)
}})
elif query_type == "type":
if len(values) > 1:
section_filter_must.append({
'bool': {
'should': [
self._get_type_filter(t)
for t in values
]
}
})
else:
section_filter_must.append(
self._get_type_filter(list(values)[0]))
else:
raise ValueError("Unknown query type")
if section_filter_must:
if len(section_filter_must) > 1:
section_should.append(
{'bool': {
'must': section_filter_must
}})
else:
section_should.append(section_filter_must[0])
if section_should:
if len(section_should) > 1:
query_filter.append({'bool': {'should': section_should}})
else:
query_filter.append(section_should[0])
# payload filters
# PB attributes prefixed by event type in ES
# we must try all possible prefixes
if seen_event_types:
# We are lucky, we only have to test some prefixes
# because there is at least one metadata filter with an event type or a tag.
payload_event_types_for_filters = seen_event_types
else:
# Bad luck, no metadata filter. We must try all event types.
payload_event_types_for_filters = event_types.keys()
payload_should = []
for payload_filter in probe.payload_filters:
payload_filter_must = []
for attribute, values in payload_filter.items.items():
if values:
def make_query_string(v):
return {
'query_string': {
'fields': [
'{}.{}'.format(event_type, attribute)
for event_type in
payload_event_types_for_filters
],
'query':
'"{}"'.format(v)
}
}
if len(values) > 1:
# TODO: escape value in query string
payload_filter_must.append({
'bool': {
'should':
[make_query_string(v) for v in values]
}
})
else:
v = list(values)[0]
payload_filter_must.append(make_query_string(v))
if payload_filter_must:
if len(payload_filter_must) > 1:
payload_should.append(
{'bool': {
'must': payload_filter_must
}})
else:
payload_should.append(payload_filter_must[0])
if payload_should:
if len(payload_should) > 1:
query_filter.append({'bool': {'should': payload_should}})
else:
query_filter.append(payload_should[0])
# search dict
if search_dict:
event_type = search_dict.pop('event_type')
for attribute, values in search_dict.items():
attribute = "{et}.{attr}".format(et=event_type, attr=attribute)
if values is None:
continue
if not isinstance(values, list):
values = [values]
elif not values:
continue
if attribute.endswith('__startswith'):
attribute = attribute.replace('__startswith', '')
if len(values) > 1:
sub_query_filter = {
'bool': {
'should': [{
'prefix': {
attribute: v
}
} for v in values]
}
}
else:
sub_query_filter = {'prefix': {attribute: values[0]}}
elif attribute.endswith('__regexp'):
attribute = attribute.replace('__regexp', '')
if len(values) > 1:
sub_query_filter = {
'bool': {
'should': [{
'regexp': {
attribute: v
}
} for v in values]
}
}
else:
sub_query_filter = {'regexp': {attribute: values[0]}}
else:
if len(values) > 1:
sub_query_filter = {'terms': {attribute: values}}
else:
sub_query_filter = {'term': {attribute: values[0]}}
query_filter.append(sub_query_filter)
return {'query': {'bool': {'filter': query_filter}}}
def _get_probe_events_body(self, probe, **search_dict):
self.wait_and_configure_if_necessary()
# TODO: doc, better args, ...
query_filter = []
# inventory and metadata filters
seen_event_types = set([])
for section, attributes in (("inventory",
(("terms",
"machine.meta_business_units.id",
"meta_business_unit_ids"),
("terms", "machine.tags.id", "tag_ids"),
("terms", "machine.platform",
"platforms"), ("terms", "machine.type",
"types"))),
("metadata",
(("type", "value", "event_types"),
("terms", "tags", "event_tags")))):
section_should = []
for section_filter in getattr(probe, "{}_filters".format(section)):
section_filter_must = []
for query_type, query_attribute, filter_attribute in attributes:
values = getattr(section_filter, filter_attribute, None)
if values:
if section == "metadata":
if filter_attribute == "event_types":
seen_event_types.update(values)
elif filter_attribute == "event_tags":
seen_event_types.update(
event_cls.event_type for v in values
for event_cls in event_tags.get(v, []))
if query_type == "terms":
section_filter_must.append(
{"terms": {
query_attribute: list(values)
}})
elif query_type == "type":
if len(values) > 1:
section_filter_must.append({
'bool': {
'should': [
self._get_type_filter(t)
for t in values
]
}
})
else:
section_filter_must.append(
self._get_type_filter(list(values)[0]))
else:
raise ValueError("Unknown query type")
if section_filter_must:
if len(section_filter_must) > 1:
section_should.append(
{'bool': {
'must': section_filter_must
}})
else:
section_should.append(section_filter_must[0])
if section_should:
if len(section_should) > 1:
query_filter.append({'bool': {'should': section_should}})
else:
query_filter.append(section_should[0])
# payload filters
# PB attributes prefixed by event type in ES
# we must try all possible prefixes
if seen_event_types:
# We are lucky, we only have to test some prefixes
# because there is at least one metadata filter with an event type or a tag.
payload_event_types_for_filters = seen_event_types
else:
# Bad luck, no metadata filter. We must try all event types.
payload_event_types_for_filters = event_types.keys()
payload_should = []
for payload_filter in probe.payload_filters:
payload_filter_must = []
for attribute, operator, values in payload_filter.items:
# the values may be casted to booleans if the field is stored as a boolean field in elasticsearch
# only json like boolean strings will be automatically interpreted as booleans
processed_values = [
v.lower() if v in ("False", "True") else v for v in values
]
attribute_queries = [{
"term": {
"{}.{}".format(event_type, attribute): value
}
} for event_type, value in product(
payload_event_types_for_filters, processed_values)]
if len(attribute_queries) > 1:
# OR for the different values
if operator == PayloadFilter.IN:
bool_attr = "should"
elif operator == PayloadFilter.NOT_IN:
bool_attr = "must_not"
payload_filter_must.append(
{'bool': {
bool_attr: attribute_queries
}})
else:
attribute_query = attribute_queries[0]
if operator == PayloadFilter.NOT_IN:
attribute_query = {
'bool': {
'must_not': attribute_query
}
}
payload_filter_must.append(attribute_query)
if payload_filter_must:
# AND for the attributes of the same payload filter
if len(payload_filter_must) > 1:
payload_should.append(
{'bool': {
'must': payload_filter_must
}})
else:
payload_should.append(payload_filter_must[0])
if payload_should:
if len(payload_should) > 1:
query_filter.append({'bool': {'should': payload_should}})
else:
query_filter.append(payload_should[0])
# search dict
if search_dict:
event_type = search_dict.pop('event_type')
for attribute, values in search_dict.items():
attribute = "{et}.{attr}".format(et=event_type, attr=attribute)
if values is None:
continue
if not isinstance(values, list):
values = [values]
elif not values:
continue
if attribute.endswith('__startswith'):
attribute = attribute.replace('__startswith', '')
if len(values) > 1:
sub_query_filter = {
'bool': {
'should': [{
'prefix': {
attribute: v
}
} for v in values]
}
}
else:
sub_query_filter = {'prefix': {attribute: values[0]}}
elif attribute.endswith('__regexp'):
attribute = attribute.replace('__regexp', '')
if len(values) > 1:
sub_query_filter = {
'bool': {
'should': [{
'regexp': {
attribute: v
}
} for v in values]
}
}
else:
sub_query_filter = {'regexp': {attribute: values[0]}}
else:
if len(values) > 1:
sub_query_filter = {'terms': {attribute: values}}
else:
sub_query_filter = {'term': {attribute: values[0]}}
query_filter.append(sub_query_filter)
return {'query': {'bool': {'filter': query_filter}}}
def _get_probe_events_body(self, probe, **search_dict):
self.wait_and_configure_if_necessary()
# TODO: doc, better args, ...
query_filter = []
# inventory and metadata filters
seen_event_types = set([])
for section, attributes in (("inventory", (("terms",
"machine.meta_business_units.id",
"meta_business_unit_ids"),
("terms", "machine.tags.id", "tag_ids"),
("terms", "machine.platform", "platforms"),
("terms", "machine.type", "types"))),
("metadata", (("type", "value", "event_types"),
("terms", "tags", "event_tags")))):
section_should = []
for section_filter in getattr(probe, "{}_filters".format(section)):
section_filter_must = []
for query_type, query_attribute, filter_attribute in attributes:
values = getattr(section_filter, filter_attribute, None)
if values:
if section == "metadata":
if filter_attribute == "event_types":
seen_event_types.update(values)
elif filter_attribute == "event_tags":
seen_event_types.update(event_cls.event_type
for v in values
for event_cls in event_tags.get(v, []))
if query_type == "terms":
section_filter_must.append({"terms": {query_attribute: list(values)}})
elif query_type == "type":
if len(values) > 1:
section_filter_must.append(
{'bool': {'should': [{'type': {'value': t}} for t in values]}})
else:
section_filter_must.append({"type": {"value": list(values)[0]}})
else:
raise ValueError("Unknown query type")
if section_filter_must:
if len(section_filter_must) > 1:
section_should.append({'bool': {'must': section_filter_must}})
else:
section_should.append(section_filter_must[0])
if section_should:
if len(section_should) > 1:
query_filter.append({'bool': {'should': section_should}})
else:
query_filter.append(section_should[0])
# payload filters
# PB attributes prefixed by event type in ES
# we must try all possible prefixes
if seen_event_types:
# We are lucky, we only have to test some prefixes
# because there is at least one metadata filter with an event type or a tag.
payload_event_types_for_filters = seen_event_types
else:
# Bad luck, no metadata filter. We must try all event types.
payload_event_types_for_filters = event_types.keys()
payload_should = []
for payload_filter in probe.payload_filters:
payload_filter_must = []
for attribute, values in payload_filter.items.items():
if values:
def make_query_string(v):
return {'query_string': {'fields': ['{}.{}'.format(event_type, attribute)
for event_type in payload_event_types_for_filters],
'query': '"{}"'.format(v)}}
if len(values) > 1:
# TODO: escape value in query string
payload_filter_must.append({'bool': {'should': [make_query_string(v) for v in values]}})
else:
v = list(values)[0]
payload_filter_must.append(make_query_string(v))
if payload_filter_must:
if len(payload_filter_must) > 1:
payload_should.append({'bool': {'must': payload_filter_must}})
else:
payload_should.append(payload_filter_must[0])
if payload_should:
if len(payload_should) > 1:
query_filter.append({'bool': {'should': payload_should}})
else:
query_filter.append(payload_should[0])
# search dict
if search_dict:
event_type = search_dict.pop('event_type')
for attribute, values in search_dict.items():
attribute = "{et}.{attr}".format(et=event_type, attr=attribute)
if not values:
continue
if not isinstance(values, list):
values = [values]
if attribute.endswith('__startswith'):
attribute = attribute.replace('__startswith', '')
if len(values) > 1:
query_filter.append({'bool': {'should': [{'prefix': {attribute: v}} for v in values]}})
else:
query_filter.append({'prefix': {attribute: values[0]}})
elif attribute.endswith('__regexp'):
attribute = attribute.replace('__regexp', '')
if len(values) > 1:
query_filter.append({'bool': {'should': [{'regexp': {attribute: v}} for v in values]}})
else:
query_filter.append({'regexp': {attribute: values[0]}})
else:
if len(values) > 1:
query_filter.append({'terms': {attribute: values}})
else:
query_filter.append({'term': {attribute: values[0]}})
return {'query': {'bool': {'filter': query_filter}}}