def check_value(term):
if term is None:
return False
if isinstance(term, list):
return any(check_value(t) for t in term)
if isinstance(term, (bool, float, int)) or utils.is_string(term):
v = value
if utils.is_string(v) and isinstance(term, (bool, int, float)):
if isinstance(v, bool):
v = v == "false"
if isinstance(term, int):
v = int(v)
elif isinstance(v, float):
v = float(v)
elif utils.is_string(term) and isinstance(
v, (bool, int, float)):
v = utils.to_unicode(v)
return compare_function(term, v)
else:
raise KqlRuntimeError("Cannot compare value {}".format(term))
def equals(cls, term, value):
if utils.is_string(term) and utils.is_string(value):
if CidrMatch.ip_compiled.match(term) and CidrMatch.cidr_compiled.match(value):
# check for an ipv4 cidr
if value not in cls.__cidr_cache:
cls.__cidr_cache[value] = CidrMatch.get_callback(None, eql.ast.String(value))
return cls.__cidr_cache[value](term)
return term == value
def _array_contains(array, value):
if array is None:
return False
if is_string(value):
value = value.lower()
for item in array:
if item == value:
return True
elif is_string(item) and item.lower() == value:
return True
return False
def walk__field(self, node):
"""Callback function to walk the AST."""
reserved = 'true', 'false', 'null'
if node.base in reserved:
if len(node.sub_fields) != 0:
raise self._error(node, "Invalid field name {base}")
elif node.base == 'true':
return Boolean(True)
elif node.base == 'false':
return Boolean(False)
elif node.base == 'null':
return Null()
else:
raise self._error(node.base, "Unhandled literal")
path = []
for sub_field in self.walk(node.sub_fields):
if is_string(sub_field) and sub_field in reserved:
raise self._error(node,
"Invalid attribute {}".format(sub_field))
path.append(sub_field)
if not path and node.base in self.preprocessor.constants:
constant = self.preprocessor.constants[node.base]
return constant.value
return Field(node.base, path)
def optimize(self):
"""Optimize the AST."""
expression = self.expression
# move all the literals to the front, preserve their ordering
literals = [v for k, v in self._get_literals().items()]
dynamic = [v for v in self.container if not isinstance(v, Literal)]
container = literals + dynamic
# check to see if a literal value is in the list of literal values
if isinstance(self.expression, Literal):
value = self.expression.value
if is_string(value):
value = value.lower()
if value in self._get_literals():
return Boolean(True)
container = dynamic
if len(container) == 0:
return Boolean(False)
elif len(container) == 1:
return Comparison(expression, Comparison.EQ,
container[0]).optimize()
elif expression in container:
return Boolean(True)
return InSet(expression, container)
def equals(x, y):
if not types_match(x, y):
return False
elif is_string(x):
return x.lower() == y.lower()
else:
return x == y
def fold(expr):
"""Test method for parsing and folding."""
if is_string(expr):
expr = parse_expression(expr)
return expr.fold()
elif isinstance(expr, Expression):
return expr.fold()
else:
raise TypeError("Unable to fold {}".format(expr))
def event_callback(self, *event_types):
"""Get a decorator that registers a function as an event callback in the engine."""
assert all(is_string(e) for e in event_types)
def event_callback_decorator(f):
for event_type in event_types:
self.add_event_callback(event_type, f)
return f
return event_callback_decorator
def from_python(cls, value):
if value is None:
return Null()
elif isinstance(value, bool):
return Boolean(value)
elif is_number(value):
return Number(value)
elif is_string(value):
return String(value)
else:
raise EqlCompileError("Unknown type {} for value {}".format(type(value).__name__, value))
def _number(arg, base=10): # type: (str, int) -> int|float
if is_number(arg):
return arg
elif is_string(arg):
if '.' in arg:
return float(arg)
if arg.startswith('0x'):
arg = arg[2:]
base = 16
try:
return int(arg, base)
except ValueError:
return None
def walk__literal(self, node):
"""Callback function to walk the AST."""
literal = self.walk(node.value)
if literal is None:
return literal
elif is_string(literal):
# If a 'raw' string is detected, then only unescape the quote character
if node.text.startswith('?'):
quote_char = node.text[-1]
literal = literal.replace("\\" + quote_char, quote_char)
else:
literal = String.unescape(literal)
return String(to_unicode(literal))
elif isinstance(literal, bool):
return Boolean(literal)
else:
return Number(literal)
def __init__(self, config=None):
"""Create the engine with an optional list of files."""
super(BaseEngine, self).__init__(config)
self.analytics = [] # type: list[EqlAnalytic]
self.preprocessor = PreProcessor()
with use_schema(self.schema):
definitions = self.get_config('definitions', [])
if is_string(definitions):
definitions = parse_definitions(definitions)
self.preprocessor.add_definitions(definitions)
for path in self.get_config('definitions_files', []):
with open(path, 'r') as f:
definitions = parse_definitions(f.read())
self.preprocessor.add_definitions(definitions)
def from_data(cls, data):
"""Load an event from a dictionary.
:param dict data: Dictionary with the event type, time, and keys.
"""
data = data.get('data_buffer', data)
timestamp = data.get('timestamp', 0)
if is_string(data.get('event_type')):
event_type = data['event_type']
elif 'event_type_full' in data:
event_type = data['event_type_full']
if event_type.endswith('_event'):
event_type = event_type[:-len('_event')]
else:
event_type = EVENT_TYPE_GENERIC
return cls(event_type, timestamp, data)
def get_reducer(query, config=None):
"""Get a reducer to aggregate results from distributed EQL queries.
:param str|dict|EqlAnalytic|PipedQuery query: The query text or parsed query
:param dict config: The configuration for PythonEngine
"""
if isinstance(query, dict):
query = parse_analytic(query)
elif is_string(query):
query = parse_query(query, implied_base=True, implied_any=True)
def reducer(inputs):
results = []
engine = PythonEngine(config)
engine.add_reducer(query)
engine.add_output_hook(results.append)
engine.reduce_events(inputs, finalize=True)
return results
return reducer
def _check_in_set(self, node): # type: (InSet) -> callable
if all(isinstance(item, Literal) for item in node.container):
values = set()
for item in node.container:
value = item.value
if is_string(value):
values.add(value.lower())
else:
values.add(value)
get_value = self.convert(node.expression)
def callback(scope): # type: (Scope) -> bool
check_value = get_value(scope)
if is_string(check_value):
check_value = check_value.lower()
return check_value in values
return callback
else:
return self.convert(node.synonym)
def get_engine(query, config=None):
"""Run an EQL query or analytic over a list of events and get the results.
:param str|dict|EqlAnalytic|PipedQuery query: The query text or parsed query
:param dict config: The configuration for PythonEngine
"""
if isinstance(query, dict):
query = parse_analytic(query)
elif is_string(query):
query = parse_query(query, implied_base=True, implied_any=True)
def run_engine(inputs):
results = []
engine = PythonEngine(config)
if isinstance(query, PipedQuery):
engine.add_query(query)
else:
engine.add_analytic(query)
engine.add_output_hook(results.append)
engine.stream_events(inputs, finalize=True)
return results
return run_engine
def callback(scope): # type: (Scope) -> bool
check_value = get_value(scope)
if is_string(check_value):
check_value = check_value.lower()
return check_value in values
def _str_substring(a,
start=None,
end=None): # type: (str, int, int) -> str
if is_string(a):
return a[start:end]
def _str_contains(a, b): # type: (str, str) -> bool
return is_string(a) and is_string(b) and b.lower() in a.lower()
def _str_ends_width(a, b): # type: (str, str) -> bool
return is_string(a) and is_string(b) and a.lower().endswith(b.lower())
def types_match(x, y):
return (type(x) == type(y) or is_string(x) and is_string(y)
or is_number(x) and is_number(y))
def _str_index_of(a, b, start=0): # type: (str, str, int) -> int
if is_string(a) and is_string(b):
a = a.lower()
b = b.lower()
if b in a[start:]:
return a.index(b, start)
def _str_starts_with(a, b): # type: (str, str) -> bool
return is_string(a) and is_string(b) and a.lower().startswith(
b.lower())
def query_multiple_events(self): # type: () -> (int, Field)
"""Get the index into the event array and query."""
if self.base == Field.EVENTS and len(self.path) >= 2:
if is_number(self.path[0]) and is_string(self.path[1]):
return self.path[0], Field(self.path[1], self.path[2:])
return 0, self
