def check_value(term): if term is None: return False if isinstance(term, list): return any(check_value(t) for t in term) if isinstance(term, (bool, float, int)) or utils.is_string(term): v = value if utils.is_string(v) and isinstance(term, (bool, int, float)): if isinstance(v, bool): v = v == "false" if isinstance(term, int): v = int(v) elif isinstance(v, float): v = float(v) elif utils.is_string(term) and isinstance( v, (bool, int, float)): v = utils.to_unicode(v) return compare_function(term, v) else: raise KqlRuntimeError("Cannot compare value {}".format(term))
def equals(cls, term, value): if utils.is_string(term) and utils.is_string(value): if CidrMatch.ip_compiled.match(term) and CidrMatch.cidr_compiled.match(value): # check for an ipv4 cidr if value not in cls.__cidr_cache: cls.__cidr_cache[value] = CidrMatch.get_callback(None, eql.ast.String(value)) return cls.__cidr_cache[value](term) return term == value
def _array_contains(array, value): if array is None: return False if is_string(value): value = value.lower() for item in array: if item == value: return True elif is_string(item) and item.lower() == value: return True return False
def walk__field(self, node): """Callback function to walk the AST.""" reserved = 'true', 'false', 'null' if node.base in reserved: if len(node.sub_fields) != 0: raise self._error(node, "Invalid field name {base}") elif node.base == 'true': return Boolean(True) elif node.base == 'false': return Boolean(False) elif node.base == 'null': return Null() else: raise self._error(node.base, "Unhandled literal") path = [] for sub_field in self.walk(node.sub_fields): if is_string(sub_field) and sub_field in reserved: raise self._error(node, "Invalid attribute {}".format(sub_field)) path.append(sub_field) if not path and node.base in self.preprocessor.constants: constant = self.preprocessor.constants[node.base] return constant.value return Field(node.base, path)
def optimize(self): """Optimize the AST.""" expression = self.expression # move all the literals to the front, preserve their ordering literals = [v for k, v in self._get_literals().items()] dynamic = [v for v in self.container if not isinstance(v, Literal)] container = literals + dynamic # check to see if a literal value is in the list of literal values if isinstance(self.expression, Literal): value = self.expression.value if is_string(value): value = value.lower() if value in self._get_literals(): return Boolean(True) container = dynamic if len(container) == 0: return Boolean(False) elif len(container) == 1: return Comparison(expression, Comparison.EQ, container[0]).optimize() elif expression in container: return Boolean(True) return InSet(expression, container)
def equals(x, y): if not types_match(x, y): return False elif is_string(x): return x.lower() == y.lower() else: return x == y
def fold(expr): """Test method for parsing and folding.""" if is_string(expr): expr = parse_expression(expr) return expr.fold() elif isinstance(expr, Expression): return expr.fold() else: raise TypeError("Unable to fold {}".format(expr))
def event_callback(self, *event_types): """Get a decorator that registers a function as an event callback in the engine.""" assert all(is_string(e) for e in event_types) def event_callback_decorator(f): for event_type in event_types: self.add_event_callback(event_type, f) return f return event_callback_decorator
def from_python(cls, value): if value is None: return Null() elif isinstance(value, bool): return Boolean(value) elif is_number(value): return Number(value) elif is_string(value): return String(value) else: raise EqlCompileError("Unknown type {} for value {}".format(type(value).__name__, value))
def _number(arg, base=10): # type: (str, int) -> int|float if is_number(arg): return arg elif is_string(arg): if '.' in arg: return float(arg) if arg.startswith('0x'): arg = arg[2:] base = 16 try: return int(arg, base) except ValueError: return None
def walk__literal(self, node): """Callback function to walk the AST.""" literal = self.walk(node.value) if literal is None: return literal elif is_string(literal): # If a 'raw' string is detected, then only unescape the quote character if node.text.startswith('?'): quote_char = node.text[-1] literal = literal.replace("\\" + quote_char, quote_char) else: literal = String.unescape(literal) return String(to_unicode(literal)) elif isinstance(literal, bool): return Boolean(literal) else: return Number(literal)
def __init__(self, config=None): """Create the engine with an optional list of files.""" super(BaseEngine, self).__init__(config) self.analytics = [] # type: list[EqlAnalytic] self.preprocessor = PreProcessor() with use_schema(self.schema): definitions = self.get_config('definitions', []) if is_string(definitions): definitions = parse_definitions(definitions) self.preprocessor.add_definitions(definitions) for path in self.get_config('definitions_files', []): with open(path, 'r') as f: definitions = parse_definitions(f.read()) self.preprocessor.add_definitions(definitions)
def from_data(cls, data): """Load an event from a dictionary. :param dict data: Dictionary with the event type, time, and keys. """ data = data.get('data_buffer', data) timestamp = data.get('timestamp', 0) if is_string(data.get('event_type')): event_type = data['event_type'] elif 'event_type_full' in data: event_type = data['event_type_full'] if event_type.endswith('_event'): event_type = event_type[:-len('_event')] else: event_type = EVENT_TYPE_GENERIC return cls(event_type, timestamp, data)
def get_reducer(query, config=None): """Get a reducer to aggregate results from distributed EQL queries. :param str|dict|EqlAnalytic|PipedQuery query: The query text or parsed query :param dict config: The configuration for PythonEngine """ if isinstance(query, dict): query = parse_analytic(query) elif is_string(query): query = parse_query(query, implied_base=True, implied_any=True) def reducer(inputs): results = [] engine = PythonEngine(config) engine.add_reducer(query) engine.add_output_hook(results.append) engine.reduce_events(inputs, finalize=True) return results return reducer
def _check_in_set(self, node): # type: (InSet) -> callable if all(isinstance(item, Literal) for item in node.container): values = set() for item in node.container: value = item.value if is_string(value): values.add(value.lower()) else: values.add(value) get_value = self.convert(node.expression) def callback(scope): # type: (Scope) -> bool check_value = get_value(scope) if is_string(check_value): check_value = check_value.lower() return check_value in values return callback else: return self.convert(node.synonym)
def get_engine(query, config=None): """Run an EQL query or analytic over a list of events and get the results. :param str|dict|EqlAnalytic|PipedQuery query: The query text or parsed query :param dict config: The configuration for PythonEngine """ if isinstance(query, dict): query = parse_analytic(query) elif is_string(query): query = parse_query(query, implied_base=True, implied_any=True) def run_engine(inputs): results = [] engine = PythonEngine(config) if isinstance(query, PipedQuery): engine.add_query(query) else: engine.add_analytic(query) engine.add_output_hook(results.append) engine.stream_events(inputs, finalize=True) return results return run_engine
def callback(scope): # type: (Scope) -> bool check_value = get_value(scope) if is_string(check_value): check_value = check_value.lower() return check_value in values
def _str_substring(a, start=None, end=None): # type: (str, int, int) -> str if is_string(a): return a[start:end]
def _str_contains(a, b): # type: (str, str) -> bool return is_string(a) and is_string(b) and b.lower() in a.lower()
def _str_ends_width(a, b): # type: (str, str) -> bool return is_string(a) and is_string(b) and a.lower().endswith(b.lower())
def types_match(x, y): return (type(x) == type(y) or is_string(x) and is_string(y) or is_number(x) and is_number(y))
def _str_index_of(a, b, start=0): # type: (str, str, int) -> int if is_string(a) and is_string(b): a = a.lower() b = b.lower() if b in a[start:]: return a.index(b, start)
def _str_starts_with(a, b): # type: (str, str) -> bool return is_string(a) and is_string(b) and a.lower().startswith( b.lower())
def query_multiple_events(self): # type: () -> (int, Field) """Get the index into the event array and query.""" if self.base == Field.EVENTS and len(self.path) >= 2: if is_number(self.path[0]) and is_string(self.path[1]): return self.path[0], Field(self.path[1], self.path[2:]) return 0, self