def validate_query(self, beats_version: str, ecs_versions: List[str]):
"""Static method to validate the query, called from the parent which contains [metadata] information."""
indexes = self.index or []
parsed = self.parsed_query
beat_types = [
index.split("-")[0] for index in indexes if "beat-*" in index
]
beat_schema = beats.get_schema_from_kql(
parsed, beat_types, version=beats_version) if beat_types else None
if not ecs_versions:
kql.parse(self.query,
schema=ecs.get_kql_schema(indexes=indexes,
beat_schema=beat_schema))
else:
for version in ecs_versions:
schema = ecs.get_kql_schema(version=version,
indexes=indexes,
beat_schema=beat_schema)
try:
kql.parse(self.query, schema=schema)
except kql.KqlParseError as exc:
message = exc.error_msg
trailer = None
if "Unknown field" in message and beat_types:
trailer = "\nTry adding event.module or event.dataset to specify beats module"
raise kql.KqlParseError(exc.error_msg,
exc.line,
exc.column,
exc.source,
len(exc.caret.lstrip()),
trailer=trailer) from None
def test_upper_tokens(self):
queries = [
"a:b AND c:d",
"a:b OR c:d",
"NOT a:b",
"a:(b OR c)",
"a:(b AND c)",
"a:(NOT b)",
]
for q in queries:
with self.assertRaises(kql.KqlParseError):
kql.parse(q)
def validate(self, as_rule=False, versioned=False):
"""Validate against a rule schema, query schema, and linting."""
self.normalize()
if as_rule:
schema_validate(self.rule_format(), as_rule=True)
else:
schema_validate(self.contents, versioned=versioned)
if self.query and self.contents['language'] == 'kuery':
# validate against all specified schemas or the latest if none specified
ecs_versions = self.metadata.get('ecs_version')
indexes = self.contents.get("index", [])
beat_types = [
index.split("-")[0] for index in indexes if "beat-*" in index
]
beat_schema = beats.get_schema_for_query(
self.parsed_kql, beat_types) if beat_types else None
if not ecs_versions:
kql.parse(self.query,
schema=ecs.get_kql_schema(indexes=indexes,
beat_schema=beat_schema))
else:
for version in ecs_versions:
try:
schema = ecs.get_kql_schema(version=version,
indexes=indexes,
beat_schema=beat_schema)
except KeyError:
raise KeyError(
'Unknown ecs schema version: {} in rule {}.\n'
'Do you need to update schemas?'.format(
version, self.name))
try:
kql.parse(self.query, schema=schema)
except kql.KqlParseError as exc:
message = exc.error_msg
trailer = None
if "Unknown field" in message and beat_types:
trailer = "\nTry adding event.module and event.dataset to specify beats module"
raise kql.KqlParseError(exc.error_msg,
exc.line,
exc.column,
exc.source,
len(exc.caret.lstrip()),
trailer=trailer)
def test_all_rule_queries_optimized(self):
"""Ensure that every rule query is in optimized form."""
for rule in self.production_rules:
if rule.contents.data.get("language") == "kql":
source = rule.contents.data.query
tree = kql.parse(source, optimize=False)
optimized = tree.optimize(recursive=True)
err_message = f'\n{self.rule_str(rule)} Query not optimized for rule\n' \
f'Expected: {optimized}\nActual: {source}'
self.assertEqual(tree, optimized, err_message)
def validate(self, data: QueryRuleData, meta: RuleMeta) -> None:
"""Static method to validate the query, called from the parent which contains [metadata] information."""
ast = self.ast
if meta.query_schema_validation is False or meta.maturity == "deprecated":
# syntax only, which is done via self.ast
return
indexes = data.index or []
beats_version = meta.beats_version or beats.get_max_version()
ecs_versions = meta.ecs_versions or [ecs.get_max_version()]
beat_types = [
index.split("-")[0] for index in indexes if "beat-*" in index
]
beat_schema = beats.get_schema_from_kql(
ast, beat_types, version=beats_version) if beat_types else None
if not ecs_versions:
kql.parse(self.query,
schema=ecs.get_kql_schema(indexes=indexes,
beat_schema=beat_schema))
else:
for version in ecs_versions:
schema = ecs.get_kql_schema(version=version,
indexes=indexes,
beat_schema=beat_schema)
try:
kql.parse(self.query, schema=schema)
except kql.KqlParseError as exc:
message = exc.error_msg
trailer = None
if "Unknown field" in message and beat_types:
trailer = "\nTry adding event.module or event.dataset to specify beats module"
raise kql.KqlParseError(exc.error_msg,
exc.line,
exc.column,
exc.source,
len(exc.caret.lstrip()),
trailer=trailer) from None
def get_unique_query_fields(cls, rule_contents):
"""Get a list of unique fields used in a rule query from rule contents."""
query = rule_contents.get('query')
language = rule_contents.get('language')
if language in ('kuery', 'eql'):
parsed = kql.parse(
query) if language == 'kuery' else eql.parse_query(query)
return sorted(
set(
str(f) for f in parsed
if isinstance(f, (eql.ast.Field, kql.ast.Field))))
def test_all_rule_queries_optimized(self):
"""Ensure that every rule query is in optimized form."""
for file_name, contents in self.rule_files.items():
rule = TOMLRule(file_name, contents)
if contents["rule"].get("langauge") == "kql":
source = contents["rule"]["query"]
tree = kql.parse(source, optimize=False)
optimized = tree.optimize(recursive=True)
err_message = f'\n{self.rule_str(rule)} Query not optimized for rule\n' \
f'Expected: {optimized}\nActual: {source}'
self.assertEqual(tree, optimized, err_message)
def validate(self, data: QueryRuleData, meta: RuleMeta) -> None:
"""Static method to validate the query, called from the parent which contains [metadata] information."""
ast = self.ast
if meta.query_schema_validation is False or meta.maturity == "deprecated":
# syntax only, which is done via self.ast
return
for stack_version, mapping in meta.get_validation_stack_versions(
).items():
beats_version = mapping['beats']
ecs_version = mapping['ecs']
err_trailer = f'stack: {stack_version}, beats: {beats_version}, ecs: {ecs_version}'
beat_types = beats.parse_beats_from_index(data.index)
beat_schema = beats.get_schema_from_kql(
ast, beat_types, version=beats_version) if beat_types else None
schema = ecs.get_kql_schema(version=ecs_version,
indexes=data.index or [],
beat_schema=beat_schema)
try:
kql.parse(self.query, schema=schema)
except kql.KqlParseError as exc:
message = exc.error_msg
trailer = err_trailer
if "Unknown field" in message and beat_types:
trailer = f"\nTry adding event.module or event.dataset to specify beats module\n\n{trailer}"
raise kql.KqlParseError(exc.error_msg,
exc.line,
exc.column,
exc.source,
len(exc.caret.lstrip()),
trailer=trailer) from None
except Exception:
print(err_trailer)
raise
def parsed_query(self):
if self.query:
if self.contents['language'] == 'kuery':
return kql.parse(self.query)
elif self.contents['language'] == 'eql':
return eql.parse_query(self.query)
def parsed_query(self) -> kql.ast.Expression:
return kql.parse(self.query)