def load_tests():
"""Load up tests from toml to prepare for pytest.mark.parametrize."""
if fold_tests or verifier_failures:
return
for name in ["test_string_functions.toml", "test_folding.toml", "test_optimizer.toml"]:
data = eql.load_dump(eql.etc.get_etc_path(name))
for test_name, contents in sorted(data.items()):
test_name = "{file}:{test}".format(file=name, test=test_name)
case_settings = []
if "case_sensitive" not in contents and "case_insensitive" not in contents:
case_sensitive = True
case_insensitive = True
else:
case_sensitive = contents.get("case_sensitive") is True
case_insensitive = contents.get("case_insensitive") is True
if case_sensitive:
case_settings.append(True)
if case_insensitive:
case_settings.append(False)
assert len(case_settings) > 0, "{test} is missing case_sensitive/case_insensitive".format(test=test_name)
extract_tests(test_name, contents, case_settings)
def build_release(config_file, update_version_lock):
"""Assemble all the rules into Kibana-ready release files."""
config = load_dump(config_file)['package']
click.echo('[+] Building package {}'.format(config.get('name')))
package = Package.from_config(config, update_version_lock=update_version_lock)
package.save()
package.get_package_hash(verbose=True)
click.echo('- {} rules included'.format(len(package.rules)))
def load_from_file(infile, directory):
"""Load rules from file(s)."""
if infile:
for rule_file in infile:
rule_path = os.path.join(RULES_DIR, os.path.basename(rule_file))
rule = Rule(rule_path, load_dump(rule_file))
rule.save(as_rule=True, verbose=True)
elif directory:
for rule_file in glob.glob(os.path.join(directory, '**', '*.*'),
recursive=True):
try:
rule_path = os.path.join(RULES_DIR,
os.path.basename(rule_file))
rule = Rule(rule_path, load_dump(rule_file))
rule.save(as_rule=True, verbose=True)
except ValueError:
click.echo('Unable to load file: {}'.format(rule_file))
else:
click.echo('No files specified!')
def kibana_commit(ctx, local_repo, github_repo, ssh, kibana_directory, base_branch, message):
"""Prep a commit and push to Kibana."""
git_exe = shutil.which("git")
package_name = load_dump(PACKAGE_FILE)['package']["name"]
release_dir = os.path.join(RELEASE_DIR, package_name)
message = message or f"[Detection Rules] Add {package_name} rules"
if not os.path.exists(release_dir):
click.secho("Release directory doesn't exist.", fg="red", err=True)
click.echo(f"Run {click.style('python -m detection_rules build-release', bold=True)} to populate", err=True)
ctx.exit(1)
if not git_exe:
click.secho("Unable to find git", err=True, fg="red")
ctx.exit(1)
try:
if not os.path.exists(local_repo):
if not click.confirm(f"Kibana repository doesn't exist at {local_repo}. Clone?"):
ctx.exit(1)
url = f"[email protected]:{github_repo}.git" if ssh else f"https://github.com/{github_repo}.git"
subprocess.check_call([git_exe, "clone", url, local_repo, "--depth", 1])
def git(*args, show_output=False):
method = subprocess.call if show_output else subprocess.check_output
return method([git_exe, "-C", local_repo] + list(args), encoding="utf-8")
git("checkout", base_branch)
git("pull")
git("checkout", "-b", f"rules/{package_name}", show_output=True)
git("rm", "-r", kibana_directory)
source_dir = os.path.join(release_dir, "rules")
target_dir = os.path.join(local_repo, kibana_directory)
os.makedirs(target_dir)
for name in os.listdir(source_dir):
_, ext = os.path.splitext(name)
path = os.path.join(source_dir, name)
if ext in (".ts", ".json"):
shutil.copyfile(path, os.path.join(target_dir, name))
git("add", kibana_directory)
git("commit", "-S", "-m", message)
git("status", show_output=True)
click.echo(f"Kibana repository {local_repo} prepped. Push changes when ready")
click.secho(f"cd {local_repo}", bold=True)
except subprocess.CalledProcessError as exc:
ctx.exit(exc.returncode)
def create_rule(path, config, required_only, rule_type):
"""Create a detection rule."""
config = load_dump(config) if config else {}
try:
return Rule.build(path,
rule_type=rule_type,
required_only=required_only,
save=True,
**config)
finally:
rule_loader.reset()
def rule_event_search(ctx,
rule_file,
rule_id,
date_range,
count,
max_results,
verbose,
elasticsearch_client: Elasticsearch = None):
"""Search using a rule file against an Elasticsearch instance."""
rule: TOMLRule
if rule_id:
rule = get_rule(rule_id, verbose=False)
elif rule_file:
rule = TOMLRule(path=rule_file,
contents=TOMLRuleContents.from_dict(
load_dump(rule_file)))
else:
client_error('Must specify a rule file or rule ID')
if isinstance(rule.contents.data, BaseQueryRuleData):
if verbose:
click.echo(f'Searching rule: {rule.name}')
data = rule.contents.data
rule_lang = data.language
if rule_lang == 'kuery':
language_flag = None
elif rule_lang == 'eql':
language_flag = True
else:
language_flag = False
index = data.index or ['*']
ctx.invoke(event_search,
query=data.query,
index=index,
language=language_flag,
date_range=date_range,
count=count,
max_results=max_results,
verbose=verbose,
elasticsearch_client=elasticsearch_client)
else:
client_error('Rule is not a query rule!')
def view_rule(rule_id, rule_file, as_api):
"""View an internal rule or specified rule file."""
if rule_id:
rule = rule_loader.get_rule(rule_id, verbose=False)
elif rule_file:
rule = Rule(rule_file, load_dump(rule_file))
else:
click.secho('Unknown rule!', fg='red')
return
if not rule:
click.secho('Unknown format!', fg='red')
return
click.echo(
toml_write(rule.rule_format()) if not as_api else json.
dumps(rule.contents, indent=2, sort_keys=True))
return rule
def rule_event_search(ctx,
rule_file,
rule_id,
date_range,
count,
max_results,
verbose,
elasticsearch_client: Elasticsearch = None):
"""Search using a rule file against an Elasticsearch instance."""
rule = None
if rule_id:
rule = get_rule(rule_id, verbose=False)
elif rule_file:
rule = Rule(rule_file, load_dump(rule_file))
else:
client_error('Must specify a rule file or rule ID')
if rule.query and rule.contents.get('language'):
if verbose:
click.echo(f'Searching rule: {rule.name}')
rule_lang = rule.contents.get('language')
if rule_lang == 'kuery':
language = None
elif rule_lang == 'eql':
language = True
else:
language = False
ctx.invoke(event_search,
query=rule.query,
index=rule.contents.get('index', ['*']),
language=language,
date_range=date_range,
count=count,
max_results=max_results,
verbose=verbose,
elasticsearch_client=elasticsearch_client)
else:
client_error('Rule is not a query rule!')
def build_release(config_file,
update_version_lock,
release=None,
verbose=True):
"""Assemble all the rules into Kibana-ready release files."""
config = load_dump(config_file)['package']
if release is not None:
config['release'] = release
if verbose:
click.echo('[+] Building package {}'.format(config.get('name')))
package = Package.from_config(config,
update_version_lock=update_version_lock,
verbose=verbose)
package.save(verbose=verbose)
if verbose:
package.get_package_hash(verbose=True)
click.echo(f'- {len(package.rules)} rules included')
return package
