def load_tests(): """Load up tests from toml to prepare for pytest.mark.parametrize.""" if fold_tests or verifier_failures: return for name in ["test_string_functions.toml", "test_folding.toml", "test_optimizer.toml"]: data = eql.load_dump(eql.etc.get_etc_path(name)) for test_name, contents in sorted(data.items()): test_name = "{file}:{test}".format(file=name, test=test_name) case_settings = [] if "case_sensitive" not in contents and "case_insensitive" not in contents: case_sensitive = True case_insensitive = True else: case_sensitive = contents.get("case_sensitive") is True case_insensitive = contents.get("case_insensitive") is True if case_sensitive: case_settings.append(True) if case_insensitive: case_settings.append(False) assert len(case_settings) > 0, "{test} is missing case_sensitive/case_insensitive".format(test=test_name) extract_tests(test_name, contents, case_settings)
def build_release(config_file, update_version_lock): """Assemble all the rules into Kibana-ready release files.""" config = load_dump(config_file)['package'] click.echo('[+] Building package {}'.format(config.get('name'))) package = Package.from_config(config, update_version_lock=update_version_lock) package.save() package.get_package_hash(verbose=True) click.echo('- {} rules included'.format(len(package.rules)))
def load_from_file(infile, directory): """Load rules from file(s).""" if infile: for rule_file in infile: rule_path = os.path.join(RULES_DIR, os.path.basename(rule_file)) rule = Rule(rule_path, load_dump(rule_file)) rule.save(as_rule=True, verbose=True) elif directory: for rule_file in glob.glob(os.path.join(directory, '**', '*.*'), recursive=True): try: rule_path = os.path.join(RULES_DIR, os.path.basename(rule_file)) rule = Rule(rule_path, load_dump(rule_file)) rule.save(as_rule=True, verbose=True) except ValueError: click.echo('Unable to load file: {}'.format(rule_file)) else: click.echo('No files specified!')
def kibana_commit(ctx, local_repo, github_repo, ssh, kibana_directory, base_branch, message): """Prep a commit and push to Kibana.""" git_exe = shutil.which("git") package_name = load_dump(PACKAGE_FILE)['package']["name"] release_dir = os.path.join(RELEASE_DIR, package_name) message = message or f"[Detection Rules] Add {package_name} rules" if not os.path.exists(release_dir): click.secho("Release directory doesn't exist.", fg="red", err=True) click.echo(f"Run {click.style('python -m detection_rules build-release', bold=True)} to populate", err=True) ctx.exit(1) if not git_exe: click.secho("Unable to find git", err=True, fg="red") ctx.exit(1) try: if not os.path.exists(local_repo): if not click.confirm(f"Kibana repository doesn't exist at {local_repo}. Clone?"): ctx.exit(1) url = f"[email protected]:{github_repo}.git" if ssh else f"https://github.com/{github_repo}.git" subprocess.check_call([git_exe, "clone", url, local_repo, "--depth", 1]) def git(*args, show_output=False): method = subprocess.call if show_output else subprocess.check_output return method([git_exe, "-C", local_repo] + list(args), encoding="utf-8") git("checkout", base_branch) git("pull") git("checkout", "-b", f"rules/{package_name}", show_output=True) git("rm", "-r", kibana_directory) source_dir = os.path.join(release_dir, "rules") target_dir = os.path.join(local_repo, kibana_directory) os.makedirs(target_dir) for name in os.listdir(source_dir): _, ext = os.path.splitext(name) path = os.path.join(source_dir, name) if ext in (".ts", ".json"): shutil.copyfile(path, os.path.join(target_dir, name)) git("add", kibana_directory) git("commit", "-S", "-m", message) git("status", show_output=True) click.echo(f"Kibana repository {local_repo} prepped. Push changes when ready") click.secho(f"cd {local_repo}", bold=True) except subprocess.CalledProcessError as exc: ctx.exit(exc.returncode)
def create_rule(path, config, required_only, rule_type): """Create a detection rule.""" config = load_dump(config) if config else {} try: return Rule.build(path, rule_type=rule_type, required_only=required_only, save=True, **config) finally: rule_loader.reset()
def rule_event_search(ctx, rule_file, rule_id, date_range, count, max_results, verbose, elasticsearch_client: Elasticsearch = None): """Search using a rule file against an Elasticsearch instance.""" rule: TOMLRule if rule_id: rule = get_rule(rule_id, verbose=False) elif rule_file: rule = TOMLRule(path=rule_file, contents=TOMLRuleContents.from_dict( load_dump(rule_file))) else: client_error('Must specify a rule file or rule ID') if isinstance(rule.contents.data, BaseQueryRuleData): if verbose: click.echo(f'Searching rule: {rule.name}') data = rule.contents.data rule_lang = data.language if rule_lang == 'kuery': language_flag = None elif rule_lang == 'eql': language_flag = True else: language_flag = False index = data.index or ['*'] ctx.invoke(event_search, query=data.query, index=index, language=language_flag, date_range=date_range, count=count, max_results=max_results, verbose=verbose, elasticsearch_client=elasticsearch_client) else: client_error('Rule is not a query rule!')
def view_rule(rule_id, rule_file, as_api): """View an internal rule or specified rule file.""" if rule_id: rule = rule_loader.get_rule(rule_id, verbose=False) elif rule_file: rule = Rule(rule_file, load_dump(rule_file)) else: click.secho('Unknown rule!', fg='red') return if not rule: click.secho('Unknown format!', fg='red') return click.echo( toml_write(rule.rule_format()) if not as_api else json. dumps(rule.contents, indent=2, sort_keys=True)) return rule
def rule_event_search(ctx, rule_file, rule_id, date_range, count, max_results, verbose, elasticsearch_client: Elasticsearch = None): """Search using a rule file against an Elasticsearch instance.""" rule = None if rule_id: rule = get_rule(rule_id, verbose=False) elif rule_file: rule = Rule(rule_file, load_dump(rule_file)) else: client_error('Must specify a rule file or rule ID') if rule.query and rule.contents.get('language'): if verbose: click.echo(f'Searching rule: {rule.name}') rule_lang = rule.contents.get('language') if rule_lang == 'kuery': language = None elif rule_lang == 'eql': language = True else: language = False ctx.invoke(event_search, query=rule.query, index=rule.contents.get('index', ['*']), language=language, date_range=date_range, count=count, max_results=max_results, verbose=verbose, elasticsearch_client=elasticsearch_client) else: client_error('Rule is not a query rule!')
def build_release(config_file, update_version_lock, release=None, verbose=True): """Assemble all the rules into Kibana-ready release files.""" config = load_dump(config_file)['package'] if release is not None: config['release'] = release if verbose: click.echo('[+] Building package {}'.format(config.get('name'))) package = Package.from_config(config, update_version_lock=update_version_lock, verbose=verbose) package.save(verbose=verbose) if verbose: package.get_package_hash(verbose=True) click.echo(f'- {len(package.rules)} rules included') return package