def run_verifier(rules_path, config_file_path):
"""Run an sigma parsing test on a dir and returns results from the run.
Args:
rules_path: the path to the rules.
config_file_path: the path to a config file that contains mapping data.
Raises:
IOError: if the path to either test or analyzer file does not exist
or if the analyzer module or class cannot be loaded.
Returns:
a tuple of lists:
- sigma_verified_rules with rules that can be added
- sigma_rules_with_problems with rules that should not be added
"""
if not os.path.isdir(rules_path):
raise IOError('Rules not found at path: {0:s}'.format(rules_path))
if not os.path.isfile(config_file_path):
raise IOError('Config file path not found at path: {0:s}'.format(
config_file_path))
sigma_config_path = config_file_path
with open(sigma_config_path, 'r') as sigma_config_file:
sigma_config_con = sigma_config_file.read()
sigma_config = sigma_configuration.SigmaConfiguration(sigma_config_con)
backend = sigma_elasticsearch.ElasticsearchQuerystringBackend(
sigma_config, {})
return_verified_rules = []
return_rules_with_problems = []
for dirpath, dirnames, files in os.walk(rules_path):
if 'deprecated' in [x.lower() for x in dirnames]:
dirnames.remove('deprecated')
logger.info('deprecated in folder / filename found - ignored')
for rule_filename in files:
if not rule_filename.lower().endswith(RULE_EXTENSIONS):
continue
# If a sub dir is found, skip it
if os.path.isdir(os.path.join(rules_path, rule_filename)):
logger.debug(
'Directory found, skipping: {0:s}'.format(rule_filename))
continue
rule_file_path = os.path.join(dirpath, rule_filename)
rule_file_path = os.path.abspath(rule_file_path)
if verify_rules_file(rule_file_path, sigma_config, backend):
return_verified_rules.append(rule_file_path)
else:
logger.info('File did not work{0:s}'.format(rule_file_path))
return_rules_with_problems.append(rule_file_path)
return return_verified_rules, return_rules_with_problems
def get_sigma_rule_by_text(rule_text, sigma_config=None):
"""Returns a JSON represenation for a rule
Args:
rule_text: Text of the sigma rule to be parsed
sigma_config: config file object
Returns:
Json representation of the parsed rule
Raises:
sigma_exceptions.SigmaParseError: Issue with parsing the given rule
yaml.parser.ParserError: Not a correct YAML text provided
NotImplementedError: A feature in the provided Sigma rule is not
implemented in Sigma for Timesketch
"""
if sigma_config is None:
sigma_config = get_sigma_config_file()
sigma_backend = sigma_es.ElasticsearchQuerystringBackend(sigma_config, {})
rule_return = {}
# TODO check if input validation is needed / useful.
try:
parser = sigma_collection.SigmaCollectionParser(
rule_text, sigma_config, None)
parsed_sigma_rules = parser.generate(sigma_backend)
rule_yaml_data = yaml.safe_load_all(rule_text)
for doc in rule_yaml_data:
rule_return.update(doc)
except NotImplementedError as exception:
logger.error('Error generating rule {0!s}'.format(exception))
raise
except sigma_exceptions.SigmaParseError as exception:
logger.error(
'Sigma parsing error generating rule {0!s}'.format(exception))
raise
except yaml.parser.ParserError as exception:
logger.error(
'Yaml parsing error generating rule {0!s}'.format(exception))
raise
sigma_es_query = ''
for sigma_rule in parsed_sigma_rules:
sigma_es_query = sigma_rule
rule_return.update({'es_query': sigma_es_query})
rule_return.update({'file_name': 'N/A'})
rule_return.update({'file_relpath': 'N/A'})
return rule_return
def run(self):
"""Entry point for the analyzer.
Returns:
String with summary of the analyzer result.
"""
sigma_backend = sigma_elasticsearch.ElasticsearchQuerystringBackend(
self.sigma_config, {})
tags_applied = {}
rules_path = os.path.join(os.path.dirname(__file__), self._RULES_PATH)
for rule_filename in os.listdir(rules_path):
tag_name, _ = rule_filename.rsplit('.')
tags_applied[tag_name] = 0
rule_file_path = os.path.join(rules_path, rule_filename)
rule_file_path = os.path.abspath(rule_file_path)
logging.info(
'[sigma] Reading rules from {0!s}'.format(rule_file_path))
with open(rule_file_path, 'r') as rule_file:
rule_file_content = rule_file.read()
parser = sigma_collection.SigmaCollectionParser(
rule_file_content, self.sigma_config, None)
try:
results = parser.generate(sigma_backend)
except NotImplementedError as exception:
logging.error(
'Error generating rule in file {0:s}: {1!s}'.format(
rule_file_path, exception))
continue
for result in results:
logging.info(
'[sigma] Generated query {0:s}'.format(result))
number_of_tagged_events = self.run_sigma_rule(
result, tag_name)
tags_applied[tag_name] += number_of_tagged_events
total_tagged_events = sum(tags_applied.values())
output_string = 'Applied {0:d} tags\n'.format(total_tagged_events)
for tag_name, number_of_tagged_events in tags_applied.items():
output_string += '* {0:s}: {1:d}'.format(tag_name,
number_of_tagged_events)
return output_string
def run(self):
"""Entry point for the analyzer.
Returns:
String with summary of the analyzer result.
"""
sigma_backend = sigma_elasticsearch.ElasticsearchQuerystringBackend(
self.sigma_config, {})
tags_applied = {}
sigma_rule_counter = 0
rules_path = os.path.join(os.path.dirname(__file__), self._RULES_PATH)
for dirpath, dirnames, files in os.walk(rules_path):
if 'deprecated' in dirnames:
dirnames.remove('deprecated')
for rule_filename in files:
if rule_filename.lower().endswith('yml'):
# if a sub dir is found, append it to be scanned for rules
if os.path.isdir(os.path.join(rules_path, rule_filename)):
logger.error(
'this is a directory, skipping: {0:s}'.format(
rule_filename))
continue
tag_name, _ = rule_filename.rsplit('.')
tags_applied[tag_name] = 0
rule_file_path = os.path.join(dirpath, rule_filename)
rule_file_path = os.path.abspath(rule_file_path)
logger.info('[sigma] Reading rules from {0!s}'.format(
rule_file_path))
with codecs.open(rule_file_path, 'r', encoding='utf-8',
errors='replace') as rule_file:
try:
rule_file_content = rule_file.read()
parser = sigma_collection.SigmaCollectionParser(
rule_file_content, self.sigma_config, None)
parsed_sigma_rules = parser.generate(sigma_backend)
except NotImplementedError as exception:
logger.error(
'Error generating rule in file {0:s}: {1!s}'
.format(rule_file_path, exception))
continue
for sigma_rule in parsed_sigma_rules:
try:
sigma_rule_counter += 1
logger.info(
'[sigma] Generated query {0:s}'
.format(sigma_rule))
tagged_events_counter = self.run_sigma_rule(
sigma_rule, tag_name)
tags_applied[tag_name] += tagged_events_counter
except elasticsearch.TransportError as e:
logger.error(
'Timeout generating rule in file {0:s}: '
'{1!s} waiting for 10 seconds'.format(
rule_file_path, e), exc_info=True)
time.sleep(10) # waiting 10 seconds
total_tagged_events = sum(tags_applied.values())
output_string = 'Applied {0:d} tags\n'.format(total_tagged_events)
for tag_name, tagged_events_counter in tags_applied.items():
output_string += '* {0:s}: {1:d}\n'.format(
tag_name, tagged_events_counter)
if sigma_rule_counter > 0:
view = self.sketch.add_view(
view_name='Sigma Rule matches', analyzer_name=self.NAME,
query_string='tag:"sigma*"')
agg_params = {
'field': 'tag',
'limit': 20,
}
agg_obj = self.sketch.add_aggregation(
name='Top 20 Sigma tags', agg_name='field_bucket',
agg_params=agg_params, view_id=view.id, chart_type='hbarchart',
description='Created by the Sigma analyzer')
story = self.sketch.add_story("Sigma Rule hits")
story.add_text(
utils.SIGMA_STORY_HEADER, skip_if_exists=True)
story.add_text(
'## Sigma Analyzer.\n\nThe Sigma '
'analyzer takes Events and matches them with Sigma rules.'
'In this timeline the analyzer discovered {0:d} '
'Sigma tags.\n\nThis is a summary of '
'it\'s findings.'.format(sigma_rule_counter))
story.add_text(
'The top 20 most commonly discovered tags were:')
story.add_aggregation(agg_obj)
story.add_text(
'And an overview of all the discovered search terms:')
story.add_view(view)
return output_string
def get_sigma_rule(filepath, sigma_config=None):
"""Returns a JSON represenation for a rule
Args:
filepath: path to the sigma rule to be parsed
sigma_config: optional argument to pass a
sigma.configuration.SigmaConfiguration object
Returns:
Json representation of the parsed rule
Raises:
ValueError: Parsing error
IsADirectoryError: If a directory is passed as filepath
"""
try:
if isinstance(sigma_config, sigma_configuration.SigmaConfiguration):
sigma_conf_obj = sigma_config
elif isinstance(sigma_config, str):
sigma_conf_obj = get_sigma_config_file(sigma_config)
else:
sigma_conf_obj = get_sigma_config_file()
except ValueError as e:
logger.error('Problem reading the Sigma config', exc_info=True)
raise ValueError('Problem reading the Sigma config') from e
sigma_backend = sigma_es.ElasticsearchQuerystringBackend(
sigma_conf_obj, {})
try:
sigma_rules_paths = get_sigma_rules_path()
except ValueError:
sigma_rules_paths = None
if not filepath.lower().endswith('.yml'):
raise ValueError(f'{filepath} does not end with .yml')
# if a sub dir is found, nothing can be parsed
if os.path.isdir(filepath):
raise IsADirectoryError(f'{filepath} is a directory - must be a file')
abs_path = os.path.abspath(filepath)
with codecs.open(abs_path, 'r', encoding='utf-8',
errors='replace') as file:
try:
rule_return = {}
rule_yaml_data = yaml.safe_load_all(file.read())
for doc in rule_yaml_data:
rule_return.update(doc)
parser = sigma_collection.SigmaCollectionParser(
yaml.safe_dump(doc), sigma_conf_obj, None)
parsed_sigma_rules = parser.generate(sigma_backend)
except NotImplementedError as exception:
logger.error('Error generating rule in file {0:s}: {1!s}'.format(
abs_path, exception))
add_problematic_rule(filepath, doc.get('id'),
'Parts of the rule not implemented in TS')
return None
except sigma_exceptions.SigmaParseError as exception:
logger.error(
'Sigma parsing error generating rule in file {0:s}: {1!s}'.
format(abs_path, exception))
add_problematic_rule(filepath, doc.get('id'),
'sigma_exceptions.SigmaParseError')
return None
except yaml.parser.ParserError as exception:
logger.error(
'Yaml parsing error generating rule in file {0:s}: {1!s}'.
format(abs_path, exception))
add_problematic_rule(filepath, None, 'yaml.parser.ParserError')
return None
sigma_es_query = ''
for sigma_rule in parsed_sigma_rules:
sigma_es_query = _sanatize_sigma_rule(sigma_rule)
rule_return.update({'es_query': sigma_es_query})
rule_return.update({'file_name': os.path.basename(filepath)})
# in case multiple folders are in the config, need to remove them
if sigma_rules_paths:
for rule_path in sigma_rules_paths:
file_relpath = os.path.relpath(filepath, rule_path)
else:
file_relpath = 'N/A'
rule_return.update({'file_relpath': file_relpath})
return rule_return
def get_sigma_rule(filepath, sigma_config=None):
""" Returns a JSON represenation for a rule
Args:
filepath: path to the sigma rule to be parsed
sigma_config: optional argument to pass a
sigma.configuration.SigmaConfiguration object
Returns:
Json representation of the parsed rule
"""
try:
if sigma_config:
sigma_conf_obj = sigma_config
else:
sigma_conf_obj = get_sigma_config_file()
except ValueError as e:
logger.error('Problem reading the Sigma config {0:s}: '.format(e),
exc_info=True)
return None
sigma_backend = sigma_es.ElasticsearchQuerystringBackend(
sigma_conf_obj, {})
try:
sigma_rules_paths = get_sigma_rules_path()
except ValueError:
sigma_rules_paths = None
if not filepath.lower().endswith('.yml'):
return None
# if a sub dir is found, nothing can be parsed
if os.path.isdir(filepath):
return None
abs_path = os.path.abspath(filepath)
with codecs.open(abs_path, 'r', encoding='utf-8',
errors='replace') as file:
try:
rule_return = {}
rule_yaml_data = yaml.safe_load_all(file.read())
for doc in rule_yaml_data:
rule_return.update(doc)
parser = sigma_collection.SigmaCollectionParser(
str(doc), sigma_config, None)
parsed_sigma_rules = parser.generate(sigma_backend)
except NotImplementedError as exception:
logger.error('Error generating rule in file {0:s}: {1!s}'.format(
abs_path, exception))
return None
except sigma_exceptions.SigmaParseError as exception:
logger.error(
'Sigma parsing error generating rule in file {0:s}: {1!s}'.
format(abs_path, exception))
return None
except yaml.parser.ParserError as exception:
logger.error(
'Yaml parsing error generating rule in file {0:s}: {1!s}'.
format(abs_path, exception))
return None
sigma_es_query = ''
for sigma_rule in parsed_sigma_rules:
sigma_es_query = sigma_rule
rule_return.update({'es_query': sigma_es_query})
rule_return.update({'file_name': os.path.basename(filepath)})
# in case multiple folders are in the config, need to remove them
if sigma_rules_paths:
for rule_path in sigma_rules_paths:
file_relpath = os.path.relpath(filepath, rule_path)
else:
file_relpath = 'N/A'
rule_return.update({'file_relpath': file_relpath})
return rule_return
def get_sigma_rule_by_text(rule_text, sigma_config=None):
"""Returns a JSON represenation for a rule
Args:
rule_text: Text of the sigma rule to be parsed
sigma_config: config file object
Returns:
Json representation of the parsed rule
Raises:
sigma_exceptions.SigmaParseError: Issue with parsing the given rule
yaml.parser.ParserError: Not a correct YAML text provided
NotImplementedError: A feature in the provided Sigma rule is not
implemented in Sigma for Timesketch
"""
try:
if isinstance(sigma_config, sigma_configuration.SigmaConfiguration):
sigma_conf_obj = sigma_config
elif isinstance(sigma_config, str):
sigma_conf_obj = get_sigma_config_file(sigma_config)
else:
sigma_conf_obj = get_sigma_config_file()
except ValueError as e:
logger.error("Problem reading the Sigma config", exc_info=True)
raise ValueError("Problem reading the Sigma config") from e
sigma_backend = sigma_es.ElasticsearchQuerystringBackend(
sigma_conf_obj, {}
)
rule_text = sanitize_incoming_sigma_rule_text(rule_text)
rule_return = {}
parsed_sigma_rules = None
# TODO check if input validation is needed / useful.
try:
rule_yaml_data = yaml.safe_load_all(rule_text)
for doc in rule_yaml_data:
parser = sigma_collection.SigmaCollectionParser(
str(doc), sigma_conf_obj, None
)
parsed_sigma_rules = parser.generate(sigma_backend)
rule_return.update(doc)
except NotImplementedError as exception:
logger.error("Error generating rule {0!s}".format(exception))
raise
except sigma_exceptions.SigmaParseError as exception:
logger.error("Sigma parsing error rule {0!s}".format(exception))
raise
except yaml.parser.ParserError as exception:
logger.error("Yaml parsing error rule {0!s}".format(exception))
raise
assert parsed_sigma_rules is not None
sigma_es_query = ""
for sigma_rule in parsed_sigma_rules:
sigma_es_query = _sanitize_query(sigma_rule)
rule_return.update({"es_query": sigma_es_query})
rule_return.update({"file_name": "N/A"})
rule_return.update({"file_relpath": "N/A"})
return rule_return
def get_sigma_rule(filepath, sigma_config=None):
"""Returns a JSON represenation for a rule
Args:
filepath: path to the sigma rule to be parsed
sigma_config: optional argument to pass a
sigma.configuration.SigmaConfiguration object
Returns:
Json representation of the parsed rule
Raises:
ValueError: Parsing error
IsADirectoryError: If a directory is passed as filepath
"""
try:
if isinstance(sigma_config, sigma_configuration.SigmaConfiguration):
sigma_conf_obj = sigma_config
elif isinstance(sigma_config, str):
sigma_conf_obj = get_sigma_config_file(sigma_config)
else:
sigma_conf_obj = get_sigma_config_file()
except ValueError as e:
logger.error("Problem reading the Sigma config", exc_info=True)
raise ValueError("Problem reading the Sigma config") from e
sigma_backend = sigma_es.ElasticsearchQuerystringBackend(
sigma_conf_obj, {}
)
try:
sigma_rules_paths = get_sigma_rules_path()
except ValueError:
sigma_rules_paths = None
if not filepath.lower().endswith(".yml"):
raise ValueError(f"{filepath} does not end with .yml")
# if a sub dir is found, nothing can be parsed
if os.path.isdir(filepath):
raise IsADirectoryError(f"{filepath} is a directory - must be a file")
if os.stat(filepath).st_size == 0:
raise ValueError(f"{filepath} file is empty")
abs_path = os.path.abspath(filepath)
parsed_sigma_rules = None
with codecs.open(
abs_path, "r", encoding="utf-8", errors="replace"
) as file:
try:
rule_return = {}
rule_file_content = file.read()
rule_file_content = sanitize_incoming_sigma_rule_text(
rule_file_content
)
rule_yaml_data = yaml.safe_load_all(rule_file_content)
for doc in rule_yaml_data:
rule_return.update(doc)
parser = sigma_collection.SigmaCollectionParser(
yaml.safe_dump(doc), sigma_conf_obj, None
)
parsed_sigma_rules = parser.generate(sigma_backend)
except NotImplementedError as exception:
logger.error("Error rule {0:s}: {1!s}".format(abs_path, exception))
add_problematic_rule(
filepath, doc.get("id"), "Part of the rule not supported in TS"
)
return None
except sigma_exceptions.SigmaParseError as exception:
logger.error(
"Sigma parsing error rule in file {0:s}: {1!s}".format(
abs_path, exception
)
)
add_problematic_rule(
filepath, doc.get("id"), "sigma_exceptions.SigmaParseError"
)
return None
except yaml.parser.ParserError as exception:
logger.error(
"Yaml parsing error rule in file {0:s}: {1!s}".format(
abs_path, exception
)
)
add_problematic_rule(filepath, None, "yaml.parser.ParserError")
return None
sigma_es_query = ""
assert parsed_sigma_rules is not None
for sigma_rule in parsed_sigma_rules:
sigma_es_query = _sanitize_query(sigma_rule)
rule_return.update({"es_query": sigma_es_query})
rule_return.update({"file_name": os.path.basename(filepath)})
# in case multiple folders are in the config, need to remove them
if sigma_rules_paths:
for rule_path in sigma_rules_paths:
file_relpath = os.path.relpath(filepath, rule_path)
else:
file_relpath = "N/A"
rule_return.update({"file_relpath": file_relpath})
return rule_return