def run_verifier(rules_path, config_file_path): """Run an sigma parsing test on a dir and returns results from the run. Args: rules_path: the path to the rules. config_file_path: the path to a config file that contains mapping data. Raises: IOError: if the path to either test or analyzer file does not exist or if the analyzer module or class cannot be loaded. Returns: a tuple of lists: - sigma_verified_rules with rules that can be added - sigma_rules_with_problems with rules that should not be added """ if not os.path.isdir(rules_path): raise IOError('Rules not found at path: {0:s}'.format(rules_path)) if not os.path.isfile(config_file_path): raise IOError('Config file path not found at path: {0:s}'.format( config_file_path)) sigma_config_path = config_file_path with open(sigma_config_path, 'r') as sigma_config_file: sigma_config_con = sigma_config_file.read() sigma_config = sigma_configuration.SigmaConfiguration(sigma_config_con) backend = sigma_elasticsearch.ElasticsearchQuerystringBackend( sigma_config, {}) return_verified_rules = [] return_rules_with_problems = [] for dirpath, dirnames, files in os.walk(rules_path): if 'deprecated' in [x.lower() for x in dirnames]: dirnames.remove('deprecated') logger.info('deprecated in folder / filename found - ignored') for rule_filename in files: if not rule_filename.lower().endswith(RULE_EXTENSIONS): continue # If a sub dir is found, skip it if os.path.isdir(os.path.join(rules_path, rule_filename)): logger.debug( 'Directory found, skipping: {0:s}'.format(rule_filename)) continue rule_file_path = os.path.join(dirpath, rule_filename) rule_file_path = os.path.abspath(rule_file_path) if verify_rules_file(rule_file_path, sigma_config, backend): return_verified_rules.append(rule_file_path) else: logger.info('File did not work{0:s}'.format(rule_file_path)) return_rules_with_problems.append(rule_file_path) return return_verified_rules, return_rules_with_problems
def get_sigma_rule_by_text(rule_text, sigma_config=None): """Returns a JSON represenation for a rule Args: rule_text: Text of the sigma rule to be parsed sigma_config: config file object Returns: Json representation of the parsed rule Raises: sigma_exceptions.SigmaParseError: Issue with parsing the given rule yaml.parser.ParserError: Not a correct YAML text provided NotImplementedError: A feature in the provided Sigma rule is not implemented in Sigma for Timesketch """ if sigma_config is None: sigma_config = get_sigma_config_file() sigma_backend = sigma_es.ElasticsearchQuerystringBackend(sigma_config, {}) rule_return = {} # TODO check if input validation is needed / useful. try: parser = sigma_collection.SigmaCollectionParser( rule_text, sigma_config, None) parsed_sigma_rules = parser.generate(sigma_backend) rule_yaml_data = yaml.safe_load_all(rule_text) for doc in rule_yaml_data: rule_return.update(doc) except NotImplementedError as exception: logger.error('Error generating rule {0!s}'.format(exception)) raise except sigma_exceptions.SigmaParseError as exception: logger.error( 'Sigma parsing error generating rule {0!s}'.format(exception)) raise except yaml.parser.ParserError as exception: logger.error( 'Yaml parsing error generating rule {0!s}'.format(exception)) raise sigma_es_query = '' for sigma_rule in parsed_sigma_rules: sigma_es_query = sigma_rule rule_return.update({'es_query': sigma_es_query}) rule_return.update({'file_name': 'N/A'}) rule_return.update({'file_relpath': 'N/A'}) return rule_return
def run(self): """Entry point for the analyzer. Returns: String with summary of the analyzer result. """ sigma_backend = sigma_elasticsearch.ElasticsearchQuerystringBackend( self.sigma_config, {}) tags_applied = {} rules_path = os.path.join(os.path.dirname(__file__), self._RULES_PATH) for rule_filename in os.listdir(rules_path): tag_name, _ = rule_filename.rsplit('.') tags_applied[tag_name] = 0 rule_file_path = os.path.join(rules_path, rule_filename) rule_file_path = os.path.abspath(rule_file_path) logging.info( '[sigma] Reading rules from {0!s}'.format(rule_file_path)) with open(rule_file_path, 'r') as rule_file: rule_file_content = rule_file.read() parser = sigma_collection.SigmaCollectionParser( rule_file_content, self.sigma_config, None) try: results = parser.generate(sigma_backend) except NotImplementedError as exception: logging.error( 'Error generating rule in file {0:s}: {1!s}'.format( rule_file_path, exception)) continue for result in results: logging.info( '[sigma] Generated query {0:s}'.format(result)) number_of_tagged_events = self.run_sigma_rule( result, tag_name) tags_applied[tag_name] += number_of_tagged_events total_tagged_events = sum(tags_applied.values()) output_string = 'Applied {0:d} tags\n'.format(total_tagged_events) for tag_name, number_of_tagged_events in tags_applied.items(): output_string += '* {0:s}: {1:d}'.format(tag_name, number_of_tagged_events) return output_string
def run(self): """Entry point for the analyzer. Returns: String with summary of the analyzer result. """ sigma_backend = sigma_elasticsearch.ElasticsearchQuerystringBackend( self.sigma_config, {}) tags_applied = {} sigma_rule_counter = 0 rules_path = os.path.join(os.path.dirname(__file__), self._RULES_PATH) for dirpath, dirnames, files in os.walk(rules_path): if 'deprecated' in dirnames: dirnames.remove('deprecated') for rule_filename in files: if rule_filename.lower().endswith('yml'): # if a sub dir is found, append it to be scanned for rules if os.path.isdir(os.path.join(rules_path, rule_filename)): logger.error( 'this is a directory, skipping: {0:s}'.format( rule_filename)) continue tag_name, _ = rule_filename.rsplit('.') tags_applied[tag_name] = 0 rule_file_path = os.path.join(dirpath, rule_filename) rule_file_path = os.path.abspath(rule_file_path) logger.info('[sigma] Reading rules from {0!s}'.format( rule_file_path)) with codecs.open(rule_file_path, 'r', encoding='utf-8', errors='replace') as rule_file: try: rule_file_content = rule_file.read() parser = sigma_collection.SigmaCollectionParser( rule_file_content, self.sigma_config, None) parsed_sigma_rules = parser.generate(sigma_backend) except NotImplementedError as exception: logger.error( 'Error generating rule in file {0:s}: {1!s}' .format(rule_file_path, exception)) continue for sigma_rule in parsed_sigma_rules: try: sigma_rule_counter += 1 logger.info( '[sigma] Generated query {0:s}' .format(sigma_rule)) tagged_events_counter = self.run_sigma_rule( sigma_rule, tag_name) tags_applied[tag_name] += tagged_events_counter except elasticsearch.TransportError as e: logger.error( 'Timeout generating rule in file {0:s}: ' '{1!s} waiting for 10 seconds'.format( rule_file_path, e), exc_info=True) time.sleep(10) # waiting 10 seconds total_tagged_events = sum(tags_applied.values()) output_string = 'Applied {0:d} tags\n'.format(total_tagged_events) for tag_name, tagged_events_counter in tags_applied.items(): output_string += '* {0:s}: {1:d}\n'.format( tag_name, tagged_events_counter) if sigma_rule_counter > 0: view = self.sketch.add_view( view_name='Sigma Rule matches', analyzer_name=self.NAME, query_string='tag:"sigma*"') agg_params = { 'field': 'tag', 'limit': 20, } agg_obj = self.sketch.add_aggregation( name='Top 20 Sigma tags', agg_name='field_bucket', agg_params=agg_params, view_id=view.id, chart_type='hbarchart', description='Created by the Sigma analyzer') story = self.sketch.add_story("Sigma Rule hits") story.add_text( utils.SIGMA_STORY_HEADER, skip_if_exists=True) story.add_text( '## Sigma Analyzer.\n\nThe Sigma ' 'analyzer takes Events and matches them with Sigma rules.' 'In this timeline the analyzer discovered {0:d} ' 'Sigma tags.\n\nThis is a summary of ' 'it\'s findings.'.format(sigma_rule_counter)) story.add_text( 'The top 20 most commonly discovered tags were:') story.add_aggregation(agg_obj) story.add_text( 'And an overview of all the discovered search terms:') story.add_view(view) return output_string
def get_sigma_rule(filepath, sigma_config=None): """Returns a JSON represenation for a rule Args: filepath: path to the sigma rule to be parsed sigma_config: optional argument to pass a sigma.configuration.SigmaConfiguration object Returns: Json representation of the parsed rule Raises: ValueError: Parsing error IsADirectoryError: If a directory is passed as filepath """ try: if isinstance(sigma_config, sigma_configuration.SigmaConfiguration): sigma_conf_obj = sigma_config elif isinstance(sigma_config, str): sigma_conf_obj = get_sigma_config_file(sigma_config) else: sigma_conf_obj = get_sigma_config_file() except ValueError as e: logger.error('Problem reading the Sigma config', exc_info=True) raise ValueError('Problem reading the Sigma config') from e sigma_backend = sigma_es.ElasticsearchQuerystringBackend( sigma_conf_obj, {}) try: sigma_rules_paths = get_sigma_rules_path() except ValueError: sigma_rules_paths = None if not filepath.lower().endswith('.yml'): raise ValueError(f'{filepath} does not end with .yml') # if a sub dir is found, nothing can be parsed if os.path.isdir(filepath): raise IsADirectoryError(f'{filepath} is a directory - must be a file') abs_path = os.path.abspath(filepath) with codecs.open(abs_path, 'r', encoding='utf-8', errors='replace') as file: try: rule_return = {} rule_yaml_data = yaml.safe_load_all(file.read()) for doc in rule_yaml_data: rule_return.update(doc) parser = sigma_collection.SigmaCollectionParser( yaml.safe_dump(doc), sigma_conf_obj, None) parsed_sigma_rules = parser.generate(sigma_backend) except NotImplementedError as exception: logger.error('Error generating rule in file {0:s}: {1!s}'.format( abs_path, exception)) add_problematic_rule(filepath, doc.get('id'), 'Parts of the rule not implemented in TS') return None except sigma_exceptions.SigmaParseError as exception: logger.error( 'Sigma parsing error generating rule in file {0:s}: {1!s}'. format(abs_path, exception)) add_problematic_rule(filepath, doc.get('id'), 'sigma_exceptions.SigmaParseError') return None except yaml.parser.ParserError as exception: logger.error( 'Yaml parsing error generating rule in file {0:s}: {1!s}'. format(abs_path, exception)) add_problematic_rule(filepath, None, 'yaml.parser.ParserError') return None sigma_es_query = '' for sigma_rule in parsed_sigma_rules: sigma_es_query = _sanatize_sigma_rule(sigma_rule) rule_return.update({'es_query': sigma_es_query}) rule_return.update({'file_name': os.path.basename(filepath)}) # in case multiple folders are in the config, need to remove them if sigma_rules_paths: for rule_path in sigma_rules_paths: file_relpath = os.path.relpath(filepath, rule_path) else: file_relpath = 'N/A' rule_return.update({'file_relpath': file_relpath}) return rule_return
def get_sigma_rule(filepath, sigma_config=None): """ Returns a JSON represenation for a rule Args: filepath: path to the sigma rule to be parsed sigma_config: optional argument to pass a sigma.configuration.SigmaConfiguration object Returns: Json representation of the parsed rule """ try: if sigma_config: sigma_conf_obj = sigma_config else: sigma_conf_obj = get_sigma_config_file() except ValueError as e: logger.error('Problem reading the Sigma config {0:s}: '.format(e), exc_info=True) return None sigma_backend = sigma_es.ElasticsearchQuerystringBackend( sigma_conf_obj, {}) try: sigma_rules_paths = get_sigma_rules_path() except ValueError: sigma_rules_paths = None if not filepath.lower().endswith('.yml'): return None # if a sub dir is found, nothing can be parsed if os.path.isdir(filepath): return None abs_path = os.path.abspath(filepath) with codecs.open(abs_path, 'r', encoding='utf-8', errors='replace') as file: try: rule_return = {} rule_yaml_data = yaml.safe_load_all(file.read()) for doc in rule_yaml_data: rule_return.update(doc) parser = sigma_collection.SigmaCollectionParser( str(doc), sigma_config, None) parsed_sigma_rules = parser.generate(sigma_backend) except NotImplementedError as exception: logger.error('Error generating rule in file {0:s}: {1!s}'.format( abs_path, exception)) return None except sigma_exceptions.SigmaParseError as exception: logger.error( 'Sigma parsing error generating rule in file {0:s}: {1!s}'. format(abs_path, exception)) return None except yaml.parser.ParserError as exception: logger.error( 'Yaml parsing error generating rule in file {0:s}: {1!s}'. format(abs_path, exception)) return None sigma_es_query = '' for sigma_rule in parsed_sigma_rules: sigma_es_query = sigma_rule rule_return.update({'es_query': sigma_es_query}) rule_return.update({'file_name': os.path.basename(filepath)}) # in case multiple folders are in the config, need to remove them if sigma_rules_paths: for rule_path in sigma_rules_paths: file_relpath = os.path.relpath(filepath, rule_path) else: file_relpath = 'N/A' rule_return.update({'file_relpath': file_relpath}) return rule_return
def get_sigma_rule_by_text(rule_text, sigma_config=None): """Returns a JSON represenation for a rule Args: rule_text: Text of the sigma rule to be parsed sigma_config: config file object Returns: Json representation of the parsed rule Raises: sigma_exceptions.SigmaParseError: Issue with parsing the given rule yaml.parser.ParserError: Not a correct YAML text provided NotImplementedError: A feature in the provided Sigma rule is not implemented in Sigma for Timesketch """ try: if isinstance(sigma_config, sigma_configuration.SigmaConfiguration): sigma_conf_obj = sigma_config elif isinstance(sigma_config, str): sigma_conf_obj = get_sigma_config_file(sigma_config) else: sigma_conf_obj = get_sigma_config_file() except ValueError as e: logger.error("Problem reading the Sigma config", exc_info=True) raise ValueError("Problem reading the Sigma config") from e sigma_backend = sigma_es.ElasticsearchQuerystringBackend( sigma_conf_obj, {} ) rule_text = sanitize_incoming_sigma_rule_text(rule_text) rule_return = {} parsed_sigma_rules = None # TODO check if input validation is needed / useful. try: rule_yaml_data = yaml.safe_load_all(rule_text) for doc in rule_yaml_data: parser = sigma_collection.SigmaCollectionParser( str(doc), sigma_conf_obj, None ) parsed_sigma_rules = parser.generate(sigma_backend) rule_return.update(doc) except NotImplementedError as exception: logger.error("Error generating rule {0!s}".format(exception)) raise except sigma_exceptions.SigmaParseError as exception: logger.error("Sigma parsing error rule {0!s}".format(exception)) raise except yaml.parser.ParserError as exception: logger.error("Yaml parsing error rule {0!s}".format(exception)) raise assert parsed_sigma_rules is not None sigma_es_query = "" for sigma_rule in parsed_sigma_rules: sigma_es_query = _sanitize_query(sigma_rule) rule_return.update({"es_query": sigma_es_query}) rule_return.update({"file_name": "N/A"}) rule_return.update({"file_relpath": "N/A"}) return rule_return
def get_sigma_rule(filepath, sigma_config=None): """Returns a JSON represenation for a rule Args: filepath: path to the sigma rule to be parsed sigma_config: optional argument to pass a sigma.configuration.SigmaConfiguration object Returns: Json representation of the parsed rule Raises: ValueError: Parsing error IsADirectoryError: If a directory is passed as filepath """ try: if isinstance(sigma_config, sigma_configuration.SigmaConfiguration): sigma_conf_obj = sigma_config elif isinstance(sigma_config, str): sigma_conf_obj = get_sigma_config_file(sigma_config) else: sigma_conf_obj = get_sigma_config_file() except ValueError as e: logger.error("Problem reading the Sigma config", exc_info=True) raise ValueError("Problem reading the Sigma config") from e sigma_backend = sigma_es.ElasticsearchQuerystringBackend( sigma_conf_obj, {} ) try: sigma_rules_paths = get_sigma_rules_path() except ValueError: sigma_rules_paths = None if not filepath.lower().endswith(".yml"): raise ValueError(f"{filepath} does not end with .yml") # if a sub dir is found, nothing can be parsed if os.path.isdir(filepath): raise IsADirectoryError(f"{filepath} is a directory - must be a file") if os.stat(filepath).st_size == 0: raise ValueError(f"{filepath} file is empty") abs_path = os.path.abspath(filepath) parsed_sigma_rules = None with codecs.open( abs_path, "r", encoding="utf-8", errors="replace" ) as file: try: rule_return = {} rule_file_content = file.read() rule_file_content = sanitize_incoming_sigma_rule_text( rule_file_content ) rule_yaml_data = yaml.safe_load_all(rule_file_content) for doc in rule_yaml_data: rule_return.update(doc) parser = sigma_collection.SigmaCollectionParser( yaml.safe_dump(doc), sigma_conf_obj, None ) parsed_sigma_rules = parser.generate(sigma_backend) except NotImplementedError as exception: logger.error("Error rule {0:s}: {1!s}".format(abs_path, exception)) add_problematic_rule( filepath, doc.get("id"), "Part of the rule not supported in TS" ) return None except sigma_exceptions.SigmaParseError as exception: logger.error( "Sigma parsing error rule in file {0:s}: {1!s}".format( abs_path, exception ) ) add_problematic_rule( filepath, doc.get("id"), "sigma_exceptions.SigmaParseError" ) return None except yaml.parser.ParserError as exception: logger.error( "Yaml parsing error rule in file {0:s}: {1!s}".format( abs_path, exception ) ) add_problematic_rule(filepath, None, "yaml.parser.ParserError") return None sigma_es_query = "" assert parsed_sigma_rules is not None for sigma_rule in parsed_sigma_rules: sigma_es_query = _sanitize_query(sigma_rule) rule_return.update({"es_query": sigma_es_query}) rule_return.update({"file_name": os.path.basename(filepath)}) # in case multiple folders are in the config, need to remove them if sigma_rules_paths: for rule_path in sigma_rules_paths: file_relpath = os.path.relpath(filepath, rule_path) else: file_relpath = "N/A" rule_return.update({"file_relpath": file_relpath}) return rule_return