Esempio n. 1
0
def run_verifier(rules_path, config_file_path):
    """Run an sigma parsing test on a dir and returns results from the run.

    Args:
        rules_path: the path to the rules.
        config_file_path: the path to a config file that contains mapping data.

    Raises:
        IOError: if the path to either test or analyzer file does not exist
                 or if the analyzer module or class cannot be loaded.

    Returns:
        a tuple of lists:
            - sigma_verified_rules with rules that can be added
            - sigma_rules_with_problems with rules that should not be added
    """
    if not os.path.isdir(rules_path):
        raise IOError('Rules not found at path: {0:s}'.format(rules_path))
    if not os.path.isfile(config_file_path):
        raise IOError('Config file path not found at path: {0:s}'.format(
            config_file_path))

    sigma_config_path = config_file_path

    with open(sigma_config_path, 'r') as sigma_config_file:
        sigma_config_con = sigma_config_file.read()
    sigma_config = sigma_configuration.SigmaConfiguration(sigma_config_con)
    backend = sigma_elasticsearch.ElasticsearchQuerystringBackend(
        sigma_config, {})
    return_verified_rules = []
    return_rules_with_problems = []

    for dirpath, dirnames, files in os.walk(rules_path):

        if 'deprecated' in [x.lower() for x in dirnames]:
            dirnames.remove('deprecated')
            logger.info('deprecated in folder / filename found - ignored')

        for rule_filename in files:
            if not rule_filename.lower().endswith(RULE_EXTENSIONS):
                continue

            # If a sub dir is found, skip it
            if os.path.isdir(os.path.join(rules_path, rule_filename)):
                logger.debug(
                    'Directory found, skipping: {0:s}'.format(rule_filename))
                continue

            rule_file_path = os.path.join(dirpath, rule_filename)
            rule_file_path = os.path.abspath(rule_file_path)

            if verify_rules_file(rule_file_path, sigma_config, backend):
                return_verified_rules.append(rule_file_path)
            else:
                logger.info('File did not work{0:s}'.format(rule_file_path))
                return_rules_with_problems.append(rule_file_path)

    return return_verified_rules, return_rules_with_problems
Esempio n. 2
0
def get_sigma_rule_by_text(rule_text, sigma_config=None):
    """Returns a JSON represenation for a rule

    Args:
        rule_text: Text of the sigma rule to be parsed
        sigma_config: config file object

    Returns:
        Json representation of the parsed rule
    Raises:
        sigma_exceptions.SigmaParseError: Issue with parsing the given rule
        yaml.parser.ParserError: Not a correct YAML text provided
        NotImplementedError: A feature in the provided Sigma rule is not
            implemented in Sigma for Timesketch
    """
    if sigma_config is None:
        sigma_config = get_sigma_config_file()

    sigma_backend = sigma_es.ElasticsearchQuerystringBackend(sigma_config, {})

    rule_return = {}

    # TODO check if input validation is needed / useful.
    try:
        parser = sigma_collection.SigmaCollectionParser(
            rule_text, sigma_config, None)
        parsed_sigma_rules = parser.generate(sigma_backend)
        rule_yaml_data = yaml.safe_load_all(rule_text)
        for doc in rule_yaml_data:
            rule_return.update(doc)

    except NotImplementedError as exception:
        logger.error('Error generating rule {0!s}'.format(exception))
        raise

    except sigma_exceptions.SigmaParseError as exception:
        logger.error(
            'Sigma parsing error generating rule {0!s}'.format(exception))
        raise

    except yaml.parser.ParserError as exception:
        logger.error(
            'Yaml parsing error generating rule {0!s}'.format(exception))
        raise

    sigma_es_query = ''

    for sigma_rule in parsed_sigma_rules:
        sigma_es_query = sigma_rule

    rule_return.update({'es_query': sigma_es_query})
    rule_return.update({'file_name': 'N/A'})
    rule_return.update({'file_relpath': 'N/A'})

    return rule_return
Esempio n. 3
0
    def run(self):
        """Entry point for the analyzer.

        Returns:
            String with summary of the analyzer result.
        """
        sigma_backend = sigma_elasticsearch.ElasticsearchQuerystringBackend(
            self.sigma_config, {})
        tags_applied = {}

        rules_path = os.path.join(os.path.dirname(__file__), self._RULES_PATH)
        for rule_filename in os.listdir(rules_path):
            tag_name, _ = rule_filename.rsplit('.')
            tags_applied[tag_name] = 0
            rule_file_path = os.path.join(rules_path, rule_filename)
            rule_file_path = os.path.abspath(rule_file_path)
            logging.info(
                '[sigma] Reading rules from {0!s}'.format(rule_file_path))
            with open(rule_file_path, 'r') as rule_file:
                rule_file_content = rule_file.read()
                parser = sigma_collection.SigmaCollectionParser(
                    rule_file_content, self.sigma_config, None)
                try:
                    results = parser.generate(sigma_backend)
                except NotImplementedError as exception:
                    logging.error(
                        'Error generating rule in file {0:s}: {1!s}'.format(
                            rule_file_path, exception))
                    continue

                for result in results:
                    logging.info(
                        '[sigma] Generated query {0:s}'.format(result))
                    number_of_tagged_events = self.run_sigma_rule(
                        result, tag_name)
                    tags_applied[tag_name] += number_of_tagged_events

        total_tagged_events = sum(tags_applied.values())
        output_string = 'Applied {0:d} tags\n'.format(total_tagged_events)
        for tag_name, number_of_tagged_events in tags_applied.items():
            output_string += '* {0:s}: {1:d}'.format(tag_name,
                                                     number_of_tagged_events)
        return output_string
Esempio n. 4
0
    def run(self):
        """Entry point for the analyzer.

        Returns:
            String with summary of the analyzer result.
        """
        sigma_backend = sigma_elasticsearch.ElasticsearchQuerystringBackend(
            self.sigma_config, {})
        tags_applied = {}

        sigma_rule_counter = 0

        rules_path = os.path.join(os.path.dirname(__file__), self._RULES_PATH)


        for dirpath, dirnames, files in os.walk(rules_path):

            if 'deprecated' in dirnames:
                dirnames.remove('deprecated')

            for rule_filename in files:
                if rule_filename.lower().endswith('yml'):

                    # if a sub dir is found, append it to be scanned for rules
                    if os.path.isdir(os.path.join(rules_path, rule_filename)):
                        logger.error(
                            'this is a directory, skipping: {0:s}'.format(
                                rule_filename))
                        continue

                    tag_name, _ = rule_filename.rsplit('.')
                    tags_applied[tag_name] = 0
                    rule_file_path = os.path.join(dirpath, rule_filename)
                    rule_file_path = os.path.abspath(rule_file_path)
                    logger.info('[sigma] Reading rules from {0!s}'.format(
                        rule_file_path))
                    with codecs.open(rule_file_path, 'r', encoding='utf-8',
                                     errors='replace') as rule_file:
                        try:
                            rule_file_content = rule_file.read()
                            parser = sigma_collection.SigmaCollectionParser(
                                rule_file_content, self.sigma_config, None)
                            parsed_sigma_rules = parser.generate(sigma_backend)
                        except NotImplementedError as exception:
                            logger.error(
                                'Error generating rule in file {0:s}: {1!s}'
                                .format(rule_file_path, exception))
                            continue

                        for sigma_rule in parsed_sigma_rules:
                            try:
                                sigma_rule_counter += 1
                                logger.info(
                                    '[sigma] Generated query {0:s}'
                                    .format(sigma_rule))
                                tagged_events_counter = self.run_sigma_rule(
                                    sigma_rule, tag_name)
                                tags_applied[tag_name] += tagged_events_counter
                            except elasticsearch.TransportError as e:
                                logger.error(
                                    'Timeout generating rule in file {0:s}: '
                                    '{1!s} waiting for 10 seconds'.format(
                                        rule_file_path, e), exc_info=True)
                                time.sleep(10) # waiting 10 seconds

        total_tagged_events = sum(tags_applied.values())
        output_string = 'Applied {0:d} tags\n'.format(total_tagged_events)
        for tag_name, tagged_events_counter in tags_applied.items():
            output_string += '* {0:s}: {1:d}\n'.format(
                tag_name, tagged_events_counter)

        if sigma_rule_counter > 0:
            view = self.sketch.add_view(
                view_name='Sigma Rule matches', analyzer_name=self.NAME,
                query_string='tag:"sigma*"')
            agg_params = {
                'field': 'tag',
                'limit': 20,
            }
            agg_obj = self.sketch.add_aggregation(
                name='Top 20 Sigma tags', agg_name='field_bucket',
                agg_params=agg_params, view_id=view.id, chart_type='hbarchart',
                description='Created by the Sigma analyzer')


            story = self.sketch.add_story("Sigma Rule hits")
            story.add_text(
                utils.SIGMA_STORY_HEADER, skip_if_exists=True)

            story.add_text(
                '## Sigma Analyzer.\n\nThe Sigma '
                'analyzer takes Events and matches them with Sigma rules.'
                'In this timeline the analyzer discovered {0:d} '
                'Sigma tags.\n\nThis is a summary of '
                'it\'s findings.'.format(sigma_rule_counter))
            story.add_text(
                'The top 20 most commonly discovered tags were:')
            story.add_aggregation(agg_obj)
            story.add_text(
                'And an overview of all the discovered search terms:')
            story.add_view(view)


        return output_string
Esempio n. 5
0
def get_sigma_rule(filepath, sigma_config=None):
    """Returns a JSON represenation for a rule
    Args:
        filepath: path to the sigma rule to be parsed
        sigma_config: optional argument to pass a
                sigma.configuration.SigmaConfiguration object
    Returns:
        Json representation of the parsed rule
    Raises:
        ValueError: Parsing error
        IsADirectoryError: If a directory is passed as filepath
    """
    try:
        if isinstance(sigma_config, sigma_configuration.SigmaConfiguration):
            sigma_conf_obj = sigma_config
        elif isinstance(sigma_config, str):
            sigma_conf_obj = get_sigma_config_file(sigma_config)
        else:
            sigma_conf_obj = get_sigma_config_file()
    except ValueError as e:
        logger.error('Problem reading the Sigma config', exc_info=True)
        raise ValueError('Problem reading the Sigma config') from e

    sigma_backend = sigma_es.ElasticsearchQuerystringBackend(
        sigma_conf_obj, {})

    try:
        sigma_rules_paths = get_sigma_rules_path()
    except ValueError:
        sigma_rules_paths = None

    if not filepath.lower().endswith('.yml'):
        raise ValueError(f'{filepath} does not end with .yml')

    # if a sub dir is found, nothing can be parsed
    if os.path.isdir(filepath):
        raise IsADirectoryError(f'{filepath} is a directory - must be a file')

    abs_path = os.path.abspath(filepath)

    with codecs.open(abs_path, 'r', encoding='utf-8',
                     errors='replace') as file:
        try:
            rule_return = {}
            rule_yaml_data = yaml.safe_load_all(file.read())
            for doc in rule_yaml_data:
                rule_return.update(doc)
                parser = sigma_collection.SigmaCollectionParser(
                    yaml.safe_dump(doc), sigma_conf_obj, None)
                parsed_sigma_rules = parser.generate(sigma_backend)

        except NotImplementedError as exception:
            logger.error('Error generating rule in file {0:s}: {1!s}'.format(
                abs_path, exception))
            add_problematic_rule(filepath, doc.get('id'),
                                 'Parts of the rule not implemented in TS')
            return None

        except sigma_exceptions.SigmaParseError as exception:
            logger.error(
                'Sigma parsing error generating rule in file {0:s}: {1!s}'.
                format(abs_path, exception))
            add_problematic_rule(filepath, doc.get('id'),
                                 'sigma_exceptions.SigmaParseError')
            return None

        except yaml.parser.ParserError as exception:
            logger.error(
                'Yaml parsing error generating rule in file {0:s}: {1!s}'.
                format(abs_path, exception))
            add_problematic_rule(filepath, None, 'yaml.parser.ParserError')
            return None

        sigma_es_query = ''

        for sigma_rule in parsed_sigma_rules:

            sigma_es_query = _sanatize_sigma_rule(sigma_rule)

        rule_return.update({'es_query': sigma_es_query})
        rule_return.update({'file_name': os.path.basename(filepath)})

        # in case multiple folders are in the config, need to remove them
        if sigma_rules_paths:
            for rule_path in sigma_rules_paths:
                file_relpath = os.path.relpath(filepath, rule_path)
        else:
            file_relpath = 'N/A'

        rule_return.update({'file_relpath': file_relpath})

        return rule_return
Esempio n. 6
0
def get_sigma_rule(filepath, sigma_config=None):
    """ Returns a JSON represenation for a rule

    Args:
        filepath: path to the sigma rule to be parsed
        sigma_config: optional argument to pass a
                sigma.configuration.SigmaConfiguration object

    Returns:
        Json representation of the parsed rule
    """
    try:
        if sigma_config:
            sigma_conf_obj = sigma_config
        else:
            sigma_conf_obj = get_sigma_config_file()
    except ValueError as e:
        logger.error('Problem reading the Sigma config {0:s}: '.format(e),
                     exc_info=True)
        return None

    sigma_backend = sigma_es.ElasticsearchQuerystringBackend(
        sigma_conf_obj, {})

    try:
        sigma_rules_paths = get_sigma_rules_path()
    except ValueError:
        sigma_rules_paths = None

    if not filepath.lower().endswith('.yml'):
        return None

    # if a sub dir is found, nothing can be parsed
    if os.path.isdir(filepath):
        return None

    abs_path = os.path.abspath(filepath)

    with codecs.open(abs_path, 'r', encoding='utf-8',
                     errors='replace') as file:
        try:
            rule_return = {}
            rule_yaml_data = yaml.safe_load_all(file.read())
            for doc in rule_yaml_data:
                rule_return.update(doc)
                parser = sigma_collection.SigmaCollectionParser(
                    str(doc), sigma_config, None)
                parsed_sigma_rules = parser.generate(sigma_backend)

        except NotImplementedError as exception:
            logger.error('Error generating rule in file {0:s}: {1!s}'.format(
                abs_path, exception))
            return None

        except sigma_exceptions.SigmaParseError as exception:
            logger.error(
                'Sigma parsing error generating rule in file {0:s}: {1!s}'.
                format(abs_path, exception))
            return None

        except yaml.parser.ParserError as exception:
            logger.error(
                'Yaml parsing error generating rule in file {0:s}: {1!s}'.
                format(abs_path, exception))
            return None

        sigma_es_query = ''

        for sigma_rule in parsed_sigma_rules:
            sigma_es_query = sigma_rule

        rule_return.update({'es_query': sigma_es_query})
        rule_return.update({'file_name': os.path.basename(filepath)})

        # in case multiple folders are in the config, need to remove them
        if sigma_rules_paths:
            for rule_path in sigma_rules_paths:
                file_relpath = os.path.relpath(filepath, rule_path)
        else:
            file_relpath = 'N/A'

        rule_return.update({'file_relpath': file_relpath})

        return rule_return
Esempio n. 7
0
def get_sigma_rule_by_text(rule_text, sigma_config=None):
    """Returns a JSON represenation for a rule

    Args:
        rule_text: Text of the sigma rule to be parsed
        sigma_config: config file object

    Returns:
        Json representation of the parsed rule
    Raises:
        sigma_exceptions.SigmaParseError: Issue with parsing the given rule
        yaml.parser.ParserError: Not a correct YAML text provided
        NotImplementedError: A feature in the provided Sigma rule is not
            implemented in Sigma for Timesketch
    """

    try:
        if isinstance(sigma_config, sigma_configuration.SigmaConfiguration):
            sigma_conf_obj = sigma_config
        elif isinstance(sigma_config, str):
            sigma_conf_obj = get_sigma_config_file(sigma_config)
        else:
            sigma_conf_obj = get_sigma_config_file()
    except ValueError as e:
        logger.error("Problem reading the Sigma config", exc_info=True)
        raise ValueError("Problem reading the Sigma config") from e

    sigma_backend = sigma_es.ElasticsearchQuerystringBackend(
        sigma_conf_obj, {}
    )

    rule_text = sanitize_incoming_sigma_rule_text(rule_text)

    rule_return = {}
    parsed_sigma_rules = None
    # TODO check if input validation is needed / useful.
    try:
        rule_yaml_data = yaml.safe_load_all(rule_text)

        for doc in rule_yaml_data:

            parser = sigma_collection.SigmaCollectionParser(
                str(doc), sigma_conf_obj, None
            )
            parsed_sigma_rules = parser.generate(sigma_backend)
            rule_return.update(doc)

    except NotImplementedError as exception:
        logger.error("Error generating rule {0!s}".format(exception))
        raise

    except sigma_exceptions.SigmaParseError as exception:
        logger.error("Sigma parsing error rule {0!s}".format(exception))
        raise

    except yaml.parser.ParserError as exception:
        logger.error("Yaml parsing error rule {0!s}".format(exception))
        raise

    assert parsed_sigma_rules is not None

    sigma_es_query = ""

    for sigma_rule in parsed_sigma_rules:
        sigma_es_query = _sanitize_query(sigma_rule)

    rule_return.update({"es_query": sigma_es_query})
    rule_return.update({"file_name": "N/A"})
    rule_return.update({"file_relpath": "N/A"})
    return rule_return
Esempio n. 8
0
def get_sigma_rule(filepath, sigma_config=None):
    """Returns a JSON represenation for a rule
    Args:
        filepath: path to the sigma rule to be parsed
        sigma_config: optional argument to pass a
                sigma.configuration.SigmaConfiguration object
    Returns:
        Json representation of the parsed rule
    Raises:
        ValueError: Parsing error
        IsADirectoryError: If a directory is passed as filepath
    """
    try:
        if isinstance(sigma_config, sigma_configuration.SigmaConfiguration):
            sigma_conf_obj = sigma_config
        elif isinstance(sigma_config, str):
            sigma_conf_obj = get_sigma_config_file(sigma_config)
        else:
            sigma_conf_obj = get_sigma_config_file()
    except ValueError as e:
        logger.error("Problem reading the Sigma config", exc_info=True)
        raise ValueError("Problem reading the Sigma config") from e

    sigma_backend = sigma_es.ElasticsearchQuerystringBackend(
        sigma_conf_obj, {}
    )

    try:
        sigma_rules_paths = get_sigma_rules_path()
    except ValueError:
        sigma_rules_paths = None

    if not filepath.lower().endswith(".yml"):
        raise ValueError(f"{filepath} does not end with .yml")

    # if a sub dir is found, nothing can be parsed
    if os.path.isdir(filepath):
        raise IsADirectoryError(f"{filepath} is a directory - must be a file")

    if os.stat(filepath).st_size == 0:
        raise ValueError(f"{filepath} file is empty")

    abs_path = os.path.abspath(filepath)
    parsed_sigma_rules = None
    with codecs.open(
        abs_path, "r", encoding="utf-8", errors="replace"
    ) as file:
        try:
            rule_return = {}
            rule_file_content = file.read()
            rule_file_content = sanitize_incoming_sigma_rule_text(
                rule_file_content
            )
            rule_yaml_data = yaml.safe_load_all(rule_file_content)

            for doc in rule_yaml_data:
                rule_return.update(doc)
                parser = sigma_collection.SigmaCollectionParser(
                    yaml.safe_dump(doc), sigma_conf_obj, None
                )
                parsed_sigma_rules = parser.generate(sigma_backend)

        except NotImplementedError as exception:
            logger.error("Error rule {0:s}: {1!s}".format(abs_path, exception))
            add_problematic_rule(
                filepath, doc.get("id"), "Part of the rule not supported in TS"
            )
            return None

        except sigma_exceptions.SigmaParseError as exception:
            logger.error(
                "Sigma parsing error rule in file {0:s}: {1!s}".format(
                    abs_path, exception
                )
            )
            add_problematic_rule(
                filepath, doc.get("id"), "sigma_exceptions.SigmaParseError"
            )
            return None

        except yaml.parser.ParserError as exception:
            logger.error(
                "Yaml parsing error rule in file {0:s}: {1!s}".format(
                    abs_path, exception
                )
            )
            add_problematic_rule(filepath, None, "yaml.parser.ParserError")
            return None

        sigma_es_query = ""

        assert parsed_sigma_rules is not None

        for sigma_rule in parsed_sigma_rules:
            sigma_es_query = _sanitize_query(sigma_rule)

        rule_return.update({"es_query": sigma_es_query})
        rule_return.update({"file_name": os.path.basename(filepath)})

        # in case multiple folders are in the config, need to remove them
        if sigma_rules_paths:
            for rule_path in sigma_rules_paths:
                file_relpath = os.path.relpath(filepath, rule_path)
        else:
            file_relpath = "N/A"

        rule_return.update({"file_relpath": file_relpath})

        return rule_return