def process(self, filein, fileout, rulemap): count = 0 for line in filein: line = line.rstrip() if not line or line.startswith("#"): print(line, file=fileout) continue pattern = self.extract_pattern(line) if not pattern: print(line, file=fileout) else: for rule in rulemap.values(): if rule.enabled: if pattern.search(str(rule)): count += 1 print("# %s" % (rule.brief()), file=fileout) print(self.replace(line, rule), file=fileout) print("", file=fileout) logger.info("Generated %d thresholds to %s." % (count, fileout.name))
def resolve_flowbits(rulemap, disabled_rules): flowbit_resolver = idstools.rule.FlowbitResolver() flowbit_enabled = set() while True: flowbits = flowbit_resolver.get_required_flowbits(rulemap) logger.debug("Found %d required flowbits.", len(flowbits)) required_rules = flowbit_resolver.get_required_rules(rulemap, flowbits) logger.debug("Found %d rules to enable to for flowbit requirements", len(required_rules)) if not required_rules: logger.debug("All required rules enabled.") break for rule in required_rules: if not rule.enabled and rule in disabled_rules: logger.debug( "Enabling previously disabled rule for flowbits: %s" % (rule.brief())) rule.enabled = True flowbit_enabled.add(rule) logger.info("Enabled %d rules for flowbit dependencies." % (len(flowbit_enabled)))
def resolve_flowbits(rulemap, disabled_rules): flowbit_resolver = idstools.rule.FlowbitResolver() flowbit_enabled = set() while True: flowbits = flowbit_resolver.get_required_flowbits(rulemap) logger.debug("Found %d required flowbits.", len(flowbits)) required_rules = flowbit_resolver.get_required_rules(rulemap, flowbits) logger.debug( "Found %d rules to enable to fullfull flowbit requirements", len(required_rules)) if not required_rules: logger.debug("All required rules enabled.") break for rule in required_rules: if not rule.enabled and rule in disabled_rules: logger.debug( "Enabling previously disabled rule for flowbits: %s" % ( rule.brief())) rule.enabled = True flowbit_enabled.add(rule) logger.info("Enabled %d rules for flowbit dependencies." % ( len(flowbit_enabled)))
def main(): conf_filenames = [arg for arg in sys.argv if arg.startswith("@")] if not conf_filenames: if os.path.exists("./rulecat.conf"): logger.info("Loading ./rulecat.conf.") sys.argv.insert(1, "@./rulecat.conf") suricata_path = idstools.suricata.get_path() parser = argparse.ArgumentParser(fromfile_prefix_chars="@") parser.add_argument("-v", "--verbose", action="store_true", default=False, help="Be more verbose") parser.add_argument("-t", "--temp-dir", default="/var/tmp/idstools-rulecat", metavar="<directory>", help="Temporary work directory") parser.add_argument("--suricata", default=suricata_path, metavar="<path>", help="Path to Suricata program (default: %s)" % suricata_path) parser.add_argument( "-f", "--force", action="store_true", default=False, help="Force operations that might otherwise be skipped") parser.add_argument("--rules-dir", metavar="<directory>", help=argparse.SUPPRESS) parser.add_argument("-o", "--output", metavar="<directory>", dest="output", help="Output rules directory.") parser.add_argument("--merged", default=None, metavar="<filename>", help="Output merged rules file") parser.add_argument("--yaml-fragment", metavar="<filename>", help="Output YAML fragment for rule inclusion") parser.add_argument("--url", metavar="<url>", action="append", default=[], help="URL to use instead of auto-generating one") parser.add_argument("--local", metavar="<filename>", help="Local rule files or directories") parser.add_argument("--sid-msg-map", metavar="<filename>", help="Generate a sid-msg.map file") parser.add_argument("--sid-msg-map-2", metavar="<filename>", help="Generate a v2 sid-msg.map file") parser.add_argument("--disable", metavar="<filename>", help="Filename of disable rule configuration") parser.add_argument("--enable", metavar="<filename>", help="Filename of enable rule configuration") parser.add_argument("--modify", metavar="<filename>", help="Filename of rule modification configuration") parser.add_argument("--drop", metavar="<filename>", help="Filename of drop rules configuration") parser.add_argument("--threshold-in", metavar="<filename>", help="Filename of rule thresholding configuration") parser.add_argument("--threshold-out", metavar="<filename>", help="Output of processed threshold configuration") parser.add_argument("--dump-sample-configs", action="store_true", default=False, help="Dump sample config files to current directory") parser.add_argument("--etpro", metavar="<etpro-code>", help="Use ET-Pro rules with provided ET-Pro code") parser.add_argument("--etopen", action="store_true", help="Use ET-Open rules (default)") parser.add_argument("-q", "--quiet", action="store_true", default=False, help="Be quiet, warning and error messages only") parser.add_argument("--post-hook", metavar="<command>", help="Command to run after update if modified") parser.add_argument("-T", "--test-command", metavar="<command>", help="Command to test Suricata configuration") args = parser.parse_args() if args.verbose: logger.setLevel(logging.DEBUG) if args.quiet: logger.setLevel(logging.WARNING) if args.dump_sample_configs: return dump_sample_configs() if args.suricata and os.path.exists(args.suricata): suricata_version = idstools.suricata.get_version(args.suricata) logger.info("Found Suricata version %s at %s." % (str(suricata_version.full), args.suricata)) else: suricata_version = None if args.etpro: args.url.append(resolve_etpro_url(args.etpro, suricata_version)) if not args.url or args.etopen: args.url.append(resolve_etopen_url(suricata_version)) args.url = set(args.url) file_tracker = FileTracker() disable_matchers = [] enable_matchers = [] modify_filters = [] drop_filters = [] if args.disable and os.path.exists(args.disable): disable_matchers += load_matchers(args.disable) if args.enable and os.path.exists(args.enable): enable_matchers += load_matchers(args.enable) if args.modify and os.path.exists(args.modify): modify_filters += load_filters(args.modify) if args.drop and os.path.exists(args.drop): drop_filters += load_drop_filters(args.drop) files = Fetch(args).run() if args.local: load_local_files(args.local, files) rules = [] for filename in files: logger.debug("Parsing %s." % (filename)) rules += idstools.rule.parse_fileobj(BytesIO(files[filename]), filename) rulemap = build_rule_map(rules) logger.info("Loaded %d rules." % (len(rules))) # Counts of user enabled and modified rules. enable_count = 0 modify_count = 0 drop_count = 0 # List of rules disabled by user. Used for counting, and to log # rules that are re-enabled to meet flowbit requirements. disabled_rules = [] for key, rule in rulemap.items(): for matcher in disable_matchers: if rule.enabled and matcher.match(rule): logger.debug("Disabling: %s" % (rule.brief())) rule.enabled = False disabled_rules.append(rule) for matcher in enable_matchers: if not rule.enabled and matcher.match(rule): logger.debug("Enabling: %s" % (rule.brief())) rule.enabled = True enable_count += 1 # Unlike enable and disable, modify returns a new instance of # the rule. for filter in modify_filters: if filter.match(rule): rulemap[rule.id] = filter.filter(rule) modify_count += 1 for filter in drop_filters: if filter.match(rule): rulemap[rule.id] = filter.filter(rule) drop_count += 1 logger.info("Disabled %d rules." % (len(disabled_rules))) logger.info("Enabled %d rules." % (enable_count)) logger.info("Modified %d rules." % (modify_count)) logger.info("Dropped %d rules." % (drop_count)) # Fixup flowbits. resolve_flowbits(rulemap, disabled_rules) if args.output: if not os.path.exists(args.output): logger.info("Making directory %s.", args.output) os.makedirs(args.output) for filename in files: file_tracker.add(os.path.join(args.output, filename)) write_to_directory(args.output, files, rulemap) if args.merged: file_tracker.add(args.merged) write_merged(args.merged, rulemap) if args.yaml_fragment: file_tracker.add(args.yaml_fragment) write_yaml_fragment(args.yaml_fragment, files) if args.sid_msg_map: write_sid_msg_map(args.sid_msg_map, rulemap, version=1) if args.sid_msg_map_2: write_sid_msg_map(args.sid_msg_map_2, rulemap, version=2) if args.threshold_in and args.threshold_out: file_tracker.add(args.threshold_out) threshold_processor = ThresholdProcessor() threshold_processor.process(open(args.threshold_in), open(args.threshold_out, "w"), rulemap) if not args.force and not file_tracker.any_modified(): logger.info( "No changes detected, will not reload rules or run post-hooks.") return 0 if args.test_command: logger.info("Testing Suricata configuration with: %s" % (args.test_command)) rc = subprocess.Popen(args.test_command, shell=True).wait() if rc != 0: logger.error("Suricata test failed, aborting.") return 1 if args.post_hook: logger.info("Running %s." % (args.post_hook)) subprocess.Popen(args.post_hook, shell=True).wait() logger.info("Done.") return 0
def main(): conf_filenames = [arg for arg in sys.argv if arg.startswith("@")] if not conf_filenames: if os.path.exists("./rulecat.conf"): logger.info("Loading ./rulecat.conf.") sys.argv.insert(1, "@./rulecat.conf") suricata_path = idstools.suricata.get_path() parser = argparse.ArgumentParser(fromfile_prefix_chars="@") parser.add_argument("-v", "--verbose", action="store_true", default=False, help="Be more verbose") parser.add_argument("-t", "--temp-dir", default="/var/tmp/idstools-rulecat", metavar="<directory>", help="Temporary work directory") parser.add_argument("--suricata", default=suricata_path, metavar="<path>", help="Path to Suricata program (default: %s)" % suricata_path) parser.add_argument("-f", "--force", action="store_true", default=False, help="Force operations that might otherwise be skipped") parser.add_argument("--rules-dir", metavar="<directory>", help=argparse.SUPPRESS) parser.add_argument("-o", "--output", metavar="<directory>", dest="output", help="Output rules directory.") parser.add_argument("--merged", default=None, metavar="<filename>", help="Output merged rules file") parser.add_argument("--yaml-fragment", metavar="<filename>", help="Output YAML fragment for rule inclusion") parser.add_argument("--url", metavar="<url>", action="append", default=[], help="URL to use instead of auto-generating one") parser.add_argument("--local", metavar="<filename>", help="Local rule files or directories") parser.add_argument("--sid-msg-map", metavar="<filename>", help="Generate a sid-msg.map file") parser.add_argument("--sid-msg-map-2", metavar="<filename>", help="Generate a v2 sid-msg.map file") parser.add_argument("--disable", metavar="<filename>", help="Filename of disable rule configuration") parser.add_argument("--enable", metavar="<filename>", help="Filename of enable rule configuration") parser.add_argument("--modify", metavar="<filename>", help="Filename of rule modification configuration") parser.add_argument("--drop", metavar="<filename>", help="Filename of drop rules configuration") parser.add_argument("--threshold-in", metavar="<filename>", help="Filename of rule thresholding configuration") parser.add_argument("--threshold-out", metavar="<filename>", help="Output of processed threshold configuration") parser.add_argument("--dump-sample-configs", action="store_true", default=False, help="Dump sample config files to current directory") parser.add_argument("--etpro", metavar="<etpro-code>", help="Use ET-Pro rules with provided ET-Pro code") parser.add_argument("--etopen", action="store_true", help="Use ET-Open rules (default)") parser.add_argument("-q", "--quiet", action="store_true", default=False, help="Be quiet, warning and error messages only") parser.add_argument("--post-hook", metavar="<command>", help="Command to run after update if modified") parser.add_argument("-T", "--test-command", metavar="<command>", help="Command to test Suricata configuration") args = parser.parse_args() if args.verbose: logger.setLevel(logging.DEBUG) if args.quiet: logger.setLevel(logging.WARNING) if args.dump_sample_configs: return dump_sample_configs() if args.suricata and os.path.exists(args.suricata): suricata_version = idstools.suricata.get_version(args.suricata) logger.info("Found Suricata version %s at %s." % ( str(suricata_version.full), args.suricata)) else: suricata_version = None if args.etpro: args.url.append(resolve_etpro_url(args.etpro, suricata_version)) if not args.url or args.etopen: args.url.append(resolve_etopen_url(suricata_version)) args.url = set(args.url) file_tracker = FileTracker() disable_matchers = [] enable_matchers = [] modify_filters = [] drop_filters = [] if args.disable and os.path.exists(args.disable): disable_matchers += load_matchers(args.disable) if args.enable and os.path.exists(args.enable): enable_matchers += load_matchers(args.enable) if args.modify and os.path.exists(args.modify): modify_filters += load_filters(args.modify) if args.drop and os.path.exists(args.drop): drop_filters += load_drop_filters(args.drop) files = Fetch(args).run() if args.local: load_local_files(args.local, files) rules = [] for filename in files: logger.debug("Parsing %s." % (filename)) rules += idstools.rule.parse_fileobj( BytesIO(files[filename]), filename) rulemap = build_rule_map(rules) logger.info("Loaded %d rules." % (len(rules))) # Counts of user enabled and modified rules. enable_count = 0 modify_count = 0 drop_count = 0 # List of rules disabled by user. Used for counting, and to log # rules that are re-enabled to meet flowbit requirements. disabled_rules = [] for key, rule in rulemap.items(): for matcher in disable_matchers: if rule.enabled and matcher.match(rule): logger.debug("Disabling: %s" % (rule.brief())) rule.enabled = False disabled_rules.append(rule) for matcher in enable_matchers: if not rule.enabled and matcher.match(rule): logger.debug("Enabling: %s" % (rule.brief())) rule.enabled = True enable_count += 1 # Unlike enable and disable, modify returns a new instance of # the rule. for filter in modify_filters: if filter.match(rule): rulemap[rule.id] = filter.filter(rule) modify_count += 1 for filter in drop_filters: if filter.match(rule): rulemap[rule.id] = filter.filter(rule) drop_count += 1 logger.info("Disabled %d rules." % (len(disabled_rules))) logger.info("Enabled %d rules." % (enable_count)) logger.info("Modified %d rules." % (modify_count)) logger.info("Dropped %d rules." % (drop_count)) # Fixup flowbits. resolve_flowbits(rulemap, disabled_rules) if args.output: if not os.path.exists(args.output): logger.info("Making directory %s.", args.output) os.makedirs(args.output) for filename in files: file_tracker.add(os.path.join(args.output, filename)) write_to_directory(args.output, files, rulemap) if args.merged: file_tracker.add(args.merged) write_merged(args.merged, rulemap) if args.yaml_fragment: file_tracker.add(args.yaml_fragment) write_yaml_fragment(args.yaml_fragment, files) if args.sid_msg_map: write_sid_msg_map(args.sid_msg_map, rulemap, version=1) if args.sid_msg_map_2: write_sid_msg_map(args.sid_msg_map_2, rulemap, version=2) if args.threshold_in and args.threshold_out: file_tracker.add(args.threshold_out) threshold_processor = ThresholdProcessor() threshold_processor.process( open(args.threshold_in), open(args.threshold_out, "w"), rulemap) if not args.force and not file_tracker.any_modified(): logger.info( "No changes detected, will not reload rules or run post-hooks.") return 0 if args.test_command: logger.info("Testing Suricata configuration with: %s" % ( args.test_command)) rc = subprocess.Popen(args.test_command, shell=True).wait() if rc != 0: logger.error("Suricata test failed, aborting.") return 1 if args.post_hook: logger.info("Running %s." % (args.post_hook)) subprocess.Popen(args.post_hook, shell=True).wait() logger.info("Done.") return 0