def build_report(prev_rulemap, rulemap): """Build a report of changes between 2 rulemaps. Returns a dict with the following keys that each contain a list of rules. - added - removed - modified """ report = { "added": [], "removed": [], "modified": [] } for key in rulemap: rule = rulemap[key] if not rule.id in prev_rulemap: report["added"].append(rule) elif rule.format() != prev_rulemap[rule.id].format(): report["modified"].append(rule) for key in prev_rulemap: rule = prev_rulemap[key] if not rule.id in rulemap: report["removed"].append(rule) return report
def filter(self, rule): modified_rule = self.pattern.sub(self.repl, rule.format()) parsed = idstools.rule.parse(modified_rule, rule.group) if parsed is None: logger.error("Modification of rule %s results in invalid rule: %s", rule.idstr, modified_rule) return rule return parsed
def process(self, filein, fileout, rulemap): count = 0 for line in filein: line = line.rstrip() if not line or line.startswith("#"): print(line, file=fileout) continue pattern = self.extract_pattern(line) if not pattern: print(line, file=fileout) else: for rule in rulemap.values(): if rule.enabled: if pattern.search(rule.format()): count += 1 print("# %s" % (rule.brief()), file=fileout) print(self.replace(line, rule), file=fileout) print("", file=fileout) logger.info("Generated %d thresholds to %s." % (count, fileout.name))
def main(): global args conf_filenames = [arg for arg in sys.argv if arg.startswith("@")] if not conf_filenames: if os.path.exists("./rulecat.conf"): logger.info("Loading ./rulecat.conf.") sys.argv.insert(1, "@./rulecat.conf") suricata_path = idstools.suricata.get_path() parser = argparse.ArgumentParser(fromfile_prefix_chars="@") parser.add_argument("-v", "--verbose", action="store_true", default=False, help="Be more verbose") parser.add_argument("-t", "--temp-dir", default="/var/tmp/idstools-rulecat", metavar="<directory>", help="Temporary work directory") parser.add_argument("--suricata", default=suricata_path, metavar="<path>", help="Path to Suricata program (default: %s)" % suricata_path) parser.add_argument("--suricata-version", metavar="<version>", help="Override Suricata version") parser.add_argument("-f", "--force", action="store_true", default=False, help="Force operations that might otherwise be skipped") parser.add_argument("--rules-dir", metavar="<directory>", help=argparse.SUPPRESS) parser.add_argument("-o", "--output", metavar="<directory>", dest="output", help="Output rules directory.") parser.add_argument("--merged", default=None, metavar="<filename>", help="Output merged rules file") parser.add_argument("--yaml-fragment", metavar="<filename>", help="Output YAML fragment for rule inclusion") parser.add_argument("--url", metavar="<url>", action="append", default=[], help="URL to use instead of auto-generating one") parser.add_argument("--local", metavar="<filename>", action="append", default=[], help="Local rule files or directories") parser.add_argument("--sid-msg-map", metavar="<filename>", help="Generate a sid-msg.map file") parser.add_argument("--sid-msg-map-2", metavar="<filename>", help="Generate a v2 sid-msg.map file") parser.add_argument("--disable", metavar="<filename>", help="Filename of disable rule configuration") parser.add_argument("--enable", metavar="<filename>", help="Filename of enable rule configuration") parser.add_argument("--modify", metavar="<filename>", help="Filename of rule modification configuration") parser.add_argument("--drop", metavar="<filename>", help="Filename of drop rules configuration") parser.add_argument("--ignore", metavar="<filename>", action="append", default=[], help="Filenames to ignore (default: *deleted.rules)") parser.add_argument("--no-ignore", action="store_true", default=False, help="Disables the ignore option.") parser.add_argument("--threshold-in", metavar="<filename>", help="Filename of rule thresholding configuration") parser.add_argument("--threshold-out", metavar="<filename>", help="Output of processed threshold configuration") parser.add_argument("--dump-sample-configs", action="store_true", default=False, help="Dump sample config files to current directory") parser.add_argument("--etpro", metavar="<etpro-code>", help="Use ET-Pro rules with provided ET-Pro code") parser.add_argument("--etopen", action="store_true", help="Use ET-Open rules (default)") parser.add_argument("-q", "--quiet", action="store_true", default=False, help="Be quiet, warning and error messages only") parser.add_argument("--post-hook", metavar="<command>", help="Command to run after update if modified") parser.add_argument("-T", "--test-command", metavar="<command>", help="Command to test Suricata configuration") parser.add_argument("-V", "--version", action="store_true", default=False, help="Display version") args = parser.parse_args() if args.version: print("idstools-rulecat version %s" % idstools.version) return 0 if args.verbose: logger.setLevel(logging.DEBUG) if args.quiet: logger.setLevel(logging.WARNING) logger.debug("This is idstools-rulecat version %s; Python: %s" % ( idstools.version, sys.version.replace("\n", "- "))) if args.dump_sample_configs: return dump_sample_configs() # If --no-ignore was provided, make sure args.ignore is # empty. Otherwise if no ignores are provided, set a sane default. if args.no_ignore: args.ignore = [] elif len(args.ignore) == 0: args.ignore.append("*deleted.rules") if args.suricata_version: suricata_version = idstools.suricata.parse_version(args.suricata_version) if not suricata_version: logger.error("Failed to parse provided Suricata version: %s" % ( suricata_version)) return 1 logger.info("Forcing Suricata version to %s." % (suricata_version.full)) elif args.suricata and os.path.exists(args.suricata): suricata_version = idstools.suricata.get_version(args.suricata) if suricata_version: logger.info("Found Suricata version %s at %s." % ( str(suricata_version.full), args.suricata)) else: logger.warn("Failed to get Suricata version.") suricata_version = None else: suricata_version = None if args.etpro: args.url.append(resolve_etpro_url(args.etpro, suricata_version)) if not args.url or args.etopen: args.url.append(resolve_etopen_url(suricata_version)) args.url = set(args.url) file_tracker = FileTracker() disable_matchers = [] enable_matchers = [] modify_filters = [] drop_filters = [] if args.disable and os.path.exists(args.disable): disable_matchers += load_matchers(args.disable) if args.enable and os.path.exists(args.enable): enable_matchers += load_matchers(args.enable) if args.modify and os.path.exists(args.modify): modify_filters += load_filters(args.modify) if args.drop and os.path.exists(args.drop): drop_filters += load_drop_filters(args.drop) files = Fetch(args).run() # Remove ignored files. for filename in list(files.keys()): if ignore_file(args.ignore, filename): logger.info("Ignoring file %s" % (filename)) del(files[filename]) for path in args.local: load_local(path, files) rules = [] for filename in files: if not filename.endswith(".rules"): continue logger.debug("Parsing %s." % (filename)) rules += idstools.rule.parse_fileobj( io.BytesIO(files[filename]), filename) rulemap = build_rule_map(rules) logger.info("Loaded %d rules." % (len(rules))) # Counts of user enabled and modified rules. enable_count = 0 modify_count = 0 drop_count = 0 # List of rules disabled by user. Used for counting, and to log # rules that are re-enabled to meet flowbit requirements. disabled_rules = [] for key, rule in rulemap.items(): for matcher in disable_matchers: if rule.enabled and matcher.match(rule): logger.debug("Disabling: %s" % (rule.brief())) rule.enabled = False disabled_rules.append(rule) for matcher in enable_matchers: if not rule.enabled and matcher.match(rule): logger.debug("Enabling: %s" % (rule.brief())) rule.enabled = True enable_count += 1 for filter in drop_filters: if filter.match(rule): rulemap[rule.id] = filter.filter(rule) drop_count += 1 # Apply modify filters. for fltr in modify_filters: for key, rule in rulemap.items(): if fltr.match(rule): new_rule = fltr.filter(rule) if new_rule and new_rule.format() != rule.format(): rulemap[rule.id] = new_rule modify_count += 1 logger.info("Disabled %d rules." % (len(disabled_rules))) logger.info("Enabled %d rules." % (enable_count)) logger.info("Modified %d rules." % (modify_count)) logger.info("Dropped %d rules." % (drop_count)) # Fixup flowbits. resolve_flowbits(rulemap, disabled_rules) if args.output: if not os.path.exists(args.output): logger.info("Making directory %s.", args.output) os.makedirs(args.output) for filename in files: file_tracker.add( os.path.join(args.output, os.path.basename(filename))) write_to_directory(args.output, files, rulemap) if args.merged: file_tracker.add(args.merged) write_merged(args.merged, rulemap) if args.yaml_fragment: file_tracker.add(args.yaml_fragment) write_yaml_fragment(args.yaml_fragment, files) if args.sid_msg_map: write_sid_msg_map(args.sid_msg_map, rulemap, version=1) if args.sid_msg_map_2: write_sid_msg_map(args.sid_msg_map_2, rulemap, version=2) if args.threshold_in and args.threshold_out: file_tracker.add(args.threshold_out) threshold_processor = ThresholdProcessor() threshold_processor.process( open(args.threshold_in), open(args.threshold_out, "w"), rulemap) if not args.force and not file_tracker.any_modified(): logger.info( "No changes detected, will not reload rules or run post-hooks.") return 0 if args.test_command: logger.info("Testing Suricata configuration with: %s" % ( args.test_command)) rc = subprocess.Popen(args.test_command, shell=True).wait() if rc != 0: logger.error("Suricata test failed, aborting.") return 1 if args.post_hook: logger.info("Running %s." % (args.post_hook)) subprocess.Popen(args.post_hook, shell=True).wait() logger.info("Done.") return 0
def main(): global args conf_filenames = [arg for arg in sys.argv if arg.startswith("@")] if not conf_filenames: if os.path.exists("./rulecat.conf"): logger.info("Loading ./rulecat.conf.") sys.argv.insert(1, "@./rulecat.conf") suricata_path = idstools.suricata.get_path() parser = argparse.ArgumentParser(fromfile_prefix_chars="@") parser.add_argument("-v", "--verbose", action="store_true", default=False, help="Be more verbose") parser.add_argument("-t", "--temp-dir", default="/var/tmp/idstools-rulecat", metavar="<directory>", help="Temporary work directory") parser.add_argument("--suricata", default=suricata_path, metavar="<path>", help="Path to Suricata program (default: %s)" % suricata_path) parser.add_argument("--suricata-version", metavar="<version>", help="Override Suricata version") parser.add_argument("-f", "--force", action="store_true", default=False, help="Force operations that might otherwise be skipped") parser.add_argument("--rules-dir", metavar="<directory>", help=argparse.SUPPRESS) parser.add_argument("-o", "--output", metavar="<directory>", dest="output", help="Output rules directory.") parser.add_argument("--merged", default=None, metavar="<filename>", help="Output merged rules file") parser.add_argument("--yaml-fragment", metavar="<filename>", help="Output YAML fragment for rule inclusion") parser.add_argument("--url", metavar="<url>", action="append", default=[], help="URL to use instead of auto-generating one") parser.add_argument("--local", metavar="<filename>", action="append", default=[], help="Local rule files or directories") parser.add_argument("--sid-msg-map", metavar="<filename>", help="Generate a sid-msg.map file") parser.add_argument("--sid-msg-map-2", metavar="<filename>", help="Generate a v2 sid-msg.map file") parser.add_argument("--disable", metavar="<filename>", help="Filename of disable rule configuration") parser.add_argument("--enable", metavar="<filename>", help="Filename of enable rule configuration") parser.add_argument("--modify", metavar="<filename>", help="Filename of rule modification configuration") parser.add_argument("--drop", metavar="<filename>", help="Filename of drop rules configuration") parser.add_argument("--ignore", metavar="<filename>", action="append", default=[], help="Filenames to ignore (default: *deleted.rules)") parser.add_argument("--no-ignore", action="store_true", default=False, help="Disables the ignore option.") parser.add_argument("--threshold-in", metavar="<filename>", help="Filename of rule thresholding configuration") parser.add_argument("--threshold-out", metavar="<filename>", help="Output of processed threshold configuration") parser.add_argument("--dump-sample-configs", action="store_true", default=False, help="Dump sample config files to current directory") parser.add_argument("--etpro", metavar="<etpro-code>", help="Use ET-Pro rules with provided ET-Pro code") parser.add_argument("--etopen", action="store_true", help="Use ET-Open rules (default)") parser.add_argument("-q", "--quiet", action="store_true", default=False, help="Be quiet, warning and error messages only") parser.add_argument("--post-hook", metavar="<command>", help="Command to run after update if modified") parser.add_argument("-T", "--test-command", metavar="<command>", help="Command to test Suricata configuration") parser.add_argument("-V", "--version", action="store_true", default=False, help="Display version") args = parser.parse_args() if args.version: print("idstools-rulecat version %s" % idstools.version) return 0 if args.verbose: logger.setLevel(logging.DEBUG) if args.quiet: logger.setLevel(logging.WARNING) logger.debug("This is idstools-rulecat version %s; Python: %s" % ( idstools.version, sys.version.replace("\n", "- "))) if args.dump_sample_configs: return dump_sample_configs() # If --no-ignore was provided, make sure args.ignore is # empty. Otherwise if no ignores are provided, set a sane default. if args.no_ignore: args.ignore = [] elif len(args.ignore) == 0: args.ignore.append("*deleted.rules") suricata_version = None if args.suricata_version: suricata_version = idstools.suricata.parse_version(args.suricata_version) if not suricata_version: logger.error("Failed to parse provided Suricata version: %s" % ( suricata_version)) return 1 logger.info("Forcing Suricata version to %s." % (suricata_version.full)) elif args.suricata and os.path.exists(args.suricata): suricata_version = idstools.suricata.get_version(args.suricata) if suricata_version: logger.info("Found Suricata version %s at %s." % ( str(suricata_version.full), args.suricata)) else: logger.warn("Failed to get Suricata version, using %s", DEFAULT_SURICATA_VERSION) if suricata_version is None: suricata_version = idstools.suricata.parse_version( DEFAULT_SURICATA_VERSION) if args.etpro: args.url.append(resolve_etpro_url(args.etpro, suricata_version)) if not args.url or args.etopen: args.url.append(resolve_etopen_url(suricata_version)) args.url = set(args.url) file_tracker = FileTracker() disable_matchers = [] enable_matchers = [] modify_filters = [] drop_filters = [] if args.disable and os.path.exists(args.disable): disable_matchers += load_matchers(args.disable) if args.enable and os.path.exists(args.enable): enable_matchers += load_matchers(args.enable) if args.modify and os.path.exists(args.modify): modify_filters += load_filters(args.modify) if args.drop and os.path.exists(args.drop): drop_filters += load_drop_filters(args.drop) files = Fetch(args).run() # Remove ignored files. for filename in list(files.keys()): if ignore_file(args.ignore, filename): logger.info("Ignoring file %s" % (filename)) del(files[filename]) for path in args.local: load_local(path, files) rules = [] for filename in files: if not filename.endswith(".rules"): continue logger.debug("Parsing %s." % (filename)) rules += idstools.rule.parse_fileobj( io.BytesIO(files[filename]), filename) rulemap = build_rule_map(rules) logger.info("Loaded %d rules." % (len(rules))) # Counts of user enabled and modified rules. enable_count = 0 modify_count = 0 drop_count = 0 # List of rules disabled by user. Used for counting, and to log # rules that are re-enabled to meet flowbit requirements. disabled_rules = [] for key, rule in rulemap.items(): for matcher in disable_matchers: if rule.enabled and matcher.match(rule): logger.debug("Disabling: %s" % (rule.brief())) rule.enabled = False disabled_rules.append(rule) for matcher in enable_matchers: if not rule.enabled and matcher.match(rule): logger.debug("Enabling: %s" % (rule.brief())) rule.enabled = True enable_count += 1 for filter in drop_filters: if filter.match(rule): rulemap[rule.id] = filter.filter(rule) drop_count += 1 # Apply modify filters. for fltr in modify_filters: for key, rule in rulemap.items(): if fltr.match(rule): new_rule = fltr.filter(rule) if new_rule and new_rule.format() != rule.format(): rulemap[rule.id] = new_rule modify_count += 1 logger.info("Disabled %d rules." % (len(disabled_rules))) logger.info("Enabled %d rules." % (enable_count)) logger.info("Modified %d rules." % (modify_count)) logger.info("Dropped %d rules." % (drop_count)) # Fixup flowbits. resolve_flowbits(rulemap, disabled_rules) if args.output: if not os.path.exists(args.output): logger.info("Making directory %s.", args.output) os.makedirs(args.output) for filename in files: file_tracker.add( os.path.join(args.output, os.path.basename(filename))) write_to_directory(args.output, files, rulemap) if args.merged: file_tracker.add(args.merged) write_merged(args.merged, rulemap) if args.yaml_fragment: file_tracker.add(args.yaml_fragment) write_yaml_fragment(args.yaml_fragment, files) if args.sid_msg_map: write_sid_msg_map(args.sid_msg_map, rulemap, version=1) if args.sid_msg_map_2: write_sid_msg_map(args.sid_msg_map_2, rulemap, version=2) if args.threshold_in and args.threshold_out: file_tracker.add(args.threshold_out) threshold_processor = ThresholdProcessor() threshold_processor.process( open(args.threshold_in), open(args.threshold_out, "w"), rulemap) if not args.force and not file_tracker.any_modified(): logger.info( "No changes detected, will not reload rules or run post-hooks.") return 0 if args.test_command: logger.info("Testing Suricata configuration with: %s" % ( args.test_command)) rc = subprocess.Popen(args.test_command, shell=True).wait() if rc != 0: logger.error("Suricata test failed, aborting.") return 1 if args.post_hook: logger.info("Running %s." % (args.post_hook)) subprocess.Popen(args.post_hook, shell=True).wait() logger.info("Done.") return 0