def build_report(prev_rulemap, rulemap):
"""Build a report of changes between 2 rulemaps.
Returns a dict with the following keys that each contain a list of
rules.
- added
- removed
- modified
"""
report = {
"added": [],
"removed": [],
"modified": []
}
for key in rulemap:
rule = rulemap[key]
if not rule.id in prev_rulemap:
report["added"].append(rule)
elif rule.format() != prev_rulemap[rule.id].format():
report["modified"].append(rule)
for key in prev_rulemap:
rule = prev_rulemap[key]
if not rule.id in rulemap:
report["removed"].append(rule)
return report
def build_report(prev_rulemap, rulemap):
"""Build a report of changes between 2 rulemaps.
Returns a dict with the following keys that each contain a list of
rules.
- added
- removed
- modified
"""
report = {
"added": [],
"removed": [],
"modified": []
}
for key in rulemap:
rule = rulemap[key]
if not rule.id in prev_rulemap:
report["added"].append(rule)
elif rule.format() != prev_rulemap[rule.id].format():
report["modified"].append(rule)
for key in prev_rulemap:
rule = prev_rulemap[key]
if not rule.id in rulemap:
report["removed"].append(rule)
return report
def filter(self, rule):
modified_rule = self.pattern.sub(self.repl, rule.format())
parsed = idstools.rule.parse(modified_rule, rule.group)
if parsed is None:
logger.error("Modification of rule %s results in invalid rule: %s",
rule.idstr, modified_rule)
return rule
return parsed
def filter(self, rule):
modified_rule = self.pattern.sub(self.repl, rule.format())
parsed = idstools.rule.parse(modified_rule, rule.group)
if parsed is None:
logger.error("Modification of rule %s results in invalid rule: %s",
rule.idstr, modified_rule)
return rule
return parsed
def process(self, filein, fileout, rulemap):
count = 0
for line in filein:
line = line.rstrip()
if not line or line.startswith("#"):
print(line, file=fileout)
continue
pattern = self.extract_pattern(line)
if not pattern:
print(line, file=fileout)
else:
for rule in rulemap.values():
if rule.enabled:
if pattern.search(rule.format()):
count += 1
print("# %s" % (rule.brief()), file=fileout)
print(self.replace(line, rule), file=fileout)
print("", file=fileout)
logger.info("Generated %d thresholds to %s." % (count, fileout.name))
def process(self, filein, fileout, rulemap):
count = 0
for line in filein:
line = line.rstrip()
if not line or line.startswith("#"):
print(line, file=fileout)
continue
pattern = self.extract_pattern(line)
if not pattern:
print(line, file=fileout)
else:
for rule in rulemap.values():
if rule.enabled:
if pattern.search(rule.format()):
count += 1
print("# %s" % (rule.brief()), file=fileout)
print(self.replace(line, rule), file=fileout)
print("", file=fileout)
logger.info("Generated %d thresholds to %s." % (count, fileout.name))
def main():
global args
conf_filenames = [arg for arg in sys.argv if arg.startswith("@")]
if not conf_filenames:
if os.path.exists("./rulecat.conf"):
logger.info("Loading ./rulecat.conf.")
sys.argv.insert(1, "@./rulecat.conf")
suricata_path = idstools.suricata.get_path()
parser = argparse.ArgumentParser(fromfile_prefix_chars="@")
parser.add_argument("-v", "--verbose", action="store_true", default=False,
help="Be more verbose")
parser.add_argument("-t", "--temp-dir", default="/var/tmp/idstools-rulecat",
metavar="<directory>",
help="Temporary work directory")
parser.add_argument("--suricata", default=suricata_path,
metavar="<path>",
help="Path to Suricata program (default: %s)" %
suricata_path)
parser.add_argument("--suricata-version", metavar="<version>",
help="Override Suricata version")
parser.add_argument("-f", "--force", action="store_true", default=False,
help="Force operations that might otherwise be skipped")
parser.add_argument("--rules-dir", metavar="<directory>",
help=argparse.SUPPRESS)
parser.add_argument("-o", "--output", metavar="<directory>",
dest="output", help="Output rules directory.")
parser.add_argument("--merged", default=None, metavar="<filename>",
help="Output merged rules file")
parser.add_argument("--yaml-fragment", metavar="<filename>",
help="Output YAML fragment for rule inclusion")
parser.add_argument("--url", metavar="<url>", action="append",
default=[],
help="URL to use instead of auto-generating one")
parser.add_argument("--local", metavar="<filename>", action="append",
default=[],
help="Local rule files or directories")
parser.add_argument("--sid-msg-map", metavar="<filename>",
help="Generate a sid-msg.map file")
parser.add_argument("--sid-msg-map-2", metavar="<filename>",
help="Generate a v2 sid-msg.map file")
parser.add_argument("--disable", metavar="<filename>",
help="Filename of disable rule configuration")
parser.add_argument("--enable", metavar="<filename>",
help="Filename of enable rule configuration")
parser.add_argument("--modify", metavar="<filename>",
help="Filename of rule modification configuration")
parser.add_argument("--drop", metavar="<filename>",
help="Filename of drop rules configuration")
parser.add_argument("--ignore", metavar="<filename>", action="append",
default=[],
help="Filenames to ignore (default: *deleted.rules)")
parser.add_argument("--no-ignore", action="store_true", default=False,
help="Disables the ignore option.")
parser.add_argument("--threshold-in", metavar="<filename>",
help="Filename of rule thresholding configuration")
parser.add_argument("--threshold-out", metavar="<filename>",
help="Output of processed threshold configuration")
parser.add_argument("--dump-sample-configs", action="store_true",
default=False,
help="Dump sample config files to current directory")
parser.add_argument("--etpro", metavar="<etpro-code>",
help="Use ET-Pro rules with provided ET-Pro code")
parser.add_argument("--etopen", action="store_true",
help="Use ET-Open rules (default)")
parser.add_argument("-q", "--quiet", action="store_true", default=False,
help="Be quiet, warning and error messages only")
parser.add_argument("--post-hook", metavar="<command>",
help="Command to run after update if modified")
parser.add_argument("-T", "--test-command", metavar="<command>",
help="Command to test Suricata configuration")
parser.add_argument("-V", "--version", action="store_true", default=False,
help="Display version")
args = parser.parse_args()
if args.version:
print("idstools-rulecat version %s" % idstools.version)
return 0
if args.verbose:
logger.setLevel(logging.DEBUG)
if args.quiet:
logger.setLevel(logging.WARNING)
logger.debug("This is idstools-rulecat version %s; Python: %s" % (
idstools.version,
sys.version.replace("\n", "- ")))
if args.dump_sample_configs:
return dump_sample_configs()
# If --no-ignore was provided, make sure args.ignore is
# empty. Otherwise if no ignores are provided, set a sane default.
if args.no_ignore:
args.ignore = []
elif len(args.ignore) == 0:
args.ignore.append("*deleted.rules")
if args.suricata_version:
suricata_version = idstools.suricata.parse_version(args.suricata_version)
if not suricata_version:
logger.error("Failed to parse provided Suricata version: %s" % (
suricata_version))
return 1
logger.info("Forcing Suricata version to %s." % (suricata_version.full))
elif args.suricata and os.path.exists(args.suricata):
suricata_version = idstools.suricata.get_version(args.suricata)
if suricata_version:
logger.info("Found Suricata version %s at %s." % (
str(suricata_version.full), args.suricata))
else:
logger.warn("Failed to get Suricata version.")
suricata_version = None
else:
suricata_version = None
if args.etpro:
args.url.append(resolve_etpro_url(args.etpro, suricata_version))
if not args.url or args.etopen:
args.url.append(resolve_etopen_url(suricata_version))
args.url = set(args.url)
file_tracker = FileTracker()
disable_matchers = []
enable_matchers = []
modify_filters = []
drop_filters = []
if args.disable and os.path.exists(args.disable):
disable_matchers += load_matchers(args.disable)
if args.enable and os.path.exists(args.enable):
enable_matchers += load_matchers(args.enable)
if args.modify and os.path.exists(args.modify):
modify_filters += load_filters(args.modify)
if args.drop and os.path.exists(args.drop):
drop_filters += load_drop_filters(args.drop)
files = Fetch(args).run()
# Remove ignored files.
for filename in list(files.keys()):
if ignore_file(args.ignore, filename):
logger.info("Ignoring file %s" % (filename))
del(files[filename])
for path in args.local:
load_local(path, files)
rules = []
for filename in files:
if not filename.endswith(".rules"):
continue
logger.debug("Parsing %s." % (filename))
rules += idstools.rule.parse_fileobj(
io.BytesIO(files[filename]), filename)
rulemap = build_rule_map(rules)
logger.info("Loaded %d rules." % (len(rules)))
# Counts of user enabled and modified rules.
enable_count = 0
modify_count = 0
drop_count = 0
# List of rules disabled by user. Used for counting, and to log
# rules that are re-enabled to meet flowbit requirements.
disabled_rules = []
for key, rule in rulemap.items():
for matcher in disable_matchers:
if rule.enabled and matcher.match(rule):
logger.debug("Disabling: %s" % (rule.brief()))
rule.enabled = False
disabled_rules.append(rule)
for matcher in enable_matchers:
if not rule.enabled and matcher.match(rule):
logger.debug("Enabling: %s" % (rule.brief()))
rule.enabled = True
enable_count += 1
for filter in drop_filters:
if filter.match(rule):
rulemap[rule.id] = filter.filter(rule)
drop_count += 1
# Apply modify filters.
for fltr in modify_filters:
for key, rule in rulemap.items():
if fltr.match(rule):
new_rule = fltr.filter(rule)
if new_rule and new_rule.format() != rule.format():
rulemap[rule.id] = new_rule
modify_count += 1
logger.info("Disabled %d rules." % (len(disabled_rules)))
logger.info("Enabled %d rules." % (enable_count))
logger.info("Modified %d rules." % (modify_count))
logger.info("Dropped %d rules." % (drop_count))
# Fixup flowbits.
resolve_flowbits(rulemap, disabled_rules)
if args.output:
if not os.path.exists(args.output):
logger.info("Making directory %s.", args.output)
os.makedirs(args.output)
for filename in files:
file_tracker.add(
os.path.join(args.output, os.path.basename(filename)))
write_to_directory(args.output, files, rulemap)
if args.merged:
file_tracker.add(args.merged)
write_merged(args.merged, rulemap)
if args.yaml_fragment:
file_tracker.add(args.yaml_fragment)
write_yaml_fragment(args.yaml_fragment, files)
if args.sid_msg_map:
write_sid_msg_map(args.sid_msg_map, rulemap, version=1)
if args.sid_msg_map_2:
write_sid_msg_map(args.sid_msg_map_2, rulemap, version=2)
if args.threshold_in and args.threshold_out:
file_tracker.add(args.threshold_out)
threshold_processor = ThresholdProcessor()
threshold_processor.process(
open(args.threshold_in), open(args.threshold_out, "w"), rulemap)
if not args.force and not file_tracker.any_modified():
logger.info(
"No changes detected, will not reload rules or run post-hooks.")
return 0
if args.test_command:
logger.info("Testing Suricata configuration with: %s" % (
args.test_command))
rc = subprocess.Popen(args.test_command, shell=True).wait()
if rc != 0:
logger.error("Suricata test failed, aborting.")
return 1
if args.post_hook:
logger.info("Running %s." % (args.post_hook))
subprocess.Popen(args.post_hook, shell=True).wait()
logger.info("Done.")
return 0
def main():
global args
conf_filenames = [arg for arg in sys.argv if arg.startswith("@")]
if not conf_filenames:
if os.path.exists("./rulecat.conf"):
logger.info("Loading ./rulecat.conf.")
sys.argv.insert(1, "@./rulecat.conf")
suricata_path = idstools.suricata.get_path()
parser = argparse.ArgumentParser(fromfile_prefix_chars="@")
parser.add_argument("-v", "--verbose", action="store_true", default=False,
help="Be more verbose")
parser.add_argument("-t", "--temp-dir", default="/var/tmp/idstools-rulecat",
metavar="<directory>",
help="Temporary work directory")
parser.add_argument("--suricata", default=suricata_path,
metavar="<path>",
help="Path to Suricata program (default: %s)" %
suricata_path)
parser.add_argument("--suricata-version", metavar="<version>",
help="Override Suricata version")
parser.add_argument("-f", "--force", action="store_true", default=False,
help="Force operations that might otherwise be skipped")
parser.add_argument("--rules-dir", metavar="<directory>",
help=argparse.SUPPRESS)
parser.add_argument("-o", "--output", metavar="<directory>",
dest="output", help="Output rules directory.")
parser.add_argument("--merged", default=None, metavar="<filename>",
help="Output merged rules file")
parser.add_argument("--yaml-fragment", metavar="<filename>",
help="Output YAML fragment for rule inclusion")
parser.add_argument("--url", metavar="<url>", action="append",
default=[],
help="URL to use instead of auto-generating one")
parser.add_argument("--local", metavar="<filename>", action="append",
default=[],
help="Local rule files or directories")
parser.add_argument("--sid-msg-map", metavar="<filename>",
help="Generate a sid-msg.map file")
parser.add_argument("--sid-msg-map-2", metavar="<filename>",
help="Generate a v2 sid-msg.map file")
parser.add_argument("--disable", metavar="<filename>",
help="Filename of disable rule configuration")
parser.add_argument("--enable", metavar="<filename>",
help="Filename of enable rule configuration")
parser.add_argument("--modify", metavar="<filename>",
help="Filename of rule modification configuration")
parser.add_argument("--drop", metavar="<filename>",
help="Filename of drop rules configuration")
parser.add_argument("--ignore", metavar="<filename>", action="append",
default=[],
help="Filenames to ignore (default: *deleted.rules)")
parser.add_argument("--no-ignore", action="store_true", default=False,
help="Disables the ignore option.")
parser.add_argument("--threshold-in", metavar="<filename>",
help="Filename of rule thresholding configuration")
parser.add_argument("--threshold-out", metavar="<filename>",
help="Output of processed threshold configuration")
parser.add_argument("--dump-sample-configs", action="store_true",
default=False,
help="Dump sample config files to current directory")
parser.add_argument("--etpro", metavar="<etpro-code>",
help="Use ET-Pro rules with provided ET-Pro code")
parser.add_argument("--etopen", action="store_true",
help="Use ET-Open rules (default)")
parser.add_argument("-q", "--quiet", action="store_true", default=False,
help="Be quiet, warning and error messages only")
parser.add_argument("--post-hook", metavar="<command>",
help="Command to run after update if modified")
parser.add_argument("-T", "--test-command", metavar="<command>",
help="Command to test Suricata configuration")
parser.add_argument("-V", "--version", action="store_true", default=False,
help="Display version")
args = parser.parse_args()
if args.version:
print("idstools-rulecat version %s" % idstools.version)
return 0
if args.verbose:
logger.setLevel(logging.DEBUG)
if args.quiet:
logger.setLevel(logging.WARNING)
logger.debug("This is idstools-rulecat version %s; Python: %s" % (
idstools.version,
sys.version.replace("\n", "- ")))
if args.dump_sample_configs:
return dump_sample_configs()
# If --no-ignore was provided, make sure args.ignore is
# empty. Otherwise if no ignores are provided, set a sane default.
if args.no_ignore:
args.ignore = []
elif len(args.ignore) == 0:
args.ignore.append("*deleted.rules")
suricata_version = None
if args.suricata_version:
suricata_version = idstools.suricata.parse_version(args.suricata_version)
if not suricata_version:
logger.error("Failed to parse provided Suricata version: %s" % (
suricata_version))
return 1
logger.info("Forcing Suricata version to %s." % (suricata_version.full))
elif args.suricata and os.path.exists(args.suricata):
suricata_version = idstools.suricata.get_version(args.suricata)
if suricata_version:
logger.info("Found Suricata version %s at %s." % (
str(suricata_version.full), args.suricata))
else:
logger.warn("Failed to get Suricata version, using %s",
DEFAULT_SURICATA_VERSION)
if suricata_version is None:
suricata_version = idstools.suricata.parse_version(
DEFAULT_SURICATA_VERSION)
if args.etpro:
args.url.append(resolve_etpro_url(args.etpro, suricata_version))
if not args.url or args.etopen:
args.url.append(resolve_etopen_url(suricata_version))
args.url = set(args.url)
file_tracker = FileTracker()
disable_matchers = []
enable_matchers = []
modify_filters = []
drop_filters = []
if args.disable and os.path.exists(args.disable):
disable_matchers += load_matchers(args.disable)
if args.enable and os.path.exists(args.enable):
enable_matchers += load_matchers(args.enable)
if args.modify and os.path.exists(args.modify):
modify_filters += load_filters(args.modify)
if args.drop and os.path.exists(args.drop):
drop_filters += load_drop_filters(args.drop)
files = Fetch(args).run()
# Remove ignored files.
for filename in list(files.keys()):
if ignore_file(args.ignore, filename):
logger.info("Ignoring file %s" % (filename))
del(files[filename])
for path in args.local:
load_local(path, files)
rules = []
for filename in files:
if not filename.endswith(".rules"):
continue
logger.debug("Parsing %s." % (filename))
rules += idstools.rule.parse_fileobj(
io.BytesIO(files[filename]), filename)
rulemap = build_rule_map(rules)
logger.info("Loaded %d rules." % (len(rules)))
# Counts of user enabled and modified rules.
enable_count = 0
modify_count = 0
drop_count = 0
# List of rules disabled by user. Used for counting, and to log
# rules that are re-enabled to meet flowbit requirements.
disabled_rules = []
for key, rule in rulemap.items():
for matcher in disable_matchers:
if rule.enabled and matcher.match(rule):
logger.debug("Disabling: %s" % (rule.brief()))
rule.enabled = False
disabled_rules.append(rule)
for matcher in enable_matchers:
if not rule.enabled and matcher.match(rule):
logger.debug("Enabling: %s" % (rule.brief()))
rule.enabled = True
enable_count += 1
for filter in drop_filters:
if filter.match(rule):
rulemap[rule.id] = filter.filter(rule)
drop_count += 1
# Apply modify filters.
for fltr in modify_filters:
for key, rule in rulemap.items():
if fltr.match(rule):
new_rule = fltr.filter(rule)
if new_rule and new_rule.format() != rule.format():
rulemap[rule.id] = new_rule
modify_count += 1
logger.info("Disabled %d rules." % (len(disabled_rules)))
logger.info("Enabled %d rules." % (enable_count))
logger.info("Modified %d rules." % (modify_count))
logger.info("Dropped %d rules." % (drop_count))
# Fixup flowbits.
resolve_flowbits(rulemap, disabled_rules)
if args.output:
if not os.path.exists(args.output):
logger.info("Making directory %s.", args.output)
os.makedirs(args.output)
for filename in files:
file_tracker.add(
os.path.join(args.output, os.path.basename(filename)))
write_to_directory(args.output, files, rulemap)
if args.merged:
file_tracker.add(args.merged)
write_merged(args.merged, rulemap)
if args.yaml_fragment:
file_tracker.add(args.yaml_fragment)
write_yaml_fragment(args.yaml_fragment, files)
if args.sid_msg_map:
write_sid_msg_map(args.sid_msg_map, rulemap, version=1)
if args.sid_msg_map_2:
write_sid_msg_map(args.sid_msg_map_2, rulemap, version=2)
if args.threshold_in and args.threshold_out:
file_tracker.add(args.threshold_out)
threshold_processor = ThresholdProcessor()
threshold_processor.process(
open(args.threshold_in), open(args.threshold_out, "w"), rulemap)
if not args.force and not file_tracker.any_modified():
logger.info(
"No changes detected, will not reload rules or run post-hooks.")
return 0
if args.test_command:
logger.info("Testing Suricata configuration with: %s" % (
args.test_command))
rc = subprocess.Popen(args.test_command, shell=True).wait()
if rc != 0:
logger.error("Suricata test failed, aborting.")
return 1
if args.post_hook:
logger.info("Running %s." % (args.post_hook))
subprocess.Popen(args.post_hook, shell=True).wait()
logger.info("Done.")
return 0
