def process(self, filein, fileout, rulemap):
count = 0
for line in filein:
line = line.rstrip()
if not line or line.startswith("#"):
print(line, file=fileout)
continue
pattern = self.extract_pattern(line)
if not pattern:
print(line, file=fileout)
else:
for rule in rulemap.values():
if rule.enabled:
if pattern.search(str(rule)):
count += 1
print("# %s" % (rule.brief()), file=fileout)
print(self.replace(line, rule), file=fileout)
print("", file=fileout)
logger.info("Generated %d thresholds to %s." % (count, fileout.name))
def process(self, filein, fileout, rulemap):
count = 0
for line in filein:
line = line.rstrip()
if not line or line.startswith("#"):
print(line, file=fileout)
continue
pattern = self.extract_pattern(line)
if not pattern:
print(line, file=fileout)
else:
for rule in rulemap.values():
if rule.enabled:
if pattern.search(str(rule)):
count += 1
print("# %s" % (rule.brief()), file=fileout)
print(self.replace(line, rule), file=fileout)
print("", file=fileout)
logger.info("Generated %d thresholds to %s." % (count, fileout.name))
def resolve_flowbits(rulemap, disabled_rules):
flowbit_resolver = idstools.rule.FlowbitResolver()
flowbit_enabled = set()
while True:
flowbits = flowbit_resolver.get_required_flowbits(rulemap)
logger.debug("Found %d required flowbits.", len(flowbits))
required_rules = flowbit_resolver.get_required_rules(rulemap, flowbits)
logger.debug("Found %d rules to enable to for flowbit requirements",
len(required_rules))
if not required_rules:
logger.debug("All required rules enabled.")
break
for rule in required_rules:
if not rule.enabled and rule in disabled_rules:
logger.debug(
"Enabling previously disabled rule for flowbits: %s" %
(rule.brief()))
rule.enabled = True
flowbit_enabled.add(rule)
logger.info("Enabled %d rules for flowbit dependencies." %
(len(flowbit_enabled)))
def resolve_flowbits(rulemap, disabled_rules):
flowbit_resolver = idstools.rule.FlowbitResolver()
flowbit_enabled = set()
while True:
flowbits = flowbit_resolver.get_required_flowbits(rulemap)
logger.debug("Found %d required flowbits.", len(flowbits))
required_rules = flowbit_resolver.get_required_rules(rulemap, flowbits)
logger.debug(
"Found %d rules to enable to fullfull flowbit requirements",
len(required_rules))
if not required_rules:
logger.debug("All required rules enabled.")
break
for rule in required_rules:
if not rule.enabled and rule in disabled_rules:
logger.debug(
"Enabling previously disabled rule for flowbits: %s" % (
rule.brief()))
rule.enabled = True
flowbit_enabled.add(rule)
logger.info("Enabled %d rules for flowbit dependencies." % (
len(flowbit_enabled)))
def main():
conf_filenames = [arg for arg in sys.argv if arg.startswith("@")]
if not conf_filenames:
if os.path.exists("./rulecat.conf"):
logger.info("Loading ./rulecat.conf.")
sys.argv.insert(1, "@./rulecat.conf")
suricata_path = idstools.suricata.get_path()
parser = argparse.ArgumentParser(fromfile_prefix_chars="@")
parser.add_argument("-v",
"--verbose",
action="store_true",
default=False,
help="Be more verbose")
parser.add_argument("-t",
"--temp-dir",
default="/var/tmp/idstools-rulecat",
metavar="<directory>",
help="Temporary work directory")
parser.add_argument("--suricata",
default=suricata_path,
metavar="<path>",
help="Path to Suricata program (default: %s)" %
suricata_path)
parser.add_argument(
"-f",
"--force",
action="store_true",
default=False,
help="Force operations that might otherwise be skipped")
parser.add_argument("--rules-dir",
metavar="<directory>",
help=argparse.SUPPRESS)
parser.add_argument("-o",
"--output",
metavar="<directory>",
dest="output",
help="Output rules directory.")
parser.add_argument("--merged",
default=None,
metavar="<filename>",
help="Output merged rules file")
parser.add_argument("--yaml-fragment",
metavar="<filename>",
help="Output YAML fragment for rule inclusion")
parser.add_argument("--url",
metavar="<url>",
action="append",
default=[],
help="URL to use instead of auto-generating one")
parser.add_argument("--local",
metavar="<filename>",
help="Local rule files or directories")
parser.add_argument("--sid-msg-map",
metavar="<filename>",
help="Generate a sid-msg.map file")
parser.add_argument("--sid-msg-map-2",
metavar="<filename>",
help="Generate a v2 sid-msg.map file")
parser.add_argument("--disable",
metavar="<filename>",
help="Filename of disable rule configuration")
parser.add_argument("--enable",
metavar="<filename>",
help="Filename of enable rule configuration")
parser.add_argument("--modify",
metavar="<filename>",
help="Filename of rule modification configuration")
parser.add_argument("--drop",
metavar="<filename>",
help="Filename of drop rules configuration")
parser.add_argument("--threshold-in",
metavar="<filename>",
help="Filename of rule thresholding configuration")
parser.add_argument("--threshold-out",
metavar="<filename>",
help="Output of processed threshold configuration")
parser.add_argument("--dump-sample-configs",
action="store_true",
default=False,
help="Dump sample config files to current directory")
parser.add_argument("--etpro",
metavar="<etpro-code>",
help="Use ET-Pro rules with provided ET-Pro code")
parser.add_argument("--etopen",
action="store_true",
help="Use ET-Open rules (default)")
parser.add_argument("-q",
"--quiet",
action="store_true",
default=False,
help="Be quiet, warning and error messages only")
parser.add_argument("--post-hook",
metavar="<command>",
help="Command to run after update if modified")
parser.add_argument("-T",
"--test-command",
metavar="<command>",
help="Command to test Suricata configuration")
args = parser.parse_args()
if args.verbose:
logger.setLevel(logging.DEBUG)
if args.quiet:
logger.setLevel(logging.WARNING)
if args.dump_sample_configs:
return dump_sample_configs()
if args.suricata and os.path.exists(args.suricata):
suricata_version = idstools.suricata.get_version(args.suricata)
logger.info("Found Suricata version %s at %s." %
(str(suricata_version.full), args.suricata))
else:
suricata_version = None
if args.etpro:
args.url.append(resolve_etpro_url(args.etpro, suricata_version))
if not args.url or args.etopen:
args.url.append(resolve_etopen_url(suricata_version))
args.url = set(args.url)
file_tracker = FileTracker()
disable_matchers = []
enable_matchers = []
modify_filters = []
drop_filters = []
if args.disable and os.path.exists(args.disable):
disable_matchers += load_matchers(args.disable)
if args.enable and os.path.exists(args.enable):
enable_matchers += load_matchers(args.enable)
if args.modify and os.path.exists(args.modify):
modify_filters += load_filters(args.modify)
if args.drop and os.path.exists(args.drop):
drop_filters += load_drop_filters(args.drop)
files = Fetch(args).run()
if args.local:
load_local_files(args.local, files)
rules = []
for filename in files:
logger.debug("Parsing %s." % (filename))
rules += idstools.rule.parse_fileobj(BytesIO(files[filename]),
filename)
rulemap = build_rule_map(rules)
logger.info("Loaded %d rules." % (len(rules)))
# Counts of user enabled and modified rules.
enable_count = 0
modify_count = 0
drop_count = 0
# List of rules disabled by user. Used for counting, and to log
# rules that are re-enabled to meet flowbit requirements.
disabled_rules = []
for key, rule in rulemap.items():
for matcher in disable_matchers:
if rule.enabled and matcher.match(rule):
logger.debug("Disabling: %s" % (rule.brief()))
rule.enabled = False
disabled_rules.append(rule)
for matcher in enable_matchers:
if not rule.enabled and matcher.match(rule):
logger.debug("Enabling: %s" % (rule.brief()))
rule.enabled = True
enable_count += 1
# Unlike enable and disable, modify returns a new instance of
# the rule.
for filter in modify_filters:
if filter.match(rule):
rulemap[rule.id] = filter.filter(rule)
modify_count += 1
for filter in drop_filters:
if filter.match(rule):
rulemap[rule.id] = filter.filter(rule)
drop_count += 1
logger.info("Disabled %d rules." % (len(disabled_rules)))
logger.info("Enabled %d rules." % (enable_count))
logger.info("Modified %d rules." % (modify_count))
logger.info("Dropped %d rules." % (drop_count))
# Fixup flowbits.
resolve_flowbits(rulemap, disabled_rules)
if args.output:
if not os.path.exists(args.output):
logger.info("Making directory %s.", args.output)
os.makedirs(args.output)
for filename in files:
file_tracker.add(os.path.join(args.output, filename))
write_to_directory(args.output, files, rulemap)
if args.merged:
file_tracker.add(args.merged)
write_merged(args.merged, rulemap)
if args.yaml_fragment:
file_tracker.add(args.yaml_fragment)
write_yaml_fragment(args.yaml_fragment, files)
if args.sid_msg_map:
write_sid_msg_map(args.sid_msg_map, rulemap, version=1)
if args.sid_msg_map_2:
write_sid_msg_map(args.sid_msg_map_2, rulemap, version=2)
if args.threshold_in and args.threshold_out:
file_tracker.add(args.threshold_out)
threshold_processor = ThresholdProcessor()
threshold_processor.process(open(args.threshold_in),
open(args.threshold_out, "w"), rulemap)
if not args.force and not file_tracker.any_modified():
logger.info(
"No changes detected, will not reload rules or run post-hooks.")
return 0
if args.test_command:
logger.info("Testing Suricata configuration with: %s" %
(args.test_command))
rc = subprocess.Popen(args.test_command, shell=True).wait()
if rc != 0:
logger.error("Suricata test failed, aborting.")
return 1
if args.post_hook:
logger.info("Running %s." % (args.post_hook))
subprocess.Popen(args.post_hook, shell=True).wait()
logger.info("Done.")
return 0
def main():
conf_filenames = [arg for arg in sys.argv if arg.startswith("@")]
if not conf_filenames:
if os.path.exists("./rulecat.conf"):
logger.info("Loading ./rulecat.conf.")
sys.argv.insert(1, "@./rulecat.conf")
suricata_path = idstools.suricata.get_path()
parser = argparse.ArgumentParser(fromfile_prefix_chars="@")
parser.add_argument("-v", "--verbose", action="store_true", default=False,
help="Be more verbose")
parser.add_argument("-t", "--temp-dir", default="/var/tmp/idstools-rulecat",
metavar="<directory>",
help="Temporary work directory")
parser.add_argument("--suricata", default=suricata_path,
metavar="<path>",
help="Path to Suricata program (default: %s)" %
suricata_path)
parser.add_argument("-f", "--force", action="store_true", default=False,
help="Force operations that might otherwise be skipped")
parser.add_argument("--rules-dir", metavar="<directory>",
help=argparse.SUPPRESS)
parser.add_argument("-o", "--output", metavar="<directory>",
dest="output", help="Output rules directory.")
parser.add_argument("--merged", default=None, metavar="<filename>",
help="Output merged rules file")
parser.add_argument("--yaml-fragment", metavar="<filename>",
help="Output YAML fragment for rule inclusion")
parser.add_argument("--url", metavar="<url>", action="append",
default=[],
help="URL to use instead of auto-generating one")
parser.add_argument("--local", metavar="<filename>",
help="Local rule files or directories")
parser.add_argument("--sid-msg-map", metavar="<filename>",
help="Generate a sid-msg.map file")
parser.add_argument("--sid-msg-map-2", metavar="<filename>",
help="Generate a v2 sid-msg.map file")
parser.add_argument("--disable", metavar="<filename>",
help="Filename of disable rule configuration")
parser.add_argument("--enable", metavar="<filename>",
help="Filename of enable rule configuration")
parser.add_argument("--modify", metavar="<filename>",
help="Filename of rule modification configuration")
parser.add_argument("--drop", metavar="<filename>",
help="Filename of drop rules configuration")
parser.add_argument("--threshold-in", metavar="<filename>",
help="Filename of rule thresholding configuration")
parser.add_argument("--threshold-out", metavar="<filename>",
help="Output of processed threshold configuration")
parser.add_argument("--dump-sample-configs", action="store_true",
default=False,
help="Dump sample config files to current directory")
parser.add_argument("--etpro", metavar="<etpro-code>",
help="Use ET-Pro rules with provided ET-Pro code")
parser.add_argument("--etopen", action="store_true",
help="Use ET-Open rules (default)")
parser.add_argument("-q", "--quiet", action="store_true", default=False,
help="Be quiet, warning and error messages only")
parser.add_argument("--post-hook", metavar="<command>",
help="Command to run after update if modified")
parser.add_argument("-T", "--test-command", metavar="<command>",
help="Command to test Suricata configuration")
args = parser.parse_args()
if args.verbose:
logger.setLevel(logging.DEBUG)
if args.quiet:
logger.setLevel(logging.WARNING)
if args.dump_sample_configs:
return dump_sample_configs()
if args.suricata and os.path.exists(args.suricata):
suricata_version = idstools.suricata.get_version(args.suricata)
logger.info("Found Suricata version %s at %s." % (
str(suricata_version.full), args.suricata))
else:
suricata_version = None
if args.etpro:
args.url.append(resolve_etpro_url(args.etpro, suricata_version))
if not args.url or args.etopen:
args.url.append(resolve_etopen_url(suricata_version))
args.url = set(args.url)
file_tracker = FileTracker()
disable_matchers = []
enable_matchers = []
modify_filters = []
drop_filters = []
if args.disable and os.path.exists(args.disable):
disable_matchers += load_matchers(args.disable)
if args.enable and os.path.exists(args.enable):
enable_matchers += load_matchers(args.enable)
if args.modify and os.path.exists(args.modify):
modify_filters += load_filters(args.modify)
if args.drop and os.path.exists(args.drop):
drop_filters += load_drop_filters(args.drop)
files = Fetch(args).run()
if args.local:
load_local_files(args.local, files)
rules = []
for filename in files:
logger.debug("Parsing %s." % (filename))
rules += idstools.rule.parse_fileobj(
BytesIO(files[filename]), filename)
rulemap = build_rule_map(rules)
logger.info("Loaded %d rules." % (len(rules)))
# Counts of user enabled and modified rules.
enable_count = 0
modify_count = 0
drop_count = 0
# List of rules disabled by user. Used for counting, and to log
# rules that are re-enabled to meet flowbit requirements.
disabled_rules = []
for key, rule in rulemap.items():
for matcher in disable_matchers:
if rule.enabled and matcher.match(rule):
logger.debug("Disabling: %s" % (rule.brief()))
rule.enabled = False
disabled_rules.append(rule)
for matcher in enable_matchers:
if not rule.enabled and matcher.match(rule):
logger.debug("Enabling: %s" % (rule.brief()))
rule.enabled = True
enable_count += 1
# Unlike enable and disable, modify returns a new instance of
# the rule.
for filter in modify_filters:
if filter.match(rule):
rulemap[rule.id] = filter.filter(rule)
modify_count += 1
for filter in drop_filters:
if filter.match(rule):
rulemap[rule.id] = filter.filter(rule)
drop_count += 1
logger.info("Disabled %d rules." % (len(disabled_rules)))
logger.info("Enabled %d rules." % (enable_count))
logger.info("Modified %d rules." % (modify_count))
logger.info("Dropped %d rules." % (drop_count))
# Fixup flowbits.
resolve_flowbits(rulemap, disabled_rules)
if args.output:
if not os.path.exists(args.output):
logger.info("Making directory %s.", args.output)
os.makedirs(args.output)
for filename in files:
file_tracker.add(os.path.join(args.output, filename))
write_to_directory(args.output, files, rulemap)
if args.merged:
file_tracker.add(args.merged)
write_merged(args.merged, rulemap)
if args.yaml_fragment:
file_tracker.add(args.yaml_fragment)
write_yaml_fragment(args.yaml_fragment, files)
if args.sid_msg_map:
write_sid_msg_map(args.sid_msg_map, rulemap, version=1)
if args.sid_msg_map_2:
write_sid_msg_map(args.sid_msg_map_2, rulemap, version=2)
if args.threshold_in and args.threshold_out:
file_tracker.add(args.threshold_out)
threshold_processor = ThresholdProcessor()
threshold_processor.process(
open(args.threshold_in), open(args.threshold_out, "w"), rulemap)
if not args.force and not file_tracker.any_modified():
logger.info(
"No changes detected, will not reload rules or run post-hooks.")
return 0
if args.test_command:
logger.info("Testing Suricata configuration with: %s" % (
args.test_command))
rc = subprocess.Popen(args.test_command, shell=True).wait()
if rc != 0:
logger.error("Suricata test failed, aborting.")
return 1
if args.post_hook:
logger.info("Running %s." % (args.post_hook))
subprocess.Popen(args.post_hook, shell=True).wait()
logger.info("Done.")
return 0
