def fetch(self, url):
tmp_filename = self.get_tmp_filename(url)
if not config.args().force and os.path.exists(tmp_filename):
if not config.args().now and \
time.time() - os.stat(tmp_filename).st_mtime < (60 * 15):
logger.info(
"Last download less than 15 minutes ago. Not downloading %s.",
url)
return self.extract_files(tmp_filename)
if self.check_checksum(tmp_filename, url):
logger.info("Remote checksum has not changed. Not fetching.")
return self.extract_files(tmp_filename)
if not os.path.exists(config.get_cache_dir()):
os.makedirs(config.get_cache_dir(), mode=0o770)
logger.info("Fetching %s." % (url))
try:
tmp_fileobj = tempfile.NamedTemporaryFile()
suricata.update.net.get(url,
tmp_fileobj,
progress_hook=self.progress_hook)
shutil.copyfile(tmp_fileobj.name, tmp_filename)
tmp_fileobj.close()
except URLError as err:
if os.path.exists(tmp_filename):
logger.warning(
"Failed to fetch %s, "
"will use latest cached version: %s", url, err)
return self.extract_files(tmp_filename)
raise err
except Exception as err:
raise err
if not config.args().quiet:
self.progress_hook_finish()
logger.info("Done.")
return self.extract_files(tmp_filename)
def fetch(self, url):
net_arg = url
checksum = url[2]
url = url[0]
tmp_filename = self.get_tmp_filename(url)
if config.args().offline:
if config.args().force:
logger.warning("Running offline, skipping download of %s", url)
logger.info("Using latest cached version of rule file: %s", url)
if not os.path.exists(tmp_filename):
logger.error("Can't proceed offline, "
"source %s has not yet been downloaded.", url)
sys.exit(1)
return self.extract_files(tmp_filename)
if not config.args().force and os.path.exists(tmp_filename):
if not config.args().now and \
time.time() - os.stat(tmp_filename).st_mtime < (60 * 15):
logger.info(
"Last download less than 15 minutes ago. Not downloading %s.",
url)
return self.extract_files(tmp_filename)
if checksum:
if self.check_checksum(tmp_filename, url):
logger.info("Remote checksum has not changed. "
"Not fetching.")
return self.extract_files(tmp_filename)
if not os.path.exists(config.get_cache_dir()):
os.makedirs(config.get_cache_dir(), mode=0o770)
logger.info("Fetching %s." % (url))
try:
tmp_fileobj = tempfile.NamedTemporaryFile()
net.get(
net_arg,
tmp_fileobj,
progress_hook=self.progress_hook)
shutil.copyfile(tmp_fileobj.name, tmp_filename)
tmp_fileobj.close()
except URLError as err:
if os.path.exists(tmp_filename):
logger.warning(
"Failed to fetch %s, "
"will use latest cached version: %s", url, err)
return self.extract_files(tmp_filename)
raise err
except IOError as err:
self.progress_hook_finish()
logger.error("Failed to copy file: %s", err)
sys.exit(1)
except Exception as err:
raise err
self.progress_hook_finish()
logger.info("Done.")
return self.extract_files(tmp_filename)
def add_source():
args = config.args()
if args.name:
name = args.name
else:
while True:
name = raw_input("Name of source: ").strip()
if name:
break
if sources.source_name_exists(name):
logger.error("A source with name %s already exists.", name)
return 1
if args.url:
url = args.url
else:
while True:
url = raw_input("URL: ").strip()
if url:
break
source_config = sources.SourceConfiguration(name, url=url)
sources.save_source_config(source_config)
def list_sources():
free_only = config.args().free
if not sources.source_index_exists(config):
logger.info("No source index found, running update-sources")
try:
update_sources()
except exceptions.ApplicationError as err:
logger.warning("%s: will use bundled index.", err)
index = sources.load_source_index(config)
for name, source in index.get_sources().items():
is_not_free = source.get("subscribe-url")
if free_only and is_not_free:
continue
print("%s: %s" % (util.bright_cyan("Name"), util.bright_magenta(name)))
print(" %s: %s" % (util.bright_cyan("Vendor"),
util.bright_magenta(source["vendor"])))
print(" %s: %s" % (util.bright_cyan("Summary"),
util.bright_magenta(source["summary"])))
print(" %s: %s" % (util.bright_cyan("License"),
util.bright_magenta(source["license"])))
if "tags" in source:
print(" %s: %s" %
(util.bright_cyan("Tags"),
util.bright_magenta(", ".join(source["tags"]))))
if "replaces" in source:
print(" %s: %s" %
(util.bright_cyan("Replaces"),
util.bright_magenta(", ".join(source["replaces"]))))
if "parameters" in source:
print(" %s: %s" %
(util.bright_cyan("Parameters"),
util.bright_magenta(", ".join(source["parameters"]))))
if "subscribe-url" in source:
print(" %s: %s" % (util.bright_cyan("Subscription"),
util.bright_magenta(source["subscribe-url"])))
def add_source():
args = config.args()
if args.name:
name = args.name
else:
while True:
name = input("Name of source: ").strip()
if name:
break
if sources.source_name_exists(name):
logger.error("A source with name %s already exists.", name)
return 1
if args.url:
url = args.url
else:
while True:
url = input("URL: ").strip()
if url:
break
checksum = args.no_checksum
header = args.http_header if args.http_header else None
source_config = sources.SourceConfiguration(
name, header=header, url=url, checksum=checksum)
sources.save_source_config(source_config)
def disable_source():
name = config.args().name
filename = sources.get_enabled_source_filename(name)
if not os.path.exists(filename):
logger.debug("Filename %s does not exist.", filename)
logger.warning("Source %s is not enabled.", name)
return 0
logger.debug("Renaming %s to %s.disabled.", filename, filename)
os.rename(filename, "%s.disabled" % (filename))
logger.info("Source %s has been disabled", name)
def progress_hook(self, content_length, bytes_read):
if config.args().quiet or not self.istty:
return
if not content_length or content_length == 0:
percent = 0
else:
percent = int((bytes_read / float(content_length)) * 100)
buf = " %3d%% - %-30s" % (percent, "%d/%d" %
(bytes_read, content_length))
sys.stdout.write(buf)
sys.stdout.flush()
sys.stdout.write("\b" * 38)
def enable_source():
name = config.args().name
# Check if source is already enabled.
enabled_source_filename = sources.get_enabled_source_filename(name)
if os.path.exists(enabled_source_filename):
logger.error("The source %s is already enabled.", name)
return 1
# First check if this source was previous disabled and then just
# re-enable it.
disabled_source_filename = sources.get_disabled_source_filename(name)
if os.path.exists(disabled_source_filename):
logger.info("Re-enabling previous disabled source for %s.", name)
os.rename(disabled_source_filename, enabled_source_filename)
return 0
if not os.path.exists(sources.get_index_filename()):
logger.warning("Source index does not exist, will use bundled one.")
logger.warning("Please run suricata-update update-sources.")
source_index = sources.load_source_index(config)
if not name in source_index.get_sources():
logger.error("Unknown source: %s", name)
return 1
# Parse key=val options.
opts = {}
for param in config.args().params:
key, val = param.split("=", 1)
opts[key] = val
source = source_index.get_sources()[name]
if "subscribe-url" in source:
print("The source %s requires a subscription. Subscribe here:" %
(name))
print(" %s" % source["subscribe-url"])
params = {}
if "parameters" in source:
for param in source["parameters"]:
if param in opts:
params[param] = opts[param]
else:
prompt = source["parameters"][param]["prompt"]
while True:
r = raw_input("%s (%s): " % (prompt, param))
r = r.strip()
if r:
break
params[param] = r.strip()
new_source = sources.SourceConfiguration(name, params=params)
# If the source directory does not exist, create it. Also create
# the default rule-source of et/open, unless the source being
# enabled replaces it.
source_directory = sources.get_source_directory()
if not os.path.exists(source_directory):
try:
logger.info("Creating directory %s", source_directory)
os.makedirs(source_directory)
except Exception as err:
logger.error("Failed to create directory %s: %s", source_directory,
err)
return 1
if "replaces" in source and default_source in source["replaces"]:
logger.debug(
"Not enabling default source as selected source replaces it")
elif new_source.name == default_source:
logger.debug(
"Not enabling default source as selected source is the default"
)
else:
logger.info("Enabling default source %s", default_source)
if not source_index.get_source_by_name(default_source):
logger.error("Default source %s not in index", default_source)
else:
default_source_config = sources.SourceConfiguration(
default_source)
write_source_config(default_source_config, True)
write_source_config(new_source, True)
logger.info("Source %s enabled", new_source.name)
if "replaces" in source:
for replaces in source["replaces"]:
filename = sources.get_enabled_source_filename(replaces)
if os.path.exists(filename):
logger.info("Removing source %s as its replaced by %s",
replaces, new_source.name)
logger.debug("Deleting %s", filename)
os.unlink(filename)
def _main():
global args
args = parsers.parse_arg()
# Go verbose or quiet sooner than later.
if args.verbose:
logger.setLevel(logging.DEBUG)
if args.quiet:
logger.setLevel(logging.WARNING)
logger.debug("This is suricata-update version %s (rev: %s); Python: %s" %
(version, revision, sys.version.replace("\n", "- ")))
config.init(args)
# Error out if any reserved/unimplemented arguments were set.
unimplemented_args = [
"disable",
"enable",
"modify",
"drop",
]
for arg in unimplemented_args:
if hasattr(args, arg) and getattr(args, arg):
logger.error("--%s not implemented", arg)
return 1
suricata_path = config.get("suricata")
# Now parse the Suricata version. If provided on the command line,
# use that, otherwise attempt to get it from Suricata.
if args.suricata_version:
# The Suricata version was passed on the command line, parse it.
suricata_version = engine.parse_version(args.suricata_version)
if not suricata_version:
logger.error("Failed to parse provided Suricata version: %s" %
(args.suricata_version))
return 1
logger.info("Forcing Suricata version to %s." %
(suricata_version.full))
elif suricata_path:
suricata_version = engine.get_version(suricata_path)
if suricata_version:
logger.info("Found Suricata version %s at %s." %
(str(suricata_version.full), suricata_path))
else:
logger.error("Failed to get Suricata version.")
return 1
else:
logger.info("Using default Suricata version of %s",
DEFAULT_SURICATA_VERSION)
suricata_version = engine.parse_version(DEFAULT_SURICATA_VERSION)
# Provide the Suricata version to the net module to add to the
# User-Agent.
net.set_user_agent_suricata_version(suricata_version.full)
if args.subcommand:
if args.subcommand == "check-versions" and hasattr(args, "func"):
return args.func(suricata_version)
elif hasattr(args, "func"):
return args.func()
elif args.subcommand != "update":
logger.error("Unknown command: %s", args.subcommand)
return 1
if args.dump_sample_configs:
return dump_sample_configs()
# If --no-ignore was provided, clear any ignores provided in the
# config.
if args.no_ignore:
config.set(config.IGNORE_KEY, [])
file_tracker = FileTracker()
disable_matchers = []
enable_matchers = []
modify_filters = []
drop_filters = []
# Load user provided disable filters.
disable_conf_filename = config.get("disable-conf")
if disable_conf_filename and os.path.exists(disable_conf_filename):
logger.info("Loading %s.", disable_conf_filename)
disable_matchers += load_matchers(disable_conf_filename)
# Load user provided enable filters.
enable_conf_filename = config.get("enable-conf")
if enable_conf_filename and os.path.exists(enable_conf_filename):
logger.info("Loading %s.", enable_conf_filename)
enable_matchers += load_matchers(enable_conf_filename)
# Load user provided modify filters.
modify_conf_filename = config.get("modify-conf")
if modify_conf_filename and os.path.exists(modify_conf_filename):
logger.info("Loading %s.", modify_conf_filename)
modify_filters += load_filters(modify_conf_filename)
# Load user provided drop filters.
drop_conf_filename = config.get("drop-conf")
if drop_conf_filename and os.path.exists(drop_conf_filename):
logger.info("Loading %s.", drop_conf_filename)
drop_filters += load_drop_filters(drop_conf_filename)
# Load the Suricata configuration if we can.
suriconf = None
if config.get("suricata-conf") and \
os.path.exists(config.get("suricata-conf")) and \
suricata_path and os.path.exists(suricata_path):
logger.info("Loading %s", config.get("suricata-conf"))
try:
suriconf = engine.Configuration.load(config.get("suricata-conf"),
suricata_path=suricata_path)
except subprocess.CalledProcessError:
return 1
# Disable rule that are for app-layers that are not enabled.
if suriconf:
for key in suriconf.keys():
m = re.match("app-layer\.protocols\.([^\.]+)\.enabled", key)
if m:
proto = m.group(1)
if not suriconf.is_true(key, ["detection-only"]):
logger.info("Disabling rules for protocol %s", proto)
disable_matchers.append(
matchers_mod.ProtoRuleMatcher(proto))
elif proto == "smb" and suriconf.build_info:
# Special case for SMB rules. For versions less
# than 5, disable smb rules if Rust is not
# available.
if suriconf.build_info["version"].major < 5:
if not "RUST" in suriconf.build_info["features"]:
logger.info(
"Disabling rules for protocol {}".format(
proto))
disable_matchers.append(
matchers_mod.ProtoRuleMatcher(proto))
# Check that the cache directory exists and is writable.
if not os.path.exists(config.get_cache_dir()):
try:
os.makedirs(config.get_cache_dir(), mode=0o770)
except Exception as err:
logger.warning(
"Cache directory does not exist and could not be created. "
"/var/tmp will be used instead.")
config.set_cache_dir("/var/tmp")
files = load_sources(suricata_version)
load_dist_rules(files)
# Remove ignored files.
for filename in list(files.keys()):
if ignore_file(config.get("ignore"), filename):
logger.info("Ignoring file %s" % (filename))
del (files[filename])
rules = []
for filename in sorted(files):
if not filename.endswith(".rules"):
continue
logger.debug("Parsing %s." % (filename))
rules += rule_mod.parse_fileobj(io.BytesIO(files[filename]), filename)
rulemap = build_rule_map(rules)
logger.info("Loaded %d rules." % (len(rules)))
# Counts of user enabled and modified rules.
enable_count = 0
modify_count = 0
drop_count = 0
# List of rules disabled by user. Used for counting, and to log
# rules that are re-enabled to meet flowbit requirements.
disabled_rules = []
for key, rule in rulemap.items():
for matcher in disable_matchers:
if rule.enabled and matcher.match(rule):
logger.debug("Disabling: %s" % (rule.brief()))
rule.enabled = False
disabled_rules.append(rule)
for matcher in enable_matchers:
if not rule.enabled and matcher.match(rule):
logger.debug("Enabling: %s" % (rule.brief()))
rule.enabled = True
enable_count += 1
for fltr in drop_filters:
if fltr.match(rule):
rulemap[rule.id] = fltr.run(rule)
drop_count += 1
# Apply modify filters.
for fltr in modify_filters:
for key, rule in rulemap.items():
if fltr.match(rule):
new_rule = fltr.run(rule)
if new_rule and new_rule.format() != rule.format():
rulemap[rule.id] = new_rule
modify_count += 1
# Check if we should disable ja3 rules.
try:
disable_ja3(suriconf, rulemap, disabled_rules)
except Exception as err:
logger.error("Failed to dynamically disable ja3 rules: %s" % (err))
# Check rule vars, disabling rules that use unknown vars.
check_vars(suriconf, rulemap)
logger.info("Disabled %d rules." % (len(disabled_rules)))
logger.info("Enabled %d rules." % (enable_count))
logger.info("Modified %d rules." % (modify_count))
logger.info("Dropped %d rules." % (drop_count))
# Fixup flowbits.
resolve_flowbits(rulemap, disabled_rules)
# Check that output directory exists, creating it if needed.
check_output_directory(config.get_output_dir())
# Check that output directory is writable.
if not os.access(config.get_output_dir(), os.W_OK):
logger.error("Output directory is not writable: %s",
config.get_output_dir())
return 1
# Backup the output directory.
logger.info("Backing up current rules.")
backup_directory = util.mktempdir()
shutil.copytree(config.get_output_dir(),
os.path.join(backup_directory, "backup"),
ignore=copytree_ignore_backup)
if not args.no_merge:
# The default, write out a merged file.
output_filename = os.path.join(config.get_output_dir(),
DEFAULT_OUTPUT_RULE_FILENAME)
file_tracker.add(output_filename)
write_merged(os.path.join(output_filename), rulemap)
else:
for filename in files:
file_tracker.add(
os.path.join(config.get_output_dir(),
os.path.basename(filename)))
write_to_directory(config.get_output_dir(), files, rulemap)
if args.yaml_fragment:
file_tracker.add(args.yaml_fragment)
write_yaml_fragment(args.yaml_fragment, files)
if args.sid_msg_map:
write_sid_msg_map(args.sid_msg_map, rulemap, version=1)
if args.sid_msg_map_2:
write_sid_msg_map(args.sid_msg_map_2, rulemap, version=2)
if args.threshold_in and args.threshold_out:
file_tracker.add(args.threshold_out)
threshold_processor = ThresholdProcessor()
threshold_processor.process(open(args.threshold_in),
open(args.threshold_out, "w"), rulemap)
if not args.force and not file_tracker.any_modified():
logger.info("No changes detected, exiting.")
notes.dump_notes()
return 0
# Set these containers to None to fee the memory before testing Suricata which
# may consume a lot of memory by itself. Ideally we should refactor this large
# function into multiple methods so these go out of scope and get removed
# automatically.
rulemap = None
rules = None
files = None
if not test_suricata(suricata_path):
logger.error("Suricata test failed, aborting.")
logger.error("Restoring previous rules.")
copytree(os.path.join(backup_directory, "backup"),
config.get_output_dir())
return 1
if not config.args().no_reload and config.get("reload-command"):
logger.info("Running %s." % (config.get("reload-command")))
rc = subprocess.Popen(config.get("reload-command"), shell=True).wait()
if rc != 0:
logger.error("Reload command exited with error: %d", rc)
logger.info("Done.")
notes.dump_notes()
return 0
def load_sources(suricata_version):
files = {}
urls = []
http_header = None
checksum = True
# Add any URLs added with the --url command line parameter.
if config.args().url:
for url in config.args().url:
urls.append((url, http_header, checksum))
# Get the new style sources.
enabled_sources = sources.get_enabled_sources()
# Convert the Suricata version to a version string.
version_string = "%d.%d.%d" % (
suricata_version.major, suricata_version.minor, suricata_version.patch)
# Construct the URL replacement parameters that are internal to
# suricata-update.
internal_params = {"__version__": version_string}
# If we have new sources, we also need to load the index.
if enabled_sources:
index_filename = sources.get_index_filename()
if not os.path.exists(index_filename):
logger.warning("No index exists, will use bundled index.")
logger.warning("Please run suricata-update update-sources.")
if os.path.exists(index_filename) and time.time() - \
os.stat(index_filename).st_mtime > INDEX_EXPIRATION_TIME:
logger.warning(
"Source index is older than 2 weeks. "
"Please update with suricata-update update-sources.")
index = sources.Index(index_filename)
for (name, source) in enabled_sources.items():
params = source["params"] if "params" in source else {}
params.update(internal_params)
if "url" in source:
# No need to go off to the index.
http_header = source.get("http_header")
checksum = source.get("checksum")
url = (source["url"] % params, http_header, checksum)
logger.debug("Resolved source %s to URL %s.", name, url[0])
else:
if not index:
raise exceptions.ApplicationError(
"Source index is required for source %s; "
"run suricata-update update-sources" %
(source["source"]))
source_config = index.get_source_by_name(name)
try:
checksum = source_config["checksum"]
except:
checksum = True
url = (index.resolve_url(name, params), http_header, checksum)
logger.debug("Resolved source %s to URL %s.", name, url)
urls.append(url)
if config.get("sources"):
for url in config.get("sources"):
if not isinstance(url, str):
raise exceptions.InvalidConfigurationError(
"Invalid datatype for source URL: %s" % (str(url)))
url = (url % internal_params, http_header, checksum)
logger.debug("Adding source %s.", url)
urls.append(url)
# If --etopen is on the command line, make sure its added. Or if
# there are no URLs, default to ET/Open.
if config.get("etopen") or not urls:
if not config.args().offline and not urls:
logger.info(
"No sources configured, will use Emerging Threats Open")
urls.append(
(sources.get_etopen_url(internal_params), http_header, checksum))
# Converting the URLs to a set removed dupes.
urls = set(urls)
# Now download each URL.
for url in urls:
Fetch().run(url, files)
# Now load local rules.
if config.get("local") is not None:
for local in config.get("local"):
load_local(local, files)
return files
def progress_hook_finish(self):
if config.args().quiet or not self.istty:
return
sys.stdout.write("\n")
sys.stdout.flush()
def _main():
global args
global_parser = argparse.ArgumentParser(add_help=False)
global_parser.add_argument("-v",
"--verbose",
action="store_true",
default=None,
help="Be more verbose")
global_parser.add_argument(
"-q",
"--quiet",
action="store_true",
default=False,
help="Be quiet, warning and error messages only")
global_parser.add_argument(
"-D",
"--data-dir",
metavar="<directory>",
dest="data_dir",
help="Data directory (default: /var/lib/suricata)")
global_parser.add_argument(
"-c",
"--config",
metavar="<filename>",
help="configuration file (default: /etc/suricata/update.yaml)")
global_parser.add_argument(
"--suricata-conf",
metavar="<filename>",
help="configuration file (default: /etc/suricata/suricata.yaml)")
global_parser.add_argument("--suricata",
metavar="<path>",
help="Path to Suricata program")
global_parser.add_argument("--suricata-version",
metavar="<version>",
help="Override Suricata version")
global_parser.add_argument("--user-agent",
metavar="<user-agent>",
help="Set custom user-agent string")
global_parser.add_argument(
"--no-check-certificate",
action="store_true",
default=None,
help="Disable server SSL/TLS certificate verification")
global_args, rem = global_parser.parse_known_args()
if not rem or rem[0].startswith("-"):
rem.insert(0, "update")
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(dest="subcommand", metavar="<command>")
# The "update" (default) sub-command parser.
update_parser = subparsers.add_parser("update",
add_help=False,
parents=[global_parser])
update_parser.add_argument("-o",
"--output",
metavar="<directory>",
dest="output",
help="Directory to write rules to")
update_parser.add_argument(
"-f",
"--force",
action="store_true",
default=False,
help="Force operations that might otherwise be skipped")
update_parser.add_argument("--yaml-fragment",
metavar="<filename>",
help="Output YAML fragment for rule inclusion")
update_parser.add_argument(
"--url",
metavar="<url>",
action="append",
default=[],
help=
"URL to use instead of auto-generating one (can be specified multiple times)"
)
update_parser.add_argument(
"--local",
metavar="<path>",
action="append",
default=[],
help="Local rule files or directories (can be specified multiple times)"
)
update_parser.add_argument("--sid-msg-map",
metavar="<filename>",
help="Generate a sid-msg.map file")
update_parser.add_argument("--sid-msg-map-2",
metavar="<filename>",
help="Generate a v2 sid-msg.map file")
update_parser.add_argument("--disable-conf",
metavar="<filename>",
help="Filename of rule disable filters")
update_parser.add_argument("--enable-conf",
metavar="<filename>",
help="Filename of rule enable filters")
update_parser.add_argument("--modify-conf",
metavar="<filename>",
help="Filename of rule modification filters")
update_parser.add_argument("--drop-conf",
metavar="<filename>",
help="Filename of drop rules filters")
update_parser.add_argument(
"--ignore",
metavar="<pattern>",
action="append",
default=[],
help=
"Filenames to ignore (can be specified multiple times; default: *deleted.rules)"
)
update_parser.add_argument("--no-ignore",
action="store_true",
default=False,
help="Disables the ignore option.")
update_parser.add_argument(
"--threshold-in",
metavar="<filename>",
help="Filename of rule thresholding configuration")
update_parser.add_argument(
"--threshold-out",
metavar="<filename>",
help="Output of processed threshold configuration")
update_parser.add_argument(
"--dump-sample-configs",
action="store_true",
default=False,
help="Dump sample config files to current directory")
update_parser.add_argument("--etopen",
action="store_true",
help="Use ET-Open rules (default)")
update_parser.add_argument("--reload-command",
metavar="<command>",
help="Command to run after update if modified")
update_parser.add_argument("--no-reload",
action="store_true",
default=False,
help="Disable reload")
update_parser.add_argument("-T",
"--test-command",
metavar="<command>",
help="Command to test Suricata configuration")
update_parser.add_argument("--no-test",
action="store_true",
default=False,
help="Disable testing rules with Suricata")
update_parser.add_argument("-V",
"--version",
action="store_true",
default=False,
help="Display version")
update_parser.add_argument(
"--no-merge",
action="store_true",
default=False,
help="Do not merge the rules into a single file")
update_parser.add_argument("-h", "--help", action="store_true")
# Hidden argument, --now to bypass the timebased bypass of
# updating a ruleset.
update_parser.add_argument("--now",
default=False,
action="store_true",
help=argparse.SUPPRESS)
# The Python 2.7 argparse module does prefix matching which can be
# undesirable. Reserve some names here that would match existing
# options to prevent prefix matching.
update_parser.add_argument("--disable",
default=False,
help=argparse.SUPPRESS)
update_parser.add_argument("--enable",
default=False,
help=argparse.SUPPRESS)
update_parser.add_argument("--modify",
default=False,
help=argparse.SUPPRESS)
update_parser.add_argument("--drop", default=False, help=argparse.SUPPRESS)
commands.listsources.register(
subparsers.add_parser("list-sources", parents=[global_parser]))
commands.listenabledsources.register(
subparsers.add_parser("list-enabled-sources", parents=[global_parser]))
commands.addsource.register(
subparsers.add_parser("add-source", parents=[global_parser]))
commands.updatesources.register(
subparsers.add_parser("update-sources", parents=[global_parser]))
commands.enablesource.register(
subparsers.add_parser("enable-source", parents=[global_parser]))
commands.disablesource.register(
subparsers.add_parser("disable-source", parents=[global_parser]))
commands.removesource.register(
subparsers.add_parser("remove-source", parents=[global_parser]))
args = parser.parse_args(rem)
# Merge global args into args.
for arg in vars(global_args):
if not hasattr(args, arg):
setattr(args, arg, getattr(global_args, arg))
elif hasattr(args, arg) and getattr(args, arg) is None:
setattr(args, arg, getattr(global_args, arg))
# Go verbose or quiet sooner than later.
if args.verbose:
logger.setLevel(logging.DEBUG)
if args.quiet:
logger.setLevel(logging.WARNING)
config.init(args)
# Error out if any reserved/unimplemented arguments were set.
unimplemented_args = [
"disable",
"enable",
"modify",
"drop",
]
for arg in unimplemented_args:
if hasattr(args, arg) and getattr(args, arg):
logger.error("--%s not implemented", arg)
return 1
logger.debug("This is suricata-update version %s (rev: %s); Python: %s" %
(version, revision, sys.version.replace("\n", "- ")))
suricata_path = config.get("suricata")
# Now parse the Suricata version. If provided on the command line,
# use that, otherwise attempt to get it from Suricata.
if args.suricata_version:
# The Suricata version was passed on the command line, parse it.
suricata_version = suricata.update.engine.parse_version(
args.suricata_version)
if not suricata_version:
logger.error("Failed to parse provided Suricata version: %s" %
(args.suricata_version))
return 1
logger.info("Forcing Suricata version to %s." %
(suricata_version.full))
elif suricata_path:
suricata_version = suricata.update.engine.get_version(suricata_path)
if suricata_version:
logger.info("Found Suricata version %s at %s." %
(str(suricata_version.full), suricata_path))
else:
logger.error("Failed to get Suricata version.")
return 1
else:
logger.info("Using default Suricata version of %s",
DEFAULT_SURICATA_VERSION)
suricata_version = suricata.update.engine.parse_version(
DEFAULT_SURICATA_VERSION)
# Provide the Suricata version to the net module to add to the
# User-Agent.
suricata.update.net.set_user_agent_suricata_version(suricata_version.full)
# Load custom user-agent-string.
user_agent = config.get("user-agent")
if user_agent:
logger.info("Using user-agent: %s.", user_agent)
suricata.update.net.set_custom_user_agent(user_agent)
if args.subcommand:
if hasattr(args, "func"):
return args.func()
elif args.subcommand != "update":
logger.error("Unknown command: %s", args.subcommand)
return 1
if args.dump_sample_configs:
return dump_sample_configs()
if args.version:
print("suricata-update version %s (rev: %s)" % (version, revision))
return 0
if args.help:
print(update_parser.format_help())
print("""other commands:
update-sources Update the source index
list-sources List available sources
enable-source Enable a source from the index
disable-source Disable an enabled source
remove-source Remove an enabled or disabled source
list-enabled-sources List all enabled sources
add-source Add a new source by URL
""")
return 0
# If --no-ignore was provided, clear any ignores provided in the
# config.
if args.no_ignore:
config.set(config.IGNORE_KEY, [])
file_tracker = FileTracker()
disable_matchers = []
enable_matchers = []
modify_filters = []
drop_filters = []
# Load user provided disable filters.
disable_conf_filename = config.get("disable-conf")
if disable_conf_filename and os.path.exists(disable_conf_filename):
logger.info("Loading %s.", disable_conf_filename)
disable_matchers += load_matchers(disable_conf_filename)
# Load user provided enable filters.
enable_conf_filename = config.get("enable-conf")
if enable_conf_filename and os.path.exists(enable_conf_filename):
logger.info("Loading %s.", enable_conf_filename)
enable_matchers += load_matchers(enable_conf_filename)
# Load user provided modify filters.
modify_conf_filename = config.get("modify-conf")
if modify_conf_filename and os.path.exists(modify_conf_filename):
logger.info("Loading %s.", modify_conf_filename)
modify_filters += load_filters(modify_conf_filename)
# Load user provided drop filters.
drop_conf_filename = config.get("drop-conf")
if drop_conf_filename and os.path.exists(drop_conf_filename):
logger.info("Loading %s.", drop_conf_filename)
drop_filters += load_drop_filters(drop_conf_filename)
if os.path.exists(config.get("suricata-conf")) and \
suricata_path and os.path.exists(suricata_path):
logger.info("Loading %s", config.get("suricata-conf"))
suriconf = suricata.update.engine.Configuration.load(
config.get("suricata-conf"), suricata_path=suricata_path)
for key in suriconf.keys():
if key.startswith("app-layer.protocols") and \
key.endswith(".enabled"):
if not suriconf.is_true(key, ["detection-only"]):
proto = key.split(".")[2]
logger.info("Disabling rules with proto %s", proto)
disable_matchers.append(ProtoRuleMatcher(proto))
# Check that the cache directory exists and is writable.
if not os.path.exists(config.get_cache_dir()):
try:
os.makedirs(config.get_cache_dir(), mode=0o770)
except Exception as err:
logger.warning(
"Cache directory does not exist and could not be created. "
"/var/tmp will be used instead.")
config.set_cache_dir("/var/tmp")
files = load_sources(suricata_version)
load_dist_rules(files)
# Remove ignored files.
for filename in list(files.keys()):
if ignore_file(config.get("ignore"), filename):
logger.info("Ignoring file %s" % (filename))
del (files[filename])
rules = []
for filename in files:
if not filename.endswith(".rules"):
continue
logger.debug("Parsing %s." % (filename))
rules += suricata.update.rule.parse_fileobj(
io.BytesIO(files[filename]), filename)
rulemap = build_rule_map(rules)
logger.info("Loaded %d rules." % (len(rules)))
# Counts of user enabled and modified rules.
enable_count = 0
modify_count = 0
drop_count = 0
# List of rules disabled by user. Used for counting, and to log
# rules that are re-enabled to meet flowbit requirements.
disabled_rules = []
for key, rule in rulemap.items():
for matcher in disable_matchers:
if rule.enabled and matcher.match(rule):
logger.debug("Disabling: %s" % (rule.brief()))
rule.enabled = False
disabled_rules.append(rule)
for matcher in enable_matchers:
if not rule.enabled and matcher.match(rule):
logger.debug("Enabling: %s" % (rule.brief()))
rule.enabled = True
enable_count += 1
for filter in drop_filters:
if filter.match(rule):
rulemap[rule.id] = filter.filter(rule)
drop_count += 1
# Apply modify filters.
for fltr in modify_filters:
for key, rule in rulemap.items():
if fltr.match(rule):
new_rule = fltr.filter(rule)
if new_rule and new_rule.format() != rule.format():
rulemap[rule.id] = new_rule
modify_count += 1
logger.info("Disabled %d rules." % (len(disabled_rules)))
logger.info("Enabled %d rules." % (enable_count))
logger.info("Modified %d rules." % (modify_count))
logger.info("Dropped %d rules." % (drop_count))
# Fixup flowbits.
resolve_flowbits(rulemap, disabled_rules)
# Check that output directory exists.
if not os.path.exists(config.get_output_dir()):
try:
os.makedirs(config.get_output_dir(), mode=0o770)
except Exception as err:
logger.error(
"Output directory does not exist and could not be created: %s",
config.get_output_dir())
return 1
# Check that output directory is writable.
if not os.access(config.get_output_dir(), os.W_OK):
logger.error("Output directory is not writable: %s",
config.get_output_dir())
return 1
# Backup the output directory.
logger.info("Backing up current rules.")
backup_directory = util.mktempdir()
shutil.copytree(config.get_output_dir(),
os.path.join(backup_directory, "backup"),
ignore=copytree_ignore_backup)
if not args.no_merge:
# The default, write out a merged file.
output_filename = os.path.join(config.get_output_dir(),
DEFAULT_OUTPUT_RULE_FILENAME)
file_tracker.add(output_filename)
write_merged(os.path.join(output_filename), rulemap)
else:
for filename in files:
file_tracker.add(
os.path.join(config.get_output_dir(),
os.path.basename(filename)))
write_to_directory(config.get_output_dir(), files, rulemap)
if args.yaml_fragment:
file_tracker.add(args.yaml_fragment)
write_yaml_fragment(args.yaml_fragment, files)
if args.sid_msg_map:
write_sid_msg_map(args.sid_msg_map, rulemap, version=1)
if args.sid_msg_map_2:
write_sid_msg_map(args.sid_msg_map_2, rulemap, version=2)
if args.threshold_in and args.threshold_out:
file_tracker.add(args.threshold_out)
threshold_processor = ThresholdProcessor()
threshold_processor.process(open(args.threshold_in),
open(args.threshold_out, "w"), rulemap)
if not args.force and not file_tracker.any_modified():
logger.info("No changes detected, exiting.")
return 0
if not test_suricata(suricata_path):
logger.error("Suricata test failed, aborting.")
logger.error("Restoring previous rules.")
copytree(os.path.join(backup_directory, "backup"),
config.get_output_dir())
return 1
if not config.args().no_reload and config.get("reload-command"):
logger.info("Running %s." % (config.get("reload-command")))
rc = subprocess.Popen(config.get("reload-command"), shell=True).wait()
if rc != 0:
logger.error("Reload command exited with error: %d", rc)
logger.info("Done.")
return 0
def load_sources(suricata_version):
files = {}
urls = []
# Add any URLs added with the --url command line parameter.
if config.args().url:
for url in config.args().url:
urls.append(url)
# Get the new style sources.
enabled_sources = sources.get_enabled_sources()
# Convert the Suricata version to a version string.
version_string = "%d.%d.%d" % (
suricata_version.major, suricata_version.minor, suricata_version.patch)
# Construct the URL replacement parameters that are internal to
# suricata-update.
internal_params = {"__version__": version_string}
# If we have new sources, we also need to load the index.
if enabled_sources:
index_filename = os.path.join(config.get_cache_dir(),
sources.SOURCE_INDEX_FILENAME)
if os.path.exists(index_filename):
index = sources.Index(index_filename)
else:
index = None
for (name, source) in enabled_sources.items():
params = source["params"] if "params" in source else {}
params.update(internal_params)
if "url" in source:
# No need to go off to the index.
url = source["url"] % params
else:
if not index:
raise exceptions.ApplicationError(
"Source index is required for source %s; "
"run suricata-update update-sources" %
(source["source"]))
url = index.resolve_url(name, params)
logger.debug("Resolved source %s to URL %s.", name, url)
urls.append(url)
if config.get("sources"):
for url in config.get("sources"):
if type(url) not in [type("")]:
raise exceptions.InvalidConfigurationError(
"Invalid datatype for source URL: %s" % (str(url)))
url = url % internal_params
logger.debug("Adding source %s.", url)
urls.append(url)
# If --etopen is on the command line, make sure its added. Or if
# there are no URLs, default to ET/Open.
if config.get("etopen") or not urls:
if not urls:
logger.info(
"No sources configured, will use Emerging Threats Open")
urls.append(sources.get_etopen_url(internal_params))
# Converting the URLs to a set removed dupes.
urls = set(urls)
# Now download each URL.
for url in urls:
Fetch().run(url, files)
# Now load local rules.
for local in config.get("local"):
load_local(local, files)
return files
def list_sources():
enabled = config.args().enabled or \
config.args().subcommand == "list-enabled-sources"
if enabled:
found = False
# First list sources from the main config.
config_sources = config.get("sources")
if config_sources:
found = True
print("From %s:" % (config.filename))
for source in config_sources:
print(" - %s" % (source))
# And local files.
local = config.get("local")
if local:
found = True
print("Local files/directories:")
for filename in local:
print(" - %s" % (filename))
enabled_sources = sources.get_enabled_sources()
if enabled_sources:
found = True
print("Enabled sources:")
for source in enabled_sources.values():
print(" - %s" % (source["source"]))
# If no enabled sources were found, log it.
if not found:
logger.warning("No enabled sources.")
return 0
free_only = config.args().free
if not sources.source_index_exists(config):
logger.info("No source index found, running update-sources")
try:
update_sources()
except exceptions.ApplicationError as err:
logger.warning("%s: will use bundled index.", err)
index = sources.load_source_index(config)
for name, source in index.get_sources().items():
is_not_free = source.get("subscribe-url")
if free_only and is_not_free:
continue
if not config.args().all:
if source.get("deprecated") is not None or \
source.get("obsolete") is not None:
continue
print("%s: %s" % (util.bright_cyan("Name"), util.bright_magenta(name)))
print(" %s: %s" % (util.bright_cyan("Vendor"),
util.bright_magenta(source["vendor"])))
print(" %s: %s" % (util.bright_cyan("Summary"),
util.bright_magenta(source["summary"])))
print(" %s: %s" % (util.bright_cyan("License"),
util.bright_magenta(source["license"])))
if "tags" in source:
print(" %s: %s" %
(util.bright_cyan("Tags"),
util.bright_magenta(", ".join(source["tags"]))))
if "replaces" in source:
print(" %s: %s" %
(util.bright_cyan("Replaces"),
util.bright_magenta(", ".join(source["replaces"]))))
if "parameters" in source:
print(" %s: %s" %
(util.bright_cyan("Parameters"),
util.bright_magenta(", ".join(source["parameters"]))))
if "subscribe-url" in source:
print(" %s: %s" % (util.bright_cyan("Subscription"),
util.bright_magenta(source["subscribe-url"])))
if "deprecated" in source:
print(" %s: %s" % (util.orange("Deprecated"),
util.bright_magenta(source["deprecated"])))
if "obsolete" in source:
print(" %s: %s" % (util.orange("Obsolete"),
util.bright_magenta(source["obsolete"])))
