def list_enabled_sources(): found = False # First list sources from the main config. config_sources = config.get("sources") if config_sources: found = True print("From %s:" % (config.filename)) for source in config_sources: print(" - %s" % (source)) # And local files. local = config.get("local") if local: found = True print("Local files/directories:") for filename in local: print(" - %s" % (filename)) enabled_sources = sources.get_enabled_sources() if enabled_sources: found = True print("Enabled sources:") for source in enabled_sources.values(): print(" - %s" % (source["source"])) # If no enabled sources were found, log it. if not found: logger.warning("No enabled sources.")
def build_user_agent(): params = [] has_custom_user_agent = config.has("user-agent") if has_custom_user_agent: user_agent = config.get("user-agent") if user_agent is None or len(user_agent.strip()) == 0: logger.debug("Suppressing HTTP User-Agent header") return None return user_agent params = [] try: params.append("OS: {}".format(platform.system())) except Exception as err: logger.error("Failed to set user-agent OS: {}".format(str(err))) try: params.append("CPU: {}".format(osinfo.arch())) except Exception as err: logger.error("Failed to set user-agent architecture: {}".format( str(err))) try: params.append("Python: {}".format(platform.python_version())) except Exception as err: logger.error("Failed to set user-agent python version: {}".format( str(err))) try: params.append("Dist: {}".format(osinfo.dist())) except Exception as err: logger.error("Failed to set user-agent distribution: {}".format( str(err))) params.append("Suricata: %s" % (user_agent_suricata_verison)) return "Suricata-Update/%s (%s)" % (version, "; ".join(params))
def get(url, fileobj, progress_hook=None): """ Perform a GET request against a URL writing the contents into the provideded file like object. :param url: The URL to fetch :param fileobj: The fileobj to write the content to :param progress_hook: The function to call with progress updates :returns: Returns a tuple containing the number of bytes read and the result of the info() function from urllib2.urlopen(). :raises: Exceptions from urllib2.urlopen() and writing to the provided fileobj may occur. """ user_agent = build_user_agent() try: # Wrap in a try as Python versions prior to 2.7.9 don't have # create_default_context, but some distros have backported it. ssl_context = ssl.create_default_context() if config.get("no-check-certificate"): logger.debug("Disabling SSL/TLS certificate verification.") ssl_context.check_hostname = False ssl_context.verify_mode = ssl.CERT_NONE opener = build_opener(HTTPSHandler(context=ssl_context)) except: opener = build_opener() if user_agent: logger.debug("Setting HTTP User-Agent to %s", user_agent) opener.addheaders = [ ("User-Agent", user_agent), ] else: opener.addheaders = [(header, value) for header, value in opener.addheaders if header.lower() != "user-agent"] remote = opener.open(url) info = remote.info() try: content_length = int(info["content-length"]) except: content_length = 0 bytes_read = 0 while True: buf = remote.read(GET_BLOCK_SIZE) if not buf: # EOF break bytes_read += len(buf) fileobj.write(buf) if progress_hook: progress_hook(content_length, bytes_read) remote.close() fileobj.flush() return bytes_read, info
def load_dist_rules(files): """Load the rule files provided by the Suricata distribution.""" # In the future hopefully we can just pull in all files from # /usr/share/suricata/rules, but for now pull in the set of files # known to have been provided by the Suricata source. filenames = [ "app-layer-events.rules", "decoder-events.rules", "dhcp-events.rules", "dnp3-events.rules", "dns-events.rules", "files.rules", "http-events.rules", "ipsec-events.rules", "kerberos-events.rules", "modbus-events.rules", "nfs-events.rules", "ntp-events.rules", "smb-events.rules", "smtp-events.rules", "stream-events.rules", "tls-events.rules", ] dist_rule_path = config.get(config.DIST_RULE_DIRECTORY_KEY) if not dist_rule_path: logger.warning("No distribution rule directory found.") return if not os.path.exists(dist_rule_path): logger.warning("Distribution rule directory not found: %s", dist_rule_path) return if os.path.exists(dist_rule_path): if not os.access(dist_rule_path, os.R_OK): logger.warning("Distribution rule path not readable: %s", dist_rule_path) return for filename in filenames: path = os.path.join(dist_rule_path, filename) if not os.path.exists(path): continue if not os.access(path, os.R_OK): logger.warning("Distribution rule file not readable: %s", path) continue logger.info("Loading distribution rule file %s", path) try: with open(path, "rb") as fileobj: files.append(SourceFile(path, fileobj.read())) except Exception as err: logger.error("Failed to open %s: %s" % (path, err)) sys.exit(1)
def test_suricata(suricata_path): if not suricata_path: logger.info("No suricata application binary found, skipping test.") return True if config.get("no-test"): logger.info("Skipping test, disabled by configuration.") return True if config.get("test-command"): test_command = config.get("test-command") logger.info("Testing Suricata configuration with: %s" % (test_command)) env = { "SURICATA_PATH": suricata_path, "OUTPUT_DIR": config.get("output"), } if not config.get("no-merge"): env["OUTPUT_FILENAME"] = os.path.join( config.get("output"), DEFAULT_OUTPUT_RULE_FILENAME) rc = subprocess.Popen(test_command, shell=True, env=env).wait() if rc != 0: return False else: logger.info("Testing with suricata -T.") suricata_conf = config.get("suricata-conf") if not config.get("no-merge"): if not suricata.update.engine.test_configuration( suricata_path, suricata_conf, os.path.join(config.get_output_dir(), DEFAULT_OUTPUT_RULE_FILENAME)): return False else: if not suricata.update.engine.test_configuration( suricata_path, suricata_conf): return False return True
def build_user_agent(): params = [] has_custom_user_agent = config.has("user-agent") if has_custom_user_agent: user_agent = config.get("user-agent") if user_agent is None or len(user_agent.strip()) == 0: logger.debug("Suppressing HTTP User-Agent header") return None return user_agent uname_system = platform.uname()[0] params.append("OS: %s" % (uname_system)) params.append("CPU: %s" % (platform.machine())) params.append("Python: %s" % (platform.python_version())) if uname_system == "Linux": distribution = platform.linux_distribution() params.append("Dist: %s/%s" % (str(distribution[0]), str(distribution[1]))) params.append("Suricata: %s" % (user_agent_suricata_verison)) return "Suricata-Update/%s (%s)" % (version, "; ".join(params))
def _main(): global args args = parsers.parse_arg() # Go verbose or quiet sooner than later. if args.verbose: logger.setLevel(logging.DEBUG) if args.quiet: logger.setLevel(logging.WARNING) logger.debug("This is suricata-update version %s (rev: %s); Python: %s" % (version, revision, sys.version.replace("\n", "- "))) config.init(args) # Error out if any reserved/unimplemented arguments were set. unimplemented_args = [ "disable", "enable", "modify", "drop", ] for arg in unimplemented_args: if hasattr(args, arg) and getattr(args, arg): logger.error("--%s not implemented", arg) return 1 suricata_path = config.get("suricata") # Now parse the Suricata version. If provided on the command line, # use that, otherwise attempt to get it from Suricata. if args.suricata_version: # The Suricata version was passed on the command line, parse it. suricata_version = engine.parse_version(args.suricata_version) if not suricata_version: logger.error("Failed to parse provided Suricata version: %s" % (args.suricata_version)) return 1 logger.info("Forcing Suricata version to %s." % (suricata_version.full)) elif suricata_path: suricata_version = engine.get_version(suricata_path) if suricata_version: logger.info("Found Suricata version %s at %s." % (str(suricata_version.full), suricata_path)) else: logger.error("Failed to get Suricata version.") return 1 else: logger.info("Using default Suricata version of %s", DEFAULT_SURICATA_VERSION) suricata_version = engine.parse_version(DEFAULT_SURICATA_VERSION) # Provide the Suricata version to the net module to add to the # User-Agent. net.set_user_agent_suricata_version(suricata_version.full) if args.subcommand: if args.subcommand == "check-versions" and hasattr(args, "func"): return args.func(suricata_version) elif hasattr(args, "func"): return args.func() elif args.subcommand != "update": logger.error("Unknown command: %s", args.subcommand) return 1 if args.dump_sample_configs: return dump_sample_configs() # If --no-ignore was provided, clear any ignores provided in the # config. if args.no_ignore: config.set(config.IGNORE_KEY, []) file_tracker = FileTracker() disable_matchers = [] enable_matchers = [] modify_filters = [] drop_filters = [] # Load user provided disable filters. disable_conf_filename = config.get("disable-conf") if disable_conf_filename and os.path.exists(disable_conf_filename): logger.info("Loading %s.", disable_conf_filename) disable_matchers += load_matchers(disable_conf_filename) # Load user provided enable filters. enable_conf_filename = config.get("enable-conf") if enable_conf_filename and os.path.exists(enable_conf_filename): logger.info("Loading %s.", enable_conf_filename) enable_matchers += load_matchers(enable_conf_filename) # Load user provided modify filters. modify_conf_filename = config.get("modify-conf") if modify_conf_filename and os.path.exists(modify_conf_filename): logger.info("Loading %s.", modify_conf_filename) modify_filters += load_filters(modify_conf_filename) # Load user provided drop filters. drop_conf_filename = config.get("drop-conf") if drop_conf_filename and os.path.exists(drop_conf_filename): logger.info("Loading %s.", drop_conf_filename) drop_filters += load_drop_filters(drop_conf_filename) # Load the Suricata configuration if we can. suriconf = None if config.get("suricata-conf") and \ os.path.exists(config.get("suricata-conf")) and \ suricata_path and os.path.exists(suricata_path): logger.info("Loading %s", config.get("suricata-conf")) try: suriconf = engine.Configuration.load(config.get("suricata-conf"), suricata_path=suricata_path) except subprocess.CalledProcessError: return 1 # Disable rule that are for app-layers that are not enabled. if suriconf: for key in suriconf.keys(): m = re.match("app-layer\.protocols\.([^\.]+)\.enabled", key) if m: proto = m.group(1) if not suriconf.is_true(key, ["detection-only"]): logger.info("Disabling rules for protocol %s", proto) disable_matchers.append( matchers_mod.ProtoRuleMatcher(proto)) elif proto == "smb" and suriconf.build_info: # Special case for SMB rules. For versions less # than 5, disable smb rules if Rust is not # available. if suriconf.build_info["version"].major < 5: if not "RUST" in suriconf.build_info["features"]: logger.info( "Disabling rules for protocol {}".format( proto)) disable_matchers.append( matchers_mod.ProtoRuleMatcher(proto)) # Check that the cache directory exists and is writable. if not os.path.exists(config.get_cache_dir()): try: os.makedirs(config.get_cache_dir(), mode=0o770) except Exception as err: logger.warning( "Cache directory does not exist and could not be created. " "/var/tmp will be used instead.") config.set_cache_dir("/var/tmp") files = load_sources(suricata_version) load_dist_rules(files) # Remove ignored files. for filename in list(files.keys()): if ignore_file(config.get("ignore"), filename): logger.info("Ignoring file %s" % (filename)) del (files[filename]) rules = [] for filename in sorted(files): if not filename.endswith(".rules"): continue logger.debug("Parsing %s." % (filename)) rules += rule_mod.parse_fileobj(io.BytesIO(files[filename]), filename) rulemap = build_rule_map(rules) logger.info("Loaded %d rules." % (len(rules))) # Counts of user enabled and modified rules. enable_count = 0 modify_count = 0 drop_count = 0 # List of rules disabled by user. Used for counting, and to log # rules that are re-enabled to meet flowbit requirements. disabled_rules = [] for key, rule in rulemap.items(): for matcher in disable_matchers: if rule.enabled and matcher.match(rule): logger.debug("Disabling: %s" % (rule.brief())) rule.enabled = False disabled_rules.append(rule) for matcher in enable_matchers: if not rule.enabled and matcher.match(rule): logger.debug("Enabling: %s" % (rule.brief())) rule.enabled = True enable_count += 1 for fltr in drop_filters: if fltr.match(rule): rulemap[rule.id] = fltr.run(rule) drop_count += 1 # Apply modify filters. for fltr in modify_filters: for key, rule in rulemap.items(): if fltr.match(rule): new_rule = fltr.run(rule) if new_rule and new_rule.format() != rule.format(): rulemap[rule.id] = new_rule modify_count += 1 # Check if we should disable ja3 rules. try: disable_ja3(suriconf, rulemap, disabled_rules) except Exception as err: logger.error("Failed to dynamically disable ja3 rules: %s" % (err)) # Check rule vars, disabling rules that use unknown vars. check_vars(suriconf, rulemap) logger.info("Disabled %d rules." % (len(disabled_rules))) logger.info("Enabled %d rules." % (enable_count)) logger.info("Modified %d rules." % (modify_count)) logger.info("Dropped %d rules." % (drop_count)) # Fixup flowbits. resolve_flowbits(rulemap, disabled_rules) # Check that output directory exists, creating it if needed. check_output_directory(config.get_output_dir()) # Check that output directory is writable. if not os.access(config.get_output_dir(), os.W_OK): logger.error("Output directory is not writable: %s", config.get_output_dir()) return 1 # Backup the output directory. logger.info("Backing up current rules.") backup_directory = util.mktempdir() shutil.copytree(config.get_output_dir(), os.path.join(backup_directory, "backup"), ignore=copytree_ignore_backup) if not args.no_merge: # The default, write out a merged file. output_filename = os.path.join(config.get_output_dir(), DEFAULT_OUTPUT_RULE_FILENAME) file_tracker.add(output_filename) write_merged(os.path.join(output_filename), rulemap) else: for filename in files: file_tracker.add( os.path.join(config.get_output_dir(), os.path.basename(filename))) write_to_directory(config.get_output_dir(), files, rulemap) if args.yaml_fragment: file_tracker.add(args.yaml_fragment) write_yaml_fragment(args.yaml_fragment, files) if args.sid_msg_map: write_sid_msg_map(args.sid_msg_map, rulemap, version=1) if args.sid_msg_map_2: write_sid_msg_map(args.sid_msg_map_2, rulemap, version=2) if args.threshold_in and args.threshold_out: file_tracker.add(args.threshold_out) threshold_processor = ThresholdProcessor() threshold_processor.process(open(args.threshold_in), open(args.threshold_out, "w"), rulemap) if not args.force and not file_tracker.any_modified(): logger.info("No changes detected, exiting.") notes.dump_notes() return 0 # Set these containers to None to fee the memory before testing Suricata which # may consume a lot of memory by itself. Ideally we should refactor this large # function into multiple methods so these go out of scope and get removed # automatically. rulemap = None rules = None files = None if not test_suricata(suricata_path): logger.error("Suricata test failed, aborting.") logger.error("Restoring previous rules.") copytree(os.path.join(backup_directory, "backup"), config.get_output_dir()) return 1 if not config.args().no_reload and config.get("reload-command"): logger.info("Running %s." % (config.get("reload-command"))) rc = subprocess.Popen(config.get("reload-command"), shell=True).wait() if rc != 0: logger.error("Reload command exited with error: %d", rc) logger.info("Done.") notes.dump_notes() return 0
def load_sources(suricata_version): files = {} urls = [] http_header = None checksum = True # Add any URLs added with the --url command line parameter. if config.args().url: for url in config.args().url: urls.append((url, http_header, checksum)) # Get the new style sources. enabled_sources = sources.get_enabled_sources() # Convert the Suricata version to a version string. version_string = "%d.%d.%d" % ( suricata_version.major, suricata_version.minor, suricata_version.patch) # Construct the URL replacement parameters that are internal to # suricata-update. internal_params = {"__version__": version_string} # If we have new sources, we also need to load the index. if enabled_sources: index_filename = sources.get_index_filename() if not os.path.exists(index_filename): logger.warning("No index exists, will use bundled index.") logger.warning("Please run suricata-update update-sources.") if os.path.exists(index_filename) and time.time() - \ os.stat(index_filename).st_mtime > INDEX_EXPIRATION_TIME: logger.warning( "Source index is older than 2 weeks. " "Please update with suricata-update update-sources.") index = sources.Index(index_filename) for (name, source) in enabled_sources.items(): params = source["params"] if "params" in source else {} params.update(internal_params) if "url" in source: # No need to go off to the index. http_header = source.get("http_header") checksum = source.get("checksum") url = (source["url"] % params, http_header, checksum) logger.debug("Resolved source %s to URL %s.", name, url[0]) else: if not index: raise exceptions.ApplicationError( "Source index is required for source %s; " "run suricata-update update-sources" % (source["source"])) source_config = index.get_source_by_name(name) try: checksum = source_config["checksum"] except: checksum = True url = (index.resolve_url(name, params), http_header, checksum) logger.debug("Resolved source %s to URL %s.", name, url) urls.append(url) if config.get("sources"): for url in config.get("sources"): if not isinstance(url, str): raise exceptions.InvalidConfigurationError( "Invalid datatype for source URL: %s" % (str(url))) url = (url % internal_params, http_header, checksum) logger.debug("Adding source %s.", url) urls.append(url) # If --etopen is on the command line, make sure its added. Or if # there are no URLs, default to ET/Open. if config.get("etopen") or not urls: if not config.args().offline and not urls: logger.info( "No sources configured, will use Emerging Threats Open") urls.append( (sources.get_etopen_url(internal_params), http_header, checksum)) # Converting the URLs to a set removed dupes. urls = set(urls) # Now download each URL. for url in urls: Fetch().run(url, files) # Now load local rules. if config.get("local") is not None: for local in config.get("local"): load_local(local, files) return files
def get(url, fileobj, progress_hook=None): """ Perform a GET request against a URL writing the contents into the provideded file like object. :param url: The URL to fetch :param fileobj: The fileobj to write the content to :param progress_hook: The function to call with progress updates :returns: Returns a tuple containing the number of bytes read and the result of the info() function from urllib2.urlopen(). :raises: Exceptions from urllib2.urlopen() and writing to the provided fileobj may occur. """ user_agent = build_user_agent() try: # Wrap in a try as Python versions prior to 2.7.9 don't have # create_default_context, but some distros have backported it. ssl_context = ssl.create_default_context() if config.get("no-check-certificate"): logger.debug("Disabling SSL/TLS certificate verification.") ssl_context.check_hostname = False ssl_context.verify_mode = ssl.CERT_NONE opener = build_opener(HTTPSHandler(context=ssl_context)) except: opener = build_opener() if user_agent: logger.debug("Setting HTTP User-Agent to %s", user_agent) http_headers = [("User-Agent", user_agent)] else: http_headers = [(header, value) for header, value in opener.addheaders if header.lower() != "user-agent"] if isinstance(url, tuple): header = url[1].split(":") if url[1] is not None else None if header and is_header_clean(header=header): name, val = header[0].strip(), header[1].strip() logger.debug("Setting HTTP header %s to %s", name, val) http_headers.append((name, val)) elif header: logger.error("Header not set as it does not meet the criteria") url = url[0] opener.addheaders = http_headers try: remote = opener.open(url, timeout=30) except ValueError as ve: logger.error(ve) else: info = remote.info() content_length = info.get("content-length") content_length = int(content_length) if content_length else 0 bytes_read = 0 while True: buf = remote.read(GET_BLOCK_SIZE) if not buf: # EOF break bytes_read += len(buf) fileobj.write(buf) if progress_hook: progress_hook(content_length, bytes_read) remote.close() fileobj.flush() return bytes_read, info
def _main(): global args global_parser = argparse.ArgumentParser(add_help=False) global_parser.add_argument("-v", "--verbose", action="store_true", default=None, help="Be more verbose") global_parser.add_argument( "-q", "--quiet", action="store_true", default=False, help="Be quiet, warning and error messages only") global_parser.add_argument( "-D", "--data-dir", metavar="<directory>", dest="data_dir", help="Data directory (default: /var/lib/suricata)") global_parser.add_argument( "-c", "--config", metavar="<filename>", help="configuration file (default: /etc/suricata/update.yaml)") global_parser.add_argument( "--suricata-conf", metavar="<filename>", help="configuration file (default: /etc/suricata/suricata.yaml)") global_parser.add_argument("--suricata", metavar="<path>", help="Path to Suricata program") global_parser.add_argument("--suricata-version", metavar="<version>", help="Override Suricata version") global_parser.add_argument("--user-agent", metavar="<user-agent>", help="Set custom user-agent string") global_parser.add_argument( "--no-check-certificate", action="store_true", default=None, help="Disable server SSL/TLS certificate verification") global_args, rem = global_parser.parse_known_args() if not rem or rem[0].startswith("-"): rem.insert(0, "update") parser = argparse.ArgumentParser() subparsers = parser.add_subparsers(dest="subcommand", metavar="<command>") # The "update" (default) sub-command parser. update_parser = subparsers.add_parser("update", add_help=False, parents=[global_parser]) update_parser.add_argument("-o", "--output", metavar="<directory>", dest="output", help="Directory to write rules to") update_parser.add_argument( "-f", "--force", action="store_true", default=False, help="Force operations that might otherwise be skipped") update_parser.add_argument("--yaml-fragment", metavar="<filename>", help="Output YAML fragment for rule inclusion") update_parser.add_argument( "--url", metavar="<url>", action="append", default=[], help= "URL to use instead of auto-generating one (can be specified multiple times)" ) update_parser.add_argument( "--local", metavar="<path>", action="append", default=[], help="Local rule files or directories (can be specified multiple times)" ) update_parser.add_argument("--sid-msg-map", metavar="<filename>", help="Generate a sid-msg.map file") update_parser.add_argument("--sid-msg-map-2", metavar="<filename>", help="Generate a v2 sid-msg.map file") update_parser.add_argument("--disable-conf", metavar="<filename>", help="Filename of rule disable filters") update_parser.add_argument("--enable-conf", metavar="<filename>", help="Filename of rule enable filters") update_parser.add_argument("--modify-conf", metavar="<filename>", help="Filename of rule modification filters") update_parser.add_argument("--drop-conf", metavar="<filename>", help="Filename of drop rules filters") update_parser.add_argument( "--ignore", metavar="<pattern>", action="append", default=[], help= "Filenames to ignore (can be specified multiple times; default: *deleted.rules)" ) update_parser.add_argument("--no-ignore", action="store_true", default=False, help="Disables the ignore option.") update_parser.add_argument( "--threshold-in", metavar="<filename>", help="Filename of rule thresholding configuration") update_parser.add_argument( "--threshold-out", metavar="<filename>", help="Output of processed threshold configuration") update_parser.add_argument( "--dump-sample-configs", action="store_true", default=False, help="Dump sample config files to current directory") update_parser.add_argument("--etopen", action="store_true", help="Use ET-Open rules (default)") update_parser.add_argument("--reload-command", metavar="<command>", help="Command to run after update if modified") update_parser.add_argument("--no-reload", action="store_true", default=False, help="Disable reload") update_parser.add_argument("-T", "--test-command", metavar="<command>", help="Command to test Suricata configuration") update_parser.add_argument("--no-test", action="store_true", default=False, help="Disable testing rules with Suricata") update_parser.add_argument("-V", "--version", action="store_true", default=False, help="Display version") update_parser.add_argument( "--no-merge", action="store_true", default=False, help="Do not merge the rules into a single file") update_parser.add_argument("-h", "--help", action="store_true") # Hidden argument, --now to bypass the timebased bypass of # updating a ruleset. update_parser.add_argument("--now", default=False, action="store_true", help=argparse.SUPPRESS) # The Python 2.7 argparse module does prefix matching which can be # undesirable. Reserve some names here that would match existing # options to prevent prefix matching. update_parser.add_argument("--disable", default=False, help=argparse.SUPPRESS) update_parser.add_argument("--enable", default=False, help=argparse.SUPPRESS) update_parser.add_argument("--modify", default=False, help=argparse.SUPPRESS) update_parser.add_argument("--drop", default=False, help=argparse.SUPPRESS) commands.listsources.register( subparsers.add_parser("list-sources", parents=[global_parser])) commands.listenabledsources.register( subparsers.add_parser("list-enabled-sources", parents=[global_parser])) commands.addsource.register( subparsers.add_parser("add-source", parents=[global_parser])) commands.updatesources.register( subparsers.add_parser("update-sources", parents=[global_parser])) commands.enablesource.register( subparsers.add_parser("enable-source", parents=[global_parser])) commands.disablesource.register( subparsers.add_parser("disable-source", parents=[global_parser])) commands.removesource.register( subparsers.add_parser("remove-source", parents=[global_parser])) args = parser.parse_args(rem) # Merge global args into args. for arg in vars(global_args): if not hasattr(args, arg): setattr(args, arg, getattr(global_args, arg)) elif hasattr(args, arg) and getattr(args, arg) is None: setattr(args, arg, getattr(global_args, arg)) # Go verbose or quiet sooner than later. if args.verbose: logger.setLevel(logging.DEBUG) if args.quiet: logger.setLevel(logging.WARNING) config.init(args) # Error out if any reserved/unimplemented arguments were set. unimplemented_args = [ "disable", "enable", "modify", "drop", ] for arg in unimplemented_args: if hasattr(args, arg) and getattr(args, arg): logger.error("--%s not implemented", arg) return 1 logger.debug("This is suricata-update version %s (rev: %s); Python: %s" % (version, revision, sys.version.replace("\n", "- "))) suricata_path = config.get("suricata") # Now parse the Suricata version. If provided on the command line, # use that, otherwise attempt to get it from Suricata. if args.suricata_version: # The Suricata version was passed on the command line, parse it. suricata_version = suricata.update.engine.parse_version( args.suricata_version) if not suricata_version: logger.error("Failed to parse provided Suricata version: %s" % (args.suricata_version)) return 1 logger.info("Forcing Suricata version to %s." % (suricata_version.full)) elif suricata_path: suricata_version = suricata.update.engine.get_version(suricata_path) if suricata_version: logger.info("Found Suricata version %s at %s." % (str(suricata_version.full), suricata_path)) else: logger.error("Failed to get Suricata version.") return 1 else: logger.info("Using default Suricata version of %s", DEFAULT_SURICATA_VERSION) suricata_version = suricata.update.engine.parse_version( DEFAULT_SURICATA_VERSION) # Provide the Suricata version to the net module to add to the # User-Agent. suricata.update.net.set_user_agent_suricata_version(suricata_version.full) # Load custom user-agent-string. user_agent = config.get("user-agent") if user_agent: logger.info("Using user-agent: %s.", user_agent) suricata.update.net.set_custom_user_agent(user_agent) if args.subcommand: if hasattr(args, "func"): return args.func() elif args.subcommand != "update": logger.error("Unknown command: %s", args.subcommand) return 1 if args.dump_sample_configs: return dump_sample_configs() if args.version: print("suricata-update version %s (rev: %s)" % (version, revision)) return 0 if args.help: print(update_parser.format_help()) print("""other commands: update-sources Update the source index list-sources List available sources enable-source Enable a source from the index disable-source Disable an enabled source remove-source Remove an enabled or disabled source list-enabled-sources List all enabled sources add-source Add a new source by URL """) return 0 # If --no-ignore was provided, clear any ignores provided in the # config. if args.no_ignore: config.set(config.IGNORE_KEY, []) file_tracker = FileTracker() disable_matchers = [] enable_matchers = [] modify_filters = [] drop_filters = [] # Load user provided disable filters. disable_conf_filename = config.get("disable-conf") if disable_conf_filename and os.path.exists(disable_conf_filename): logger.info("Loading %s.", disable_conf_filename) disable_matchers += load_matchers(disable_conf_filename) # Load user provided enable filters. enable_conf_filename = config.get("enable-conf") if enable_conf_filename and os.path.exists(enable_conf_filename): logger.info("Loading %s.", enable_conf_filename) enable_matchers += load_matchers(enable_conf_filename) # Load user provided modify filters. modify_conf_filename = config.get("modify-conf") if modify_conf_filename and os.path.exists(modify_conf_filename): logger.info("Loading %s.", modify_conf_filename) modify_filters += load_filters(modify_conf_filename) # Load user provided drop filters. drop_conf_filename = config.get("drop-conf") if drop_conf_filename and os.path.exists(drop_conf_filename): logger.info("Loading %s.", drop_conf_filename) drop_filters += load_drop_filters(drop_conf_filename) if os.path.exists(config.get("suricata-conf")) and \ suricata_path and os.path.exists(suricata_path): logger.info("Loading %s", config.get("suricata-conf")) suriconf = suricata.update.engine.Configuration.load( config.get("suricata-conf"), suricata_path=suricata_path) for key in suriconf.keys(): if key.startswith("app-layer.protocols") and \ key.endswith(".enabled"): if not suriconf.is_true(key, ["detection-only"]): proto = key.split(".")[2] logger.info("Disabling rules with proto %s", proto) disable_matchers.append(ProtoRuleMatcher(proto)) # Check that the cache directory exists and is writable. if not os.path.exists(config.get_cache_dir()): try: os.makedirs(config.get_cache_dir(), mode=0o770) except Exception as err: logger.warning( "Cache directory does not exist and could not be created. " "/var/tmp will be used instead.") config.set_cache_dir("/var/tmp") files = load_sources(suricata_version) load_dist_rules(files) # Remove ignored files. for filename in list(files.keys()): if ignore_file(config.get("ignore"), filename): logger.info("Ignoring file %s" % (filename)) del (files[filename]) rules = [] for filename in files: if not filename.endswith(".rules"): continue logger.debug("Parsing %s." % (filename)) rules += suricata.update.rule.parse_fileobj( io.BytesIO(files[filename]), filename) rulemap = build_rule_map(rules) logger.info("Loaded %d rules." % (len(rules))) # Counts of user enabled and modified rules. enable_count = 0 modify_count = 0 drop_count = 0 # List of rules disabled by user. Used for counting, and to log # rules that are re-enabled to meet flowbit requirements. disabled_rules = [] for key, rule in rulemap.items(): for matcher in disable_matchers: if rule.enabled and matcher.match(rule): logger.debug("Disabling: %s" % (rule.brief())) rule.enabled = False disabled_rules.append(rule) for matcher in enable_matchers: if not rule.enabled and matcher.match(rule): logger.debug("Enabling: %s" % (rule.brief())) rule.enabled = True enable_count += 1 for filter in drop_filters: if filter.match(rule): rulemap[rule.id] = filter.filter(rule) drop_count += 1 # Apply modify filters. for fltr in modify_filters: for key, rule in rulemap.items(): if fltr.match(rule): new_rule = fltr.filter(rule) if new_rule and new_rule.format() != rule.format(): rulemap[rule.id] = new_rule modify_count += 1 logger.info("Disabled %d rules." % (len(disabled_rules))) logger.info("Enabled %d rules." % (enable_count)) logger.info("Modified %d rules." % (modify_count)) logger.info("Dropped %d rules." % (drop_count)) # Fixup flowbits. resolve_flowbits(rulemap, disabled_rules) # Check that output directory exists. if not os.path.exists(config.get_output_dir()): try: os.makedirs(config.get_output_dir(), mode=0o770) except Exception as err: logger.error( "Output directory does not exist and could not be created: %s", config.get_output_dir()) return 1 # Check that output directory is writable. if not os.access(config.get_output_dir(), os.W_OK): logger.error("Output directory is not writable: %s", config.get_output_dir()) return 1 # Backup the output directory. logger.info("Backing up current rules.") backup_directory = util.mktempdir() shutil.copytree(config.get_output_dir(), os.path.join(backup_directory, "backup"), ignore=copytree_ignore_backup) if not args.no_merge: # The default, write out a merged file. output_filename = os.path.join(config.get_output_dir(), DEFAULT_OUTPUT_RULE_FILENAME) file_tracker.add(output_filename) write_merged(os.path.join(output_filename), rulemap) else: for filename in files: file_tracker.add( os.path.join(config.get_output_dir(), os.path.basename(filename))) write_to_directory(config.get_output_dir(), files, rulemap) if args.yaml_fragment: file_tracker.add(args.yaml_fragment) write_yaml_fragment(args.yaml_fragment, files) if args.sid_msg_map: write_sid_msg_map(args.sid_msg_map, rulemap, version=1) if args.sid_msg_map_2: write_sid_msg_map(args.sid_msg_map_2, rulemap, version=2) if args.threshold_in and args.threshold_out: file_tracker.add(args.threshold_out) threshold_processor = ThresholdProcessor() threshold_processor.process(open(args.threshold_in), open(args.threshold_out, "w"), rulemap) if not args.force and not file_tracker.any_modified(): logger.info("No changes detected, exiting.") return 0 if not test_suricata(suricata_path): logger.error("Suricata test failed, aborting.") logger.error("Restoring previous rules.") copytree(os.path.join(backup_directory, "backup"), config.get_output_dir()) return 1 if not config.args().no_reload and config.get("reload-command"): logger.info("Running %s." % (config.get("reload-command"))) rc = subprocess.Popen(config.get("reload-command"), shell=True).wait() if rc != 0: logger.error("Reload command exited with error: %d", rc) logger.info("Done.") return 0
def load_sources(suricata_version): files = {} urls = [] # Add any URLs added with the --url command line parameter. if config.args().url: for url in config.args().url: urls.append(url) # Get the new style sources. enabled_sources = sources.get_enabled_sources() # Convert the Suricata version to a version string. version_string = "%d.%d.%d" % ( suricata_version.major, suricata_version.minor, suricata_version.patch) # Construct the URL replacement parameters that are internal to # suricata-update. internal_params = {"__version__": version_string} # If we have new sources, we also need to load the index. if enabled_sources: index_filename = os.path.join(config.get_cache_dir(), sources.SOURCE_INDEX_FILENAME) if os.path.exists(index_filename): index = sources.Index(index_filename) else: index = None for (name, source) in enabled_sources.items(): params = source["params"] if "params" in source else {} params.update(internal_params) if "url" in source: # No need to go off to the index. url = source["url"] % params else: if not index: raise exceptions.ApplicationError( "Source index is required for source %s; " "run suricata-update update-sources" % (source["source"])) url = index.resolve_url(name, params) logger.debug("Resolved source %s to URL %s.", name, url) urls.append(url) if config.get("sources"): for url in config.get("sources"): if type(url) not in [type("")]: raise exceptions.InvalidConfigurationError( "Invalid datatype for source URL: %s" % (str(url))) url = url % internal_params logger.debug("Adding source %s.", url) urls.append(url) # If --etopen is on the command line, make sure its added. Or if # there are no URLs, default to ET/Open. if config.get("etopen") or not urls: if not urls: logger.info( "No sources configured, will use Emerging Threats Open") urls.append(sources.get_etopen_url(internal_params)) # Converting the URLs to a set removed dupes. urls = set(urls) # Now download each URL. for url in urls: Fetch().run(url, files) # Now load local rules. for local in config.get("local"): load_local(local, files) return files
def list_sources(): enabled = config.args().enabled or \ config.args().subcommand == "list-enabled-sources" if enabled: found = False # First list sources from the main config. config_sources = config.get("sources") if config_sources: found = True print("From %s:" % (config.filename)) for source in config_sources: print(" - %s" % (source)) # And local files. local = config.get("local") if local: found = True print("Local files/directories:") for filename in local: print(" - %s" % (filename)) enabled_sources = sources.get_enabled_sources() if enabled_sources: found = True print("Enabled sources:") for source in enabled_sources.values(): print(" - %s" % (source["source"])) # If no enabled sources were found, log it. if not found: logger.warning("No enabled sources.") return 0 free_only = config.args().free if not sources.source_index_exists(config): logger.info("No source index found, running update-sources") try: update_sources() except exceptions.ApplicationError as err: logger.warning("%s: will use bundled index.", err) index = sources.load_source_index(config) for name, source in index.get_sources().items(): is_not_free = source.get("subscribe-url") if free_only and is_not_free: continue if not config.args().all: if source.get("deprecated") is not None or \ source.get("obsolete") is not None: continue print("%s: %s" % (util.bright_cyan("Name"), util.bright_magenta(name))) print(" %s: %s" % (util.bright_cyan("Vendor"), util.bright_magenta(source["vendor"]))) print(" %s: %s" % (util.bright_cyan("Summary"), util.bright_magenta(source["summary"]))) print(" %s: %s" % (util.bright_cyan("License"), util.bright_magenta(source["license"]))) if "tags" in source: print(" %s: %s" % (util.bright_cyan("Tags"), util.bright_magenta(", ".join(source["tags"])))) if "replaces" in source: print(" %s: %s" % (util.bright_cyan("Replaces"), util.bright_magenta(", ".join(source["replaces"])))) if "parameters" in source: print(" %s: %s" % (util.bright_cyan("Parameters"), util.bright_magenta(", ".join(source["parameters"])))) if "subscribe-url" in source: print(" %s: %s" % (util.bright_cyan("Subscription"), util.bright_magenta(source["subscribe-url"]))) if "deprecated" in source: print(" %s: %s" % (util.orange("Deprecated"), util.bright_magenta(source["deprecated"]))) if "obsolete" in source: print(" %s: %s" % (util.orange("Obsolete"), util.bright_magenta(source["obsolete"])))