def check_output_directory(output_dir): """ Check that the output directory exists, creating it if it doesn't. """ if not os.path.exists(output_dir): logger.info("Creating directory %s." % (output_dir)) try: os.makedirs(output_dir, mode=0o770) except Exception as err: raise exceptions.ApplicationError( "Failed to create directory %s: %s" % (output_dir, err))
def update_sources(): global local_index_filename local_index_filename = sources.get_index_filename() initial_content = get_initial_content() with io.BytesIO() as fileobj: url = sources.get_source_index_url() logger.info("Downloading %s", url) try: net.get(url, fileobj) except Exception as err: raise exceptions.ApplicationError( "Failed to download index: %s: %s" % (url, err)) if not os.path.exists(config.get_cache_dir()): try: os.makedirs(config.get_cache_dir()) except Exception as err: logger.error("Failed to create directory %s: %s", config.get_cache_dir(), err) return 1 write_and_compare(initial_content=initial_content, fileobj=fileobj)
def load_filters(filename): filters = [] with open(filename) as fileobj: for line in fileobj: line = line.strip() if not line or line.startswith("#"): continue line = line.rsplit(" #")[0] line = re.sub(r'\\\$', '$', line) # needed to escape $ in pp try: rule_filter = matchers_mod.ModifyRuleFilter.parse(line) filters.append(rule_filter) except Exception as err: raise exceptions.ApplicationError( "Failed to parse modify filter: {}".format(line)) return filters
def update_sources(): local_index_filename = sources.get_index_filename() with io.BytesIO() as fileobj: url = sources.get_source_index_url() logger.info("Downloading %s", url) try: net.get(url, fileobj) except Exception as err: raise exceptions.ApplicationError( "Failed to download index: %s: %s" % (url, err)) if not os.path.exists(config.get_cache_dir()): try: os.makedirs(config.get_cache_dir()) except Exception as err: logger.error("Failed to create directory %s: %s", config.get_cache_dir(), err) return 1 with open(local_index_filename, "wb") as outobj: outobj.write(fileobj.getvalue()) logger.info("Saved %s", local_index_filename)
def load_sources(suricata_version): files = {} urls = [] http_header = None checksum = True # Add any URLs added with the --url command line parameter. if config.args().url: for url in config.args().url: urls.append((url, http_header, checksum)) # Get the new style sources. enabled_sources = sources.get_enabled_sources() # Convert the Suricata version to a version string. version_string = "%d.%d.%d" % ( suricata_version.major, suricata_version.minor, suricata_version.patch) # Construct the URL replacement parameters that are internal to # suricata-update. internal_params = {"__version__": version_string} # If we have new sources, we also need to load the index. if enabled_sources: index_filename = sources.get_index_filename() if not os.path.exists(index_filename): logger.warning("No index exists, will use bundled index.") logger.warning("Please run suricata-update update-sources.") if os.path.exists(index_filename) and time.time() - \ os.stat(index_filename).st_mtime > INDEX_EXPIRATION_TIME: logger.warning( "Source index is older than 2 weeks. " "Please update with suricata-update update-sources.") index = sources.Index(index_filename) for (name, source) in enabled_sources.items(): params = source["params"] if "params" in source else {} params.update(internal_params) if "url" in source: # No need to go off to the index. http_header = source.get("http_header") checksum = source.get("checksum") url = (source["url"] % params, http_header, checksum) logger.debug("Resolved source %s to URL %s.", name, url[0]) else: if not index: raise exceptions.ApplicationError( "Source index is required for source %s; " "run suricata-update update-sources" % (source["source"])) source_config = index.get_source_by_name(name) try: checksum = source_config["checksum"] except: checksum = True url = (index.resolve_url(name, params), http_header, checksum) logger.debug("Resolved source %s to URL %s.", name, url) urls.append(url) if config.get("sources"): for url in config.get("sources"): if not isinstance(url, str): raise exceptions.InvalidConfigurationError( "Invalid datatype for source URL: %s" % (str(url))) url = (url % internal_params, http_header, checksum) logger.debug("Adding source %s.", url) urls.append(url) # If --etopen is on the command line, make sure its added. Or if # there are no URLs, default to ET/Open. if config.get("etopen") or not urls: if not config.args().offline and not urls: logger.info( "No sources configured, will use Emerging Threats Open") urls.append( (sources.get_etopen_url(internal_params), http_header, checksum)) # Converting the URLs to a set removed dupes. urls = set(urls) # Now download each URL. for url in urls: Fetch().run(url, files) # Now load local rules. if config.get("local") is not None: for local in config.get("local"): load_local(local, files) return files
def load_sources(suricata_version): files = {} urls = [] # Add any URLs added with the --url command line parameter. if config.args().url: for url in config.args().url: urls.append(url) # Get the new style sources. enabled_sources = sources.get_enabled_sources() # Convert the Suricata version to a version string. version_string = "%d.%d.%d" % ( suricata_version.major, suricata_version.minor, suricata_version.patch) # Construct the URL replacement parameters that are internal to # suricata-update. internal_params = {"__version__": version_string} # If we have new sources, we also need to load the index. if enabled_sources: index_filename = os.path.join(config.get_cache_dir(), sources.SOURCE_INDEX_FILENAME) if os.path.exists(index_filename): index = sources.Index(index_filename) else: index = None for (name, source) in enabled_sources.items(): params = source["params"] if "params" in source else {} params.update(internal_params) if "url" in source: # No need to go off to the index. url = source["url"] % params else: if not index: raise exceptions.ApplicationError( "Source index is required for source %s; " "run suricata-update update-sources" % (source["source"])) url = index.resolve_url(name, params) logger.debug("Resolved source %s to URL %s.", name, url) urls.append(url) if config.get("sources"): for url in config.get("sources"): if type(url) not in [type("")]: raise exceptions.InvalidConfigurationError( "Invalid datatype for source URL: %s" % (str(url))) url = url % internal_params logger.debug("Adding source %s.", url) urls.append(url) # If --etopen is on the command line, make sure its added. Or if # there are no URLs, default to ET/Open. if config.get("etopen") or not urls: if not urls: logger.info( "No sources configured, will use Emerging Threats Open") urls.append(sources.get_etopen_url(internal_params)) # Converting the URLs to a set removed dupes. urls = set(urls) # Now download each URL. for url in urls: Fetch().run(url, files) # Now load local rules. for local in config.get("local"): load_local(local, files) return files