class Maps(object): def __init__(self, config): self.config = config # pull in the classification/sig maps self.sigmap_timestamp = None self.genmap_timestamp = None self.classmap_timestamp = None self.rehash() # Creates new sig and class maps def rehash(self): try: self.sigmap = maps.SignatureMap() self.sigmap.load_generator_map( open(self.config['global']['generator_map'])) self.sigmap.load_signature_map( open(self.config['global']['signature_map'])) self.sigmap_timestamp = os.path.getmtime( self.config['global']['signature_map']) self.genmap_timestamp = os.path.getmtime( self.config['global']['generator_map']) except Exception, e: raise CharlotteConfigError("Error reading signature maps: %s" % e) try: self.classmap = maps.ClassificationMap() self.classmap.load_from_file( open(self.config['global']['classification_map'])) self.classmap_timestamp = os.path.getmtime( self.config['global']['classification_map']) except Exception, e: raise CharlotteConfigError("Error reading classification map: %s" % e)
def main(): msgmap = maps.SignatureMap() classmap = maps.ClassificationMap() parser = argparse.ArgumentParser( fromfile_prefix_chars='@', epilog=epilog) parser.add_argument( "-C", dest="classification_path", metavar="<classification.config>", help="path to classification config") parser.add_argument( "-S", dest="sidmsgmap_path", metavar="<msg-msg.map>", help="path to sid-msg.map") parser.add_argument( "-G", dest="genmsgmap_path", metavar="<gen-msg.map>", help="path to gen-msg.map") parser.add_argument( "--snort-conf", dest="snort_conf", metavar="<snort.conf>", help="attempt to load classifications and map files based on the " "location of the snort.conf") parser.add_argument( "--directory", metavar="<spool directory>", help="spool directory (eg: /var/log/snort)") parser.add_argument( "--prefix", metavar="<spool file prefix>", help="spool filename prefix (eg: unified2.log)") parser.add_argument( "--bookmark", action="store_true", default=False, help="enable bookmarking") parser.add_argument( "--follow", action="store_true", default=False, help="follow files/continuous mode (spool mode only)") parser.add_argument( "--delete", action="store_true", default=False, help="delete spool files") parser.add_argument( "-o", "--output", metavar="<filename>", help="output filename (eg: /var/log/snort/alerts.json") parser.add_argument( "--stdout", action="store_true", default=False, help="also log to stdout if --output is a file") parser.add_argument( "--packet-printable", action="store_true", default=False, help="add packet_printable field to events") parser.add_argument( "--packet-hex", action="store_true", default=False, help="add packet_hex field to events") parser.add_argument( "filenames", nargs="*") args = parser.parse_args() if args.snort_conf: load_from_snort_conf(args.snort_conf, classmap, msgmap) if args.classification_path: classmap.load_from_file( open(os.path.expanduser(args.classification_path))) if args.genmsgmap_path: msgmap.load_generator_map(open(os.path.expanduser(args.genmsgmap_path))) if args.sidmsgmap_path: msgmap.load_signature_map(open(os.path.expanduser(args.sidmsgmap_path))) if msgmap.size() == 0: LOG.warning("WARNING: No alert message map entries loaded.") else: LOG.info("Loaded %s rule message map entries.", msgmap.size()) if classmap.size() == 0: LOG.warning("WARNING: No classifications loaded.") else: LOG.info("Loaded %s classifications.", classmap.size()) eve_filter = EveFilter( msgmap, classmap, packet_printable=args.packet_printable, packet_hex=args.packet_hex) outputs = [] if args.output: outputs.append(OutputWrapper(args.output)) if args.stdout: outputs.append(OutputWrapper("-", sys.stdout)) else: outputs.append(OutputWrapper("-", sys.stdout)) writer = Writer(outputs, eve_filter) bookmark = None if args.directory and args.prefix: init_filename, init_offset = None, None if args.bookmark: bookmark = unified2.Unified2Bookmark( args.directory, args.prefix) init_filename, init_offset = bookmark.get() rollover_handler = RolloverHandler(args.delete) reader = unified2.SpoolRecordReader( directory=args.directory, prefix=args.prefix, follow=args.follow, init_filename=init_filename, init_offset=init_offset, rollover_hook=rollover_handler.on_rollover) elif args.filenames: if args.bookmark: LOG.error("Bookmarking not supported in file mode, exiting.") return 1 reader = unified2.FileRecordReader(*args.filenames) else: print("nothing to do.") return event = None last_record_time = time.time() queue = [] while True: flush = False record = reader.next() done = False if not record: if event and time.time() - last_record_time > 1.0: queue.append(event) event = None flush = True else: if args.follow: time.sleep(0.01) else: if event: queue.append(event) flush = True done = True else: last_record_time = time.time() if isinstance(record, unified2.Event): if event is not None: queue.append(event) flush = True event = record elif isinstance(record, unified2.ExtraData): if not event: continue event["extra-data"].append(record) elif isinstance(record, unified2.Packet): if not event: queue.append(record) flush = True else: if "packet" in event: queue.append(record) else: event["packet"] = record if flush: for record in queue: writer.write(record) if args.bookmark and bookmark: location = reader.tell() bookmark.update(*location) queue = [] if done: break
def main(): msgmap = maps.SignatureMap() classmap = maps.ClassificationMap() parser = argparse.ArgumentParser(fromfile_prefix_chars='@', epilog=epilog) parser.add_argument("-C", dest="classification_path", metavar="<classification.config>", help="path to classification config") parser.add_argument("-S", dest="sidmsgmap_path", metavar="<msg-msg.map>", help="path to sid-msg.map") parser.add_argument("-G", dest="genmsgmap_path", metavar="<gen-msg.map>", help="path to gen-msg.map") parser.add_argument( "--snort-conf", dest="snort_conf", metavar="<snort.conf>", help="attempt to load classifications and map files based on the " "location of the snort.conf") parser.add_argument("--directory", metavar="<spool directory>", help="spool directory (eg: /var/log/snort)") parser.add_argument("--prefix", metavar="<spool file prefix>", help="spool filename prefix (eg: unified2.log)") parser.add_argument("--bookmark", metavar="<filename>", help="enable bookmarking") parser.add_argument("--follow", action="store_true", default=False, help="follow files/continuous mode (spool mode only)") parser.add_argument("--cs", metavar="<cybersift ip>", help="Specify the CyberSift Server IP Address") parser.add_argument("--delete", action="store_true", default=False, help="delete spool files") parser.add_argument("--output", metavar="<filename>", help="output filename (eg: /var/log/snort/alerts.json") parser.add_argument("--stdout", action="store_true", default=False, help="also log to stdout if --output is a file") parser.add_argument( "--sort-keys", dest="sort_keys", action="store_true", default=False, help="the output of dictionaries will be sorted by key") parser.add_argument("--verbose", action="store_true", default=False, help="be more verbose") parser.add_argument("filenames", nargs="*") args = parser.parse_args() if args.verbose: LOG.setLevel(logging.DEBUG) if args.snort_conf: load_from_snort_conf(args.snort_conf, classmap, msgmap) if args.cs: elastic_ip = args.cs es = Elasticsearch( ["http://" + elastic_ip + ":80/cybersift_elasticsearch/"], timeout=600) else: LOG.error("Cannot proceed without a valid CyberSift IP") sys.exit(1) if args.classification_path: classmap.load_from_file( open(os.path.expanduser(args.classification_path))) if args.genmsgmap_path: msgmap.load_generator_map(open(os.path.expanduser( args.genmsgmap_path))) if args.sidmsgmap_path: msgmap.load_signature_map(open(os.path.expanduser( args.sidmsgmap_path))) if msgmap.size() == 0: LOG.warn("No alert message map entries loaded.") else: LOG.info("Loaded %s rule message map entries.", msgmap.size()) if classmap.size() == 0: LOG.warn("No classifications loaded.") else: LOG.info("Loaded %s classifications.", classmap.size()) outputs = [] if args.output: outputs.append(OutputWrapper(args.output)) if args.stdout: outputs.append(OutputWrapper("-", sys.stdout)) else: outputs.append(OutputWrapper("-", sys.stdout)) bookmark = None if args.filenames: if args.bookmark: LOG.error("Bookmarking not valid in file mode.") return 1 if args.follow: LOG.error("Follow not valid in file mode.") return 1 if args.delete: LOG.error("Delete not valid in file mode.") return 1 reader = unified2.FileRecordReader(*args.filenames) elif args.directory and args.prefix: if args.bookmark: current_snort_pid = str(check_output(["pgrep", "-u", "snort"])).strip() bookmark = unified2.Unified2Bookmark(filename=args.bookmark + '_' + current_snort_pid) init_filename, init_offset = bookmark.get() else: init_filename = None init_offset = None reader = unified2.SpoolRecordReader( directory=args.directory, prefix=args.prefix, follow=args.follow, rollover_hook=rollover_hook if args.delete else None, init_filename=init_filename, init_offset=init_offset) else: LOG.error("No spool or files provided.") return 1 formatter = Formatter(msgmap=msgmap, classmap=classmap) count = 0 record = True try: while record is not None: record = reader.next() if record is not None: try: as_json = formatter.format(record) if 'event' in as_json: create_snort_module_alert(as_json, es) count += 1 except Exception as err: LOG.error("Failed to encode record as JSON: %s: %s" % (str(err), str(record))) if bookmark: filename, offset = reader.tell() bookmark.update(filename, offset) except unified2.UnknownRecordType as err: if count == 0: LOG.error("%s: Is this a unified2 file?" % (err)) else: LOG.error(err)
def main(): msgmap = maps.SignatureMap() classmap = maps.ClassificationMap() parser = argparse.ArgumentParser( fromfile_prefix_chars='@', epilog=epilog) parser.add_argument( "-C", dest="classification_path", metavar="<classification.config>", help="path to classification config") parser.add_argument( "-S", dest="sidmsgmap_path", metavar="<msg-msg.map>", help="path to sid-msg.map") parser.add_argument( "-G", dest="genmsgmap_path", metavar="<gen-msg.map>", help="path to gen-msg.map") parser.add_argument( "--snort-conf", dest="snort_conf", metavar="<snort.conf>", help="attempt to load classifications and map files based on the " "location of the snort.conf") parser.add_argument( "--directory", metavar="<spool directory>", help="spool directory (eg: /var/log/snort)") parser.add_argument( "--prefix", metavar="<spool file prefix>", help="spool filename prefix (eg: unified2.log)") parser.add_argument( "--bookmark", metavar="<filename>", help="enable bookmarking") parser.add_argument( "--follow", action="store_true", default=False, help="follow files/continuous mode (spool mode only)") parser.add_argument( "--delete", action="store_true", default=False, help="delete spool files") parser.add_argument( "--output", metavar="<filename>", help="output filename (eg: /var/log/snort/alerts.json") parser.add_argument( "--stdout", action="store_true", default=False, help="also log to stdout if --output is a file") parser.add_argument( "--verbose", action="store_true", default=False, help="be more verbose") parser.add_argument( "filenames", nargs="*") args = parser.parse_args() if args.verbose: LOG.setLevel(logging.DEBUG) if args.snort_conf: load_from_snort_conf(args.snort_conf, classmap, msgmap) if args.classification_path: classmap.load_from_file( open(os.path.expanduser(args.classification_path))) if args.genmsgmap_path: msgmap.load_generator_map(open(os.path.expanduser(args.genmsgmap_path))) if args.sidmsgmap_path: msgmap.load_signature_map(open(os.path.expanduser(args.sidmsgmap_path))) if msgmap.size() == 0: LOG.warn("No alert message map entries loaded.") else: LOG.info("Loaded %s rule message map entries.", msgmap.size()) if classmap.size() == 0: LOG.warn("No classifications loaded.") else: LOG.info("Loaded %s classifications.", classmap.size()) outputs = [] if args.output: outputs.append(OutputWrapper(args.output)) if args.stdout: outputs.append(OutputWrapper("-", sys.stdout)) else: outputs.append(OutputWrapper("-", sys.stdout)) bookmark = None if args.filenames: if args.bookmark: LOG.error("Bookmarking not valid in file mode.") return 1 if args.follow: LOG.error("Follow not valid in file mode.") return 1 if args.delete: LOG.error("Delete not valid in file mode.") return 1 reader = unified2.FileRecordReader(*args.filenames) elif args.directory and args.prefix: if args.bookmark: bookmark = unified2.Unified2Bookmark(filename=args.bookmark) init_filename, init_offset = bookmark.get() else: init_filename = None init_offset = None reader = unified2.SpoolRecordReader( directory=args.directory, prefix=args.prefix, follow=args.follow, rollover_hook=rollover_hook if args.delete else None, init_filename=init_filename, init_offset=init_offset) else: LOG.error("No spool or files provided.") return 1 formatter = Formatter(msgmap=msgmap, classmap=classmap) count = 0 try: for record in reader: try: as_json = json.dumps(formatter.format(record)) for out in outputs: out.write(as_json) count += 1 except Exception as err: LOG.error("Failed to encode record as JSON: %s: %s" % ( str(err), str(record))) if bookmark: filename, offset = reader.tell() bookmark.update(filename, offset) except unified2.UnknownRecordType as err: if count == 0: LOG.error("%s: Is this a unified2 file?" % (err)) else: LOG.error(err)
def main(): msgmap = maps.SignatureMap() classmap = maps.ClassificationMap() parser = argparse.ArgumentParser(fromfile_prefix_chars='@') parser.add_argument("-C", dest="classification_path", metavar="<classification.config>", help="path to classification config") parser.add_argument("-S", dest="sidmsgmap_path", metavar="<msg-msg.map>", help="path to sid-msg.map") parser.add_argument("-G", dest="genmsgmap_path", metavar="<gen-msg.map>", help="path to gen-msg.map") parser.add_argument( "--snort-conf", dest="snort_conf", metavar="<snort.conf>", help="attempt to load classifications and map files based on the " "location of the snort.conf") parser.add_argument("--directory", metavar="<spool directory>", help="spool directory (eg: /var/log/snort)") parser.add_argument("--prefix", metavar="<spool file prefix>", help="spool filename prefix (eg: unified2.log)") parser.add_argument("--bookmark", action="store_true", default=False, help="enable bookmarking") parser.add_argument("--follow", action="store_true", default=False, help="follow files/continuous mode (spool mode only)") parser.add_argument("filenames", nargs="*") args = parser.parse_args() if args.snort_conf: load_from_snort_conf(args.snort_conf, classmap, msgmap) if args.classification_path: classmap.load_from_file( open(os.path.expanduser(args.classification_path))) if args.genmsgmap_path: msgmap.load_generator_map(open(os.path.expanduser( args.genmsgmap_path))) if args.sidmsgmap_path: msgmap.load_signature_map(open(os.path.expanduser( args.sidmsgmap_path))) if msgmap.size() == 0: LOG.warn("WARNING: No alert message map entries loaded.") else: LOG.info("Loaded %s rule message map entries.", msgmap.size()) if classmap.size() == 0: LOG.warn("WARNING: No classifications loaded.") else: LOG.info("Loaded %s classifications.", classmap.size()) if args.directory and args.prefix: reader = unified2.SpoolEventReader(directory=args.directory, prefix=args.prefix, follow=args.follow, bookmark=args.bookmark) for event in reader: print_event(event, msgmap, classmap) elif args.filenames: reader = unified2.FileEventReader(*args.filenames) for event in reader: print_event(event, msgmap, classmap) else: parser.print_help() return 1
def __init__(self, config_path): self.classmap = maps.ClassificationMap() self.msgmap = maps.SignatureMap() self.load_from_config_path(config_path)
type=int, help="Spool to the output file every threshold_seconds, 20 by default") return parser parser = create_parser() args = parser.parse_args() assert os.path.exists( args.directory), args.directory + ': no such file or directory' assert os.path.exists(os.path.dirname( args.output)), os.path.dirname(args.output) + ': no such file or directory' msgmap = maps.SignatureMap() classmap = maps.ClassificationMap() formatter = Formatter(msgmap=msgmap, classmap=classmap) reader = unified2.SpoolRecordReader(directory=args.directory, prefix=args.prefix, follow=True, init_filename=None, init_offset=None) labels = [ 'generator-id', 'signature-id', 'blocked', 'source-ip', 'dport-icode' ] alert_count = {} text_collector = {} last_time = time.time()
def main(): msgmap = maps.SignatureMap() classmap = maps.ClassificationMap() parser = argparse.ArgumentParser(fromfile_prefix_chars='@', epilog=epilog) parser.add_argument("-C", dest="classification_path", metavar="<classification.config>", help="path to classification config") parser.add_argument("-S", dest="sidmsgmap_path", metavar="<msg-msg.map>", help="path to sid-msg.map") parser.add_argument("-G", dest="genmsgmap_path", metavar="<gen-msg.map>", help="path to gen-msg.map") parser.add_argument( "--snort-conf", dest="snort_conf", metavar="<snort.conf>", help="attempt to load classifications and map files based on the " "location of the snort.conf") parser.add_argument("--directory", metavar="<spool directory>", help="spool directory (eg: /var/log/snort)") parser.add_argument("--prefix", metavar="<spool file prefix>", help="spool filename prefix (eg: unified2.log)") parser.add_argument("--bookmark", action="store_true", default=False, help="enable bookmarking") parser.add_argument("--follow", action="store_true", default=False, help="follow files/continuous mode (spool mode only)") parser.add_argument("--delete", action="store_true", default=False, help="delete spool files") parser.add_argument("--output", metavar="<filename>", help="output filename (eg: /var/log/snort/alerts.json") parser.add_argument("--stdout", action="store_true", default=False, help="also log to stdout if --output is a file") parser.add_argument("filenames", nargs="*") args = parser.parse_args() if args.snort_conf: load_from_snort_conf(args.snort_conf, classmap, msgmap) if args.classification_path: classmap.load_from_file( open(os.path.expanduser(args.classification_path))) if args.genmsgmap_path: msgmap.load_generator_map(open(os.path.expanduser( args.genmsgmap_path))) if args.sidmsgmap_path: msgmap.load_signature_map(open(os.path.expanduser( args.sidmsgmap_path))) if msgmap.size() == 0: LOG.warn("WARNING: No alert message map entries loaded.") else: LOG.info("Loaded %s rule message map entries.", msgmap.size()) if classmap.size() == 0: LOG.warn("WARNING: No classifications loaded.") else: LOG.info("Loaded %s classifications.", classmap.size()) eve_filter = EveFilter(msgmap, classmap) outputs = [] if args.output: outputs.append(OutputWrapper(args.output)) if args.stdout: outputs.append(OutputWrapper("-", sys.stdout)) else: outputs.append(OutputWrapper("-", sys.stdout)) if args.directory and args.prefix: reader = unified2.SpoolEventReader(directory=args.directory, prefix=args.prefix, follow=args.follow, delete=args.delete, bookmark=args.bookmark) elif args.filenames: reader = unified2.FileEventReader(*args.filenames) else: print("nothing to do.") return for event in reader: try: encoded = json.dumps(eve_filter.filter(event)) for out in outputs: out.write(encoded) except Exception as err: LOG.error("Failed to encode record as JSON: %s: %s" % (str(err), str(event)))