def test_bookmarking(self): """Test that when bookmarking is used, a second invocation of the SpoolEventReader picks up where it left off. """ for i in range(2): with open("%s/unified2.log.%04d" % (self.tmpdir, i), "ab") as outfile: with open(self.test_filename, "rb") as test_file: outfile.write(test_file.read()) reader = unified2.SpoolEventReader(self.tmpdir, "unified2", bookmark=True) event = reader.next() self.assertIsNotNone(event) print(reader.bookmark.get()) bookmark_filename, bookmark_offset = reader.bookmark.get() self.assertEqual(bookmark_filename, "unified2.log.0000") # The offset should be the offset at end of the first event, # even though the offset of the underlying file has moved on. self.assertEqual(bookmark_offset, 38950) self.assertEqual(reader.reader.tell()[1], 68) # Now create a new SpoolEventReader, the underlying offset # should be our bookmark locations. reader = unified2.SpoolEventReader(self.tmpdir, "unified2", bookmark=True) bookmark_filename, bookmark_offset = reader.bookmark.get() underlying_filename, underlying_offset = reader.reader.tell() self.assertEqual(bookmark_filename, "unified2.log.0000") self.assertEqual(bookmark_offset, 38950) self.assertEqual(bookmark_filename, os.path.basename(underlying_filename)) self.assertEqual(bookmark_offset, underlying_offset) # Read the next and final event, and check bookmark location. self.assertIsNotNone(reader.next()) self.assertIsNone(reader.next()) bookmark_filename, bookmark_offset = reader.bookmark.get() self.assertEqual(bookmark_filename, "unified2.log.0001") self.assertEqual(bookmark_offset, 38950) # As this was the last event, the underlying location should # be the same as the bookmark. bookmark_filename, bookmark_offset = reader.bookmark.get() underlying_filename, underlying_offset = reader.reader.tell() self.assertEqual(bookmark_filename, os.path.basename(underlying_filename)) self.assertEqual(bookmark_offset, underlying_offset)
def test_eof(self): """ Basic test of the SpoolEventReader aggregrating spool reader. """ reader = unified2.SpoolEventReader(self.tmpdir, "unified2") shutil.copy( self.test_filename, "%s/unified2.log.1382627900" % self.tmpdir) self.assertTrue(isinstance(reader.next(), unified2.Event)) self.assertTrue(reader.next() is None)
def test_with_file_rotation(self): reader = unified2.SpoolEventReader(self.tmpdir, "unified2") for i in range(2): open("%s/unified2.log.%04d" % (self.tmpdir, i), "ab").write( open(self.test_filename, "rb").read()) self.assertTrue(isinstance(reader.next(), unified2.Event)) self.assertTrue(reader.next() is None)
def read_ids_events(self): reader = unified2.SpoolEventReader(self.directory, self.prefix, follow=True, bookmark=True, delete=self.delete_files) for alert in reader: asyncio.run_coroutine_threadsafe(self.process_alert(alert), loop=self.loop)
def test_with_growing_file(self): reader = unified2.SpoolEventReader(self.tmpdir, "unified2") log_file = open("%s/unified2.log.0001" % (self.tmpdir), "ab") log_file.write(open(self.test_filename, "rb").read()) log_file.close() self.assertTrue(isinstance(reader.next(), unified2.Event)) self.assertTrue(reader.next() is None) log_file = open("%s/unified2.log.0001" % (self.tmpdir), "ab") log_file.write(open(self.test_filename, "rb").read()) log_file.close() self.assertTrue(isinstance(reader.next(), unified2.Event)) self.assertTrue(reader.next() is None)
from idstools import unified2 import re import json import string import random import memcache mc = memcache.Client(['127.0.0.1:11211'], debug=1) reader = unified2.SpoolEventReader("/opt/telepath/suricata/logs/", "unified2.alert", follow=True) rules = {} with open("/opt/telepath/suricata/suricata.rules") as f: content = f.readlines() for line in content: result = re.match(r'.*msg:\"([^\"]+)\".*sid:([^;]+)', line) if result: rules[result.groups()[1]] = result.groups()[0] #print rules for record in reader: if 'signature-id' in record and 'destination-ip' in record and 'source-ip' in record and 'event-second' in record: out = { "ts": record['event-second'], "rule": rules[str(record['signature-id'])],
def main(): msgmap = maps.SignatureMap() classmap = maps.ClassificationMap() parser = argparse.ArgumentParser(fromfile_prefix_chars='@') parser.add_argument("-C", dest="classification_path", metavar="<classification.config>", help="path to classification config") parser.add_argument("-S", dest="sidmsgmap_path", metavar="<msg-msg.map>", help="path to sid-msg.map") parser.add_argument("-G", dest="genmsgmap_path", metavar="<gen-msg.map>", help="path to gen-msg.map") parser.add_argument( "--snort-conf", dest="snort_conf", metavar="<snort.conf>", help="attempt to load classifications and map files based on the " "location of the snort.conf") parser.add_argument("--directory", metavar="<spool directory>", help="spool directory (eg: /var/log/snort)") parser.add_argument("--prefix", metavar="<spool file prefix>", help="spool filename prefix (eg: unified2.log)") parser.add_argument("--bookmark", action="store_true", default=False, help="enable bookmarking") parser.add_argument("--follow", action="store_true", default=False, help="follow files/continuous mode (spool mode only)") parser.add_argument("filenames", nargs="*") args = parser.parse_args() if args.snort_conf: load_from_snort_conf(args.snort_conf, classmap, msgmap) if args.classification_path: classmap.load_from_file( open(os.path.expanduser(args.classification_path))) if args.genmsgmap_path: msgmap.load_generator_map(open(os.path.expanduser( args.genmsgmap_path))) if args.sidmsgmap_path: msgmap.load_signature_map(open(os.path.expanduser( args.sidmsgmap_path))) if msgmap.size() == 0: LOG.warn("WARNING: No alert message map entries loaded.") else: LOG.info("Loaded %s rule message map entries.", msgmap.size()) if classmap.size() == 0: LOG.warn("WARNING: No classifications loaded.") else: LOG.info("Loaded %s classifications.", classmap.size()) if args.directory and args.prefix: reader = unified2.SpoolEventReader(directory=args.directory, prefix=args.prefix, follow=args.follow, bookmark=args.bookmark) for event in reader: print_event(event, msgmap, classmap) elif args.filenames: reader = unified2.FileEventReader(*args.filenames) for event in reader: print_event(event, msgmap, classmap) else: parser.print_help() return 1
def main(): msgmap = maps.SignatureMap() classmap = maps.ClassificationMap() parser = argparse.ArgumentParser(fromfile_prefix_chars='@', epilog=epilog) parser.add_argument("-C", dest="classification_path", metavar="<classification.config>", help="path to classification config") parser.add_argument("-S", dest="sidmsgmap_path", metavar="<msg-msg.map>", help="path to sid-msg.map") parser.add_argument("-G", dest="genmsgmap_path", metavar="<gen-msg.map>", help="path to gen-msg.map") parser.add_argument( "--snort-conf", dest="snort_conf", metavar="<snort.conf>", help="attempt to load classifications and map files based on the " "location of the snort.conf") parser.add_argument("--directory", metavar="<spool directory>", help="spool directory (eg: /var/log/snort)") parser.add_argument("--prefix", metavar="<spool file prefix>", help="spool filename prefix (eg: unified2.log)") parser.add_argument("--bookmark", action="store_true", default=False, help="enable bookmarking") parser.add_argument("--follow", action="store_true", default=False, help="follow files/continuous mode (spool mode only)") parser.add_argument("--delete", action="store_true", default=False, help="delete spool files") parser.add_argument("--output", metavar="<filename>", help="output filename (eg: /var/log/snort/alerts.json") parser.add_argument("--stdout", action="store_true", default=False, help="also log to stdout if --output is a file") parser.add_argument("filenames", nargs="*") args = parser.parse_args() if args.snort_conf: load_from_snort_conf(args.snort_conf, classmap, msgmap) if args.classification_path: classmap.load_from_file( open(os.path.expanduser(args.classification_path))) if args.genmsgmap_path: msgmap.load_generator_map(open(os.path.expanduser( args.genmsgmap_path))) if args.sidmsgmap_path: msgmap.load_signature_map(open(os.path.expanduser( args.sidmsgmap_path))) if msgmap.size() == 0: LOG.warn("WARNING: No alert message map entries loaded.") else: LOG.info("Loaded %s rule message map entries.", msgmap.size()) if classmap.size() == 0: LOG.warn("WARNING: No classifications loaded.") else: LOG.info("Loaded %s classifications.", classmap.size()) eve_filter = EveFilter(msgmap, classmap) outputs = [] if args.output: outputs.append(OutputWrapper(args.output)) if args.stdout: outputs.append(OutputWrapper("-", sys.stdout)) else: outputs.append(OutputWrapper("-", sys.stdout)) if args.directory and args.prefix: reader = unified2.SpoolEventReader(directory=args.directory, prefix=args.prefix, follow=args.follow, delete=args.delete, bookmark=args.bookmark) elif args.filenames: reader = unified2.FileEventReader(*args.filenames) else: print("nothing to do.") return for event in reader: try: encoded = json.dumps(eve_filter.filter(event)) for out in outputs: out.write(encoded) except Exception as err: LOG.error("Failed to encode record as JSON: %s: %s" % (str(err), str(event)))
#Init GEOIP data for IP details geo_lite_city = pygeoip.GeoIP('/usr/local/lookups/GeoLiteCity.dat') geo_ip_asn = pygeoip.GeoIP('/usr/local/lookups/GeoIPASNum.dat') logging.warning('Loaded latest ASN and City info') nowtimedom = datetime.datetime.now() updatedurationdom = datetime.timedelta(minutes=5) updatetimedom = nowtimedom + updatedurationdom # FOR ASN and City Info nowtime = datetime.datetime.now() updateduration = datetime.timedelta(hours=6) updatetime = nowtime + updateduration reader = unified2.SpoolEventReader("/var/log/snort", "snort.u2.*", follow=True, delete=False, bookmark=True) httplist = [] max_buffer_size = 1024 nowtime = datetime.datetime.now() timeduration = datetime.timedelta(seconds=5) endtime = nowtime + timeduration dlog = DnifLogger(AsyncHttpConsumer(url, buffer_size=max_buffer_size)) dlog.start() try: for event in reader: if datetime.datetime.now() > updatetime: try: geo_lite_city = pygeoip.GeoIP('GeoLiteCity.dat') geo_ip_asn = pygeoip.GeoIP('GeoIPASNum.dat')