def test_create_contract(self): contract = Contract() # validate initial fields self.failUnless(hasattr(contract, 'field_input') == True) self.failUnless(contract.field_input.fieldname == 'input') self.failUnless(contract.field_input.fieldtype == IdentityField) self.failUnless(contract.field_input.guarantees == True) self.failUnless(contract.field_input.ephemeral == False) self.failUnless(hasattr(contract, 'field_hostname') == True) self.failUnless(contract.field_hostname.fieldname == 'hostname') self.failUnless(contract.field_hostname.fieldtype == IdentityField) self.failUnless(contract.field_hostname.guarantees == True) self.failUnless(contract.field_hostname.ephemeral == False) self.failUnless(hasattr(contract, 'field_message') == True) self.failUnless(contract.field_message.fieldname == 'message') self.failUnless(contract.field_message.fieldtype == TextField) self.failUnless(contract.field_message.guarantees == True) self.failUnless(contract.field_message.ephemeral == False) # add a custom field contract.addAssertion('test', IdentityField, guarantees=True, ephemeral=False) self.failUnless(hasattr(contract, 'field_test') == True) self.failUnless(contract.field_test.fieldname == 'test') self.failUnless(contract.field_test.fieldtype == IdentityField) self.failUnless(contract.field_test.guarantees == True) self.failUnless(contract.field_test.ephemeral == False)
class RFC2822DatetimeFilter(Filter): """ Given an event field with a RFC2822-compatible datetime string, convert the string to a datetime.datetime and store it as the event ts. """ implements(IFilter) def configure(self, section): self._fieldname = section.getString('source field', 'rfc2822_date') self._expected = section.getBoolean('expects source', False) self._guaranteed = section.getBoolean('guarantees source', False) self._contract = Contract() self._contract.addAssertion( unicode(self._fieldname), u'text', expects=self._expected, guarantees=self._guaranteed, ephemeral=True) self._assertion = getattr(self._contract, unicode('field_' + self._fieldname)) self._contract.sign() def getContract(self): return self._contract def filter(self, event): try: ts = email.utils.parsedate_tz(event[self._assertion]) event.ts = datetime.datetime.fromtimestamp(time.mktime(ts)) return event except Exception, e: raise FilterError("failed to update ts: %s" % e)
def configure(self, section): self._fieldname = section.getString('source field', 'syslog_date') self._expected = section.getBoolean('expects source', False) self._guaranteed = section.getBoolean('guarantees source', False) self._contract = Contract() self._contract.addAssertion(unicode(self._fieldname), u'text', expects=self._expected, guarantees=self._guaranteed, ephemeral=True) self._assertion = getattr(self._contract, unicode('field_' + self._fieldname)) self._contract.sign()
def configure(self, section): self._linematcher = re.compile( r'(?P<ts>[A-Za-z]{3} [ \d]\d \d\d:\d\d\:\d\d) (?P<hostname>\S*) (?P<msg>.*)' ) self._tagmatcher = re.compile(r'^(\S+)\[(\d+)\]:$|^(\S+):$') self._contract = Contract() self._contract.addAssertion(u'syslog_pid', u'int', guarantees=False) self._contract.addAssertion(u'syslog_tag', u'text', guarantees=False) self._contract.sign()
def configure(self, section): self._contract = Contract() self._contract.addAssertion(u'dt_year', u'int', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'dt_month', u'int', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'dt_day', u'int', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'dt_hour', u'int', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'dt_minute', u'int', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'dt_second', u'int', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'dt_weekday', u'int', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'dt_yearday', u'int', expects=False, guarantees=True, ephemeral=False) self._contract.sign()
def configure(self, section): self._regex = re.compile( r''' (?P<remotehost>[\d.]+)\ (?P<remotelog>\S+)\ (?P<remoteuser>\S+)\ \[(?P<date>[\w:/]+\s[+\-]\d{4})\]\ \"(?P<request>.+?)\"\ (?P<status>\d{3})\ (?P<byteswritten>\d+)\ \"(?P<referrer>[^\"]+)\"\ \"(?P<useragent>[^\"]+)\"''', re.VERBOSE) self._contract = Contract() self._contract.addAssertion(u'remotehost', u'text', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'remotelog', u'literal', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'remoteuser', u'literal', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'request', u'text', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'status', u'int', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'byteswritten', u'int', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'referrer', u'text', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'useragent', u'text', expects=False, guarantees=True, ephemeral=False) self._contract.sign()
def test_check_contract(self): contract = Contract() contract.addAssertion('test', IdentityField, guarantees=True, ephemeral=False) # test len method self.failUnless(len(contract) == 4) # test iter method for asrt in contract: self.failUnless(isinstance(asrt, Assertion)) self.failUnlessIn(asrt.fieldname, ('input','hostname','message','test')) # test field iteration method for fieldname,fieldtype in contract.fields(): self.failUnlessIn(fieldname, ('input','hostname','message','test'))
def configure(self, section): self._linematcher = re.compile(r'(?P<ts>[A-Za-z]{3} [ \d]\d \d\d:\d\d\:\d\d) (?P<hostname>\S*) (?P<msg>.*)') self._tagmatcher = re.compile(r'^(\S+)\[(\d+)\]:$|^(\S+):$') self._contract = Contract() self._contract.addAssertion(u'syslog_pid', u'int', guarantees=False) self._contract.addAssertion(u'syslog_tag', u'text', guarantees=False) self._contract.sign()
def configure(self, section): self._contract = Contract() self._contract.addAssertion(u'nagios_evtype', u'literal', guarantees=True) self._contract.addAssertion(u'nagios_host', u'text', guarantees=False) self._contract.addAssertion(u'nagios_service', u'text', guarantees=False) self._contract.addAssertion(u'nagios_status', u'text', guarantees=False) self._contract.addAssertion(u'nagios_state', u'text', guarantees=False) self._contract.addAssertion(u'nagios_attempt', u'int', guarantees=False) self._contract.sign()
def test_dynamic_schema(self): index = self.output.getIndex() schema = index.getSchema() contract = Contract().sign() ts,offset,message = Output_Store_Tests.test_data[0] event = Event(ts, offset) event[contract.field_message] = message self.output.receiveEvent(event) for fieldname,fieldtype,fieldvalue in event: self.assertTrue(schema.hasField(fieldname, fieldtype))
class SyslogDatetimeFilter(Filter): """ Given an event field with a syslog-compatible datetime string, convert the string to a datetime.datetime and store as the event ts. """ implements(IFilter) def configure(self, section): self._fieldname = section.getString('source field', 'syslog_date') self._expected = section.getBoolean('expects source', False) self._guaranteed = section.getBoolean('guarantees source', False) self._contract = Contract() self._contract.addAssertion(unicode(self._fieldname), u'text', expects=self._expected, guarantees=self._guaranteed, ephemeral=True) self._assertion = getattr(self._contract, unicode('field_' + self._fieldname)) self._contract.sign() def getContract(self): return self._contract def filter(self, event): try: date = event[self._assertion] # if there is no leading zero in front of the day, then add it if date[4] == ' ': date = list(date) date[4] = '0' date = ''.join(date) # append the year to the date string date += ' %i' % time.localtime()[0] # parse the date string into a struct_time ts = time.strptime(date, "%b %d %H:%M:%S %Y") event.ts = datetime.datetime.fromtimestamp(time.mktime(ts)) return event except Exception, e: raise FilterError("failed to update ts: %s" % e)
class SyslogFilter(Filter): implements(IFilter) def configure(self, section): self._linematcher = re.compile( r'(?P<ts>[A-Za-z]{3} [ \d]\d \d\d:\d\d\:\d\d) (?P<hostname>\S*) (?P<msg>.*)' ) self._tagmatcher = re.compile(r'^(\S+)\[(\d+)\]:$|^(\S+):$') self._contract = Contract() self._contract.addAssertion(u'syslog_pid', u'int', guarantees=False) self._contract.addAssertion(u'syslog_tag', u'text', guarantees=False) self._contract.sign() def getContract(self): return self._contract def filter(self, event): # split the line into timestamp, hostname, and message m = self._linematcher.match(event[self._contract.field_message]) if m == None: raise FilterError("[filter:%s] line is not in syslog format" % self.name) ts, hostname, msg = m.group('ts', 'hostname', 'msg') if ts == None or hostname == None or msg == None: raise FilterError("[filter:%s] line is not in syslog format" % self.name) # parse the timestamp try: event.ts = dateutil.parser.parse(ts) except Exception, e: raise FilterError("[filter:%s] failed to parse ts '%s': %s" % (self.name, ts, e)) event[self._contract.field_hostname] = hostname # split the message into tag and content tag, content = msg.split(' ', 1) m = self._tagmatcher.match(tag) if m == None: raise FilterError("[filter:%s] line has an invalid tag" % self.name) data = m.groups() if data[0] != None and data[1] != None: event[self._contract.field_syslog_tag] = data[0] event[self._contract.field_syslog_pid] = int(data[1]) elif data[2] != None: event[self._contract.field_syslog_tag] = data[2] else: raise FilterError("[filter:%s] line has an invalid tag" % self.name) event[self._contract.field_message] = content return event
def configure(self, section): self._regex = re.compile(r''' (?P<remotehost>[\d.]+)\ (?P<remotelog>\S+)\ (?P<remoteuser>\S+)\ \[(?P<date>[\w:/]+\s[+\-]\d{4})\]\ \"(?P<request>.+?)\"\ (?P<status>\d{3})\ (?P<byteswritten>\d+)''', re.VERBOSE ) self._contract = Contract() self._contract.addAssertion(u'remotehost', u'text', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'remotelog', u'text', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'remoteuser', u'literal', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'request', u'literal', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'status', u'int', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'byteswritten', u'int', expects=False, guarantees=True, ephemeral=False) self._contract.sign()
def test_read_write_index(self): contract = Contract().sign() for ts,offset,message in Output_Store_Tests.test_data: event = Event(ts, offset) event[contract.field_message] = message self.output.receiveEvent(event) # read back from the index index = self.output.getIndex() startId = EVID.fromDatetime(*Output_Store_Tests.test_data[0][0:2]) endId = EVID.fromDatetime(*Output_Store_Tests.test_data[-1][0:2]) searcher = index.newSearcher() try: npostings = searcher.postingsLength(None, None, startId, endId) self.assertTrue(npostings == len(Output_Store_Tests.test_data)) i = searcher.iterPostings(None, None, startId, endId) postings = [] while True: posting = i.nextPosting() if posting == (None, None, None): break postings.append(posting) finally: searcher.close()
class SyslogFilter(Filter): implements(IFilter) def configure(self, section): self._linematcher = re.compile(r'(?P<ts>[A-Za-z]{3} [ \d]\d \d\d:\d\d\:\d\d) (?P<hostname>\S*) (?P<msg>.*)') self._tagmatcher = re.compile(r'^(\S+)\[(\d+)\]:$|^(\S+):$') self._contract = Contract() self._contract.addAssertion(u'syslog_pid', u'int', guarantees=False) self._contract.addAssertion(u'syslog_tag', u'text', guarantees=False) self._contract.sign() def getContract(self): return self._contract def filter(self, event): # split the line into timestamp, hostname, and message m = self._linematcher.match(event[self._contract.field_message]) if m == None: raise FilterError("[filter:%s] line is not in syslog format" % self.name) ts,hostname,msg = m.group('ts','hostname','msg') if ts == None or hostname == None or msg == None: raise FilterError("[filter:%s] line is not in syslog format" % self.name) # parse the timestamp try: event.ts = dateutil.parser.parse(ts) except Exception, e: raise FilterError("[filter:%s] failed to parse ts '%s': %s" % (self.name, ts, e)) event[self._contract.field_hostname] = hostname # split the message into tag and content tag,content = msg.split(' ', 1) m = self._tagmatcher.match(tag) if m == None: raise FilterError("[filter:%s] line has an invalid tag" % self.name) data = m.groups() if data[0] != None and data[1] != None: event[self._contract.field_syslog_tag] = data[0] event[self._contract.field_syslog_pid] = int(data[1]) elif data[2] != None: event[self._contract.field_syslog_tag] = data[2] else: raise FilterError("[filter:%s] line has an invalid tag" % self.name) event[self._contract.field_message] = content return event
def setUp(self): datadir = os.path.abspath(self.mktemp()) os.mkdir(datadir) settings = _UnittestSettings() settings.load({ 'plugin:output:store': { 'data directory': datadir, }, 'output:test': { 'type': 'store', } }) self.plugin = StoreOutputPlugin() self.plugin.configure(settings.section('plugin:output:store')) self.output = StoreOutput(self.plugin, 'test', MockFieldStore()) self.output.configure(settings.section('output:test')) self.plugin.startService() self.output.startService() contract = Contract().sign() # write events to the index for ts, offset, message in Bier_Searching_Tests.test_data: event = Event(ts, offset) event[contract.field_message] = message self.output.receiveEvent(event)
def test_validate_contract_succeeds(self): contract = Contract().sign() prior = Contract().sign() contract.validateContract(prior)
class NagiosFilter(Filter): implements(IFilter) def configure(self, section): self._contract = Contract() self._contract.addAssertion(u'nagios_evtype', u'literal', guarantees=True) self._contract.addAssertion(u'nagios_host', u'text', guarantees=False) self._contract.addAssertion(u'nagios_service', u'text', guarantees=False) self._contract.addAssertion(u'nagios_status', u'text', guarantees=False) self._contract.addAssertion(u'nagios_state', u'text', guarantees=False) self._contract.addAssertion(u'nagios_attempt', u'int', guarantees=False) self._contract.sign() def getContract(self): return self._contract def filter(self, event): line = event[self._contract.field_message] # all lines should start with '[' if line[0] != '[': raise FilterError( "incoming line '%s' didn't start with timestamp" % line) # parse the event timestamp try: ts, line = line[1:].split(']', 1) except: raise FilterError( "incoming line '%s' didn't start with timestamp" % line) try: event.ts = datetime.fromtimestamp(float(ts)) except Exception, e: raise FilterError("%s cannot be converted into a timestamp: %s" % (ts, e)) # determine the event type try: evtype, line = line.split(':', 1) evtype = evtype.strip() except: raise StopFiltering() # set the nagios_event type field event[self._contract.field_nagios_evtype] = evtype # parse the rest of the line if evtype == 'HOST ALERT': return self._hostAlert(line, event) if evtype == 'SERVICE ALERT': return self._serviceAlert(line, event) if evtype == 'Error': return self._error(line, event) if evtype == 'Warning': return self._warning(line, event) raise StopFiltering()
class NagiosFilter(Filter): implements(IFilter) def configure(self, section): self._contract = Contract() self._contract.addAssertion(u'nagios_evtype', u'literal', guarantees=True) self._contract.addAssertion(u'nagios_host', u'text', guarantees=False) self._contract.addAssertion(u'nagios_service', u'text', guarantees=False) self._contract.addAssertion(u'nagios_status', u'text', guarantees=False) self._contract.addAssertion(u'nagios_state', u'text', guarantees=False) self._contract.addAssertion(u'nagios_attempt', u'int', guarantees=False) self._contract.sign() def getContract(self): return self._contract def filter(self, event): line = event[self._contract.field_message] # all lines should start with '[' if line[0] != '[': raise FilterError("incoming line '%s' didn't start with timestamp" % line) # parse the event timestamp try: ts,line = line[1:].split(']', 1) except: raise FilterError("incoming line '%s' didn't start with timestamp" % line) try: event.ts = datetime.fromtimestamp(float(ts)) except Exception, e: raise FilterError("%s cannot be converted into a timestamp: %s" % (ts, e)) # determine the event type try: evtype,line = line.split(':', 1) evtype = evtype.strip() except: raise StopFiltering() # set the nagios_event type field event[self._contract.field_nagios_evtype] = evtype # parse the rest of the line if evtype == 'HOST ALERT': return self._hostAlert(line, event) if evtype == 'SERVICE ALERT': return self._serviceAlert(line, event) if evtype == 'Error': return self._error(line, event) if evtype == 'Warning': return self._warning(line, event) raise StopFiltering()
def configure(self, section): self._regex = re.compile( r'(?P<date>\d{6})\w+(?P<time>\d\d:\d\d\:\d\d)\w+(?P<msg>.*)') self._contract = Contract().sign()
def __init__(self, plugin, name, fieldstore): self._plugin = plugin self.setName(name) self._fieldstore = fieldstore self._index = None self._contract = Contract().sign()
def test_get_contract(self): contract = self.output.getContract() prior = Contract().sign() contract.validateContract(prior)
class ApacheCommonFilter(Filter): implements(IFilter) def configure(self, section): self._regex = re.compile( r''' (?P<remotehost>[\d.]+)\ (?P<remotelog>\S+)\ (?P<remoteuser>\S+)\ \[(?P<date>[\w:/]+\s[+\-]\d{4})\]\ \"(?P<request>.+?)\"\ (?P<status>\d{3})\ (?P<byteswritten>\d+)''', re.VERBOSE) self._contract = Contract() self._contract.addAssertion(u'remotehost', u'text', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'remotelog', u'text', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'remoteuser', u'literal', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'request', u'literal', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'status', u'int', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'byteswritten', u'int', expects=False, guarantees=True, ephemeral=False) self._contract.sign() def getContract(self): return self._contract def filter(self, event): line = event[self._contract.field_message] m = self._regex.match(line) if m == None: raise FilterError("incoming line '%s' didn't match regex" % line) date = m.group('date') if date == None: raise FilterError("regex did not match 'date'") # parse the timestamp try: event.ts = dateutil.parser.parse(date, dayfirst=True, fuzzy=True) except Exception, e: raise FilterError("failed to parse date '%s': %s" % (date, e)) # extract each field for assertion in self._contract: if assertion.fieldname in ('message', 'hostname', 'input'): continue value = m.group(assertion.fieldname) if value == None: raise FilterError("regex did not match '%s'" % assertion.fieldname) if assertion.fieldtype == u'int': value = int(value) event[assertion] = value return event
class ApacheCombinedFilter(Filter): implements(IFilter) def configure(self, section): self._regex = re.compile(r''' (?P<remotehost>[\d.]+)\ (?P<remotelog>\S+)\ (?P<remoteuser>\S+)\ \[(?P<date>[\w:/]+\s[+\-]\d{4})\]\ \"(?P<request>.+?)\"\ (?P<status>\d{3})\ (?P<byteswritten>\d+)\ \"(?P<referrer>[^\"]+)\"\ \"(?P<useragent>[^\"]+)\"''', re.VERBOSE ) self._contract = Contract() self._contract.addAssertion(u'remotehost', u'text', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'remotelog', u'literal', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'remoteuser', u'literal', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'request', u'text', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'status', u'int', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'byteswritten', u'int', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'referrer', u'text', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'useragent', u'text', expects=False, guarantees=True, ephemeral=False) self._contract.sign() def getContract(self): return self._contract def filter(self, event): line = event[self._contract.message] m = self._regex.match(line) if m == None: raise FilterError("incoming line '%s' didn't match regex" % line) date = m.group('date') if date == None: raise FilterError("regex did not match 'date'") # parse the timestamp try: event.ts = dateutil.parser.parse(date, dayfirst=True, fuzzy=True) except Exception, e: raise FilterError("failed to parse date '%s': %s" % (date, e)) # extract each field for assertion in self._contract: if assertion.fieldname in ('message', 'hostname', 'input'): continue value = m.group(assertion.fieldname) if value == None: raise FilterError("regex did not match '%s'" % assertion.fieldname) if assertion.fieldtype == u'int': value = int(value) event[assertion] = value return event
def test_sign_contract(self): contract = Contract() contract.addAssertion('test', IdentityField, guarantees=True, ephemeral=False) # sign the contract, no more modifications allowed contract.sign() self.failUnlessRaises(Exception, contract.addAssertion, 'fails', TextField, guarantees=True, ephemeral=False)
class DatetimeExpanderFilter(Filter): """ Expand the event ts into separate fields for each datetime component. """ implements(IFilter) def configure(self, section): self._contract = Contract() self._contract.addAssertion(u'dt_year', u'int', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'dt_month', u'int', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'dt_day', u'int', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'dt_hour', u'int', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'dt_minute', u'int', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'dt_second', u'int', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'dt_weekday', u'int', expects=False, guarantees=True, ephemeral=False) self._contract.addAssertion(u'dt_yearday', u'int', expects=False, guarantees=True, ephemeral=False) self._contract.sign() def getContract(self): return self._contract def filter(self, event): try: tm = event.ts.timetuple() event[self._contract.field_dt_year] = tm.tm_year event[self._contract.field_dt_month] = tm.tm_mon event[self._contract.field_dt_day] = tm.tm_mday event[self._contract.field_dt_hour] = tm.tm_hour event[self._contract.field_dt_minute] = tm.tm_min event[self._contract.field_dt_second] = tm.tm_sec event[self._contract.field_dt_weekday] = tm.tm_wday event[self._contract.field_dt_yearday] = tm.tm_yday return event except Exception, e: raise FilterError("failed to expand ts: %s" % e)
def test_validate_contract_expects_missing_field(self): prior = Contract().sign() contract = Contract().addAssertion('test', TextField, expects=True).sign() self.failUnlessRaises(Exception, contract.validatesAgainst, prior)